Show More
@@ -1,568 +1,571 | |||||
1 | # encoding: utf-8 |
|
1 | # encoding: utf-8 | |
2 | """ |
|
2 | """ | |
3 | A base class for a configurable application. |
|
3 | A base class for a configurable application. | |
4 |
|
4 | |||
5 | Authors: |
|
5 | Authors: | |
6 |
|
6 | |||
7 | * Brian Granger |
|
7 | * Brian Granger | |
8 | * Min RK |
|
8 | * Min RK | |
9 | """ |
|
9 | """ | |
10 |
|
10 | |||
11 | #----------------------------------------------------------------------------- |
|
11 | #----------------------------------------------------------------------------- | |
12 | # Copyright (C) 2008-2011 The IPython Development Team |
|
12 | # Copyright (C) 2008-2011 The IPython Development Team | |
13 | # |
|
13 | # | |
14 | # Distributed under the terms of the BSD License. The full license is in |
|
14 | # Distributed under the terms of the BSD License. The full license is in | |
15 | # the file COPYING, distributed as part of this software. |
|
15 | # the file COPYING, distributed as part of this software. | |
16 | #----------------------------------------------------------------------------- |
|
16 | #----------------------------------------------------------------------------- | |
17 |
|
17 | |||
18 | #----------------------------------------------------------------------------- |
|
18 | #----------------------------------------------------------------------------- | |
19 | # Imports |
|
19 | # Imports | |
20 | #----------------------------------------------------------------------------- |
|
20 | #----------------------------------------------------------------------------- | |
21 |
|
21 | |||
22 | import logging |
|
22 | import logging | |
23 | import os |
|
23 | import os | |
24 | import re |
|
24 | import re | |
25 | import sys |
|
25 | import sys | |
26 | from copy import deepcopy |
|
26 | from copy import deepcopy | |
27 | from collections import defaultdict |
|
27 | from collections import defaultdict | |
28 |
|
28 | |||
29 | from IPython.external.decorator import decorator |
|
29 | from IPython.external.decorator import decorator | |
30 |
|
30 | |||
31 | from IPython.config.configurable import SingletonConfigurable |
|
31 | from IPython.config.configurable import SingletonConfigurable | |
32 | from IPython.config.loader import ( |
|
32 | from IPython.config.loader import ( | |
33 | KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound, |
|
33 | KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound, | |
34 | ) |
|
34 | ) | |
35 |
|
35 | |||
36 | from IPython.utils.traitlets import ( |
|
36 | from IPython.utils.traitlets import ( | |
37 | Unicode, List, Enum, Dict, Instance, TraitError |
|
37 | Unicode, List, Enum, Dict, Instance, TraitError | |
38 | ) |
|
38 | ) | |
39 | from IPython.utils.importstring import import_item |
|
39 | from IPython.utils.importstring import import_item | |
40 | from IPython.utils.text import indent, wrap_paragraphs, dedent |
|
40 | from IPython.utils.text import indent, wrap_paragraphs, dedent | |
41 |
|
41 | |||
42 | #----------------------------------------------------------------------------- |
|
42 | #----------------------------------------------------------------------------- | |
43 | # function for re-wrapping a helpstring |
|
43 | # function for re-wrapping a helpstring | |
44 | #----------------------------------------------------------------------------- |
|
44 | #----------------------------------------------------------------------------- | |
45 |
|
45 | |||
46 | #----------------------------------------------------------------------------- |
|
46 | #----------------------------------------------------------------------------- | |
47 | # Descriptions for the various sections |
|
47 | # Descriptions for the various sections | |
48 | #----------------------------------------------------------------------------- |
|
48 | #----------------------------------------------------------------------------- | |
49 |
|
49 | |||
50 | # merge flags&aliases into options |
|
50 | # merge flags&aliases into options | |
51 | option_description = """ |
|
51 | option_description = """ | |
52 | Arguments that take values are actually convenience aliases to full |
|
52 | Arguments that take values are actually convenience aliases to full | |
53 | Configurables, whose aliases are listed on the help line. For more information |
|
53 | Configurables, whose aliases are listed on the help line. For more information | |
54 | on full configurables, see '--help-all'. |
|
54 | on full configurables, see '--help-all'. | |
55 | """.strip() # trim newlines of front and back |
|
55 | """.strip() # trim newlines of front and back | |
56 |
|
56 | |||
57 | keyvalue_description = """ |
|
57 | keyvalue_description = """ | |
58 | Parameters are set from command-line arguments of the form: |
|
58 | Parameters are set from command-line arguments of the form: | |
59 | `--Class.trait=value`. |
|
59 | `--Class.trait=value`. | |
60 | This line is evaluated in Python, so simple expressions are allowed, e.g.:: |
|
60 | This line is evaluated in Python, so simple expressions are allowed, e.g.:: | |
61 | `--C.a='range(3)'` For setting C.a=[0,1,2]. |
|
61 | `--C.a='range(3)'` For setting C.a=[0,1,2]. | |
62 | """.strip() # trim newlines of front and back |
|
62 | """.strip() # trim newlines of front and back | |
63 |
|
63 | |||
64 | subcommand_description = """ |
|
64 | subcommand_description = """ | |
65 | Subcommands are launched as `{app} cmd [args]`. For information on using |
|
65 | Subcommands are launched as `{app} cmd [args]`. For information on using | |
66 | subcommand 'cmd', do: `{app} cmd -h`. |
|
66 | subcommand 'cmd', do: `{app} cmd -h`. | |
67 | """.strip().format(app=os.path.basename(sys.argv[0])) |
|
67 | """.strip().format(app=os.path.basename(sys.argv[0])) | |
68 | # get running program name |
|
68 | # get running program name | |
69 |
|
69 | |||
70 | #----------------------------------------------------------------------------- |
|
70 | #----------------------------------------------------------------------------- | |
71 | # Application class |
|
71 | # Application class | |
72 | #----------------------------------------------------------------------------- |
|
72 | #----------------------------------------------------------------------------- | |
73 |
|
73 | |||
74 | @decorator |
|
74 | @decorator | |
75 | def catch_config_error(method, app, *args, **kwargs): |
|
75 | def catch_config_error(method, app, *args, **kwargs): | |
76 | """Method decorator for catching invalid config (Trait/ArgumentErrors) during init. |
|
76 | """Method decorator for catching invalid config (Trait/ArgumentErrors) during init. | |
77 |
|
77 | |||
78 | On a TraitError (generally caused by bad config), this will print the trait's |
|
78 | On a TraitError (generally caused by bad config), this will print the trait's | |
79 | message, and exit the app. |
|
79 | message, and exit the app. | |
80 |
|
80 | |||
81 | For use on init methods, to prevent invoking excepthook on invalid input. |
|
81 | For use on init methods, to prevent invoking excepthook on invalid input. | |
82 | """ |
|
82 | """ | |
83 | try: |
|
83 | try: | |
84 | return method(app, *args, **kwargs) |
|
84 | return method(app, *args, **kwargs) | |
85 | except (TraitError, ArgumentError) as e: |
|
85 | except (TraitError, ArgumentError) as e: | |
86 | app.print_description() |
|
86 | app.print_description() | |
87 | app.print_help() |
|
87 | app.print_help() | |
88 | app.print_examples() |
|
88 | app.print_examples() | |
89 | app.log.fatal("Bad config encountered during initialization:") |
|
89 | app.log.fatal("Bad config encountered during initialization:") | |
90 | app.log.fatal(str(e)) |
|
90 | app.log.fatal(str(e)) | |
91 | app.log.debug("Config at the time: %s", app.config) |
|
91 | app.log.debug("Config at the time: %s", app.config) | |
92 | app.exit(1) |
|
92 | app.exit(1) | |
93 |
|
93 | |||
94 |
|
94 | |||
95 | class ApplicationError(Exception): |
|
95 | class ApplicationError(Exception): | |
96 | pass |
|
96 | pass | |
97 |
|
97 | |||
98 | class LevelFormatter(logging.Formatter): |
|
98 | class LevelFormatter(logging.Formatter): | |
99 | """Formatter with additional `highlevel` record |
|
99 | """Formatter with additional `highlevel` record | |
100 |
|
100 | |||
101 | This field is empty if log level is less than highlevel_limit, |
|
101 | This field is empty if log level is less than highlevel_limit, | |
102 | otherwise it is formatted with self.highlevel_format. |
|
102 | otherwise it is formatted with self.highlevel_format. | |
103 |
|
103 | |||
104 | Useful for adding 'WARNING' to warning messages, |
|
104 | Useful for adding 'WARNING' to warning messages, | |
105 | without adding 'INFO' to info, etc. |
|
105 | without adding 'INFO' to info, etc. | |
106 | """ |
|
106 | """ | |
107 | highlevel_limit = logging.WARN |
|
107 | highlevel_limit = logging.WARN | |
108 | highlevel_format = " %(levelname)s |" |
|
108 | highlevel_format = " %(levelname)s |" | |
109 |
|
109 | |||
110 | def format(self, record): |
|
110 | def format(self, record): | |
111 | if record.levelno >= self.highlevel_limit: |
|
111 | if record.levelno >= self.highlevel_limit: | |
112 | record.highlevel = self.highlevel_format % record.__dict__ |
|
112 | record.highlevel = self.highlevel_format % record.__dict__ | |
113 | else: |
|
113 | else: | |
114 | record.highlevel = "" |
|
114 | record.highlevel = "" | |
115 |
|
115 | |||
116 | return super(LevelFormatter, self).format(record) |
|
116 | return super(LevelFormatter, self).format(record) | |
117 |
|
117 | |||
118 |
|
118 | |||
119 | class Application(SingletonConfigurable): |
|
119 | class Application(SingletonConfigurable): | |
120 | """A singleton application with full configuration support.""" |
|
120 | """A singleton application with full configuration support.""" | |
121 |
|
121 | |||
122 | # The name of the application, will usually match the name of the command |
|
122 | # The name of the application, will usually match the name of the command | |
123 | # line application |
|
123 | # line application | |
124 | name = Unicode(u'application') |
|
124 | name = Unicode(u'application') | |
125 |
|
125 | |||
126 | # The description of the application that is printed at the beginning |
|
126 | # The description of the application that is printed at the beginning | |
127 | # of the help. |
|
127 | # of the help. | |
128 | description = Unicode(u'This is an application.') |
|
128 | description = Unicode(u'This is an application.') | |
129 | # default section descriptions |
|
129 | # default section descriptions | |
130 | option_description = Unicode(option_description) |
|
130 | option_description = Unicode(option_description) | |
131 | keyvalue_description = Unicode(keyvalue_description) |
|
131 | keyvalue_description = Unicode(keyvalue_description) | |
132 | subcommand_description = Unicode(subcommand_description) |
|
132 | subcommand_description = Unicode(subcommand_description) | |
133 |
|
133 | |||
134 | # The usage and example string that goes at the end of the help string. |
|
134 | # The usage and example string that goes at the end of the help string. | |
135 | examples = Unicode() |
|
135 | examples = Unicode() | |
136 |
|
136 | |||
137 | # A sequence of Configurable subclasses whose config=True attributes will |
|
137 | # A sequence of Configurable subclasses whose config=True attributes will | |
138 | # be exposed at the command line. |
|
138 | # be exposed at the command line. | |
139 | classes = List([]) |
|
139 | classes = List([]) | |
140 |
|
140 | |||
141 | # The version string of this application. |
|
141 | # The version string of this application. | |
142 | version = Unicode(u'0.0') |
|
142 | version = Unicode(u'0.0') | |
143 |
|
143 | |||
144 | # The log level for the application |
|
144 | # The log level for the application | |
145 | log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'), |
|
145 | log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'), | |
146 | default_value=logging.WARN, |
|
146 | default_value=logging.WARN, | |
147 | config=True, |
|
147 | config=True, | |
148 | help="Set the log level by value or name.") |
|
148 | help="Set the log level by value or name.") | |
149 | def _log_level_changed(self, name, old, new): |
|
149 | def _log_level_changed(self, name, old, new): | |
150 | """Adjust the log level when log_level is set.""" |
|
150 | """Adjust the log level when log_level is set.""" | |
151 | if isinstance(new, basestring): |
|
151 | if isinstance(new, basestring): | |
152 | new = getattr(logging, new) |
|
152 | new = getattr(logging, new) | |
153 | self.log_level = new |
|
153 | self.log_level = new | |
154 | self.log.setLevel(new) |
|
154 | self.log.setLevel(new) | |
155 |
|
155 | |||
156 | log_datefmt = Unicode("%Y-%m-%d %H:%M:%S", config=True, |
|
156 | log_datefmt = Unicode("%Y-%m-%d %H:%M:%S", config=True, | |
157 | help="The date format used by logging formatters for %(asctime)s" |
|
157 | help="The date format used by logging formatters for %(asctime)s" | |
158 | ) |
|
158 | ) | |
159 | def _log_datefmt_changed(self, name, old, new): |
|
159 | def _log_datefmt_changed(self, name, old, new): | |
160 | self._log_format_changed() |
|
160 | self._log_format_changed() | |
161 |
|
161 | |||
162 | log_format = Unicode("[%(name)s]%(highlevel)s %(message)s", config=True, |
|
162 | log_format = Unicode("[%(name)s]%(highlevel)s %(message)s", config=True, | |
163 | help="The Logging format template", |
|
163 | help="The Logging format template", | |
164 | ) |
|
164 | ) | |
165 | def _log_format_changed(self, name, old, new): |
|
165 | def _log_format_changed(self, name, old, new): | |
166 | """Change the log formatter when log_format is set.""" |
|
166 | """Change the log formatter when log_format is set.""" | |
167 | _log_handler = self.log.handlers[0] |
|
167 | _log_handler = self.log.handlers[0] | |
168 | _log_formatter = LevelFormatter(new, datefmt=self.log_datefmt) |
|
168 | _log_formatter = LevelFormatter(new, datefmt=self.log_datefmt) | |
169 | _log_handler.setFormatter(_log_formatter) |
|
169 | _log_handler.setFormatter(_log_formatter) | |
170 |
|
170 | |||
171 | log = Instance(logging.Logger) |
|
171 | log = Instance(logging.Logger) | |
172 | def _log_default(self): |
|
172 | def _log_default(self): | |
173 | """Start logging for this application. |
|
173 | """Start logging for this application. | |
174 |
|
174 | |||
175 | The default is to log to stderr using a StreamHandler, if no default |
|
175 | The default is to log to stderr using a StreamHandler, if no default | |
176 | handler already exists. The log level starts at logging.WARN, but this |
|
176 | handler already exists. The log level starts at logging.WARN, but this | |
177 | can be adjusted by setting the ``log_level`` attribute. |
|
177 | can be adjusted by setting the ``log_level`` attribute. | |
178 | """ |
|
178 | """ | |
179 | log = logging.getLogger(self.__class__.__name__) |
|
179 | log = logging.getLogger(self.__class__.__name__) | |
180 | log.setLevel(self.log_level) |
|
180 | log.setLevel(self.log_level) | |
181 | log.propagate = False |
|
181 | log.propagate = False | |
182 | _log = log # copied from Logger.hasHandlers() (new in Python 3.2) |
|
182 | _log = log # copied from Logger.hasHandlers() (new in Python 3.2) | |
183 | while _log: |
|
183 | while _log: | |
184 | if _log.handlers: |
|
184 | if _log.handlers: | |
185 | return log |
|
185 | return log | |
186 | if not _log.propagate: |
|
186 | if not _log.propagate: | |
187 | break |
|
187 | break | |
188 | else: |
|
188 | else: | |
189 | _log = _log.parent |
|
189 | _log = _log.parent | |
190 | if sys.executable.endswith('pythonw.exe'): |
|
190 | if sys.executable.endswith('pythonw.exe'): | |
191 | # this should really go to a file, but file-logging is only |
|
191 | # this should really go to a file, but file-logging is only | |
192 | # hooked up in parallel applications |
|
192 | # hooked up in parallel applications | |
193 | _log_handler = logging.StreamHandler(open(os.devnull, 'w')) |
|
193 | _log_handler = logging.StreamHandler(open(os.devnull, 'w')) | |
194 | else: |
|
194 | else: | |
195 | _log_handler = logging.StreamHandler() |
|
195 | _log_handler = logging.StreamHandler() | |
196 | _log_formatter = LevelFormatter(self.log_format, datefmt=self.log_datefmt) |
|
196 | _log_formatter = LevelFormatter(self.log_format, datefmt=self.log_datefmt) | |
197 | _log_handler.setFormatter(_log_formatter) |
|
197 | _log_handler.setFormatter(_log_formatter) | |
198 | log.addHandler(_log_handler) |
|
198 | log.addHandler(_log_handler) | |
199 | return log |
|
199 | return log | |
200 |
|
200 | |||
201 | # the alias map for configurables |
|
201 | # the alias map for configurables | |
202 | aliases = Dict({'log-level' : 'Application.log_level'}) |
|
202 | aliases = Dict({'log-level' : 'Application.log_level'}) | |
203 |
|
203 | |||
204 | # flags for loading Configurables or store_const style flags |
|
204 | # flags for loading Configurables or store_const style flags | |
205 | # flags are loaded from this dict by '--key' flags |
|
205 | # flags are loaded from this dict by '--key' flags | |
206 | # this must be a dict of two-tuples, the first element being the Config/dict |
|
206 | # this must be a dict of two-tuples, the first element being the Config/dict | |
207 | # and the second being the help string for the flag |
|
207 | # and the second being the help string for the flag | |
208 | flags = Dict() |
|
208 | flags = Dict() | |
209 | def _flags_changed(self, name, old, new): |
|
209 | def _flags_changed(self, name, old, new): | |
210 | """ensure flags dict is valid""" |
|
210 | """ensure flags dict is valid""" | |
211 | for key,value in new.iteritems(): |
|
211 | for key,value in new.iteritems(): | |
212 | assert len(value) == 2, "Bad flag: %r:%s"%(key,value) |
|
212 | assert len(value) == 2, "Bad flag: %r:%s"%(key,value) | |
213 | assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value) |
|
213 | assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value) | |
214 | assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value) |
|
214 | assert isinstance(value[1], basestring), "Bad flag: %r:%s"%(key,value) | |
215 |
|
215 | |||
216 |
|
216 | |||
217 | # subcommands for launching other applications |
|
217 | # subcommands for launching other applications | |
218 | # if this is not empty, this will be a parent Application |
|
218 | # if this is not empty, this will be a parent Application | |
219 | # this must be a dict of two-tuples, |
|
219 | # this must be a dict of two-tuples, | |
220 | # the first element being the application class/import string |
|
220 | # the first element being the application class/import string | |
221 | # and the second being the help string for the subcommand |
|
221 | # and the second being the help string for the subcommand | |
222 | subcommands = Dict() |
|
222 | subcommands = Dict() | |
223 | # parse_command_line will initialize a subapp, if requested |
|
223 | # parse_command_line will initialize a subapp, if requested | |
224 | subapp = Instance('IPython.config.application.Application', allow_none=True) |
|
224 | subapp = Instance('IPython.config.application.Application', allow_none=True) | |
225 |
|
225 | |||
226 | # extra command-line arguments that don't set config values |
|
226 | # extra command-line arguments that don't set config values | |
227 | extra_args = List(Unicode) |
|
227 | extra_args = List(Unicode) | |
228 |
|
228 | |||
229 |
|
229 | |||
230 | def __init__(self, **kwargs): |
|
230 | def __init__(self, **kwargs): | |
231 | SingletonConfigurable.__init__(self, **kwargs) |
|
231 | SingletonConfigurable.__init__(self, **kwargs) | |
232 | # Ensure my class is in self.classes, so my attributes appear in command line |
|
232 | # Ensure my class is in self.classes, so my attributes appear in command line | |
233 | # options and config files. |
|
233 | # options and config files. | |
234 | if self.__class__ not in self.classes: |
|
234 | if self.__class__ not in self.classes: | |
235 | self.classes.insert(0, self.__class__) |
|
235 | self.classes.insert(0, self.__class__) | |
236 |
|
236 | |||
237 | def _config_changed(self, name, old, new): |
|
237 | def _config_changed(self, name, old, new): | |
238 | SingletonConfigurable._config_changed(self, name, old, new) |
|
238 | SingletonConfigurable._config_changed(self, name, old, new) | |
239 | self.log.debug('Config changed:') |
|
239 | self.log.debug('Config changed:') | |
240 | self.log.debug(repr(new)) |
|
240 | self.log.debug(repr(new)) | |
241 |
|
241 | |||
242 | @catch_config_error |
|
242 | @catch_config_error | |
243 | def initialize(self, argv=None): |
|
243 | def initialize(self, argv=None): | |
244 | """Do the basic steps to configure me. |
|
244 | """Do the basic steps to configure me. | |
245 |
|
245 | |||
246 | Override in subclasses. |
|
246 | Override in subclasses. | |
247 | """ |
|
247 | """ | |
248 | self.parse_command_line(argv) |
|
248 | self.parse_command_line(argv) | |
249 |
|
249 | |||
250 |
|
250 | |||
251 | def start(self): |
|
251 | def start(self): | |
252 | """Start the app mainloop. |
|
252 | """Start the app mainloop. | |
253 |
|
253 | |||
254 | Override in subclasses. |
|
254 | Override in subclasses. | |
255 | """ |
|
255 | """ | |
256 | if self.subapp is not None: |
|
256 | if self.subapp is not None: | |
257 | return self.subapp.start() |
|
257 | return self.subapp.start() | |
258 |
|
258 | |||
259 | def print_alias_help(self): |
|
259 | def print_alias_help(self): | |
260 | """Print the alias part of the help.""" |
|
260 | """Print the alias part of the help.""" | |
261 | if not self.aliases: |
|
261 | if not self.aliases: | |
262 | return |
|
262 | return | |
263 |
|
263 | |||
264 | lines = [] |
|
264 | lines = [] | |
265 | classdict = {} |
|
265 | classdict = {} | |
266 | for cls in self.classes: |
|
266 | for cls in self.classes: | |
267 | # include all parents (up to, but excluding Configurable) in available names |
|
267 | # include all parents (up to, but excluding Configurable) in available names | |
268 | for c in cls.mro()[:-3]: |
|
268 | for c in cls.mro()[:-3]: | |
269 | classdict[c.__name__] = c |
|
269 | classdict[c.__name__] = c | |
270 |
|
270 | |||
271 | for alias, longname in self.aliases.iteritems(): |
|
271 | for alias, longname in self.aliases.iteritems(): | |
272 | classname, traitname = longname.split('.',1) |
|
272 | classname, traitname = longname.split('.',1) | |
273 | cls = classdict[classname] |
|
273 | cls = classdict[classname] | |
274 |
|
274 | |||
275 | trait = cls.class_traits(config=True)[traitname] |
|
275 | trait = cls.class_traits(config=True)[traitname] | |
276 | help = cls.class_get_trait_help(trait).splitlines() |
|
276 | help = cls.class_get_trait_help(trait).splitlines() | |
277 | # reformat first line |
|
277 | # reformat first line | |
278 | help[0] = help[0].replace(longname, alias) + ' (%s)'%longname |
|
278 | help[0] = help[0].replace(longname, alias) + ' (%s)'%longname | |
279 | if len(alias) == 1: |
|
279 | if len(alias) == 1: | |
280 | help[0] = help[0].replace('--%s='%alias, '-%s '%alias) |
|
280 | help[0] = help[0].replace('--%s='%alias, '-%s '%alias) | |
281 | lines.extend(help) |
|
281 | lines.extend(help) | |
282 | # lines.append('') |
|
282 | # lines.append('') | |
283 | print os.linesep.join(lines) |
|
283 | print os.linesep.join(lines) | |
284 |
|
284 | |||
285 | def print_flag_help(self): |
|
285 | def print_flag_help(self): | |
286 | """Print the flag part of the help.""" |
|
286 | """Print the flag part of the help.""" | |
287 | if not self.flags: |
|
287 | if not self.flags: | |
288 | return |
|
288 | return | |
289 |
|
289 | |||
290 | lines = [] |
|
290 | lines = [] | |
291 | for m, (cfg,help) in self.flags.iteritems(): |
|
291 | for m, (cfg,help) in self.flags.iteritems(): | |
292 | prefix = '--' if len(m) > 1 else '-' |
|
292 | prefix = '--' if len(m) > 1 else '-' | |
293 | lines.append(prefix+m) |
|
293 | lines.append(prefix+m) | |
294 | lines.append(indent(dedent(help.strip()))) |
|
294 | lines.append(indent(dedent(help.strip()))) | |
295 | # lines.append('') |
|
295 | # lines.append('') | |
296 | print os.linesep.join(lines) |
|
296 | print os.linesep.join(lines) | |
297 |
|
297 | |||
298 | def print_options(self): |
|
298 | def print_options(self): | |
299 | if not self.flags and not self.aliases: |
|
299 | if not self.flags and not self.aliases: | |
300 | return |
|
300 | return | |
301 | lines = ['Options'] |
|
301 | lines = ['Options'] | |
302 | lines.append('-'*len(lines[0])) |
|
302 | lines.append('-'*len(lines[0])) | |
303 | lines.append('') |
|
303 | lines.append('') | |
304 | for p in wrap_paragraphs(self.option_description): |
|
304 | for p in wrap_paragraphs(self.option_description): | |
305 | lines.append(p) |
|
305 | lines.append(p) | |
306 | lines.append('') |
|
306 | lines.append('') | |
307 | print os.linesep.join(lines) |
|
307 | print os.linesep.join(lines) | |
308 | self.print_flag_help() |
|
308 | self.print_flag_help() | |
309 | self.print_alias_help() |
|
309 | self.print_alias_help() | |
310 |
|
310 | |||
311 |
|
311 | |||
312 | def print_subcommands(self): |
|
312 | def print_subcommands(self): | |
313 | """Print the subcommand part of the help.""" |
|
313 | """Print the subcommand part of the help.""" | |
314 | if not self.subcommands: |
|
314 | if not self.subcommands: | |
315 | return |
|
315 | return | |
316 |
|
316 | |||
317 | lines = ["Subcommands"] |
|
317 | lines = ["Subcommands"] | |
318 | lines.append('-'*len(lines[0])) |
|
318 | lines.append('-'*len(lines[0])) | |
319 | lines.append('') |
|
319 | lines.append('') | |
320 | for p in wrap_paragraphs(self.subcommand_description): |
|
320 | for p in wrap_paragraphs(self.subcommand_description): | |
321 | lines.append(p) |
|
321 | lines.append(p) | |
322 | lines.append('') |
|
322 | lines.append('') | |
323 | for subc, (cls, help) in self.subcommands.iteritems(): |
|
323 | for subc, (cls, help) in self.subcommands.iteritems(): | |
324 | lines.append(subc) |
|
324 | lines.append(subc) | |
325 | if help: |
|
325 | if help: | |
326 | lines.append(indent(dedent(help.strip()))) |
|
326 | lines.append(indent(dedent(help.strip()))) | |
327 | lines.append('') |
|
327 | lines.append('') | |
328 | print os.linesep.join(lines) |
|
328 | print os.linesep.join(lines) | |
329 |
|
329 | |||
330 | def print_help(self, classes=False): |
|
330 | def print_help(self, classes=False): | |
331 | """Print the help for each Configurable class in self.classes. |
|
331 | """Print the help for each Configurable class in self.classes. | |
332 |
|
332 | |||
333 | If classes=False (the default), only flags and aliases are printed. |
|
333 | If classes=False (the default), only flags and aliases are printed. | |
334 | """ |
|
334 | """ | |
335 | self.print_subcommands() |
|
335 | self.print_subcommands() | |
336 | self.print_options() |
|
336 | self.print_options() | |
337 |
|
337 | |||
338 | if classes: |
|
338 | if classes: | |
339 | if self.classes: |
|
339 | if self.classes: | |
340 | print "Class parameters" |
|
340 | print "Class parameters" | |
341 | print "----------------" |
|
341 | print "----------------" | |
342 |
|
342 | |||
343 | for p in wrap_paragraphs(self.keyvalue_description): |
|
343 | for p in wrap_paragraphs(self.keyvalue_description): | |
344 | print p |
|
344 | print p | |
345 |
|
345 | |||
346 |
|
346 | |||
347 | for cls in self.classes: |
|
347 | for cls in self.classes: | |
348 | cls.class_print_help() |
|
348 | cls.class_print_help() | |
349 |
|
349 | |||
350 | else: |
|
350 | else: | |
351 | print "To see all available configurables, use `--help-all`" |
|
351 | print "To see all available configurables, use `--help-all`" | |
352 |
|
352 | |||
353 |
|
353 | |||
354 | def print_description(self): |
|
354 | def print_description(self): | |
355 | """Print the application description.""" |
|
355 | """Print the application description.""" | |
356 | for p in wrap_paragraphs(self.description): |
|
356 | for p in wrap_paragraphs(self.description): | |
357 | print p |
|
357 | print p | |
358 |
|
358 | |||
359 |
|
359 | |||
360 | def print_examples(self): |
|
360 | def print_examples(self): | |
361 | """Print usage and examples. |
|
361 | """Print usage and examples. | |
362 |
|
362 | |||
363 | This usage string goes at the end of the command line help string |
|
363 | This usage string goes at the end of the command line help string | |
364 | and should contain examples of the application's usage. |
|
364 | and should contain examples of the application's usage. | |
365 | """ |
|
365 | """ | |
366 | if self.examples: |
|
366 | if self.examples: | |
367 | print "Examples" |
|
367 | print "Examples" | |
368 | print "--------" |
|
368 | print "--------" | |
369 |
|
369 | |||
370 | print indent(dedent(self.examples.strip())) |
|
370 | print indent(dedent(self.examples.strip())) | |
371 |
|
371 | |||
372 |
|
372 | |||
373 | def print_version(self): |
|
373 | def print_version(self): | |
374 | """Print the version string.""" |
|
374 | """Print the version string.""" | |
375 | print self.version |
|
375 | print self.version | |
376 |
|
376 | |||
377 | def update_config(self, config): |
|
377 | def update_config(self, config): | |
378 | """Fire the traits events when the config is updated.""" |
|
378 | """Fire the traits events when the config is updated.""" | |
379 | # Save a copy of the current config. |
|
379 | # Save a copy of the current config. | |
380 | newconfig = deepcopy(self.config) |
|
380 | newconfig = deepcopy(self.config) | |
381 | # Merge the new config into the current one. |
|
381 | # Merge the new config into the current one. | |
382 | newconfig.merge(config) |
|
382 | newconfig.merge(config) | |
383 | # Save the combined config as self.config, which triggers the traits |
|
383 | # Save the combined config as self.config, which triggers the traits | |
384 | # events. |
|
384 | # events. | |
385 | self.config = newconfig |
|
385 | self.config = newconfig | |
386 |
|
386 | |||
387 | @catch_config_error |
|
387 | @catch_config_error | |
388 | def initialize_subcommand(self, subc, argv=None): |
|
388 | def initialize_subcommand(self, subc, argv=None): | |
389 | """Initialize a subcommand with argv.""" |
|
389 | """Initialize a subcommand with argv.""" | |
390 | subapp,help = self.subcommands.get(subc) |
|
390 | subapp,help = self.subcommands.get(subc) | |
391 |
|
391 | |||
392 | if isinstance(subapp, basestring): |
|
392 | if isinstance(subapp, basestring): | |
393 | subapp = import_item(subapp) |
|
393 | subapp = import_item(subapp) | |
394 |
|
394 | |||
395 | # clear existing instances |
|
395 | # clear existing instances | |
396 | self.__class__.clear_instance() |
|
396 | self.__class__.clear_instance() | |
397 | # instantiate |
|
397 | # instantiate | |
398 | self.subapp = subapp.instance(config=self.config) |
|
398 | self.subapp = subapp.instance(config=self.config) | |
399 | # and initialize subapp |
|
399 | # and initialize subapp | |
400 | self.subapp.initialize(argv) |
|
400 | self.subapp.initialize(argv) | |
401 |
|
401 | |||
402 | def flatten_flags(self): |
|
402 | def flatten_flags(self): | |
403 | """flatten flags and aliases, so cl-args override as expected. |
|
403 | """flatten flags and aliases, so cl-args override as expected. | |
404 |
|
404 | |||
405 | This prevents issues such as an alias pointing to InteractiveShell, |
|
405 | This prevents issues such as an alias pointing to InteractiveShell, | |
406 | but a config file setting the same trait in TerminalInteraciveShell |
|
406 | but a config file setting the same trait in TerminalInteraciveShell | |
407 | getting inappropriate priority over the command-line arg. |
|
407 | getting inappropriate priority over the command-line arg. | |
408 |
|
408 | |||
409 | Only aliases with exactly one descendent in the class list |
|
409 | Only aliases with exactly one descendent in the class list | |
410 | will be promoted. |
|
410 | will be promoted. | |
411 |
|
411 | |||
412 | """ |
|
412 | """ | |
413 | # build a tree of classes in our list that inherit from a particular |
|
413 | # build a tree of classes in our list that inherit from a particular | |
414 | # it will be a dict by parent classname of classes in our list |
|
414 | # it will be a dict by parent classname of classes in our list | |
415 | # that are descendents |
|
415 | # that are descendents | |
416 | mro_tree = defaultdict(list) |
|
416 | mro_tree = defaultdict(list) | |
417 | for cls in self.classes: |
|
417 | for cls in self.classes: | |
418 | clsname = cls.__name__ |
|
418 | clsname = cls.__name__ | |
419 | for parent in cls.mro()[1:-3]: |
|
419 | for parent in cls.mro()[1:-3]: | |
420 | # exclude cls itself and Configurable,HasTraits,object |
|
420 | # exclude cls itself and Configurable,HasTraits,object | |
421 | mro_tree[parent.__name__].append(clsname) |
|
421 | mro_tree[parent.__name__].append(clsname) | |
422 | # flatten aliases, which have the form: |
|
422 | # flatten aliases, which have the form: | |
423 | # { 'alias' : 'Class.trait' } |
|
423 | # { 'alias' : 'Class.trait' } | |
424 | aliases = {} |
|
424 | aliases = {} | |
425 | for alias, cls_trait in self.aliases.iteritems(): |
|
425 | for alias, cls_trait in self.aliases.iteritems(): | |
426 | cls,trait = cls_trait.split('.',1) |
|
426 | cls,trait = cls_trait.split('.',1) | |
427 | children = mro_tree[cls] |
|
427 | children = mro_tree[cls] | |
428 | if len(children) == 1: |
|
428 | if len(children) == 1: | |
429 | # exactly one descendent, promote alias |
|
429 | # exactly one descendent, promote alias | |
430 | cls = children[0] |
|
430 | cls = children[0] | |
431 | aliases[alias] = '.'.join([cls,trait]) |
|
431 | aliases[alias] = '.'.join([cls,trait]) | |
432 |
|
432 | |||
433 | # flatten flags, which are of the form: |
|
433 | # flatten flags, which are of the form: | |
434 | # { 'key' : ({'Cls' : {'trait' : value}}, 'help')} |
|
434 | # { 'key' : ({'Cls' : {'trait' : value}}, 'help')} | |
435 | flags = {} |
|
435 | flags = {} | |
436 | for key, (flagdict, help) in self.flags.iteritems(): |
|
436 | for key, (flagdict, help) in self.flags.iteritems(): | |
437 | newflag = {} |
|
437 | newflag = {} | |
438 | for cls, subdict in flagdict.iteritems(): |
|
438 | for cls, subdict in flagdict.iteritems(): | |
439 | children = mro_tree[cls] |
|
439 | children = mro_tree[cls] | |
440 | # exactly one descendent, promote flag section |
|
440 | # exactly one descendent, promote flag section | |
441 | if len(children) == 1: |
|
441 | if len(children) == 1: | |
442 | cls = children[0] |
|
442 | cls = children[0] | |
443 | newflag[cls] = subdict |
|
443 | newflag[cls] = subdict | |
444 | flags[key] = (newflag, help) |
|
444 | flags[key] = (newflag, help) | |
445 | return flags, aliases |
|
445 | return flags, aliases | |
446 |
|
446 | |||
447 | @catch_config_error |
|
447 | @catch_config_error | |
448 | def parse_command_line(self, argv=None): |
|
448 | def parse_command_line(self, argv=None): | |
449 | """Parse the command line arguments.""" |
|
449 | """Parse the command line arguments.""" | |
450 | argv = sys.argv[1:] if argv is None else argv |
|
450 | argv = sys.argv[1:] if argv is None else argv | |
451 |
|
451 | |||
452 | if argv and argv[0] == 'help': |
|
452 | if argv and argv[0] == 'help': | |
453 | # turn `ipython help notebook` into `ipython notebook -h` |
|
453 | # turn `ipython help notebook` into `ipython notebook -h` | |
454 | argv = argv[1:] + ['-h'] |
|
454 | argv = argv[1:] + ['-h'] | |
455 |
|
455 | |||
456 | if self.subcommands and len(argv) > 0: |
|
456 | if self.subcommands and len(argv) > 0: | |
457 | # we have subcommands, and one may have been specified |
|
457 | # we have subcommands, and one may have been specified | |
458 | subc, subargv = argv[0], argv[1:] |
|
458 | subc, subargv = argv[0], argv[1:] | |
459 | if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands: |
|
459 | if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands: | |
460 | # it's a subcommand, and *not* a flag or class parameter |
|
460 | # it's a subcommand, and *not* a flag or class parameter | |
461 | return self.initialize_subcommand(subc, subargv) |
|
461 | return self.initialize_subcommand(subc, subargv) | |
462 |
|
462 | |||
463 | # Arguments after a '--' argument are for the script IPython may be |
|
463 | # Arguments after a '--' argument are for the script IPython may be | |
464 | # about to run, not IPython iteslf. For arguments parsed here (help and |
|
464 | # about to run, not IPython iteslf. For arguments parsed here (help and | |
465 | # version), we want to only search the arguments up to the first |
|
465 | # version), we want to only search the arguments up to the first | |
466 | # occurrence of '--', which we're calling interpreted_argv. |
|
466 | # occurrence of '--', which we're calling interpreted_argv. | |
467 | try: |
|
467 | try: | |
468 | interpreted_argv = argv[:argv.index('--')] |
|
468 | interpreted_argv = argv[:argv.index('--')] | |
469 | except ValueError: |
|
469 | except ValueError: | |
470 | interpreted_argv = argv |
|
470 | interpreted_argv = argv | |
471 |
|
471 | |||
472 | if any(x in interpreted_argv for x in ('-h', '--help-all', '--help')): |
|
472 | if any(x in interpreted_argv for x in ('-h', '--help-all', '--help')): | |
473 | self.print_description() |
|
473 | self.print_description() | |
474 | self.print_help('--help-all' in interpreted_argv) |
|
474 | self.print_help('--help-all' in interpreted_argv) | |
475 | self.print_examples() |
|
475 | self.print_examples() | |
476 | self.exit(0) |
|
476 | self.exit(0) | |
477 |
|
477 | |||
478 | if '--version' in interpreted_argv or '-V' in interpreted_argv: |
|
478 | if '--version' in interpreted_argv or '-V' in interpreted_argv: | |
479 | self.print_version() |
|
479 | self.print_version() | |
480 | self.exit(0) |
|
480 | self.exit(0) | |
481 |
|
481 | |||
482 | # flatten flags&aliases, so cl-args get appropriate priority: |
|
482 | # flatten flags&aliases, so cl-args get appropriate priority: | |
483 | flags,aliases = self.flatten_flags() |
|
483 | flags,aliases = self.flatten_flags() | |
484 |
|
484 | |||
485 | loader = KVArgParseConfigLoader(argv=argv, aliases=aliases, |
|
485 | loader = KVArgParseConfigLoader(argv=argv, aliases=aliases, | |
486 | flags=flags) |
|
486 | flags=flags) | |
487 | config = loader.load_config() |
|
487 | config = loader.load_config() | |
488 | self.update_config(config) |
|
488 | self.update_config(config) | |
489 | # store unparsed args in extra_args |
|
489 | # store unparsed args in extra_args | |
490 | self.extra_args = loader.extra_args |
|
490 | self.extra_args = loader.extra_args | |
491 |
|
491 | |||
492 | @catch_config_error |
|
492 | @catch_config_error | |
493 | def load_config_file(self, filename, path=None): |
|
493 | def load_config_file(self, filename, path=None): | |
494 | """Load a .py based config file by filename and path.""" |
|
494 | """Load a .py based config file by filename and path.""" | |
495 | loader = PyFileConfigLoader(filename, path=path) |
|
495 | loader = PyFileConfigLoader(filename, path=path) | |
496 | try: |
|
496 | try: | |
497 | config = loader.load_config() |
|
497 | config = loader.load_config() | |
498 | except ConfigFileNotFound: |
|
498 | except ConfigFileNotFound: | |
499 | # problem finding the file, raise |
|
499 | # problem finding the file, raise | |
500 | raise |
|
500 | raise | |
501 | except Exception: |
|
501 | except Exception: | |
502 | # try to get the full filename, but it will be empty in the |
|
502 | # try to get the full filename, but it will be empty in the | |
503 | # unlikely event that the error raised before filefind finished |
|
503 | # unlikely event that the error raised before filefind finished | |
504 | filename = loader.full_filename or filename |
|
504 | filename = loader.full_filename or filename | |
505 | # problem while running the file |
|
505 | # problem while running the file | |
506 | self.log.error("Exception while loading config file %s", |
|
506 | self.log.error("Exception while loading config file %s", | |
507 | filename, exc_info=True) |
|
507 | filename, exc_info=True) | |
508 | else: |
|
508 | else: | |
509 | self.log.debug("Loaded config file: %s", loader.full_filename) |
|
509 | self.log.debug("Loaded config file: %s", loader.full_filename) | |
510 | self.update_config(config) |
|
510 | self.update_config(config) | |
511 |
|
511 | |||
512 | def generate_config_file(self): |
|
512 | def generate_config_file(self): | |
513 | """generate default config file from Configurables""" |
|
513 | """generate default config file from Configurables""" | |
514 | lines = ["# Configuration file for %s."%self.name] |
|
514 | lines = ["# Configuration file for %s."%self.name] | |
515 | lines.append('') |
|
515 | lines.append('') | |
516 | lines.append('c = get_config()') |
|
516 | lines.append('c = get_config()') | |
517 | lines.append('') |
|
517 | lines.append('') | |
518 | for cls in self.classes: |
|
518 | for cls in self.classes: | |
519 | lines.append(cls.class_config_section()) |
|
519 | lines.append(cls.class_config_section()) | |
520 | return '\n'.join(lines) |
|
520 | return '\n'.join(lines) | |
521 |
|
521 | |||
522 | def exit(self, exit_status=0): |
|
522 | def exit(self, exit_status=0): | |
523 | self.log.debug("Exiting application: %s" % self.name) |
|
523 | self.log.debug("Exiting application: %s" % self.name) | |
524 | sys.exit(exit_status) |
|
524 | sys.exit(exit_status) | |
525 |
|
525 | |||
526 | @classmethod |
|
526 | @classmethod | |
527 |
def launch_ |
|
527 | def launch_instance(cls, argv=None, **kwargs): | |
528 |
"""Launch a global instance of this Application |
|
528 | """Launch a global instance of this Application | |
|
529 | ||||
|
530 | If a global instance already exists, this reinitializes and starts it | |||
|
531 | """ | |||
529 | app = cls.instance(**kwargs) |
|
532 | app = cls.instance(**kwargs) | |
530 | app.initialize(argv) |
|
533 | app.initialize(argv) | |
531 | app.start() |
|
534 | app.start() | |
532 |
|
535 | |||
533 | #----------------------------------------------------------------------------- |
|
536 | #----------------------------------------------------------------------------- | |
534 | # utility functions, for convenience |
|
537 | # utility functions, for convenience | |
535 | #----------------------------------------------------------------------------- |
|
538 | #----------------------------------------------------------------------------- | |
536 |
|
539 | |||
537 | def boolean_flag(name, configurable, set_help='', unset_help=''): |
|
540 | def boolean_flag(name, configurable, set_help='', unset_help=''): | |
538 | """Helper for building basic --trait, --no-trait flags. |
|
541 | """Helper for building basic --trait, --no-trait flags. | |
539 |
|
542 | |||
540 | Parameters |
|
543 | Parameters | |
541 | ---------- |
|
544 | ---------- | |
542 |
|
545 | |||
543 | name : str |
|
546 | name : str | |
544 | The name of the flag. |
|
547 | The name of the flag. | |
545 | configurable : str |
|
548 | configurable : str | |
546 | The 'Class.trait' string of the trait to be set/unset with the flag |
|
549 | The 'Class.trait' string of the trait to be set/unset with the flag | |
547 | set_help : unicode |
|
550 | set_help : unicode | |
548 | help string for --name flag |
|
551 | help string for --name flag | |
549 | unset_help : unicode |
|
552 | unset_help : unicode | |
550 | help string for --no-name flag |
|
553 | help string for --no-name flag | |
551 |
|
554 | |||
552 | Returns |
|
555 | Returns | |
553 | ------- |
|
556 | ------- | |
554 |
|
557 | |||
555 | cfg : dict |
|
558 | cfg : dict | |
556 | A dict with two keys: 'name', and 'no-name', for setting and unsetting |
|
559 | A dict with two keys: 'name', and 'no-name', for setting and unsetting | |
557 | the trait, respectively. |
|
560 | the trait, respectively. | |
558 | """ |
|
561 | """ | |
559 | # default helpstrings |
|
562 | # default helpstrings | |
560 | set_help = set_help or "set %s=True"%configurable |
|
563 | set_help = set_help or "set %s=True"%configurable | |
561 | unset_help = unset_help or "set %s=False"%configurable |
|
564 | unset_help = unset_help or "set %s=False"%configurable | |
562 |
|
565 | |||
563 | cls,trait = configurable.split('.') |
|
566 | cls,trait = configurable.split('.') | |
564 |
|
567 | |||
565 | setter = {cls : {trait : True}} |
|
568 | setter = {cls : {trait : True}} | |
566 | unsetter = {cls : {trait : False}} |
|
569 | unsetter = {cls : {trait : False}} | |
567 | return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)} |
|
570 | return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)} | |
568 |
|
571 |
@@ -1,749 +1,749 | |||||
1 | # coding: utf-8 |
|
1 | # coding: utf-8 | |
2 | """A tornado based IPython notebook server. |
|
2 | """A tornado based IPython notebook server. | |
3 |
|
3 | |||
4 | Authors: |
|
4 | Authors: | |
5 |
|
5 | |||
6 | * Brian Granger |
|
6 | * Brian Granger | |
7 | """ |
|
7 | """ | |
8 | #----------------------------------------------------------------------------- |
|
8 | #----------------------------------------------------------------------------- | |
9 | # Copyright (C) 2013 The IPython Development Team |
|
9 | # Copyright (C) 2013 The IPython Development Team | |
10 | # |
|
10 | # | |
11 | # Distributed under the terms of the BSD License. The full license is in |
|
11 | # Distributed under the terms of the BSD License. The full license is in | |
12 | # the file COPYING, distributed as part of this software. |
|
12 | # the file COPYING, distributed as part of this software. | |
13 | #----------------------------------------------------------------------------- |
|
13 | #----------------------------------------------------------------------------- | |
14 |
|
14 | |||
15 | #----------------------------------------------------------------------------- |
|
15 | #----------------------------------------------------------------------------- | |
16 | # Imports |
|
16 | # Imports | |
17 | #----------------------------------------------------------------------------- |
|
17 | #----------------------------------------------------------------------------- | |
18 |
|
18 | |||
19 | # stdlib |
|
19 | # stdlib | |
20 | import errno |
|
20 | import errno | |
21 | import logging |
|
21 | import logging | |
22 | import os |
|
22 | import os | |
23 | import random |
|
23 | import random | |
24 | import select |
|
24 | import select | |
25 | import signal |
|
25 | import signal | |
26 | import socket |
|
26 | import socket | |
27 | import sys |
|
27 | import sys | |
28 | import threading |
|
28 | import threading | |
29 | import time |
|
29 | import time | |
30 | import webbrowser |
|
30 | import webbrowser | |
31 |
|
31 | |||
32 |
|
32 | |||
33 | # Third party |
|
33 | # Third party | |
34 | # check for pyzmq 2.1.11 |
|
34 | # check for pyzmq 2.1.11 | |
35 | from IPython.utils.zmqrelated import check_for_zmq |
|
35 | from IPython.utils.zmqrelated import check_for_zmq | |
36 | check_for_zmq('2.1.11', 'IPython.html') |
|
36 | check_for_zmq('2.1.11', 'IPython.html') | |
37 |
|
37 | |||
38 | from jinja2 import Environment, FileSystemLoader |
|
38 | from jinja2 import Environment, FileSystemLoader | |
39 |
|
39 | |||
40 | # Install the pyzmq ioloop. This has to be done before anything else from |
|
40 | # Install the pyzmq ioloop. This has to be done before anything else from | |
41 | # tornado is imported. |
|
41 | # tornado is imported. | |
42 | from zmq.eventloop import ioloop |
|
42 | from zmq.eventloop import ioloop | |
43 | ioloop.install() |
|
43 | ioloop.install() | |
44 |
|
44 | |||
45 | # check for tornado 2.1.0 |
|
45 | # check for tornado 2.1.0 | |
46 | msg = "The IPython Notebook requires tornado >= 2.1.0" |
|
46 | msg = "The IPython Notebook requires tornado >= 2.1.0" | |
47 | try: |
|
47 | try: | |
48 | import tornado |
|
48 | import tornado | |
49 | except ImportError: |
|
49 | except ImportError: | |
50 | raise ImportError(msg) |
|
50 | raise ImportError(msg) | |
51 | try: |
|
51 | try: | |
52 | version_info = tornado.version_info |
|
52 | version_info = tornado.version_info | |
53 | except AttributeError: |
|
53 | except AttributeError: | |
54 | raise ImportError(msg + ", but you have < 1.1.0") |
|
54 | raise ImportError(msg + ", but you have < 1.1.0") | |
55 | if version_info < (2,1,0): |
|
55 | if version_info < (2,1,0): | |
56 | raise ImportError(msg + ", but you have %s" % tornado.version) |
|
56 | raise ImportError(msg + ", but you have %s" % tornado.version) | |
57 |
|
57 | |||
58 | from tornado import httpserver |
|
58 | from tornado import httpserver | |
59 | from tornado import web |
|
59 | from tornado import web | |
60 |
|
60 | |||
61 | # Our own libraries |
|
61 | # Our own libraries | |
62 | from IPython.html import DEFAULT_STATIC_FILES_PATH |
|
62 | from IPython.html import DEFAULT_STATIC_FILES_PATH | |
63 |
|
63 | |||
64 | from .services.kernels.kernelmanager import MappingKernelManager |
|
64 | from .services.kernels.kernelmanager import MappingKernelManager | |
65 | from .services.notebooks.nbmanager import NotebookManager |
|
65 | from .services.notebooks.nbmanager import NotebookManager | |
66 | from .services.notebooks.filenbmanager import FileNotebookManager |
|
66 | from .services.notebooks.filenbmanager import FileNotebookManager | |
67 | from .services.clusters.clustermanager import ClusterManager |
|
67 | from .services.clusters.clustermanager import ClusterManager | |
68 |
|
68 | |||
69 | from .base.handlers import AuthenticatedFileHandler, FileFindHandler |
|
69 | from .base.handlers import AuthenticatedFileHandler, FileFindHandler | |
70 |
|
70 | |||
71 | from IPython.config.application import catch_config_error, boolean_flag |
|
71 | from IPython.config.application import catch_config_error, boolean_flag | |
72 | from IPython.core.application import BaseIPythonApplication |
|
72 | from IPython.core.application import BaseIPythonApplication | |
73 | from IPython.consoleapp import IPythonConsoleApp |
|
73 | from IPython.consoleapp import IPythonConsoleApp | |
74 | from IPython.kernel import swallow_argv |
|
74 | from IPython.kernel import swallow_argv | |
75 | from IPython.kernel.zmq.session import default_secure |
|
75 | from IPython.kernel.zmq.session import default_secure | |
76 | from IPython.kernel.zmq.kernelapp import ( |
|
76 | from IPython.kernel.zmq.kernelapp import ( | |
77 | kernel_flags, |
|
77 | kernel_flags, | |
78 | kernel_aliases, |
|
78 | kernel_aliases, | |
79 | ) |
|
79 | ) | |
80 | from IPython.utils.importstring import import_item |
|
80 | from IPython.utils.importstring import import_item | |
81 | from IPython.utils.localinterfaces import LOCALHOST |
|
81 | from IPython.utils.localinterfaces import LOCALHOST | |
82 | from IPython.utils import submodule |
|
82 | from IPython.utils import submodule | |
83 | from IPython.utils.traitlets import ( |
|
83 | from IPython.utils.traitlets import ( | |
84 | Dict, Unicode, Integer, List, Bool, Bytes, |
|
84 | Dict, Unicode, Integer, List, Bool, Bytes, | |
85 | DottedObjectName |
|
85 | DottedObjectName | |
86 | ) |
|
86 | ) | |
87 | from IPython.utils import py3compat |
|
87 | from IPython.utils import py3compat | |
88 | from IPython.utils.path import filefind |
|
88 | from IPython.utils.path import filefind | |
89 |
|
89 | |||
90 | from .utils import url_path_join |
|
90 | from .utils import url_path_join | |
91 |
|
91 | |||
92 | #----------------------------------------------------------------------------- |
|
92 | #----------------------------------------------------------------------------- | |
93 | # Module globals |
|
93 | # Module globals | |
94 | #----------------------------------------------------------------------------- |
|
94 | #----------------------------------------------------------------------------- | |
95 |
|
95 | |||
96 | _examples = """ |
|
96 | _examples = """ | |
97 | ipython notebook # start the notebook |
|
97 | ipython notebook # start the notebook | |
98 | ipython notebook --profile=sympy # use the sympy profile |
|
98 | ipython notebook --profile=sympy # use the sympy profile | |
99 | ipython notebook --pylab=inline # pylab in inline plotting mode |
|
99 | ipython notebook --pylab=inline # pylab in inline plotting mode | |
100 | ipython notebook --certfile=mycert.pem # use SSL/TLS certificate |
|
100 | ipython notebook --certfile=mycert.pem # use SSL/TLS certificate | |
101 | ipython notebook --port=5555 --ip=* # Listen on port 5555, all interfaces |
|
101 | ipython notebook --port=5555 --ip=* # Listen on port 5555, all interfaces | |
102 | """ |
|
102 | """ | |
103 |
|
103 | |||
104 | #----------------------------------------------------------------------------- |
|
104 | #----------------------------------------------------------------------------- | |
105 | # Helper functions |
|
105 | # Helper functions | |
106 | #----------------------------------------------------------------------------- |
|
106 | #----------------------------------------------------------------------------- | |
107 |
|
107 | |||
108 | def random_ports(port, n): |
|
108 | def random_ports(port, n): | |
109 | """Generate a list of n random ports near the given port. |
|
109 | """Generate a list of n random ports near the given port. | |
110 |
|
110 | |||
111 | The first 5 ports will be sequential, and the remaining n-5 will be |
|
111 | The first 5 ports will be sequential, and the remaining n-5 will be | |
112 | randomly selected in the range [port-2*n, port+2*n]. |
|
112 | randomly selected in the range [port-2*n, port+2*n]. | |
113 | """ |
|
113 | """ | |
114 | for i in range(min(5, n)): |
|
114 | for i in range(min(5, n)): | |
115 | yield port + i |
|
115 | yield port + i | |
116 | for i in range(n-5): |
|
116 | for i in range(n-5): | |
117 | yield port + random.randint(-2*n, 2*n) |
|
117 | yield port + random.randint(-2*n, 2*n) | |
118 |
|
118 | |||
119 | def load_handlers(name): |
|
119 | def load_handlers(name): | |
120 | """Load the (URL pattern, handler) tuples for each component.""" |
|
120 | """Load the (URL pattern, handler) tuples for each component.""" | |
121 | name = 'IPython.html.' + name |
|
121 | name = 'IPython.html.' + name | |
122 | mod = __import__(name, fromlist=['default_handlers']) |
|
122 | mod = __import__(name, fromlist=['default_handlers']) | |
123 | return mod.default_handlers |
|
123 | return mod.default_handlers | |
124 |
|
124 | |||
125 | #----------------------------------------------------------------------------- |
|
125 | #----------------------------------------------------------------------------- | |
126 | # The Tornado web application |
|
126 | # The Tornado web application | |
127 | #----------------------------------------------------------------------------- |
|
127 | #----------------------------------------------------------------------------- | |
128 |
|
128 | |||
129 | class NotebookWebApplication(web.Application): |
|
129 | class NotebookWebApplication(web.Application): | |
130 |
|
130 | |||
131 | def __init__(self, ipython_app, kernel_manager, notebook_manager, |
|
131 | def __init__(self, ipython_app, kernel_manager, notebook_manager, | |
132 | cluster_manager, log, |
|
132 | cluster_manager, log, | |
133 | base_project_url, settings_overrides): |
|
133 | base_project_url, settings_overrides): | |
134 |
|
134 | |||
135 | settings = self.init_settings( |
|
135 | settings = self.init_settings( | |
136 | ipython_app, kernel_manager, notebook_manager, cluster_manager, |
|
136 | ipython_app, kernel_manager, notebook_manager, cluster_manager, | |
137 | log, base_project_url, settings_overrides) |
|
137 | log, base_project_url, settings_overrides) | |
138 | handlers = self.init_handlers(settings) |
|
138 | handlers = self.init_handlers(settings) | |
139 |
|
139 | |||
140 | super(NotebookWebApplication, self).__init__(handlers, **settings) |
|
140 | super(NotebookWebApplication, self).__init__(handlers, **settings) | |
141 |
|
141 | |||
142 | def init_settings(self, ipython_app, kernel_manager, notebook_manager, |
|
142 | def init_settings(self, ipython_app, kernel_manager, notebook_manager, | |
143 | cluster_manager, log, |
|
143 | cluster_manager, log, | |
144 | base_project_url, settings_overrides): |
|
144 | base_project_url, settings_overrides): | |
145 | # Python < 2.6.5 doesn't accept unicode keys in f(**kwargs), and |
|
145 | # Python < 2.6.5 doesn't accept unicode keys in f(**kwargs), and | |
146 | # base_project_url will always be unicode, which will in turn |
|
146 | # base_project_url will always be unicode, which will in turn | |
147 | # make the patterns unicode, and ultimately result in unicode |
|
147 | # make the patterns unicode, and ultimately result in unicode | |
148 | # keys in kwargs to handler._execute(**kwargs) in tornado. |
|
148 | # keys in kwargs to handler._execute(**kwargs) in tornado. | |
149 | # This enforces that base_project_url be ascii in that situation. |
|
149 | # This enforces that base_project_url be ascii in that situation. | |
150 | # |
|
150 | # | |
151 | # Note that the URLs these patterns check against are escaped, |
|
151 | # Note that the URLs these patterns check against are escaped, | |
152 | # and thus guaranteed to be ASCII: 'hΓ©llo' is really 'h%C3%A9llo'. |
|
152 | # and thus guaranteed to be ASCII: 'hΓ©llo' is really 'h%C3%A9llo'. | |
153 | base_project_url = py3compat.unicode_to_str(base_project_url, 'ascii') |
|
153 | base_project_url = py3compat.unicode_to_str(base_project_url, 'ascii') | |
154 | template_path = os.path.join(os.path.dirname(__file__), "templates") |
|
154 | template_path = os.path.join(os.path.dirname(__file__), "templates") | |
155 | settings = dict( |
|
155 | settings = dict( | |
156 | # basics |
|
156 | # basics | |
157 | base_project_url=base_project_url, |
|
157 | base_project_url=base_project_url, | |
158 | base_kernel_url=ipython_app.base_kernel_url, |
|
158 | base_kernel_url=ipython_app.base_kernel_url, | |
159 | template_path=template_path, |
|
159 | template_path=template_path, | |
160 | static_path=ipython_app.static_file_path, |
|
160 | static_path=ipython_app.static_file_path, | |
161 | static_handler_class = FileFindHandler, |
|
161 | static_handler_class = FileFindHandler, | |
162 | static_url_prefix = url_path_join(base_project_url,'/static/'), |
|
162 | static_url_prefix = url_path_join(base_project_url,'/static/'), | |
163 |
|
163 | |||
164 | # authentication |
|
164 | # authentication | |
165 | cookie_secret=ipython_app.cookie_secret, |
|
165 | cookie_secret=ipython_app.cookie_secret, | |
166 | login_url=url_path_join(base_project_url,'/login'), |
|
166 | login_url=url_path_join(base_project_url,'/login'), | |
167 | read_only=ipython_app.read_only, |
|
167 | read_only=ipython_app.read_only, | |
168 | password=ipython_app.password, |
|
168 | password=ipython_app.password, | |
169 |
|
169 | |||
170 | # managers |
|
170 | # managers | |
171 | kernel_manager=kernel_manager, |
|
171 | kernel_manager=kernel_manager, | |
172 | notebook_manager=notebook_manager, |
|
172 | notebook_manager=notebook_manager, | |
173 | cluster_manager=cluster_manager, |
|
173 | cluster_manager=cluster_manager, | |
174 |
|
174 | |||
175 | # IPython stuff |
|
175 | # IPython stuff | |
176 | mathjax_url=ipython_app.mathjax_url, |
|
176 | mathjax_url=ipython_app.mathjax_url, | |
177 | max_msg_size=ipython_app.max_msg_size, |
|
177 | max_msg_size=ipython_app.max_msg_size, | |
178 | config=ipython_app.config, |
|
178 | config=ipython_app.config, | |
179 | use_less=ipython_app.use_less, |
|
179 | use_less=ipython_app.use_less, | |
180 | jinja2_env=Environment(loader=FileSystemLoader(template_path)), |
|
180 | jinja2_env=Environment(loader=FileSystemLoader(template_path)), | |
181 | ) |
|
181 | ) | |
182 |
|
182 | |||
183 | # allow custom overrides for the tornado web app. |
|
183 | # allow custom overrides for the tornado web app. | |
184 | settings.update(settings_overrides) |
|
184 | settings.update(settings_overrides) | |
185 | return settings |
|
185 | return settings | |
186 |
|
186 | |||
187 | def init_handlers(self, settings): |
|
187 | def init_handlers(self, settings): | |
188 | # Load the (URL pattern, handler) tuples for each component. |
|
188 | # Load the (URL pattern, handler) tuples for each component. | |
189 | handlers = [] |
|
189 | handlers = [] | |
190 | handlers.extend(load_handlers('base.handlers')) |
|
190 | handlers.extend(load_handlers('base.handlers')) | |
191 | handlers.extend(load_handlers('tree.handlers')) |
|
191 | handlers.extend(load_handlers('tree.handlers')) | |
192 | handlers.extend(load_handlers('auth.login')) |
|
192 | handlers.extend(load_handlers('auth.login')) | |
193 | handlers.extend(load_handlers('auth.logout')) |
|
193 | handlers.extend(load_handlers('auth.logout')) | |
194 | handlers.extend(load_handlers('notebook.handlers')) |
|
194 | handlers.extend(load_handlers('notebook.handlers')) | |
195 | handlers.extend(load_handlers('services.kernels.handlers')) |
|
195 | handlers.extend(load_handlers('services.kernels.handlers')) | |
196 | handlers.extend(load_handlers('services.notebooks.handlers')) |
|
196 | handlers.extend(load_handlers('services.notebooks.handlers')) | |
197 | handlers.extend(load_handlers('services.clusters.handlers')) |
|
197 | handlers.extend(load_handlers('services.clusters.handlers')) | |
198 | handlers.extend([ |
|
198 | handlers.extend([ | |
199 | (r"/files/(.*)", AuthenticatedFileHandler, {'path' : settings['notebook_manager'].notebook_dir}), |
|
199 | (r"/files/(.*)", AuthenticatedFileHandler, {'path' : settings['notebook_manager'].notebook_dir}), | |
200 | ]) |
|
200 | ]) | |
201 | # prepend base_project_url onto the patterns that we match |
|
201 | # prepend base_project_url onto the patterns that we match | |
202 | new_handlers = [] |
|
202 | new_handlers = [] | |
203 | for handler in handlers: |
|
203 | for handler in handlers: | |
204 | pattern = url_path_join(settings['base_project_url'], handler[0]) |
|
204 | pattern = url_path_join(settings['base_project_url'], handler[0]) | |
205 | new_handler = tuple([pattern] + list(handler[1:])) |
|
205 | new_handler = tuple([pattern] + list(handler[1:])) | |
206 | new_handlers.append(new_handler) |
|
206 | new_handlers.append(new_handler) | |
207 | return new_handlers |
|
207 | return new_handlers | |
208 |
|
208 | |||
209 |
|
209 | |||
210 |
|
210 | |||
211 | #----------------------------------------------------------------------------- |
|
211 | #----------------------------------------------------------------------------- | |
212 | # Aliases and Flags |
|
212 | # Aliases and Flags | |
213 | #----------------------------------------------------------------------------- |
|
213 | #----------------------------------------------------------------------------- | |
214 |
|
214 | |||
215 | flags = dict(kernel_flags) |
|
215 | flags = dict(kernel_flags) | |
216 | flags['no-browser']=( |
|
216 | flags['no-browser']=( | |
217 | {'NotebookApp' : {'open_browser' : False}}, |
|
217 | {'NotebookApp' : {'open_browser' : False}}, | |
218 | "Don't open the notebook in a browser after startup." |
|
218 | "Don't open the notebook in a browser after startup." | |
219 | ) |
|
219 | ) | |
220 | flags['no-mathjax']=( |
|
220 | flags['no-mathjax']=( | |
221 | {'NotebookApp' : {'enable_mathjax' : False}}, |
|
221 | {'NotebookApp' : {'enable_mathjax' : False}}, | |
222 | """Disable MathJax |
|
222 | """Disable MathJax | |
223 |
|
223 | |||
224 | MathJax is the javascript library IPython uses to render math/LaTeX. It is |
|
224 | MathJax is the javascript library IPython uses to render math/LaTeX. It is | |
225 | very large, so you may want to disable it if you have a slow internet |
|
225 | very large, so you may want to disable it if you have a slow internet | |
226 | connection, or for offline use of the notebook. |
|
226 | connection, or for offline use of the notebook. | |
227 |
|
227 | |||
228 | When disabled, equations etc. will appear as their untransformed TeX source. |
|
228 | When disabled, equations etc. will appear as their untransformed TeX source. | |
229 | """ |
|
229 | """ | |
230 | ) |
|
230 | ) | |
231 | flags['read-only'] = ( |
|
231 | flags['read-only'] = ( | |
232 | {'NotebookApp' : {'read_only' : True}}, |
|
232 | {'NotebookApp' : {'read_only' : True}}, | |
233 | """Allow read-only access to notebooks. |
|
233 | """Allow read-only access to notebooks. | |
234 |
|
234 | |||
235 | When using a password to protect the notebook server, this flag |
|
235 | When using a password to protect the notebook server, this flag | |
236 | allows unauthenticated clients to view the notebook list, and |
|
236 | allows unauthenticated clients to view the notebook list, and | |
237 | individual notebooks, but not edit them, start kernels, or run |
|
237 | individual notebooks, but not edit them, start kernels, or run | |
238 | code. |
|
238 | code. | |
239 |
|
239 | |||
240 | If no password is set, the server will be entirely read-only. |
|
240 | If no password is set, the server will be entirely read-only. | |
241 | """ |
|
241 | """ | |
242 | ) |
|
242 | ) | |
243 |
|
243 | |||
244 | # Add notebook manager flags |
|
244 | # Add notebook manager flags | |
245 | flags.update(boolean_flag('script', 'FileNotebookManager.save_script', |
|
245 | flags.update(boolean_flag('script', 'FileNotebookManager.save_script', | |
246 | 'Auto-save a .py script everytime the .ipynb notebook is saved', |
|
246 | 'Auto-save a .py script everytime the .ipynb notebook is saved', | |
247 | 'Do not auto-save .py scripts for every notebook')) |
|
247 | 'Do not auto-save .py scripts for every notebook')) | |
248 |
|
248 | |||
249 | # the flags that are specific to the frontend |
|
249 | # the flags that are specific to the frontend | |
250 | # these must be scrubbed before being passed to the kernel, |
|
250 | # these must be scrubbed before being passed to the kernel, | |
251 | # or it will raise an error on unrecognized flags |
|
251 | # or it will raise an error on unrecognized flags | |
252 | notebook_flags = ['no-browser', 'no-mathjax', 'read-only', 'script', 'no-script'] |
|
252 | notebook_flags = ['no-browser', 'no-mathjax', 'read-only', 'script', 'no-script'] | |
253 |
|
253 | |||
254 | aliases = dict(kernel_aliases) |
|
254 | aliases = dict(kernel_aliases) | |
255 |
|
255 | |||
256 | aliases.update({ |
|
256 | aliases.update({ | |
257 | 'ip': 'NotebookApp.ip', |
|
257 | 'ip': 'NotebookApp.ip', | |
258 | 'port': 'NotebookApp.port', |
|
258 | 'port': 'NotebookApp.port', | |
259 | 'port-retries': 'NotebookApp.port_retries', |
|
259 | 'port-retries': 'NotebookApp.port_retries', | |
260 | 'transport': 'KernelManager.transport', |
|
260 | 'transport': 'KernelManager.transport', | |
261 | 'keyfile': 'NotebookApp.keyfile', |
|
261 | 'keyfile': 'NotebookApp.keyfile', | |
262 | 'certfile': 'NotebookApp.certfile', |
|
262 | 'certfile': 'NotebookApp.certfile', | |
263 | 'notebook-dir': 'NotebookManager.notebook_dir', |
|
263 | 'notebook-dir': 'NotebookManager.notebook_dir', | |
264 | 'browser': 'NotebookApp.browser', |
|
264 | 'browser': 'NotebookApp.browser', | |
265 | }) |
|
265 | }) | |
266 |
|
266 | |||
267 | # remove ipkernel flags that are singletons, and don't make sense in |
|
267 | # remove ipkernel flags that are singletons, and don't make sense in | |
268 | # multi-kernel evironment: |
|
268 | # multi-kernel evironment: | |
269 | aliases.pop('f', None) |
|
269 | aliases.pop('f', None) | |
270 |
|
270 | |||
271 | notebook_aliases = [u'port', u'port-retries', u'ip', u'keyfile', u'certfile', |
|
271 | notebook_aliases = [u'port', u'port-retries', u'ip', u'keyfile', u'certfile', | |
272 | u'notebook-dir'] |
|
272 | u'notebook-dir'] | |
273 |
|
273 | |||
274 | #----------------------------------------------------------------------------- |
|
274 | #----------------------------------------------------------------------------- | |
275 | # NotebookApp |
|
275 | # NotebookApp | |
276 | #----------------------------------------------------------------------------- |
|
276 | #----------------------------------------------------------------------------- | |
277 |
|
277 | |||
278 | class NotebookApp(BaseIPythonApplication): |
|
278 | class NotebookApp(BaseIPythonApplication): | |
279 |
|
279 | |||
280 | name = 'ipython-notebook' |
|
280 | name = 'ipython-notebook' | |
281 | default_config_file_name='ipython_notebook_config.py' |
|
281 | default_config_file_name='ipython_notebook_config.py' | |
282 |
|
282 | |||
283 | description = """ |
|
283 | description = """ | |
284 | The IPython HTML Notebook. |
|
284 | The IPython HTML Notebook. | |
285 |
|
285 | |||
286 | This launches a Tornado based HTML Notebook Server that serves up an |
|
286 | This launches a Tornado based HTML Notebook Server that serves up an | |
287 | HTML5/Javascript Notebook client. |
|
287 | HTML5/Javascript Notebook client. | |
288 | """ |
|
288 | """ | |
289 | examples = _examples |
|
289 | examples = _examples | |
290 |
|
290 | |||
291 | classes = IPythonConsoleApp.classes + [MappingKernelManager, NotebookManager, |
|
291 | classes = IPythonConsoleApp.classes + [MappingKernelManager, NotebookManager, | |
292 | FileNotebookManager] |
|
292 | FileNotebookManager] | |
293 | flags = Dict(flags) |
|
293 | flags = Dict(flags) | |
294 | aliases = Dict(aliases) |
|
294 | aliases = Dict(aliases) | |
295 |
|
295 | |||
296 | kernel_argv = List(Unicode) |
|
296 | kernel_argv = List(Unicode) | |
297 |
|
297 | |||
298 | max_msg_size = Integer(65536, config=True, help=""" |
|
298 | max_msg_size = Integer(65536, config=True, help=""" | |
299 | The max raw message size accepted from the browser |
|
299 | The max raw message size accepted from the browser | |
300 | over a WebSocket connection. |
|
300 | over a WebSocket connection. | |
301 | """) |
|
301 | """) | |
302 |
|
302 | |||
303 | def _log_level_default(self): |
|
303 | def _log_level_default(self): | |
304 | return logging.INFO |
|
304 | return logging.INFO | |
305 |
|
305 | |||
306 | def _log_format_default(self): |
|
306 | def _log_format_default(self): | |
307 | """override default log format to include time""" |
|
307 | """override default log format to include time""" | |
308 | return u"%(asctime)s.%(msecs).03d [%(name)s]%(highlevel)s %(message)s" |
|
308 | return u"%(asctime)s.%(msecs).03d [%(name)s]%(highlevel)s %(message)s" | |
309 |
|
309 | |||
310 | # create requested profiles by default, if they don't exist: |
|
310 | # create requested profiles by default, if they don't exist: | |
311 | auto_create = Bool(True) |
|
311 | auto_create = Bool(True) | |
312 |
|
312 | |||
313 | # file to be opened in the notebook server |
|
313 | # file to be opened in the notebook server | |
314 | file_to_run = Unicode('') |
|
314 | file_to_run = Unicode('') | |
315 |
|
315 | |||
316 | # Network related information. |
|
316 | # Network related information. | |
317 |
|
317 | |||
318 | ip = Unicode(LOCALHOST, config=True, |
|
318 | ip = Unicode(LOCALHOST, config=True, | |
319 | help="The IP address the notebook server will listen on." |
|
319 | help="The IP address the notebook server will listen on." | |
320 | ) |
|
320 | ) | |
321 |
|
321 | |||
322 | def _ip_changed(self, name, old, new): |
|
322 | def _ip_changed(self, name, old, new): | |
323 | if new == u'*': self.ip = u'' |
|
323 | if new == u'*': self.ip = u'' | |
324 |
|
324 | |||
325 | port = Integer(8888, config=True, |
|
325 | port = Integer(8888, config=True, | |
326 | help="The port the notebook server will listen on." |
|
326 | help="The port the notebook server will listen on." | |
327 | ) |
|
327 | ) | |
328 | port_retries = Integer(50, config=True, |
|
328 | port_retries = Integer(50, config=True, | |
329 | help="The number of additional ports to try if the specified port is not available." |
|
329 | help="The number of additional ports to try if the specified port is not available." | |
330 | ) |
|
330 | ) | |
331 |
|
331 | |||
332 | certfile = Unicode(u'', config=True, |
|
332 | certfile = Unicode(u'', config=True, | |
333 | help="""The full path to an SSL/TLS certificate file.""" |
|
333 | help="""The full path to an SSL/TLS certificate file.""" | |
334 | ) |
|
334 | ) | |
335 |
|
335 | |||
336 | keyfile = Unicode(u'', config=True, |
|
336 | keyfile = Unicode(u'', config=True, | |
337 | help="""The full path to a private key file for usage with SSL/TLS.""" |
|
337 | help="""The full path to a private key file for usage with SSL/TLS.""" | |
338 | ) |
|
338 | ) | |
339 |
|
339 | |||
340 | cookie_secret = Bytes(b'', config=True, |
|
340 | cookie_secret = Bytes(b'', config=True, | |
341 | help="""The random bytes used to secure cookies. |
|
341 | help="""The random bytes used to secure cookies. | |
342 | By default this is a new random number every time you start the Notebook. |
|
342 | By default this is a new random number every time you start the Notebook. | |
343 | Set it to a value in a config file to enable logins to persist across server sessions. |
|
343 | Set it to a value in a config file to enable logins to persist across server sessions. | |
344 |
|
344 | |||
345 | Note: Cookie secrets should be kept private, do not share config files with |
|
345 | Note: Cookie secrets should be kept private, do not share config files with | |
346 | cookie_secret stored in plaintext (you can read the value from a file). |
|
346 | cookie_secret stored in plaintext (you can read the value from a file). | |
347 | """ |
|
347 | """ | |
348 | ) |
|
348 | ) | |
349 | def _cookie_secret_default(self): |
|
349 | def _cookie_secret_default(self): | |
350 | return os.urandom(1024) |
|
350 | return os.urandom(1024) | |
351 |
|
351 | |||
352 | password = Unicode(u'', config=True, |
|
352 | password = Unicode(u'', config=True, | |
353 | help="""Hashed password to use for web authentication. |
|
353 | help="""Hashed password to use for web authentication. | |
354 |
|
354 | |||
355 | To generate, type in a python/IPython shell: |
|
355 | To generate, type in a python/IPython shell: | |
356 |
|
356 | |||
357 | from IPython.lib import passwd; passwd() |
|
357 | from IPython.lib import passwd; passwd() | |
358 |
|
358 | |||
359 | The string should be of the form type:salt:hashed-password. |
|
359 | The string should be of the form type:salt:hashed-password. | |
360 | """ |
|
360 | """ | |
361 | ) |
|
361 | ) | |
362 |
|
362 | |||
363 | open_browser = Bool(True, config=True, |
|
363 | open_browser = Bool(True, config=True, | |
364 | help="""Whether to open in a browser after starting. |
|
364 | help="""Whether to open in a browser after starting. | |
365 | The specific browser used is platform dependent and |
|
365 | The specific browser used is platform dependent and | |
366 | determined by the python standard library `webbrowser` |
|
366 | determined by the python standard library `webbrowser` | |
367 | module, unless it is overridden using the --browser |
|
367 | module, unless it is overridden using the --browser | |
368 | (NotebookApp.browser) configuration option. |
|
368 | (NotebookApp.browser) configuration option. | |
369 | """) |
|
369 | """) | |
370 |
|
370 | |||
371 | browser = Unicode(u'', config=True, |
|
371 | browser = Unicode(u'', config=True, | |
372 | help="""Specify what command to use to invoke a web |
|
372 | help="""Specify what command to use to invoke a web | |
373 | browser when opening the notebook. If not specified, the |
|
373 | browser when opening the notebook. If not specified, the | |
374 | default browser will be determined by the `webbrowser` |
|
374 | default browser will be determined by the `webbrowser` | |
375 | standard library module, which allows setting of the |
|
375 | standard library module, which allows setting of the | |
376 | BROWSER environment variable to override it. |
|
376 | BROWSER environment variable to override it. | |
377 | """) |
|
377 | """) | |
378 |
|
378 | |||
379 | read_only = Bool(False, config=True, |
|
379 | read_only = Bool(False, config=True, | |
380 | help="Whether to prevent editing/execution of notebooks." |
|
380 | help="Whether to prevent editing/execution of notebooks." | |
381 | ) |
|
381 | ) | |
382 |
|
382 | |||
383 | use_less = Bool(False, config=True, |
|
383 | use_less = Bool(False, config=True, | |
384 | help="""Wether to use Browser Side less-css parsing |
|
384 | help="""Wether to use Browser Side less-css parsing | |
385 | instead of compiled css version in templates that allows |
|
385 | instead of compiled css version in templates that allows | |
386 | it. This is mainly convenient when working on the less |
|
386 | it. This is mainly convenient when working on the less | |
387 | file to avoid a build step, or if user want to overwrite |
|
387 | file to avoid a build step, or if user want to overwrite | |
388 | some of the less variables without having to recompile |
|
388 | some of the less variables without having to recompile | |
389 | everything. |
|
389 | everything. | |
390 |
|
390 | |||
391 | You will need to install the less.js component in the static directory |
|
391 | You will need to install the less.js component in the static directory | |
392 | either in the source tree or in your profile folder. |
|
392 | either in the source tree or in your profile folder. | |
393 | """) |
|
393 | """) | |
394 |
|
394 | |||
395 | webapp_settings = Dict(config=True, |
|
395 | webapp_settings = Dict(config=True, | |
396 | help="Supply overrides for the tornado.web.Application that the " |
|
396 | help="Supply overrides for the tornado.web.Application that the " | |
397 | "IPython notebook uses.") |
|
397 | "IPython notebook uses.") | |
398 |
|
398 | |||
399 | enable_mathjax = Bool(True, config=True, |
|
399 | enable_mathjax = Bool(True, config=True, | |
400 | help="""Whether to enable MathJax for typesetting math/TeX |
|
400 | help="""Whether to enable MathJax for typesetting math/TeX | |
401 |
|
401 | |||
402 | MathJax is the javascript library IPython uses to render math/LaTeX. It is |
|
402 | MathJax is the javascript library IPython uses to render math/LaTeX. It is | |
403 | very large, so you may want to disable it if you have a slow internet |
|
403 | very large, so you may want to disable it if you have a slow internet | |
404 | connection, or for offline use of the notebook. |
|
404 | connection, or for offline use of the notebook. | |
405 |
|
405 | |||
406 | When disabled, equations etc. will appear as their untransformed TeX source. |
|
406 | When disabled, equations etc. will appear as their untransformed TeX source. | |
407 | """ |
|
407 | """ | |
408 | ) |
|
408 | ) | |
409 | def _enable_mathjax_changed(self, name, old, new): |
|
409 | def _enable_mathjax_changed(self, name, old, new): | |
410 | """set mathjax url to empty if mathjax is disabled""" |
|
410 | """set mathjax url to empty if mathjax is disabled""" | |
411 | if not new: |
|
411 | if not new: | |
412 | self.mathjax_url = u'' |
|
412 | self.mathjax_url = u'' | |
413 |
|
413 | |||
414 | base_project_url = Unicode('/', config=True, |
|
414 | base_project_url = Unicode('/', config=True, | |
415 | help='''The base URL for the notebook server. |
|
415 | help='''The base URL for the notebook server. | |
416 |
|
416 | |||
417 | Leading and trailing slashes can be omitted, |
|
417 | Leading and trailing slashes can be omitted, | |
418 | and will automatically be added. |
|
418 | and will automatically be added. | |
419 | ''') |
|
419 | ''') | |
420 | def _base_project_url_changed(self, name, old, new): |
|
420 | def _base_project_url_changed(self, name, old, new): | |
421 | if not new.startswith('/'): |
|
421 | if not new.startswith('/'): | |
422 | self.base_project_url = '/'+new |
|
422 | self.base_project_url = '/'+new | |
423 | elif not new.endswith('/'): |
|
423 | elif not new.endswith('/'): | |
424 | self.base_project_url = new+'/' |
|
424 | self.base_project_url = new+'/' | |
425 |
|
425 | |||
426 | base_kernel_url = Unicode('/', config=True, |
|
426 | base_kernel_url = Unicode('/', config=True, | |
427 | help='''The base URL for the kernel server |
|
427 | help='''The base URL for the kernel server | |
428 |
|
428 | |||
429 | Leading and trailing slashes can be omitted, |
|
429 | Leading and trailing slashes can be omitted, | |
430 | and will automatically be added. |
|
430 | and will automatically be added. | |
431 | ''') |
|
431 | ''') | |
432 | def _base_kernel_url_changed(self, name, old, new): |
|
432 | def _base_kernel_url_changed(self, name, old, new): | |
433 | if not new.startswith('/'): |
|
433 | if not new.startswith('/'): | |
434 | self.base_kernel_url = '/'+new |
|
434 | self.base_kernel_url = '/'+new | |
435 | elif not new.endswith('/'): |
|
435 | elif not new.endswith('/'): | |
436 | self.base_kernel_url = new+'/' |
|
436 | self.base_kernel_url = new+'/' | |
437 |
|
437 | |||
438 | websocket_url = Unicode("", config=True, |
|
438 | websocket_url = Unicode("", config=True, | |
439 | help="""The base URL for the websocket server, |
|
439 | help="""The base URL for the websocket server, | |
440 | if it differs from the HTTP server (hint: it almost certainly doesn't). |
|
440 | if it differs from the HTTP server (hint: it almost certainly doesn't). | |
441 |
|
441 | |||
442 | Should be in the form of an HTTP origin: ws[s]://hostname[:port] |
|
442 | Should be in the form of an HTTP origin: ws[s]://hostname[:port] | |
443 | """ |
|
443 | """ | |
444 | ) |
|
444 | ) | |
445 |
|
445 | |||
446 | extra_static_paths = List(Unicode, config=True, |
|
446 | extra_static_paths = List(Unicode, config=True, | |
447 | help="""Extra paths to search for serving static files. |
|
447 | help="""Extra paths to search for serving static files. | |
448 |
|
448 | |||
449 | This allows adding javascript/css to be available from the notebook server machine, |
|
449 | This allows adding javascript/css to be available from the notebook server machine, | |
450 | or overriding individual files in the IPython""" |
|
450 | or overriding individual files in the IPython""" | |
451 | ) |
|
451 | ) | |
452 | def _extra_static_paths_default(self): |
|
452 | def _extra_static_paths_default(self): | |
453 | return [os.path.join(self.profile_dir.location, 'static')] |
|
453 | return [os.path.join(self.profile_dir.location, 'static')] | |
454 |
|
454 | |||
455 | @property |
|
455 | @property | |
456 | def static_file_path(self): |
|
456 | def static_file_path(self): | |
457 | """return extra paths + the default location""" |
|
457 | """return extra paths + the default location""" | |
458 | return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH] |
|
458 | return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH] | |
459 |
|
459 | |||
460 | mathjax_url = Unicode("", config=True, |
|
460 | mathjax_url = Unicode("", config=True, | |
461 | help="""The url for MathJax.js.""" |
|
461 | help="""The url for MathJax.js.""" | |
462 | ) |
|
462 | ) | |
463 | def _mathjax_url_default(self): |
|
463 | def _mathjax_url_default(self): | |
464 | if not self.enable_mathjax: |
|
464 | if not self.enable_mathjax: | |
465 | return u'' |
|
465 | return u'' | |
466 | static_url_prefix = self.webapp_settings.get("static_url_prefix", |
|
466 | static_url_prefix = self.webapp_settings.get("static_url_prefix", | |
467 | url_path_join(self.base_project_url, "static") |
|
467 | url_path_join(self.base_project_url, "static") | |
468 | ) |
|
468 | ) | |
469 | try: |
|
469 | try: | |
470 | mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), self.static_file_path) |
|
470 | mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), self.static_file_path) | |
471 | except IOError: |
|
471 | except IOError: | |
472 | if self.certfile: |
|
472 | if self.certfile: | |
473 | # HTTPS: load from Rackspace CDN, because SSL certificate requires it |
|
473 | # HTTPS: load from Rackspace CDN, because SSL certificate requires it | |
474 | base = u"https://c328740.ssl.cf1.rackcdn.com" |
|
474 | base = u"https://c328740.ssl.cf1.rackcdn.com" | |
475 | else: |
|
475 | else: | |
476 | base = u"http://cdn.mathjax.org" |
|
476 | base = u"http://cdn.mathjax.org" | |
477 |
|
477 | |||
478 | url = base + u"/mathjax/latest/MathJax.js" |
|
478 | url = base + u"/mathjax/latest/MathJax.js" | |
479 | self.log.info("Using MathJax from CDN: %s", url) |
|
479 | self.log.info("Using MathJax from CDN: %s", url) | |
480 | return url |
|
480 | return url | |
481 | else: |
|
481 | else: | |
482 | self.log.info("Using local MathJax from %s" % mathjax) |
|
482 | self.log.info("Using local MathJax from %s" % mathjax) | |
483 | return url_path_join(static_url_prefix, u"mathjax/MathJax.js") |
|
483 | return url_path_join(static_url_prefix, u"mathjax/MathJax.js") | |
484 |
|
484 | |||
485 | def _mathjax_url_changed(self, name, old, new): |
|
485 | def _mathjax_url_changed(self, name, old, new): | |
486 | if new and not self.enable_mathjax: |
|
486 | if new and not self.enable_mathjax: | |
487 | # enable_mathjax=False overrides mathjax_url |
|
487 | # enable_mathjax=False overrides mathjax_url | |
488 | self.mathjax_url = u'' |
|
488 | self.mathjax_url = u'' | |
489 | else: |
|
489 | else: | |
490 | self.log.info("Using MathJax: %s", new) |
|
490 | self.log.info("Using MathJax: %s", new) | |
491 |
|
491 | |||
492 | notebook_manager_class = DottedObjectName('IPython.html.services.notebooks.filenbmanager.FileNotebookManager', |
|
492 | notebook_manager_class = DottedObjectName('IPython.html.services.notebooks.filenbmanager.FileNotebookManager', | |
493 | config=True, |
|
493 | config=True, | |
494 | help='The notebook manager class to use.') |
|
494 | help='The notebook manager class to use.') | |
495 |
|
495 | |||
496 | trust_xheaders = Bool(False, config=True, |
|
496 | trust_xheaders = Bool(False, config=True, | |
497 | help=("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers" |
|
497 | help=("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers" | |
498 | "sent by the upstream reverse proxy. Neccesary if the proxy handles SSL") |
|
498 | "sent by the upstream reverse proxy. Neccesary if the proxy handles SSL") | |
499 | ) |
|
499 | ) | |
500 |
|
500 | |||
501 | def parse_command_line(self, argv=None): |
|
501 | def parse_command_line(self, argv=None): | |
502 | super(NotebookApp, self).parse_command_line(argv) |
|
502 | super(NotebookApp, self).parse_command_line(argv) | |
503 | if argv is None: |
|
503 | if argv is None: | |
504 | argv = sys.argv[1:] |
|
504 | argv = sys.argv[1:] | |
505 |
|
505 | |||
506 | # Scrub frontend-specific flags |
|
506 | # Scrub frontend-specific flags | |
507 | self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags) |
|
507 | self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags) | |
508 | # Kernel should inherit default config file from frontend |
|
508 | # Kernel should inherit default config file from frontend | |
509 | self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name) |
|
509 | self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name) | |
510 |
|
510 | |||
511 | if self.extra_args: |
|
511 | if self.extra_args: | |
512 | f = os.path.abspath(self.extra_args[0]) |
|
512 | f = os.path.abspath(self.extra_args[0]) | |
513 | if os.path.isdir(f): |
|
513 | if os.path.isdir(f): | |
514 | nbdir = f |
|
514 | nbdir = f | |
515 | else: |
|
515 | else: | |
516 | self.file_to_run = f |
|
516 | self.file_to_run = f | |
517 | nbdir = os.path.dirname(f) |
|
517 | nbdir = os.path.dirname(f) | |
518 | self.config.NotebookManager.notebook_dir = nbdir |
|
518 | self.config.NotebookManager.notebook_dir = nbdir | |
519 |
|
519 | |||
520 | def init_configurables(self): |
|
520 | def init_configurables(self): | |
521 | # force Session default to be secure |
|
521 | # force Session default to be secure | |
522 | default_secure(self.config) |
|
522 | default_secure(self.config) | |
523 | self.kernel_manager = MappingKernelManager( |
|
523 | self.kernel_manager = MappingKernelManager( | |
524 | parent=self, log=self.log, kernel_argv=self.kernel_argv, |
|
524 | parent=self, log=self.log, kernel_argv=self.kernel_argv, | |
525 | connection_dir = self.profile_dir.security_dir, |
|
525 | connection_dir = self.profile_dir.security_dir, | |
526 | ) |
|
526 | ) | |
527 | kls = import_item(self.notebook_manager_class) |
|
527 | kls = import_item(self.notebook_manager_class) | |
528 | self.notebook_manager = kls(parent=self, log=self.log) |
|
528 | self.notebook_manager = kls(parent=self, log=self.log) | |
529 | self.notebook_manager.load_notebook_names() |
|
529 | self.notebook_manager.load_notebook_names() | |
530 | self.cluster_manager = ClusterManager(parent=self, log=self.log) |
|
530 | self.cluster_manager = ClusterManager(parent=self, log=self.log) | |
531 | self.cluster_manager.update_profiles() |
|
531 | self.cluster_manager.update_profiles() | |
532 |
|
532 | |||
533 | def init_logging(self): |
|
533 | def init_logging(self): | |
534 | # This prevents double log messages because tornado use a root logger that |
|
534 | # This prevents double log messages because tornado use a root logger that | |
535 | # self.log is a child of. The logging module dipatches log messages to a log |
|
535 | # self.log is a child of. The logging module dipatches log messages to a log | |
536 | # and all of its ancenstors until propagate is set to False. |
|
536 | # and all of its ancenstors until propagate is set to False. | |
537 | self.log.propagate = False |
|
537 | self.log.propagate = False | |
538 |
|
538 | |||
539 | # hook up tornado 3's loggers to our app handlers |
|
539 | # hook up tornado 3's loggers to our app handlers | |
540 | for name in ('access', 'application', 'general'): |
|
540 | for name in ('access', 'application', 'general'): | |
541 | logging.getLogger('tornado.%s' % name).handlers = self.log.handlers |
|
541 | logging.getLogger('tornado.%s' % name).handlers = self.log.handlers | |
542 |
|
542 | |||
543 | def init_webapp(self): |
|
543 | def init_webapp(self): | |
544 | """initialize tornado webapp and httpserver""" |
|
544 | """initialize tornado webapp and httpserver""" | |
545 | self.web_app = NotebookWebApplication( |
|
545 | self.web_app = NotebookWebApplication( | |
546 | self, self.kernel_manager, self.notebook_manager, |
|
546 | self, self.kernel_manager, self.notebook_manager, | |
547 | self.cluster_manager, self.log, |
|
547 | self.cluster_manager, self.log, | |
548 | self.base_project_url, self.webapp_settings |
|
548 | self.base_project_url, self.webapp_settings | |
549 | ) |
|
549 | ) | |
550 | if self.certfile: |
|
550 | if self.certfile: | |
551 | ssl_options = dict(certfile=self.certfile) |
|
551 | ssl_options = dict(certfile=self.certfile) | |
552 | if self.keyfile: |
|
552 | if self.keyfile: | |
553 | ssl_options['keyfile'] = self.keyfile |
|
553 | ssl_options['keyfile'] = self.keyfile | |
554 | else: |
|
554 | else: | |
555 | ssl_options = None |
|
555 | ssl_options = None | |
556 | self.web_app.password = self.password |
|
556 | self.web_app.password = self.password | |
557 | self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options, |
|
557 | self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options, | |
558 | xheaders=self.trust_xheaders) |
|
558 | xheaders=self.trust_xheaders) | |
559 | if not self.ip: |
|
559 | if not self.ip: | |
560 | warning = "WARNING: The notebook server is listening on all IP addresses" |
|
560 | warning = "WARNING: The notebook server is listening on all IP addresses" | |
561 | if ssl_options is None: |
|
561 | if ssl_options is None: | |
562 | self.log.critical(warning + " and not using encryption. This " |
|
562 | self.log.critical(warning + " and not using encryption. This " | |
563 | "is not recommended.") |
|
563 | "is not recommended.") | |
564 | if not self.password and not self.read_only: |
|
564 | if not self.password and not self.read_only: | |
565 | self.log.critical(warning + " and not using authentication. " |
|
565 | self.log.critical(warning + " and not using authentication. " | |
566 | "This is highly insecure and not recommended.") |
|
566 | "This is highly insecure and not recommended.") | |
567 | success = None |
|
567 | success = None | |
568 | for port in random_ports(self.port, self.port_retries+1): |
|
568 | for port in random_ports(self.port, self.port_retries+1): | |
569 | try: |
|
569 | try: | |
570 | self.http_server.listen(port, self.ip) |
|
570 | self.http_server.listen(port, self.ip) | |
571 | except socket.error as e: |
|
571 | except socket.error as e: | |
572 | # XXX: remove the e.errno == -9 block when we require |
|
572 | # XXX: remove the e.errno == -9 block when we require | |
573 | # tornado >= 3.0 |
|
573 | # tornado >= 3.0 | |
574 | if e.errno == -9 and tornado.version_info[0] < 3: |
|
574 | if e.errno == -9 and tornado.version_info[0] < 3: | |
575 | # The flags passed to socket.getaddrinfo from |
|
575 | # The flags passed to socket.getaddrinfo from | |
576 | # tornado.netutils.bind_sockets can cause "gaierror: |
|
576 | # tornado.netutils.bind_sockets can cause "gaierror: | |
577 | # [Errno -9] Address family for hostname not supported" |
|
577 | # [Errno -9] Address family for hostname not supported" | |
578 | # when the interface is not associated, for example. |
|
578 | # when the interface is not associated, for example. | |
579 | # Changing the flags to exclude socket.AI_ADDRCONFIG does |
|
579 | # Changing the flags to exclude socket.AI_ADDRCONFIG does | |
580 | # not cause this error, but the only way to do this is to |
|
580 | # not cause this error, but the only way to do this is to | |
581 | # monkeypatch socket to remove the AI_ADDRCONFIG attribute |
|
581 | # monkeypatch socket to remove the AI_ADDRCONFIG attribute | |
582 | saved_AI_ADDRCONFIG = socket.AI_ADDRCONFIG |
|
582 | saved_AI_ADDRCONFIG = socket.AI_ADDRCONFIG | |
583 | self.log.warn('Monkeypatching socket to fix tornado bug') |
|
583 | self.log.warn('Monkeypatching socket to fix tornado bug') | |
584 | del(socket.AI_ADDRCONFIG) |
|
584 | del(socket.AI_ADDRCONFIG) | |
585 | try: |
|
585 | try: | |
586 | # retry the tornado call without AI_ADDRCONFIG flags |
|
586 | # retry the tornado call without AI_ADDRCONFIG flags | |
587 | self.http_server.listen(port, self.ip) |
|
587 | self.http_server.listen(port, self.ip) | |
588 | except socket.error as e2: |
|
588 | except socket.error as e2: | |
589 | e = e2 |
|
589 | e = e2 | |
590 | else: |
|
590 | else: | |
591 | self.port = port |
|
591 | self.port = port | |
592 | success = True |
|
592 | success = True | |
593 | break |
|
593 | break | |
594 | # restore the monekypatch |
|
594 | # restore the monekypatch | |
595 | socket.AI_ADDRCONFIG = saved_AI_ADDRCONFIG |
|
595 | socket.AI_ADDRCONFIG = saved_AI_ADDRCONFIG | |
596 | if e.errno != errno.EADDRINUSE: |
|
596 | if e.errno != errno.EADDRINUSE: | |
597 | raise |
|
597 | raise | |
598 | self.log.info('The port %i is already in use, trying another random port.' % port) |
|
598 | self.log.info('The port %i is already in use, trying another random port.' % port) | |
599 | else: |
|
599 | else: | |
600 | self.port = port |
|
600 | self.port = port | |
601 | success = True |
|
601 | success = True | |
602 | break |
|
602 | break | |
603 | if not success: |
|
603 | if not success: | |
604 | self.log.critical('ERROR: the notebook server could not be started because ' |
|
604 | self.log.critical('ERROR: the notebook server could not be started because ' | |
605 | 'no available port could be found.') |
|
605 | 'no available port could be found.') | |
606 | self.exit(1) |
|
606 | self.exit(1) | |
607 |
|
607 | |||
608 | def init_signal(self): |
|
608 | def init_signal(self): | |
609 | if not sys.platform.startswith('win'): |
|
609 | if not sys.platform.startswith('win'): | |
610 | signal.signal(signal.SIGINT, self._handle_sigint) |
|
610 | signal.signal(signal.SIGINT, self._handle_sigint) | |
611 | signal.signal(signal.SIGTERM, self._signal_stop) |
|
611 | signal.signal(signal.SIGTERM, self._signal_stop) | |
612 | if hasattr(signal, 'SIGUSR1'): |
|
612 | if hasattr(signal, 'SIGUSR1'): | |
613 | # Windows doesn't support SIGUSR1 |
|
613 | # Windows doesn't support SIGUSR1 | |
614 | signal.signal(signal.SIGUSR1, self._signal_info) |
|
614 | signal.signal(signal.SIGUSR1, self._signal_info) | |
615 | if hasattr(signal, 'SIGINFO'): |
|
615 | if hasattr(signal, 'SIGINFO'): | |
616 | # only on BSD-based systems |
|
616 | # only on BSD-based systems | |
617 | signal.signal(signal.SIGINFO, self._signal_info) |
|
617 | signal.signal(signal.SIGINFO, self._signal_info) | |
618 |
|
618 | |||
619 | def _handle_sigint(self, sig, frame): |
|
619 | def _handle_sigint(self, sig, frame): | |
620 | """SIGINT handler spawns confirmation dialog""" |
|
620 | """SIGINT handler spawns confirmation dialog""" | |
621 | # register more forceful signal handler for ^C^C case |
|
621 | # register more forceful signal handler for ^C^C case | |
622 | signal.signal(signal.SIGINT, self._signal_stop) |
|
622 | signal.signal(signal.SIGINT, self._signal_stop) | |
623 | # request confirmation dialog in bg thread, to avoid |
|
623 | # request confirmation dialog in bg thread, to avoid | |
624 | # blocking the App |
|
624 | # blocking the App | |
625 | thread = threading.Thread(target=self._confirm_exit) |
|
625 | thread = threading.Thread(target=self._confirm_exit) | |
626 | thread.daemon = True |
|
626 | thread.daemon = True | |
627 | thread.start() |
|
627 | thread.start() | |
628 |
|
628 | |||
629 | def _restore_sigint_handler(self): |
|
629 | def _restore_sigint_handler(self): | |
630 | """callback for restoring original SIGINT handler""" |
|
630 | """callback for restoring original SIGINT handler""" | |
631 | signal.signal(signal.SIGINT, self._handle_sigint) |
|
631 | signal.signal(signal.SIGINT, self._handle_sigint) | |
632 |
|
632 | |||
633 | def _confirm_exit(self): |
|
633 | def _confirm_exit(self): | |
634 | """confirm shutdown on ^C |
|
634 | """confirm shutdown on ^C | |
635 |
|
635 | |||
636 | A second ^C, or answering 'y' within 5s will cause shutdown, |
|
636 | A second ^C, or answering 'y' within 5s will cause shutdown, | |
637 | otherwise original SIGINT handler will be restored. |
|
637 | otherwise original SIGINT handler will be restored. | |
638 |
|
638 | |||
639 | This doesn't work on Windows. |
|
639 | This doesn't work on Windows. | |
640 | """ |
|
640 | """ | |
641 | # FIXME: remove this delay when pyzmq dependency is >= 2.1.11 |
|
641 | # FIXME: remove this delay when pyzmq dependency is >= 2.1.11 | |
642 | time.sleep(0.1) |
|
642 | time.sleep(0.1) | |
643 | info = self.log.info |
|
643 | info = self.log.info | |
644 | info('interrupted') |
|
644 | info('interrupted') | |
645 | print self.notebook_info() |
|
645 | print self.notebook_info() | |
646 | sys.stdout.write("Shutdown this notebook server (y/[n])? ") |
|
646 | sys.stdout.write("Shutdown this notebook server (y/[n])? ") | |
647 | sys.stdout.flush() |
|
647 | sys.stdout.flush() | |
648 | r,w,x = select.select([sys.stdin], [], [], 5) |
|
648 | r,w,x = select.select([sys.stdin], [], [], 5) | |
649 | if r: |
|
649 | if r: | |
650 | line = sys.stdin.readline() |
|
650 | line = sys.stdin.readline() | |
651 | if line.lower().startswith('y'): |
|
651 | if line.lower().startswith('y'): | |
652 | self.log.critical("Shutdown confirmed") |
|
652 | self.log.critical("Shutdown confirmed") | |
653 | ioloop.IOLoop.instance().stop() |
|
653 | ioloop.IOLoop.instance().stop() | |
654 | return |
|
654 | return | |
655 | else: |
|
655 | else: | |
656 | print "No answer for 5s:", |
|
656 | print "No answer for 5s:", | |
657 | print "resuming operation..." |
|
657 | print "resuming operation..." | |
658 | # no answer, or answer is no: |
|
658 | # no answer, or answer is no: | |
659 | # set it back to original SIGINT handler |
|
659 | # set it back to original SIGINT handler | |
660 | # use IOLoop.add_callback because signal.signal must be called |
|
660 | # use IOLoop.add_callback because signal.signal must be called | |
661 | # from main thread |
|
661 | # from main thread | |
662 | ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler) |
|
662 | ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler) | |
663 |
|
663 | |||
664 | def _signal_stop(self, sig, frame): |
|
664 | def _signal_stop(self, sig, frame): | |
665 | self.log.critical("received signal %s, stopping", sig) |
|
665 | self.log.critical("received signal %s, stopping", sig) | |
666 | ioloop.IOLoop.instance().stop() |
|
666 | ioloop.IOLoop.instance().stop() | |
667 |
|
667 | |||
668 | def _signal_info(self, sig, frame): |
|
668 | def _signal_info(self, sig, frame): | |
669 | print self.notebook_info() |
|
669 | print self.notebook_info() | |
670 |
|
670 | |||
671 | def init_components(self): |
|
671 | def init_components(self): | |
672 | """Check the components submodule, and warn if it's unclean""" |
|
672 | """Check the components submodule, and warn if it's unclean""" | |
673 | status = submodule.check_submodule_status() |
|
673 | status = submodule.check_submodule_status() | |
674 | if status == 'missing': |
|
674 | if status == 'missing': | |
675 | self.log.warn("components submodule missing, running `git submodule update`") |
|
675 | self.log.warn("components submodule missing, running `git submodule update`") | |
676 | submodule.update_submodules(submodule.ipython_parent()) |
|
676 | submodule.update_submodules(submodule.ipython_parent()) | |
677 | elif status == 'unclean': |
|
677 | elif status == 'unclean': | |
678 | self.log.warn("components submodule unclean, you may see 404s on static/components") |
|
678 | self.log.warn("components submodule unclean, you may see 404s on static/components") | |
679 | self.log.warn("run `setup.py submodule` or `git submodule update` to update") |
|
679 | self.log.warn("run `setup.py submodule` or `git submodule update` to update") | |
680 |
|
680 | |||
681 |
|
681 | |||
682 | @catch_config_error |
|
682 | @catch_config_error | |
683 | def initialize(self, argv=None): |
|
683 | def initialize(self, argv=None): | |
684 | self.init_logging() |
|
684 | self.init_logging() | |
685 | super(NotebookApp, self).initialize(argv) |
|
685 | super(NotebookApp, self).initialize(argv) | |
686 | self.init_configurables() |
|
686 | self.init_configurables() | |
687 | self.init_components() |
|
687 | self.init_components() | |
688 | self.init_webapp() |
|
688 | self.init_webapp() | |
689 | self.init_signal() |
|
689 | self.init_signal() | |
690 |
|
690 | |||
691 | def cleanup_kernels(self): |
|
691 | def cleanup_kernels(self): | |
692 | """Shutdown all kernels. |
|
692 | """Shutdown all kernels. | |
693 |
|
693 | |||
694 | The kernels will shutdown themselves when this process no longer exists, |
|
694 | The kernels will shutdown themselves when this process no longer exists, | |
695 | but explicit shutdown allows the KernelManagers to cleanup the connection files. |
|
695 | but explicit shutdown allows the KernelManagers to cleanup the connection files. | |
696 | """ |
|
696 | """ | |
697 | self.log.info('Shutting down kernels') |
|
697 | self.log.info('Shutting down kernels') | |
698 | self.kernel_manager.shutdown_all() |
|
698 | self.kernel_manager.shutdown_all() | |
699 |
|
699 | |||
700 | def notebook_info(self): |
|
700 | def notebook_info(self): | |
701 | "Return the current working directory and the server url information" |
|
701 | "Return the current working directory and the server url information" | |
702 | mgr_info = self.notebook_manager.info_string() + "\n" |
|
702 | mgr_info = self.notebook_manager.info_string() + "\n" | |
703 | return mgr_info +"The IPython Notebook is running at: %s" % self._url |
|
703 | return mgr_info +"The IPython Notebook is running at: %s" % self._url | |
704 |
|
704 | |||
705 | def start(self): |
|
705 | def start(self): | |
706 | """ Start the IPython Notebook server app, after initialization |
|
706 | """ Start the IPython Notebook server app, after initialization | |
707 |
|
707 | |||
708 | This method takes no arguments so all configuration and initialization |
|
708 | This method takes no arguments so all configuration and initialization | |
709 | must be done prior to calling this method.""" |
|
709 | must be done prior to calling this method.""" | |
710 | ip = self.ip if self.ip else '[all ip addresses on your system]' |
|
710 | ip = self.ip if self.ip else '[all ip addresses on your system]' | |
711 | proto = 'https' if self.certfile else 'http' |
|
711 | proto = 'https' if self.certfile else 'http' | |
712 | info = self.log.info |
|
712 | info = self.log.info | |
713 | self._url = "%s://%s:%i%s" % (proto, ip, self.port, |
|
713 | self._url = "%s://%s:%i%s" % (proto, ip, self.port, | |
714 | self.base_project_url) |
|
714 | self.base_project_url) | |
715 | for line in self.notebook_info().split("\n"): |
|
715 | for line in self.notebook_info().split("\n"): | |
716 | info(line) |
|
716 | info(line) | |
717 | info("Use Control-C to stop this server and shut down all kernels.") |
|
717 | info("Use Control-C to stop this server and shut down all kernels.") | |
718 |
|
718 | |||
719 | if self.open_browser or self.file_to_run: |
|
719 | if self.open_browser or self.file_to_run: | |
720 | ip = self.ip or LOCALHOST |
|
720 | ip = self.ip or LOCALHOST | |
721 | try: |
|
721 | try: | |
722 | browser = webbrowser.get(self.browser or None) |
|
722 | browser = webbrowser.get(self.browser or None) | |
723 | except webbrowser.Error as e: |
|
723 | except webbrowser.Error as e: | |
724 | self.log.warn('No web browser found: %s.' % e) |
|
724 | self.log.warn('No web browser found: %s.' % e) | |
725 | browser = None |
|
725 | browser = None | |
726 |
|
726 | |||
727 | if self.file_to_run: |
|
727 | if self.file_to_run: | |
728 | name, _ = os.path.splitext(os.path.basename(self.file_to_run)) |
|
728 | name, _ = os.path.splitext(os.path.basename(self.file_to_run)) | |
729 | url = self.notebook_manager.rev_mapping.get(name, '') |
|
729 | url = self.notebook_manager.rev_mapping.get(name, '') | |
730 | else: |
|
730 | else: | |
731 | url = '' |
|
731 | url = '' | |
732 | if browser: |
|
732 | if browser: | |
733 | b = lambda : browser.open("%s://%s:%i%s%s" % (proto, ip, |
|
733 | b = lambda : browser.open("%s://%s:%i%s%s" % (proto, ip, | |
734 | self.port, self.base_project_url, url), new=2) |
|
734 | self.port, self.base_project_url, url), new=2) | |
735 | threading.Thread(target=b).start() |
|
735 | threading.Thread(target=b).start() | |
736 | try: |
|
736 | try: | |
737 | ioloop.IOLoop.instance().start() |
|
737 | ioloop.IOLoop.instance().start() | |
738 | except KeyboardInterrupt: |
|
738 | except KeyboardInterrupt: | |
739 | info("Interrupted...") |
|
739 | info("Interrupted...") | |
740 | finally: |
|
740 | finally: | |
741 | self.cleanup_kernels() |
|
741 | self.cleanup_kernels() | |
742 |
|
742 | |||
743 |
|
743 | |||
744 | #----------------------------------------------------------------------------- |
|
744 | #----------------------------------------------------------------------------- | |
745 | # Main entry point |
|
745 | # Main entry point | |
746 | #----------------------------------------------------------------------------- |
|
746 | #----------------------------------------------------------------------------- | |
747 |
|
747 | |||
748 |
launch_new_instance = NotebookApp.launch_ |
|
748 | launch_new_instance = NotebookApp.launch_instance | |
749 |
|
749 |
@@ -1,207 +1,207 | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | """NBConvert is a utility for conversion of IPYNB files. |
|
2 | """NBConvert is a utility for conversion of IPYNB files. | |
3 |
|
3 | |||
4 | Commandline interface for the NBConvert conversion utility. Read the |
|
4 | Commandline interface for the NBConvert conversion utility. Read the | |
5 | readme.rst for usage information |
|
5 | readme.rst for usage information | |
6 | """ |
|
6 | """ | |
7 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
8 | #Copyright (c) 2013, the IPython Development Team. |
|
8 | #Copyright (c) 2013, the IPython Development Team. | |
9 | # |
|
9 | # | |
10 | #Distributed under the terms of the Modified BSD License. |
|
10 | #Distributed under the terms of the Modified BSD License. | |
11 | # |
|
11 | # | |
12 | #The full license is in the file COPYING.txt, distributed with this software. |
|
12 | #The full license is in the file COPYING.txt, distributed with this software. | |
13 | #----------------------------------------------------------------------------- |
|
13 | #----------------------------------------------------------------------------- | |
14 |
|
14 | |||
15 | #----------------------------------------------------------------------------- |
|
15 | #----------------------------------------------------------------------------- | |
16 | #Imports |
|
16 | #Imports | |
17 | #----------------------------------------------------------------------------- |
|
17 | #----------------------------------------------------------------------------- | |
18 |
|
18 | |||
19 | #Stdlib imports |
|
19 | #Stdlib imports | |
20 | from __future__ import print_function |
|
20 | from __future__ import print_function | |
21 | import sys |
|
21 | import sys | |
22 | import io |
|
22 | import io | |
23 | import os |
|
23 | import os | |
24 |
|
24 | |||
25 | #From IPython |
|
25 | #From IPython | |
26 | from IPython.config.application import Application |
|
26 | from IPython.config.application import Application | |
27 | from IPython.utils.traitlets import Bool |
|
27 | from IPython.utils.traitlets import Bool | |
28 |
|
28 | |||
29 | from .exporters.export import export_by_name |
|
29 | from .exporters.export import export_by_name | |
30 | from .exporters.exporter import Exporter |
|
30 | from .exporters.exporter import Exporter | |
31 | from .transformers import extractfigure |
|
31 | from .transformers import extractfigure | |
32 | from .utils.config import GlobalConfigurable |
|
32 | from .utils.config import GlobalConfigurable | |
33 |
|
33 | |||
34 | #----------------------------------------------------------------------------- |
|
34 | #----------------------------------------------------------------------------- | |
35 | #Globals and constants |
|
35 | #Globals and constants | |
36 | #----------------------------------------------------------------------------- |
|
36 | #----------------------------------------------------------------------------- | |
37 |
|
37 | |||
38 | #'Keys in resources' user prompt. |
|
38 | #'Keys in resources' user prompt. | |
39 | KEYS_PROMPT_HEAD = "====================== Keys in Resources ==================================" |
|
39 | KEYS_PROMPT_HEAD = "====================== Keys in Resources ==================================" | |
40 | KEYS_PROMPT_BODY = """ |
|
40 | KEYS_PROMPT_BODY = """ | |
41 | =========================================================================== |
|
41 | =========================================================================== | |
42 | You are responsible for writting these files into the appropriate |
|
42 | You are responsible for writting these files into the appropriate | |
43 | directorie(s) if need be. If you do not want to see this message, enable |
|
43 | directorie(s) if need be. If you do not want to see this message, enable | |
44 | the 'write' (boolean) flag of the converter. |
|
44 | the 'write' (boolean) flag of the converter. | |
45 | =========================================================================== |
|
45 | =========================================================================== | |
46 | """ |
|
46 | """ | |
47 |
|
47 | |||
48 | #----------------------------------------------------------------------------- |
|
48 | #----------------------------------------------------------------------------- | |
49 | #Classes and functions |
|
49 | #Classes and functions | |
50 | #----------------------------------------------------------------------------- |
|
50 | #----------------------------------------------------------------------------- | |
51 |
|
51 | |||
52 | class NbConvertApp(Application): |
|
52 | class NbConvertApp(Application): | |
53 | """Application used to convert to and from notebook file type (*.ipynb)""" |
|
53 | """Application used to convert to and from notebook file type (*.ipynb)""" | |
54 |
|
54 | |||
55 | stdout = Bool( |
|
55 | stdout = Bool( | |
56 | False, config=True, |
|
56 | False, config=True, | |
57 | help="""Whether to print the converted IPYNB file to stdout |
|
57 | help="""Whether to print the converted IPYNB file to stdout | |
58 | use full do diff files without actually writing a new file""" |
|
58 | use full do diff files without actually writing a new file""" | |
59 | ) |
|
59 | ) | |
60 |
|
60 | |||
61 | write = Bool( |
|
61 | write = Bool( | |
62 | True, config=True, |
|
62 | True, config=True, | |
63 | help="""Should the converted notebook file be written to disk |
|
63 | help="""Should the converted notebook file be written to disk | |
64 | along with potential extracted resources.""" |
|
64 | along with potential extracted resources.""" | |
65 | ) |
|
65 | ) | |
66 |
|
66 | |||
67 | aliases = { |
|
67 | aliases = { | |
68 | 'stdout':'NbConvertApp.stdout', |
|
68 | 'stdout':'NbConvertApp.stdout', | |
69 | 'write':'NbConvertApp.write', |
|
69 | 'write':'NbConvertApp.write', | |
70 | } |
|
70 | } | |
71 |
|
71 | |||
72 | flags = {} |
|
72 | flags = {} | |
73 |
|
73 | |||
74 | flags['stdout'] = ( |
|
74 | flags['stdout'] = ( | |
75 | {'NbConvertApp' : {'stdout' : True}}, |
|
75 | {'NbConvertApp' : {'stdout' : True}}, | |
76 | """Print converted file to stdout, equivalent to --stdout=True |
|
76 | """Print converted file to stdout, equivalent to --stdout=True | |
77 | """ |
|
77 | """ | |
78 | ) |
|
78 | ) | |
79 |
|
79 | |||
80 | flags['no-write'] = ( |
|
80 | flags['no-write'] = ( | |
81 | {'NbConvertApp' : {'write' : True}}, |
|
81 | {'NbConvertApp' : {'write' : True}}, | |
82 | """Do not write to disk, equivalent to --write=False |
|
82 | """Do not write to disk, equivalent to --write=False | |
83 | """ |
|
83 | """ | |
84 | ) |
|
84 | ) | |
85 |
|
85 | |||
86 |
|
86 | |||
87 | def __init__(self, **kwargs): |
|
87 | def __init__(self, **kwargs): | |
88 | """Public constructor""" |
|
88 | """Public constructor""" | |
89 |
|
89 | |||
90 | #Call base class |
|
90 | #Call base class | |
91 | super(NbConvertApp, self).__init__(**kwargs) |
|
91 | super(NbConvertApp, self).__init__(**kwargs) | |
92 |
|
92 | |||
93 | #Register class here to have help with help all |
|
93 | #Register class here to have help with help all | |
94 | self.classes.insert(0, Exporter) |
|
94 | self.classes.insert(0, Exporter) | |
95 | self.classes.insert(0, GlobalConfigurable) |
|
95 | self.classes.insert(0, GlobalConfigurable) | |
96 |
|
96 | |||
97 |
|
97 | |||
98 | def start(self, argv=None): |
|
98 | def start(self, argv=None): | |
99 | """Entrypoint of NbConvert application. |
|
99 | """Entrypoint of NbConvert application. | |
100 |
|
100 | |||
101 | Parameters |
|
101 | Parameters | |
102 | ---------- |
|
102 | ---------- | |
103 | argv : list |
|
103 | argv : list | |
104 | Commandline arguments |
|
104 | Commandline arguments | |
105 | """ |
|
105 | """ | |
106 |
|
106 | |||
107 | #Parse the commandline options. |
|
107 | #Parse the commandline options. | |
108 | self.parse_command_line(argv) |
|
108 | self.parse_command_line(argv) | |
109 |
|
109 | |||
110 | #Call base |
|
110 | #Call base | |
111 | super(NbConvertApp, self).start() |
|
111 | super(NbConvertApp, self).start() | |
112 |
|
112 | |||
113 | #The last arguments in list will be used by nbconvert |
|
113 | #The last arguments in list will be used by nbconvert | |
114 | if len(self.extra_args) is not 3: |
|
114 | if len(self.extra_args) is not 3: | |
115 | print( "Wrong number of arguments, use --help flag for usage", file=sys.stderr) |
|
115 | print( "Wrong number of arguments, use --help flag for usage", file=sys.stderr) | |
116 | sys.exit(-1) |
|
116 | sys.exit(-1) | |
117 | export_type = (self.extra_args)[1] |
|
117 | export_type = (self.extra_args)[1] | |
118 | ipynb_file = (self.extra_args)[2] |
|
118 | ipynb_file = (self.extra_args)[2] | |
119 |
|
119 | |||
120 | #Export |
|
120 | #Export | |
121 | return_value = export_by_name(export_type, ipynb_file) |
|
121 | return_value = export_by_name(export_type, ipynb_file) | |
122 | if return_value is None: |
|
122 | if return_value is None: | |
123 | print("Error: '%s' template not found." % export_type) |
|
123 | print("Error: '%s' template not found." % export_type) | |
124 | return |
|
124 | return | |
125 | else: |
|
125 | else: | |
126 | (output, resources, exporter) = return_value |
|
126 | (output, resources, exporter) = return_value | |
127 |
|
127 | |||
128 | #TODO: Allow user to set output directory and file. |
|
128 | #TODO: Allow user to set output directory and file. | |
129 | destination_filename = None |
|
129 | destination_filename = None | |
130 | destination_directory = None |
|
130 | destination_directory = None | |
131 | if self.write: |
|
131 | if self.write: | |
132 |
|
132 | |||
133 | #Get the file name without the '.ipynb' (6 chars) extension and then |
|
133 | #Get the file name without the '.ipynb' (6 chars) extension and then | |
134 | #remove any addition periods and spaces. The resulting name will |
|
134 | #remove any addition periods and spaces. The resulting name will | |
135 | #be used to create the directory that the files will be exported |
|
135 | #be used to create the directory that the files will be exported | |
136 | #into. |
|
136 | #into. | |
137 | out_root = ipynb_file[:-6].replace('.', '_').replace(' ', '_') |
|
137 | out_root = ipynb_file[:-6].replace('.', '_').replace(' ', '_') | |
138 | destination_filename = os.path.join(out_root+'.'+exporter.file_extension) |
|
138 | destination_filename = os.path.join(out_root+'.'+exporter.file_extension) | |
139 |
|
139 | |||
140 | destination_directory = out_root+'_files' |
|
140 | destination_directory = out_root+'_files' | |
141 | if not os.path.exists(destination_directory): |
|
141 | if not os.path.exists(destination_directory): | |
142 | os.mkdir(destination_directory) |
|
142 | os.mkdir(destination_directory) | |
143 |
|
143 | |||
144 | #Write the results |
|
144 | #Write the results | |
145 | if self.stdout or not (destination_filename is None and destination_directory is None): |
|
145 | if self.stdout or not (destination_filename is None and destination_directory is None): | |
146 | self._write_results(output, resources, destination_filename, destination_directory) |
|
146 | self._write_results(output, resources, destination_filename, destination_directory) | |
147 |
|
147 | |||
148 |
|
148 | |||
149 | def _write_results(self, output, resources, destination_filename=None, destination_directory=None): |
|
149 | def _write_results(self, output, resources, destination_filename=None, destination_directory=None): | |
150 | """Output the conversion results to the console and/or filesystem |
|
150 | """Output the conversion results to the console and/or filesystem | |
151 |
|
151 | |||
152 | Parameters |
|
152 | Parameters | |
153 | ---------- |
|
153 | ---------- | |
154 | output : str |
|
154 | output : str | |
155 | Output of conversion |
|
155 | Output of conversion | |
156 | resources : dictionary |
|
156 | resources : dictionary | |
157 | Additional input/output used by the transformers. For |
|
157 | Additional input/output used by the transformers. For | |
158 | example, the ExtractFigure transformer outputs the |
|
158 | example, the ExtractFigure transformer outputs the | |
159 | figures it extracts into this dictionary. This method |
|
159 | figures it extracts into this dictionary. This method | |
160 | relies on the figures being in this dictionary when |
|
160 | relies on the figures being in this dictionary when | |
161 | attempting to write the figures to the file system. |
|
161 | attempting to write the figures to the file system. | |
162 | destination_filename : str, Optional |
|
162 | destination_filename : str, Optional | |
163 | Filename to write output into. If None, output is not |
|
163 | Filename to write output into. If None, output is not | |
164 | written to a file. |
|
164 | written to a file. | |
165 | destination_directory : str, Optional |
|
165 | destination_directory : str, Optional | |
166 | Directory to write notebook data (i.e. figures) to. If |
|
166 | Directory to write notebook data (i.e. figures) to. If | |
167 | None, figures are not written to the file system. |
|
167 | None, figures are not written to the file system. | |
168 | """ |
|
168 | """ | |
169 |
|
169 | |||
170 | if self.stdout: |
|
170 | if self.stdout: | |
171 | print(output.encode('utf-8')) |
|
171 | print(output.encode('utf-8')) | |
172 |
|
172 | |||
173 | #Write file output from conversion. |
|
173 | #Write file output from conversion. | |
174 | if not destination_filename is None: |
|
174 | if not destination_filename is None: | |
175 | with io.open(destination_filename, 'w') as f: |
|
175 | with io.open(destination_filename, 'w') as f: | |
176 | f.write(output) |
|
176 | f.write(output) | |
177 |
|
177 | |||
178 | #Get the key names used by the extract figure transformer |
|
178 | #Get the key names used by the extract figure transformer | |
179 | figures_key = extractfigure.FIGURES_KEY |
|
179 | figures_key = extractfigure.FIGURES_KEY | |
180 | binary_key = extractfigure.BINARY_KEY |
|
180 | binary_key = extractfigure.BINARY_KEY | |
181 | text_key = extractfigure.TEXT_KEY |
|
181 | text_key = extractfigure.TEXT_KEY | |
182 |
|
182 | |||
183 | #Output any associate figures into the same "root" directory. |
|
183 | #Output any associate figures into the same "root" directory. | |
184 | binkeys = resources.get(figures_key, {}).get(binary_key,{}).keys() |
|
184 | binkeys = resources.get(figures_key, {}).get(binary_key,{}).keys() | |
185 | textkeys = resources.get(figures_key, {}).get(text_key,{}).keys() |
|
185 | textkeys = resources.get(figures_key, {}).get(text_key,{}).keys() | |
186 | if binkeys or textkeys : |
|
186 | if binkeys or textkeys : | |
187 | if not destination_directory is None: |
|
187 | if not destination_directory is None: | |
188 | for key in binkeys: |
|
188 | for key in binkeys: | |
189 | with io.open(os.path.join(destination_directory, key), 'wb') as f: |
|
189 | with io.open(os.path.join(destination_directory, key), 'wb') as f: | |
190 | f.write(resources[figures_key][binary_key][key]) |
|
190 | f.write(resources[figures_key][binary_key][key]) | |
191 | for key in textkeys: |
|
191 | for key in textkeys: | |
192 | with io.open(os.path.join(destination_directory, key), 'w') as f: |
|
192 | with io.open(os.path.join(destination_directory, key), 'w') as f: | |
193 | f.write(resources[figures_key][text_key][key]) |
|
193 | f.write(resources[figures_key][text_key][key]) | |
194 |
|
194 | |||
195 | #Figures that weren't exported which will need to be created by the |
|
195 | #Figures that weren't exported which will need to be created by the | |
196 | #user. Tell the user what figures these are. |
|
196 | #user. Tell the user what figures these are. | |
197 | if self.stdout: |
|
197 | if self.stdout: | |
198 | print(KEYS_PROMPT_HEAD, file=sys.stderr) |
|
198 | print(KEYS_PROMPT_HEAD, file=sys.stderr) | |
199 | print(resources[figures_key].keys(), file=sys.stderr) |
|
199 | print(resources[figures_key].keys(), file=sys.stderr) | |
200 | print(KEYS_PROMPT_BODY , file=sys.stderr) |
|
200 | print(KEYS_PROMPT_BODY , file=sys.stderr) | |
201 |
|
201 | |||
202 | #----------------------------------------------------------------------------- |
|
202 | #----------------------------------------------------------------------------- | |
203 | # Main entry point |
|
203 | # Main entry point | |
204 | #----------------------------------------------------------------------------- |
|
204 | #----------------------------------------------------------------------------- | |
205 |
|
205 | |||
206 |
launch_new_instance = NbConvertApp.launch_ |
|
206 | launch_new_instance = NbConvertApp.launch_instance | |
207 |
|
207 |
@@ -1,615 +1,615 | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | The ipcluster application. |
|
4 | The ipcluster application. | |
5 |
|
5 | |||
6 | Authors: |
|
6 | Authors: | |
7 |
|
7 | |||
8 | * Brian Granger |
|
8 | * Brian Granger | |
9 | * MinRK |
|
9 | * MinRK | |
10 |
|
10 | |||
11 | """ |
|
11 | """ | |
12 |
|
12 | |||
13 | #----------------------------------------------------------------------------- |
|
13 | #----------------------------------------------------------------------------- | |
14 | # Copyright (C) 2008-2011 The IPython Development Team |
|
14 | # Copyright (C) 2008-2011 The IPython Development Team | |
15 | # |
|
15 | # | |
16 | # Distributed under the terms of the BSD License. The full license is in |
|
16 | # Distributed under the terms of the BSD License. The full license is in | |
17 | # the file COPYING, distributed as part of this software. |
|
17 | # the file COPYING, distributed as part of this software. | |
18 | #----------------------------------------------------------------------------- |
|
18 | #----------------------------------------------------------------------------- | |
19 |
|
19 | |||
20 | #----------------------------------------------------------------------------- |
|
20 | #----------------------------------------------------------------------------- | |
21 | # Imports |
|
21 | # Imports | |
22 | #----------------------------------------------------------------------------- |
|
22 | #----------------------------------------------------------------------------- | |
23 |
|
23 | |||
24 | import errno |
|
24 | import errno | |
25 | import logging |
|
25 | import logging | |
26 | import os |
|
26 | import os | |
27 | import re |
|
27 | import re | |
28 | import signal |
|
28 | import signal | |
29 |
|
29 | |||
30 | from subprocess import check_call, CalledProcessError, PIPE |
|
30 | from subprocess import check_call, CalledProcessError, PIPE | |
31 | import zmq |
|
31 | import zmq | |
32 | from zmq.eventloop import ioloop |
|
32 | from zmq.eventloop import ioloop | |
33 |
|
33 | |||
34 | from IPython.config.application import Application, boolean_flag, catch_config_error |
|
34 | from IPython.config.application import Application, boolean_flag, catch_config_error | |
35 | from IPython.config.loader import Config |
|
35 | from IPython.config.loader import Config | |
36 | from IPython.core.application import BaseIPythonApplication |
|
36 | from IPython.core.application import BaseIPythonApplication | |
37 | from IPython.core.profiledir import ProfileDir |
|
37 | from IPython.core.profiledir import ProfileDir | |
38 | from IPython.utils.daemonize import daemonize |
|
38 | from IPython.utils.daemonize import daemonize | |
39 | from IPython.utils.importstring import import_item |
|
39 | from IPython.utils.importstring import import_item | |
40 | from IPython.utils.sysinfo import num_cpus |
|
40 | from IPython.utils.sysinfo import num_cpus | |
41 | from IPython.utils.traitlets import (Integer, Unicode, Bool, CFloat, Dict, List, Any, |
|
41 | from IPython.utils.traitlets import (Integer, Unicode, Bool, CFloat, Dict, List, Any, | |
42 | DottedObjectName) |
|
42 | DottedObjectName) | |
43 |
|
43 | |||
44 | from IPython.parallel.apps.baseapp import ( |
|
44 | from IPython.parallel.apps.baseapp import ( | |
45 | BaseParallelApplication, |
|
45 | BaseParallelApplication, | |
46 | PIDFileError, |
|
46 | PIDFileError, | |
47 | base_flags, base_aliases |
|
47 | base_flags, base_aliases | |
48 | ) |
|
48 | ) | |
49 |
|
49 | |||
50 |
|
50 | |||
51 | #----------------------------------------------------------------------------- |
|
51 | #----------------------------------------------------------------------------- | |
52 | # Module level variables |
|
52 | # Module level variables | |
53 | #----------------------------------------------------------------------------- |
|
53 | #----------------------------------------------------------------------------- | |
54 |
|
54 | |||
55 |
|
55 | |||
56 | default_config_file_name = u'ipcluster_config.py' |
|
56 | default_config_file_name = u'ipcluster_config.py' | |
57 |
|
57 | |||
58 |
|
58 | |||
59 | _description = """Start an IPython cluster for parallel computing. |
|
59 | _description = """Start an IPython cluster for parallel computing. | |
60 |
|
60 | |||
61 | An IPython cluster consists of 1 controller and 1 or more engines. |
|
61 | An IPython cluster consists of 1 controller and 1 or more engines. | |
62 | This command automates the startup of these processes using a wide range of |
|
62 | This command automates the startup of these processes using a wide range of | |
63 | startup methods (SSH, local processes, PBS, mpiexec, SGE, LSF, HTCondor, |
|
63 | startup methods (SSH, local processes, PBS, mpiexec, SGE, LSF, HTCondor, | |
64 | Windows HPC Server 2008). To start a cluster with 4 engines on your |
|
64 | Windows HPC Server 2008). To start a cluster with 4 engines on your | |
65 | local host simply do 'ipcluster start --n=4'. For more complex usage |
|
65 | local host simply do 'ipcluster start --n=4'. For more complex usage | |
66 | you will typically do 'ipython profile create mycluster --parallel', then edit |
|
66 | you will typically do 'ipython profile create mycluster --parallel', then edit | |
67 | configuration files, followed by 'ipcluster start --profile=mycluster --n=4'. |
|
67 | configuration files, followed by 'ipcluster start --profile=mycluster --n=4'. | |
68 | """ |
|
68 | """ | |
69 |
|
69 | |||
70 | _main_examples = """ |
|
70 | _main_examples = """ | |
71 | ipcluster start --n=4 # start a 4 node cluster on localhost |
|
71 | ipcluster start --n=4 # start a 4 node cluster on localhost | |
72 | ipcluster start -h # show the help string for the start subcmd |
|
72 | ipcluster start -h # show the help string for the start subcmd | |
73 |
|
73 | |||
74 | ipcluster stop -h # show the help string for the stop subcmd |
|
74 | ipcluster stop -h # show the help string for the stop subcmd | |
75 | ipcluster engines -h # show the help string for the engines subcmd |
|
75 | ipcluster engines -h # show the help string for the engines subcmd | |
76 | """ |
|
76 | """ | |
77 |
|
77 | |||
78 | _start_examples = """ |
|
78 | _start_examples = """ | |
79 | ipython profile create mycluster --parallel # create mycluster profile |
|
79 | ipython profile create mycluster --parallel # create mycluster profile | |
80 | ipcluster start --profile=mycluster --n=4 # start mycluster with 4 nodes |
|
80 | ipcluster start --profile=mycluster --n=4 # start mycluster with 4 nodes | |
81 | """ |
|
81 | """ | |
82 |
|
82 | |||
83 | _stop_examples = """ |
|
83 | _stop_examples = """ | |
84 | ipcluster stop --profile=mycluster # stop a running cluster by profile name |
|
84 | ipcluster stop --profile=mycluster # stop a running cluster by profile name | |
85 | """ |
|
85 | """ | |
86 |
|
86 | |||
87 | _engines_examples = """ |
|
87 | _engines_examples = """ | |
88 | ipcluster engines --profile=mycluster --n=4 # start 4 engines only |
|
88 | ipcluster engines --profile=mycluster --n=4 # start 4 engines only | |
89 | """ |
|
89 | """ | |
90 |
|
90 | |||
91 |
|
91 | |||
92 | # Exit codes for ipcluster |
|
92 | # Exit codes for ipcluster | |
93 |
|
93 | |||
94 | # This will be the exit code if the ipcluster appears to be running because |
|
94 | # This will be the exit code if the ipcluster appears to be running because | |
95 | # a .pid file exists |
|
95 | # a .pid file exists | |
96 | ALREADY_STARTED = 10 |
|
96 | ALREADY_STARTED = 10 | |
97 |
|
97 | |||
98 |
|
98 | |||
99 | # This will be the exit code if ipcluster stop is run, but there is not .pid |
|
99 | # This will be the exit code if ipcluster stop is run, but there is not .pid | |
100 | # file to be found. |
|
100 | # file to be found. | |
101 | ALREADY_STOPPED = 11 |
|
101 | ALREADY_STOPPED = 11 | |
102 |
|
102 | |||
103 | # This will be the exit code if ipcluster engines is run, but there is not .pid |
|
103 | # This will be the exit code if ipcluster engines is run, but there is not .pid | |
104 | # file to be found. |
|
104 | # file to be found. | |
105 | NO_CLUSTER = 12 |
|
105 | NO_CLUSTER = 12 | |
106 |
|
106 | |||
107 |
|
107 | |||
108 | #----------------------------------------------------------------------------- |
|
108 | #----------------------------------------------------------------------------- | |
109 | # Utilities |
|
109 | # Utilities | |
110 | #----------------------------------------------------------------------------- |
|
110 | #----------------------------------------------------------------------------- | |
111 |
|
111 | |||
112 | def find_launcher_class(clsname, kind): |
|
112 | def find_launcher_class(clsname, kind): | |
113 | """Return a launcher for a given clsname and kind. |
|
113 | """Return a launcher for a given clsname and kind. | |
114 |
|
114 | |||
115 | Parameters |
|
115 | Parameters | |
116 | ========== |
|
116 | ========== | |
117 | clsname : str |
|
117 | clsname : str | |
118 | The full name of the launcher class, either with or without the |
|
118 | The full name of the launcher class, either with or without the | |
119 | module path, or an abbreviation (MPI, SSH, SGE, PBS, LSF, HTCondor |
|
119 | module path, or an abbreviation (MPI, SSH, SGE, PBS, LSF, HTCondor | |
120 | WindowsHPC). |
|
120 | WindowsHPC). | |
121 | kind : str |
|
121 | kind : str | |
122 | Either 'EngineSet' or 'Controller'. |
|
122 | Either 'EngineSet' or 'Controller'. | |
123 | """ |
|
123 | """ | |
124 | if '.' not in clsname: |
|
124 | if '.' not in clsname: | |
125 | # not a module, presume it's the raw name in apps.launcher |
|
125 | # not a module, presume it's the raw name in apps.launcher | |
126 | if kind and kind not in clsname: |
|
126 | if kind and kind not in clsname: | |
127 | # doesn't match necessary full class name, assume it's |
|
127 | # doesn't match necessary full class name, assume it's | |
128 | # just 'PBS' or 'MPI' etc prefix: |
|
128 | # just 'PBS' or 'MPI' etc prefix: | |
129 | clsname = clsname + kind + 'Launcher' |
|
129 | clsname = clsname + kind + 'Launcher' | |
130 | clsname = 'IPython.parallel.apps.launcher.'+clsname |
|
130 | clsname = 'IPython.parallel.apps.launcher.'+clsname | |
131 | klass = import_item(clsname) |
|
131 | klass = import_item(clsname) | |
132 | return klass |
|
132 | return klass | |
133 |
|
133 | |||
134 | #----------------------------------------------------------------------------- |
|
134 | #----------------------------------------------------------------------------- | |
135 | # Main application |
|
135 | # Main application | |
136 | #----------------------------------------------------------------------------- |
|
136 | #----------------------------------------------------------------------------- | |
137 |
|
137 | |||
138 | start_help = """Start an IPython cluster for parallel computing |
|
138 | start_help = """Start an IPython cluster for parallel computing | |
139 |
|
139 | |||
140 | Start an ipython cluster by its profile name or cluster |
|
140 | Start an ipython cluster by its profile name or cluster | |
141 | directory. Cluster directories contain configuration, log and |
|
141 | directory. Cluster directories contain configuration, log and | |
142 | security related files and are named using the convention |
|
142 | security related files and are named using the convention | |
143 | 'profile_<name>' and should be creating using the 'start' |
|
143 | 'profile_<name>' and should be creating using the 'start' | |
144 | subcommand of 'ipcluster'. If your cluster directory is in |
|
144 | subcommand of 'ipcluster'. If your cluster directory is in | |
145 | the cwd or the ipython directory, you can simply refer to it |
|
145 | the cwd or the ipython directory, you can simply refer to it | |
146 | using its profile name, 'ipcluster start --n=4 --profile=<profile>`, |
|
146 | using its profile name, 'ipcluster start --n=4 --profile=<profile>`, | |
147 | otherwise use the 'profile-dir' option. |
|
147 | otherwise use the 'profile-dir' option. | |
148 | """ |
|
148 | """ | |
149 | stop_help = """Stop a running IPython cluster |
|
149 | stop_help = """Stop a running IPython cluster | |
150 |
|
150 | |||
151 | Stop a running ipython cluster by its profile name or cluster |
|
151 | Stop a running ipython cluster by its profile name or cluster | |
152 | directory. Cluster directories are named using the convention |
|
152 | directory. Cluster directories are named using the convention | |
153 | 'profile_<name>'. If your cluster directory is in |
|
153 | 'profile_<name>'. If your cluster directory is in | |
154 | the cwd or the ipython directory, you can simply refer to it |
|
154 | the cwd or the ipython directory, you can simply refer to it | |
155 | using its profile name, 'ipcluster stop --profile=<profile>`, otherwise |
|
155 | using its profile name, 'ipcluster stop --profile=<profile>`, otherwise | |
156 | use the '--profile-dir' option. |
|
156 | use the '--profile-dir' option. | |
157 | """ |
|
157 | """ | |
158 | engines_help = """Start engines connected to an existing IPython cluster |
|
158 | engines_help = """Start engines connected to an existing IPython cluster | |
159 |
|
159 | |||
160 | Start one or more engines to connect to an existing Cluster |
|
160 | Start one or more engines to connect to an existing Cluster | |
161 | by profile name or cluster directory. |
|
161 | by profile name or cluster directory. | |
162 | Cluster directories contain configuration, log and |
|
162 | Cluster directories contain configuration, log and | |
163 | security related files and are named using the convention |
|
163 | security related files and are named using the convention | |
164 | 'profile_<name>' and should be creating using the 'start' |
|
164 | 'profile_<name>' and should be creating using the 'start' | |
165 | subcommand of 'ipcluster'. If your cluster directory is in |
|
165 | subcommand of 'ipcluster'. If your cluster directory is in | |
166 | the cwd or the ipython directory, you can simply refer to it |
|
166 | the cwd or the ipython directory, you can simply refer to it | |
167 | using its profile name, 'ipcluster engines --n=4 --profile=<profile>`, |
|
167 | using its profile name, 'ipcluster engines --n=4 --profile=<profile>`, | |
168 | otherwise use the 'profile-dir' option. |
|
168 | otherwise use the 'profile-dir' option. | |
169 | """ |
|
169 | """ | |
170 | stop_aliases = dict( |
|
170 | stop_aliases = dict( | |
171 | signal='IPClusterStop.signal', |
|
171 | signal='IPClusterStop.signal', | |
172 | ) |
|
172 | ) | |
173 | stop_aliases.update(base_aliases) |
|
173 | stop_aliases.update(base_aliases) | |
174 |
|
174 | |||
175 | class IPClusterStop(BaseParallelApplication): |
|
175 | class IPClusterStop(BaseParallelApplication): | |
176 | name = u'ipcluster' |
|
176 | name = u'ipcluster' | |
177 | description = stop_help |
|
177 | description = stop_help | |
178 | examples = _stop_examples |
|
178 | examples = _stop_examples | |
179 | config_file_name = Unicode(default_config_file_name) |
|
179 | config_file_name = Unicode(default_config_file_name) | |
180 |
|
180 | |||
181 | signal = Integer(signal.SIGINT, config=True, |
|
181 | signal = Integer(signal.SIGINT, config=True, | |
182 | help="signal to use for stopping processes.") |
|
182 | help="signal to use for stopping processes.") | |
183 |
|
183 | |||
184 | aliases = Dict(stop_aliases) |
|
184 | aliases = Dict(stop_aliases) | |
185 |
|
185 | |||
186 | def start(self): |
|
186 | def start(self): | |
187 | """Start the app for the stop subcommand.""" |
|
187 | """Start the app for the stop subcommand.""" | |
188 | try: |
|
188 | try: | |
189 | pid = self.get_pid_from_file() |
|
189 | pid = self.get_pid_from_file() | |
190 | except PIDFileError: |
|
190 | except PIDFileError: | |
191 | self.log.critical( |
|
191 | self.log.critical( | |
192 | 'Could not read pid file, cluster is probably not running.' |
|
192 | 'Could not read pid file, cluster is probably not running.' | |
193 | ) |
|
193 | ) | |
194 | # Here I exit with a unusual exit status that other processes |
|
194 | # Here I exit with a unusual exit status that other processes | |
195 | # can watch for to learn how I existed. |
|
195 | # can watch for to learn how I existed. | |
196 | self.remove_pid_file() |
|
196 | self.remove_pid_file() | |
197 | self.exit(ALREADY_STOPPED) |
|
197 | self.exit(ALREADY_STOPPED) | |
198 |
|
198 | |||
199 | if not self.check_pid(pid): |
|
199 | if not self.check_pid(pid): | |
200 | self.log.critical( |
|
200 | self.log.critical( | |
201 | 'Cluster [pid=%r] is not running.' % pid |
|
201 | 'Cluster [pid=%r] is not running.' % pid | |
202 | ) |
|
202 | ) | |
203 | self.remove_pid_file() |
|
203 | self.remove_pid_file() | |
204 | # Here I exit with a unusual exit status that other processes |
|
204 | # Here I exit with a unusual exit status that other processes | |
205 | # can watch for to learn how I existed. |
|
205 | # can watch for to learn how I existed. | |
206 | self.exit(ALREADY_STOPPED) |
|
206 | self.exit(ALREADY_STOPPED) | |
207 |
|
207 | |||
208 | elif os.name=='posix': |
|
208 | elif os.name=='posix': | |
209 | sig = self.signal |
|
209 | sig = self.signal | |
210 | self.log.info( |
|
210 | self.log.info( | |
211 | "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig) |
|
211 | "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig) | |
212 | ) |
|
212 | ) | |
213 | try: |
|
213 | try: | |
214 | os.kill(pid, sig) |
|
214 | os.kill(pid, sig) | |
215 | except OSError: |
|
215 | except OSError: | |
216 | self.log.error("Stopping cluster failed, assuming already dead.", |
|
216 | self.log.error("Stopping cluster failed, assuming already dead.", | |
217 | exc_info=True) |
|
217 | exc_info=True) | |
218 | self.remove_pid_file() |
|
218 | self.remove_pid_file() | |
219 | elif os.name=='nt': |
|
219 | elif os.name=='nt': | |
220 | try: |
|
220 | try: | |
221 | # kill the whole tree |
|
221 | # kill the whole tree | |
222 | p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE) |
|
222 | p = check_call(['taskkill', '-pid', str(pid), '-t', '-f'], stdout=PIPE,stderr=PIPE) | |
223 | except (CalledProcessError, OSError): |
|
223 | except (CalledProcessError, OSError): | |
224 | self.log.error("Stopping cluster failed, assuming already dead.", |
|
224 | self.log.error("Stopping cluster failed, assuming already dead.", | |
225 | exc_info=True) |
|
225 | exc_info=True) | |
226 | self.remove_pid_file() |
|
226 | self.remove_pid_file() | |
227 |
|
227 | |||
228 | engine_aliases = {} |
|
228 | engine_aliases = {} | |
229 | engine_aliases.update(base_aliases) |
|
229 | engine_aliases.update(base_aliases) | |
230 | engine_aliases.update(dict( |
|
230 | engine_aliases.update(dict( | |
231 | n='IPClusterEngines.n', |
|
231 | n='IPClusterEngines.n', | |
232 | engines = 'IPClusterEngines.engine_launcher_class', |
|
232 | engines = 'IPClusterEngines.engine_launcher_class', | |
233 | daemonize = 'IPClusterEngines.daemonize', |
|
233 | daemonize = 'IPClusterEngines.daemonize', | |
234 | )) |
|
234 | )) | |
235 | engine_flags = {} |
|
235 | engine_flags = {} | |
236 | engine_flags.update(base_flags) |
|
236 | engine_flags.update(base_flags) | |
237 |
|
237 | |||
238 | engine_flags.update(dict( |
|
238 | engine_flags.update(dict( | |
239 | daemonize=( |
|
239 | daemonize=( | |
240 | {'IPClusterEngines' : {'daemonize' : True}}, |
|
240 | {'IPClusterEngines' : {'daemonize' : True}}, | |
241 | """run the cluster into the background (not available on Windows)""", |
|
241 | """run the cluster into the background (not available on Windows)""", | |
242 | ) |
|
242 | ) | |
243 | )) |
|
243 | )) | |
244 | class IPClusterEngines(BaseParallelApplication): |
|
244 | class IPClusterEngines(BaseParallelApplication): | |
245 |
|
245 | |||
246 | name = u'ipcluster' |
|
246 | name = u'ipcluster' | |
247 | description = engines_help |
|
247 | description = engines_help | |
248 | examples = _engines_examples |
|
248 | examples = _engines_examples | |
249 | usage = None |
|
249 | usage = None | |
250 | config_file_name = Unicode(default_config_file_name) |
|
250 | config_file_name = Unicode(default_config_file_name) | |
251 | default_log_level = logging.INFO |
|
251 | default_log_level = logging.INFO | |
252 | classes = List() |
|
252 | classes = List() | |
253 | def _classes_default(self): |
|
253 | def _classes_default(self): | |
254 | from IPython.parallel.apps import launcher |
|
254 | from IPython.parallel.apps import launcher | |
255 | launchers = launcher.all_launchers |
|
255 | launchers = launcher.all_launchers | |
256 | eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__] |
|
256 | eslaunchers = [ l for l in launchers if 'EngineSet' in l.__name__] | |
257 | return [ProfileDir]+eslaunchers |
|
257 | return [ProfileDir]+eslaunchers | |
258 |
|
258 | |||
259 | n = Integer(num_cpus(), config=True, |
|
259 | n = Integer(num_cpus(), config=True, | |
260 | help="""The number of engines to start. The default is to use one for each |
|
260 | help="""The number of engines to start. The default is to use one for each | |
261 | CPU on your machine""") |
|
261 | CPU on your machine""") | |
262 |
|
262 | |||
263 | engine_launcher = Any(config=True, help="Deprecated, use engine_launcher_class") |
|
263 | engine_launcher = Any(config=True, help="Deprecated, use engine_launcher_class") | |
264 | def _engine_launcher_changed(self, name, old, new): |
|
264 | def _engine_launcher_changed(self, name, old, new): | |
265 | if isinstance(new, basestring): |
|
265 | if isinstance(new, basestring): | |
266 | self.log.warn("WARNING: %s.engine_launcher is deprecated as of 0.12," |
|
266 | self.log.warn("WARNING: %s.engine_launcher is deprecated as of 0.12," | |
267 | " use engine_launcher_class" % self.__class__.__name__) |
|
267 | " use engine_launcher_class" % self.__class__.__name__) | |
268 | self.engine_launcher_class = new |
|
268 | self.engine_launcher_class = new | |
269 | engine_launcher_class = DottedObjectName('LocalEngineSetLauncher', |
|
269 | engine_launcher_class = DottedObjectName('LocalEngineSetLauncher', | |
270 | config=True, |
|
270 | config=True, | |
271 | help="""The class for launching a set of Engines. Change this value |
|
271 | help="""The class for launching a set of Engines. Change this value | |
272 | to use various batch systems to launch your engines, such as PBS,SGE,MPI,etc. |
|
272 | to use various batch systems to launch your engines, such as PBS,SGE,MPI,etc. | |
273 | Each launcher class has its own set of configuration options, for making sure |
|
273 | Each launcher class has its own set of configuration options, for making sure | |
274 | it will work in your environment. |
|
274 | it will work in your environment. | |
275 |
|
275 | |||
276 | You can also write your own launcher, and specify it's absolute import path, |
|
276 | You can also write your own launcher, and specify it's absolute import path, | |
277 | as in 'mymodule.launcher.FTLEnginesLauncher`. |
|
277 | as in 'mymodule.launcher.FTLEnginesLauncher`. | |
278 |
|
278 | |||
279 | IPython's bundled examples include: |
|
279 | IPython's bundled examples include: | |
280 |
|
280 | |||
281 | Local : start engines locally as subprocesses [default] |
|
281 | Local : start engines locally as subprocesses [default] | |
282 | MPI : use mpiexec to launch engines in an MPI environment |
|
282 | MPI : use mpiexec to launch engines in an MPI environment | |
283 | PBS : use PBS (qsub) to submit engines to a batch queue |
|
283 | PBS : use PBS (qsub) to submit engines to a batch queue | |
284 | SGE : use SGE (qsub) to submit engines to a batch queue |
|
284 | SGE : use SGE (qsub) to submit engines to a batch queue | |
285 | LSF : use LSF (bsub) to submit engines to a batch queue |
|
285 | LSF : use LSF (bsub) to submit engines to a batch queue | |
286 | SSH : use SSH to start the controller |
|
286 | SSH : use SSH to start the controller | |
287 | Note that SSH does *not* move the connection files |
|
287 | Note that SSH does *not* move the connection files | |
288 | around, so you will likely have to do this manually |
|
288 | around, so you will likely have to do this manually | |
289 | unless the machines are on a shared file system. |
|
289 | unless the machines are on a shared file system. | |
290 | HTCondor : use HTCondor to submit engines to a batch queue |
|
290 | HTCondor : use HTCondor to submit engines to a batch queue | |
291 | WindowsHPC : use Windows HPC |
|
291 | WindowsHPC : use Windows HPC | |
292 |
|
292 | |||
293 | If you are using one of IPython's builtin launchers, you can specify just the |
|
293 | If you are using one of IPython's builtin launchers, you can specify just the | |
294 | prefix, e.g: |
|
294 | prefix, e.g: | |
295 |
|
295 | |||
296 | c.IPClusterEngines.engine_launcher_class = 'SSH' |
|
296 | c.IPClusterEngines.engine_launcher_class = 'SSH' | |
297 |
|
297 | |||
298 | or: |
|
298 | or: | |
299 |
|
299 | |||
300 | ipcluster start --engines=MPI |
|
300 | ipcluster start --engines=MPI | |
301 |
|
301 | |||
302 | """ |
|
302 | """ | |
303 | ) |
|
303 | ) | |
304 | daemonize = Bool(False, config=True, |
|
304 | daemonize = Bool(False, config=True, | |
305 | help="""Daemonize the ipcluster program. This implies --log-to-file. |
|
305 | help="""Daemonize the ipcluster program. This implies --log-to-file. | |
306 | Not available on Windows. |
|
306 | Not available on Windows. | |
307 | """) |
|
307 | """) | |
308 |
|
308 | |||
309 | def _daemonize_changed(self, name, old, new): |
|
309 | def _daemonize_changed(self, name, old, new): | |
310 | if new: |
|
310 | if new: | |
311 | self.log_to_file = True |
|
311 | self.log_to_file = True | |
312 |
|
312 | |||
313 | early_shutdown = Integer(30, config=True, help="The timeout (in seconds)") |
|
313 | early_shutdown = Integer(30, config=True, help="The timeout (in seconds)") | |
314 | _stopping = False |
|
314 | _stopping = False | |
315 |
|
315 | |||
316 | aliases = Dict(engine_aliases) |
|
316 | aliases = Dict(engine_aliases) | |
317 | flags = Dict(engine_flags) |
|
317 | flags = Dict(engine_flags) | |
318 |
|
318 | |||
319 | @catch_config_error |
|
319 | @catch_config_error | |
320 | def initialize(self, argv=None): |
|
320 | def initialize(self, argv=None): | |
321 | super(IPClusterEngines, self).initialize(argv) |
|
321 | super(IPClusterEngines, self).initialize(argv) | |
322 | self.init_signal() |
|
322 | self.init_signal() | |
323 | self.init_launchers() |
|
323 | self.init_launchers() | |
324 |
|
324 | |||
325 | def init_launchers(self): |
|
325 | def init_launchers(self): | |
326 | self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet') |
|
326 | self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet') | |
327 |
|
327 | |||
328 | def init_signal(self): |
|
328 | def init_signal(self): | |
329 | # Setup signals |
|
329 | # Setup signals | |
330 | signal.signal(signal.SIGINT, self.sigint_handler) |
|
330 | signal.signal(signal.SIGINT, self.sigint_handler) | |
331 |
|
331 | |||
332 | def build_launcher(self, clsname, kind=None): |
|
332 | def build_launcher(self, clsname, kind=None): | |
333 | """import and instantiate a Launcher based on importstring""" |
|
333 | """import and instantiate a Launcher based on importstring""" | |
334 | try: |
|
334 | try: | |
335 | klass = find_launcher_class(clsname, kind) |
|
335 | klass = find_launcher_class(clsname, kind) | |
336 | except (ImportError, KeyError): |
|
336 | except (ImportError, KeyError): | |
337 | self.log.fatal("Could not import launcher class: %r"%clsname) |
|
337 | self.log.fatal("Could not import launcher class: %r"%clsname) | |
338 | self.exit(1) |
|
338 | self.exit(1) | |
339 |
|
339 | |||
340 | launcher = klass( |
|
340 | launcher = klass( | |
341 | work_dir=u'.', parent=self, log=self.log, |
|
341 | work_dir=u'.', parent=self, log=self.log, | |
342 | profile_dir=self.profile_dir.location, cluster_id=self.cluster_id, |
|
342 | profile_dir=self.profile_dir.location, cluster_id=self.cluster_id, | |
343 | ) |
|
343 | ) | |
344 | return launcher |
|
344 | return launcher | |
345 |
|
345 | |||
346 | def engines_started_ok(self): |
|
346 | def engines_started_ok(self): | |
347 | self.log.info("Engines appear to have started successfully") |
|
347 | self.log.info("Engines appear to have started successfully") | |
348 | self.early_shutdown = 0 |
|
348 | self.early_shutdown = 0 | |
349 |
|
349 | |||
350 | def start_engines(self): |
|
350 | def start_engines(self): | |
351 | # Some EngineSetLaunchers ignore `n` and use their own engine count, such as SSH: |
|
351 | # Some EngineSetLaunchers ignore `n` and use their own engine count, such as SSH: | |
352 | n = getattr(self.engine_launcher, 'engine_count', self.n) |
|
352 | n = getattr(self.engine_launcher, 'engine_count', self.n) | |
353 | self.log.info("Starting %s Engines with %s", n, self.engine_launcher_class) |
|
353 | self.log.info("Starting %s Engines with %s", n, self.engine_launcher_class) | |
354 | self.engine_launcher.start(self.n) |
|
354 | self.engine_launcher.start(self.n) | |
355 | self.engine_launcher.on_stop(self.engines_stopped_early) |
|
355 | self.engine_launcher.on_stop(self.engines_stopped_early) | |
356 | if self.early_shutdown: |
|
356 | if self.early_shutdown: | |
357 | ioloop.DelayedCallback(self.engines_started_ok, self.early_shutdown*1000, self.loop).start() |
|
357 | ioloop.DelayedCallback(self.engines_started_ok, self.early_shutdown*1000, self.loop).start() | |
358 |
|
358 | |||
359 | def engines_stopped_early(self, r): |
|
359 | def engines_stopped_early(self, r): | |
360 | if self.early_shutdown and not self._stopping: |
|
360 | if self.early_shutdown and not self._stopping: | |
361 | self.log.error(""" |
|
361 | self.log.error(""" | |
362 | Engines shutdown early, they probably failed to connect. |
|
362 | Engines shutdown early, they probably failed to connect. | |
363 |
|
363 | |||
364 | Check the engine log files for output. |
|
364 | Check the engine log files for output. | |
365 |
|
365 | |||
366 | If your controller and engines are not on the same machine, you probably |
|
366 | If your controller and engines are not on the same machine, you probably | |
367 | have to instruct the controller to listen on an interface other than localhost. |
|
367 | have to instruct the controller to listen on an interface other than localhost. | |
368 |
|
368 | |||
369 | You can set this by adding "--ip='*'" to your ControllerLauncher.controller_args. |
|
369 | You can set this by adding "--ip='*'" to your ControllerLauncher.controller_args. | |
370 |
|
370 | |||
371 | Be sure to read our security docs before instructing your controller to listen on |
|
371 | Be sure to read our security docs before instructing your controller to listen on | |
372 | a public interface. |
|
372 | a public interface. | |
373 | """) |
|
373 | """) | |
374 | self.stop_launchers() |
|
374 | self.stop_launchers() | |
375 |
|
375 | |||
376 | return self.engines_stopped(r) |
|
376 | return self.engines_stopped(r) | |
377 |
|
377 | |||
378 | def engines_stopped(self, r): |
|
378 | def engines_stopped(self, r): | |
379 | return self.loop.stop() |
|
379 | return self.loop.stop() | |
380 |
|
380 | |||
381 | def stop_engines(self): |
|
381 | def stop_engines(self): | |
382 | if self.engine_launcher.running: |
|
382 | if self.engine_launcher.running: | |
383 | self.log.info("Stopping Engines...") |
|
383 | self.log.info("Stopping Engines...") | |
384 | d = self.engine_launcher.stop() |
|
384 | d = self.engine_launcher.stop() | |
385 | return d |
|
385 | return d | |
386 | else: |
|
386 | else: | |
387 | return None |
|
387 | return None | |
388 |
|
388 | |||
389 | def stop_launchers(self, r=None): |
|
389 | def stop_launchers(self, r=None): | |
390 | if not self._stopping: |
|
390 | if not self._stopping: | |
391 | self._stopping = True |
|
391 | self._stopping = True | |
392 | self.log.error("IPython cluster: stopping") |
|
392 | self.log.error("IPython cluster: stopping") | |
393 | self.stop_engines() |
|
393 | self.stop_engines() | |
394 | # Wait a few seconds to let things shut down. |
|
394 | # Wait a few seconds to let things shut down. | |
395 | dc = ioloop.DelayedCallback(self.loop.stop, 3000, self.loop) |
|
395 | dc = ioloop.DelayedCallback(self.loop.stop, 3000, self.loop) | |
396 | dc.start() |
|
396 | dc.start() | |
397 |
|
397 | |||
398 | def sigint_handler(self, signum, frame): |
|
398 | def sigint_handler(self, signum, frame): | |
399 | self.log.debug("SIGINT received, stopping launchers...") |
|
399 | self.log.debug("SIGINT received, stopping launchers...") | |
400 | self.stop_launchers() |
|
400 | self.stop_launchers() | |
401 |
|
401 | |||
402 | def start_logging(self): |
|
402 | def start_logging(self): | |
403 | # Remove old log files of the controller and engine |
|
403 | # Remove old log files of the controller and engine | |
404 | if self.clean_logs: |
|
404 | if self.clean_logs: | |
405 | log_dir = self.profile_dir.log_dir |
|
405 | log_dir = self.profile_dir.log_dir | |
406 | for f in os.listdir(log_dir): |
|
406 | for f in os.listdir(log_dir): | |
407 | if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f): |
|
407 | if re.match(r'ip(engine|controller)z-\d+\.(log|err|out)',f): | |
408 | os.remove(os.path.join(log_dir, f)) |
|
408 | os.remove(os.path.join(log_dir, f)) | |
409 | # This will remove old log files for ipcluster itself |
|
409 | # This will remove old log files for ipcluster itself | |
410 | # super(IPBaseParallelApplication, self).start_logging() |
|
410 | # super(IPBaseParallelApplication, self).start_logging() | |
411 |
|
411 | |||
412 | def start(self): |
|
412 | def start(self): | |
413 | """Start the app for the engines subcommand.""" |
|
413 | """Start the app for the engines subcommand.""" | |
414 | self.log.info("IPython cluster: started") |
|
414 | self.log.info("IPython cluster: started") | |
415 | # First see if the cluster is already running |
|
415 | # First see if the cluster is already running | |
416 |
|
416 | |||
417 | # Now log and daemonize |
|
417 | # Now log and daemonize | |
418 | self.log.info( |
|
418 | self.log.info( | |
419 | 'Starting engines with [daemon=%r]' % self.daemonize |
|
419 | 'Starting engines with [daemon=%r]' % self.daemonize | |
420 | ) |
|
420 | ) | |
421 | # TODO: Get daemonize working on Windows or as a Windows Server. |
|
421 | # TODO: Get daemonize working on Windows or as a Windows Server. | |
422 | if self.daemonize: |
|
422 | if self.daemonize: | |
423 | if os.name=='posix': |
|
423 | if os.name=='posix': | |
424 | daemonize() |
|
424 | daemonize() | |
425 |
|
425 | |||
426 | dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop) |
|
426 | dc = ioloop.DelayedCallback(self.start_engines, 0, self.loop) | |
427 | dc.start() |
|
427 | dc.start() | |
428 | # Now write the new pid file AFTER our new forked pid is active. |
|
428 | # Now write the new pid file AFTER our new forked pid is active. | |
429 | # self.write_pid_file() |
|
429 | # self.write_pid_file() | |
430 | try: |
|
430 | try: | |
431 | self.loop.start() |
|
431 | self.loop.start() | |
432 | except KeyboardInterrupt: |
|
432 | except KeyboardInterrupt: | |
433 | pass |
|
433 | pass | |
434 | except zmq.ZMQError as e: |
|
434 | except zmq.ZMQError as e: | |
435 | if e.errno == errno.EINTR: |
|
435 | if e.errno == errno.EINTR: | |
436 | pass |
|
436 | pass | |
437 | else: |
|
437 | else: | |
438 | raise |
|
438 | raise | |
439 |
|
439 | |||
440 | start_aliases = {} |
|
440 | start_aliases = {} | |
441 | start_aliases.update(engine_aliases) |
|
441 | start_aliases.update(engine_aliases) | |
442 | start_aliases.update(dict( |
|
442 | start_aliases.update(dict( | |
443 | delay='IPClusterStart.delay', |
|
443 | delay='IPClusterStart.delay', | |
444 | controller = 'IPClusterStart.controller_launcher_class', |
|
444 | controller = 'IPClusterStart.controller_launcher_class', | |
445 | )) |
|
445 | )) | |
446 | start_aliases['clean-logs'] = 'IPClusterStart.clean_logs' |
|
446 | start_aliases['clean-logs'] = 'IPClusterStart.clean_logs' | |
447 |
|
447 | |||
448 | class IPClusterStart(IPClusterEngines): |
|
448 | class IPClusterStart(IPClusterEngines): | |
449 |
|
449 | |||
450 | name = u'ipcluster' |
|
450 | name = u'ipcluster' | |
451 | description = start_help |
|
451 | description = start_help | |
452 | examples = _start_examples |
|
452 | examples = _start_examples | |
453 | default_log_level = logging.INFO |
|
453 | default_log_level = logging.INFO | |
454 | auto_create = Bool(True, config=True, |
|
454 | auto_create = Bool(True, config=True, | |
455 | help="whether to create the profile_dir if it doesn't exist") |
|
455 | help="whether to create the profile_dir if it doesn't exist") | |
456 | classes = List() |
|
456 | classes = List() | |
457 | def _classes_default(self,): |
|
457 | def _classes_default(self,): | |
458 | from IPython.parallel.apps import launcher |
|
458 | from IPython.parallel.apps import launcher | |
459 | return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers |
|
459 | return [ProfileDir] + [IPClusterEngines] + launcher.all_launchers | |
460 |
|
460 | |||
461 | clean_logs = Bool(True, config=True, |
|
461 | clean_logs = Bool(True, config=True, | |
462 | help="whether to cleanup old logs before starting") |
|
462 | help="whether to cleanup old logs before starting") | |
463 |
|
463 | |||
464 | delay = CFloat(1., config=True, |
|
464 | delay = CFloat(1., config=True, | |
465 | help="delay (in s) between starting the controller and the engines") |
|
465 | help="delay (in s) between starting the controller and the engines") | |
466 |
|
466 | |||
467 | controller_launcher = Any(config=True, help="Deprecated, use controller_launcher_class") |
|
467 | controller_launcher = Any(config=True, help="Deprecated, use controller_launcher_class") | |
468 | def _controller_launcher_changed(self, name, old, new): |
|
468 | def _controller_launcher_changed(self, name, old, new): | |
469 | if isinstance(new, basestring): |
|
469 | if isinstance(new, basestring): | |
470 | # old 0.11-style config |
|
470 | # old 0.11-style config | |
471 | self.log.warn("WARNING: %s.controller_launcher is deprecated as of 0.12," |
|
471 | self.log.warn("WARNING: %s.controller_launcher is deprecated as of 0.12," | |
472 | " use controller_launcher_class" % self.__class__.__name__) |
|
472 | " use controller_launcher_class" % self.__class__.__name__) | |
473 | self.controller_launcher_class = new |
|
473 | self.controller_launcher_class = new | |
474 | controller_launcher_class = DottedObjectName('LocalControllerLauncher', |
|
474 | controller_launcher_class = DottedObjectName('LocalControllerLauncher', | |
475 | config=True, |
|
475 | config=True, | |
476 | help="""The class for launching a Controller. Change this value if you want |
|
476 | help="""The class for launching a Controller. Change this value if you want | |
477 | your controller to also be launched by a batch system, such as PBS,SGE,MPI,etc. |
|
477 | your controller to also be launched by a batch system, such as PBS,SGE,MPI,etc. | |
478 |
|
478 | |||
479 | Each launcher class has its own set of configuration options, for making sure |
|
479 | Each launcher class has its own set of configuration options, for making sure | |
480 | it will work in your environment. |
|
480 | it will work in your environment. | |
481 |
|
481 | |||
482 | Note that using a batch launcher for the controller *does not* put it |
|
482 | Note that using a batch launcher for the controller *does not* put it | |
483 | in the same batch job as the engines, so they will still start separately. |
|
483 | in the same batch job as the engines, so they will still start separately. | |
484 |
|
484 | |||
485 | IPython's bundled examples include: |
|
485 | IPython's bundled examples include: | |
486 |
|
486 | |||
487 | Local : start engines locally as subprocesses |
|
487 | Local : start engines locally as subprocesses | |
488 | MPI : use mpiexec to launch the controller in an MPI universe |
|
488 | MPI : use mpiexec to launch the controller in an MPI universe | |
489 | PBS : use PBS (qsub) to submit the controller to a batch queue |
|
489 | PBS : use PBS (qsub) to submit the controller to a batch queue | |
490 | SGE : use SGE (qsub) to submit the controller to a batch queue |
|
490 | SGE : use SGE (qsub) to submit the controller to a batch queue | |
491 | LSF : use LSF (bsub) to submit the controller to a batch queue |
|
491 | LSF : use LSF (bsub) to submit the controller to a batch queue | |
492 | HTCondor : use HTCondor to submit the controller to a batch queue |
|
492 | HTCondor : use HTCondor to submit the controller to a batch queue | |
493 | SSH : use SSH to start the controller |
|
493 | SSH : use SSH to start the controller | |
494 | WindowsHPC : use Windows HPC |
|
494 | WindowsHPC : use Windows HPC | |
495 |
|
495 | |||
496 | If you are using one of IPython's builtin launchers, you can specify just the |
|
496 | If you are using one of IPython's builtin launchers, you can specify just the | |
497 | prefix, e.g: |
|
497 | prefix, e.g: | |
498 |
|
498 | |||
499 | c.IPClusterStart.controller_launcher_class = 'SSH' |
|
499 | c.IPClusterStart.controller_launcher_class = 'SSH' | |
500 |
|
500 | |||
501 | or: |
|
501 | or: | |
502 |
|
502 | |||
503 | ipcluster start --controller=MPI |
|
503 | ipcluster start --controller=MPI | |
504 |
|
504 | |||
505 | """ |
|
505 | """ | |
506 | ) |
|
506 | ) | |
507 | reset = Bool(False, config=True, |
|
507 | reset = Bool(False, config=True, | |
508 | help="Whether to reset config files as part of '--create'." |
|
508 | help="Whether to reset config files as part of '--create'." | |
509 | ) |
|
509 | ) | |
510 |
|
510 | |||
511 | # flags = Dict(flags) |
|
511 | # flags = Dict(flags) | |
512 | aliases = Dict(start_aliases) |
|
512 | aliases = Dict(start_aliases) | |
513 |
|
513 | |||
514 | def init_launchers(self): |
|
514 | def init_launchers(self): | |
515 | self.controller_launcher = self.build_launcher(self.controller_launcher_class, 'Controller') |
|
515 | self.controller_launcher = self.build_launcher(self.controller_launcher_class, 'Controller') | |
516 | self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet') |
|
516 | self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet') | |
517 |
|
517 | |||
518 | def engines_stopped(self, r): |
|
518 | def engines_stopped(self, r): | |
519 | """prevent parent.engines_stopped from stopping everything on engine shutdown""" |
|
519 | """prevent parent.engines_stopped from stopping everything on engine shutdown""" | |
520 | pass |
|
520 | pass | |
521 |
|
521 | |||
522 | def start_controller(self): |
|
522 | def start_controller(self): | |
523 | self.log.info("Starting Controller with %s", self.controller_launcher_class) |
|
523 | self.log.info("Starting Controller with %s", self.controller_launcher_class) | |
524 | self.controller_launcher.on_stop(self.stop_launchers) |
|
524 | self.controller_launcher.on_stop(self.stop_launchers) | |
525 | self.controller_launcher.start() |
|
525 | self.controller_launcher.start() | |
526 |
|
526 | |||
527 | def stop_controller(self): |
|
527 | def stop_controller(self): | |
528 | # self.log.info("In stop_controller") |
|
528 | # self.log.info("In stop_controller") | |
529 | if self.controller_launcher and self.controller_launcher.running: |
|
529 | if self.controller_launcher and self.controller_launcher.running: | |
530 | return self.controller_launcher.stop() |
|
530 | return self.controller_launcher.stop() | |
531 |
|
531 | |||
532 | def stop_launchers(self, r=None): |
|
532 | def stop_launchers(self, r=None): | |
533 | if not self._stopping: |
|
533 | if not self._stopping: | |
534 | self.stop_controller() |
|
534 | self.stop_controller() | |
535 | super(IPClusterStart, self).stop_launchers() |
|
535 | super(IPClusterStart, self).stop_launchers() | |
536 |
|
536 | |||
537 | def start(self): |
|
537 | def start(self): | |
538 | """Start the app for the start subcommand.""" |
|
538 | """Start the app for the start subcommand.""" | |
539 | # First see if the cluster is already running |
|
539 | # First see if the cluster is already running | |
540 | try: |
|
540 | try: | |
541 | pid = self.get_pid_from_file() |
|
541 | pid = self.get_pid_from_file() | |
542 | except PIDFileError: |
|
542 | except PIDFileError: | |
543 | pass |
|
543 | pass | |
544 | else: |
|
544 | else: | |
545 | if self.check_pid(pid): |
|
545 | if self.check_pid(pid): | |
546 | self.log.critical( |
|
546 | self.log.critical( | |
547 | 'Cluster is already running with [pid=%s]. ' |
|
547 | 'Cluster is already running with [pid=%s]. ' | |
548 | 'use "ipcluster stop" to stop the cluster.' % pid |
|
548 | 'use "ipcluster stop" to stop the cluster.' % pid | |
549 | ) |
|
549 | ) | |
550 | # Here I exit with a unusual exit status that other processes |
|
550 | # Here I exit with a unusual exit status that other processes | |
551 | # can watch for to learn how I existed. |
|
551 | # can watch for to learn how I existed. | |
552 | self.exit(ALREADY_STARTED) |
|
552 | self.exit(ALREADY_STARTED) | |
553 | else: |
|
553 | else: | |
554 | self.remove_pid_file() |
|
554 | self.remove_pid_file() | |
555 |
|
555 | |||
556 |
|
556 | |||
557 | # Now log and daemonize |
|
557 | # Now log and daemonize | |
558 | self.log.info( |
|
558 | self.log.info( | |
559 | 'Starting ipcluster with [daemon=%r]' % self.daemonize |
|
559 | 'Starting ipcluster with [daemon=%r]' % self.daemonize | |
560 | ) |
|
560 | ) | |
561 | # TODO: Get daemonize working on Windows or as a Windows Server. |
|
561 | # TODO: Get daemonize working on Windows or as a Windows Server. | |
562 | if self.daemonize: |
|
562 | if self.daemonize: | |
563 | if os.name=='posix': |
|
563 | if os.name=='posix': | |
564 | daemonize() |
|
564 | daemonize() | |
565 |
|
565 | |||
566 | dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop) |
|
566 | dc = ioloop.DelayedCallback(self.start_controller, 0, self.loop) | |
567 | dc.start() |
|
567 | dc.start() | |
568 | dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop) |
|
568 | dc = ioloop.DelayedCallback(self.start_engines, 1000*self.delay, self.loop) | |
569 | dc.start() |
|
569 | dc.start() | |
570 | # Now write the new pid file AFTER our new forked pid is active. |
|
570 | # Now write the new pid file AFTER our new forked pid is active. | |
571 | self.write_pid_file() |
|
571 | self.write_pid_file() | |
572 | try: |
|
572 | try: | |
573 | self.loop.start() |
|
573 | self.loop.start() | |
574 | except KeyboardInterrupt: |
|
574 | except KeyboardInterrupt: | |
575 | pass |
|
575 | pass | |
576 | except zmq.ZMQError as e: |
|
576 | except zmq.ZMQError as e: | |
577 | if e.errno == errno.EINTR: |
|
577 | if e.errno == errno.EINTR: | |
578 | pass |
|
578 | pass | |
579 | else: |
|
579 | else: | |
580 | raise |
|
580 | raise | |
581 | finally: |
|
581 | finally: | |
582 | self.remove_pid_file() |
|
582 | self.remove_pid_file() | |
583 |
|
583 | |||
584 | base='IPython.parallel.apps.ipclusterapp.IPCluster' |
|
584 | base='IPython.parallel.apps.ipclusterapp.IPCluster' | |
585 |
|
585 | |||
586 | class IPClusterApp(BaseIPythonApplication): |
|
586 | class IPClusterApp(BaseIPythonApplication): | |
587 | name = u'ipcluster' |
|
587 | name = u'ipcluster' | |
588 | description = _description |
|
588 | description = _description | |
589 | examples = _main_examples |
|
589 | examples = _main_examples | |
590 |
|
590 | |||
591 | subcommands = { |
|
591 | subcommands = { | |
592 | 'start' : (base+'Start', start_help), |
|
592 | 'start' : (base+'Start', start_help), | |
593 | 'stop' : (base+'Stop', stop_help), |
|
593 | 'stop' : (base+'Stop', stop_help), | |
594 | 'engines' : (base+'Engines', engines_help), |
|
594 | 'engines' : (base+'Engines', engines_help), | |
595 | } |
|
595 | } | |
596 |
|
596 | |||
597 | # no aliases or flags for parent App |
|
597 | # no aliases or flags for parent App | |
598 | aliases = Dict() |
|
598 | aliases = Dict() | |
599 | flags = Dict() |
|
599 | flags = Dict() | |
600 |
|
600 | |||
601 | def start(self): |
|
601 | def start(self): | |
602 | if self.subapp is None: |
|
602 | if self.subapp is None: | |
603 | print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys()) |
|
603 | print "No subcommand specified. Must specify one of: %s"%(self.subcommands.keys()) | |
604 |
|
604 | |||
605 | self.print_description() |
|
605 | self.print_description() | |
606 | self.print_subcommands() |
|
606 | self.print_subcommands() | |
607 | self.exit(1) |
|
607 | self.exit(1) | |
608 | else: |
|
608 | else: | |
609 | return self.subapp.start() |
|
609 | return self.subapp.start() | |
610 |
|
610 | |||
611 |
launch_new_instance = IPClusterApp.launch_ |
|
611 | launch_new_instance = IPClusterApp.launch_instance | |
612 |
|
612 | |||
613 | if __name__ == '__main__': |
|
613 | if __name__ == '__main__': | |
614 | launch_new_instance() |
|
614 | launch_new_instance() | |
615 |
|
615 |
@@ -1,551 +1,551 | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | The IPython controller application. |
|
4 | The IPython controller application. | |
5 |
|
5 | |||
6 | Authors: |
|
6 | Authors: | |
7 |
|
7 | |||
8 | * Brian Granger |
|
8 | * Brian Granger | |
9 | * MinRK |
|
9 | * MinRK | |
10 |
|
10 | |||
11 | """ |
|
11 | """ | |
12 |
|
12 | |||
13 | #----------------------------------------------------------------------------- |
|
13 | #----------------------------------------------------------------------------- | |
14 | # Copyright (C) 2008-2011 The IPython Development Team |
|
14 | # Copyright (C) 2008-2011 The IPython Development Team | |
15 | # |
|
15 | # | |
16 | # Distributed under the terms of the BSD License. The full license is in |
|
16 | # Distributed under the terms of the BSD License. The full license is in | |
17 | # the file COPYING, distributed as part of this software. |
|
17 | # the file COPYING, distributed as part of this software. | |
18 | #----------------------------------------------------------------------------- |
|
18 | #----------------------------------------------------------------------------- | |
19 |
|
19 | |||
20 | #----------------------------------------------------------------------------- |
|
20 | #----------------------------------------------------------------------------- | |
21 | # Imports |
|
21 | # Imports | |
22 | #----------------------------------------------------------------------------- |
|
22 | #----------------------------------------------------------------------------- | |
23 |
|
23 | |||
24 | from __future__ import with_statement |
|
24 | from __future__ import with_statement | |
25 |
|
25 | |||
26 | import json |
|
26 | import json | |
27 | import os |
|
27 | import os | |
28 | import stat |
|
28 | import stat | |
29 | import sys |
|
29 | import sys | |
30 |
|
30 | |||
31 | from multiprocessing import Process |
|
31 | from multiprocessing import Process | |
32 | from signal import signal, SIGINT, SIGABRT, SIGTERM |
|
32 | from signal import signal, SIGINT, SIGABRT, SIGTERM | |
33 |
|
33 | |||
34 | import zmq |
|
34 | import zmq | |
35 | from zmq.devices import ProcessMonitoredQueue |
|
35 | from zmq.devices import ProcessMonitoredQueue | |
36 | from zmq.log.handlers import PUBHandler |
|
36 | from zmq.log.handlers import PUBHandler | |
37 |
|
37 | |||
38 | from IPython.core.profiledir import ProfileDir |
|
38 | from IPython.core.profiledir import ProfileDir | |
39 |
|
39 | |||
40 | from IPython.parallel.apps.baseapp import ( |
|
40 | from IPython.parallel.apps.baseapp import ( | |
41 | BaseParallelApplication, |
|
41 | BaseParallelApplication, | |
42 | base_aliases, |
|
42 | base_aliases, | |
43 | base_flags, |
|
43 | base_flags, | |
44 | catch_config_error, |
|
44 | catch_config_error, | |
45 | ) |
|
45 | ) | |
46 | from IPython.utils.importstring import import_item |
|
46 | from IPython.utils.importstring import import_item | |
47 | from IPython.utils.localinterfaces import LOCALHOST, PUBLIC_IPS |
|
47 | from IPython.utils.localinterfaces import LOCALHOST, PUBLIC_IPS | |
48 | from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict, TraitError |
|
48 | from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict, TraitError | |
49 |
|
49 | |||
50 | from IPython.kernel.zmq.session import ( |
|
50 | from IPython.kernel.zmq.session import ( | |
51 | Session, session_aliases, session_flags, default_secure |
|
51 | Session, session_aliases, session_flags, default_secure | |
52 | ) |
|
52 | ) | |
53 |
|
53 | |||
54 | from IPython.parallel.controller.heartmonitor import HeartMonitor |
|
54 | from IPython.parallel.controller.heartmonitor import HeartMonitor | |
55 | from IPython.parallel.controller.hub import HubFactory |
|
55 | from IPython.parallel.controller.hub import HubFactory | |
56 | from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler |
|
56 | from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler | |
57 | from IPython.parallel.controller.dictdb import DictDB |
|
57 | from IPython.parallel.controller.dictdb import DictDB | |
58 |
|
58 | |||
59 | from IPython.parallel.util import split_url, disambiguate_url, set_hwm |
|
59 | from IPython.parallel.util import split_url, disambiguate_url, set_hwm | |
60 |
|
60 | |||
61 | # conditional import of SQLiteDB / MongoDB backend class |
|
61 | # conditional import of SQLiteDB / MongoDB backend class | |
62 | real_dbs = [] |
|
62 | real_dbs = [] | |
63 |
|
63 | |||
64 | try: |
|
64 | try: | |
65 | from IPython.parallel.controller.sqlitedb import SQLiteDB |
|
65 | from IPython.parallel.controller.sqlitedb import SQLiteDB | |
66 | except ImportError: |
|
66 | except ImportError: | |
67 | pass |
|
67 | pass | |
68 | else: |
|
68 | else: | |
69 | real_dbs.append(SQLiteDB) |
|
69 | real_dbs.append(SQLiteDB) | |
70 |
|
70 | |||
71 | try: |
|
71 | try: | |
72 | from IPython.parallel.controller.mongodb import MongoDB |
|
72 | from IPython.parallel.controller.mongodb import MongoDB | |
73 | except ImportError: |
|
73 | except ImportError: | |
74 | pass |
|
74 | pass | |
75 | else: |
|
75 | else: | |
76 | real_dbs.append(MongoDB) |
|
76 | real_dbs.append(MongoDB) | |
77 |
|
77 | |||
78 |
|
78 | |||
79 |
|
79 | |||
80 | #----------------------------------------------------------------------------- |
|
80 | #----------------------------------------------------------------------------- | |
81 | # Module level variables |
|
81 | # Module level variables | |
82 | #----------------------------------------------------------------------------- |
|
82 | #----------------------------------------------------------------------------- | |
83 |
|
83 | |||
84 |
|
84 | |||
85 | #: The default config file name for this application |
|
85 | #: The default config file name for this application | |
86 | default_config_file_name = u'ipcontroller_config.py' |
|
86 | default_config_file_name = u'ipcontroller_config.py' | |
87 |
|
87 | |||
88 |
|
88 | |||
89 | _description = """Start the IPython controller for parallel computing. |
|
89 | _description = """Start the IPython controller for parallel computing. | |
90 |
|
90 | |||
91 | The IPython controller provides a gateway between the IPython engines and |
|
91 | The IPython controller provides a gateway between the IPython engines and | |
92 | clients. The controller needs to be started before the engines and can be |
|
92 | clients. The controller needs to be started before the engines and can be | |
93 | configured using command line options or using a cluster directory. Cluster |
|
93 | configured using command line options or using a cluster directory. Cluster | |
94 | directories contain config, log and security files and are usually located in |
|
94 | directories contain config, log and security files and are usually located in | |
95 | your ipython directory and named as "profile_name". See the `profile` |
|
95 | your ipython directory and named as "profile_name". See the `profile` | |
96 | and `profile-dir` options for details. |
|
96 | and `profile-dir` options for details. | |
97 | """ |
|
97 | """ | |
98 |
|
98 | |||
99 | _examples = """ |
|
99 | _examples = """ | |
100 | ipcontroller --ip=192.168.0.1 --port=1000 # listen on ip, port for engines |
|
100 | ipcontroller --ip=192.168.0.1 --port=1000 # listen on ip, port for engines | |
101 | ipcontroller --scheme=pure # use the pure zeromq scheduler |
|
101 | ipcontroller --scheme=pure # use the pure zeromq scheduler | |
102 | """ |
|
102 | """ | |
103 |
|
103 | |||
104 |
|
104 | |||
105 | #----------------------------------------------------------------------------- |
|
105 | #----------------------------------------------------------------------------- | |
106 | # The main application |
|
106 | # The main application | |
107 | #----------------------------------------------------------------------------- |
|
107 | #----------------------------------------------------------------------------- | |
108 | flags = {} |
|
108 | flags = {} | |
109 | flags.update(base_flags) |
|
109 | flags.update(base_flags) | |
110 | flags.update({ |
|
110 | flags.update({ | |
111 | 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}}, |
|
111 | 'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}}, | |
112 | 'Use threads instead of processes for the schedulers'), |
|
112 | 'Use threads instead of processes for the schedulers'), | |
113 | 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}}, |
|
113 | 'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}}, | |
114 | 'use the SQLiteDB backend'), |
|
114 | 'use the SQLiteDB backend'), | |
115 | 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}}, |
|
115 | 'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}}, | |
116 | 'use the MongoDB backend'), |
|
116 | 'use the MongoDB backend'), | |
117 | 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}}, |
|
117 | 'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}}, | |
118 | 'use the in-memory DictDB backend'), |
|
118 | 'use the in-memory DictDB backend'), | |
119 | 'nodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.NoDB'}}, |
|
119 | 'nodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.NoDB'}}, | |
120 | """use dummy DB backend, which doesn't store any information. |
|
120 | """use dummy DB backend, which doesn't store any information. | |
121 |
|
121 | |||
122 | This is the default as of IPython 0.13. |
|
122 | This is the default as of IPython 0.13. | |
123 |
|
123 | |||
124 | To enable delayed or repeated retrieval of results from the Hub, |
|
124 | To enable delayed or repeated retrieval of results from the Hub, | |
125 | select one of the true db backends. |
|
125 | select one of the true db backends. | |
126 | """), |
|
126 | """), | |
127 | 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}}, |
|
127 | 'reuse' : ({'IPControllerApp' : {'reuse_files' : True}}, | |
128 | 'reuse existing json connection files'), |
|
128 | 'reuse existing json connection files'), | |
129 | 'restore' : ({'IPControllerApp' : {'restore_engines' : True, 'reuse_files' : True}}, |
|
129 | 'restore' : ({'IPControllerApp' : {'restore_engines' : True, 'reuse_files' : True}}, | |
130 | 'Attempt to restore engines from a JSON file. ' |
|
130 | 'Attempt to restore engines from a JSON file. ' | |
131 | 'For use when resuming a crashed controller'), |
|
131 | 'For use when resuming a crashed controller'), | |
132 | }) |
|
132 | }) | |
133 |
|
133 | |||
134 | flags.update(session_flags) |
|
134 | flags.update(session_flags) | |
135 |
|
135 | |||
136 | aliases = dict( |
|
136 | aliases = dict( | |
137 | ssh = 'IPControllerApp.ssh_server', |
|
137 | ssh = 'IPControllerApp.ssh_server', | |
138 | enginessh = 'IPControllerApp.engine_ssh_server', |
|
138 | enginessh = 'IPControllerApp.engine_ssh_server', | |
139 | location = 'IPControllerApp.location', |
|
139 | location = 'IPControllerApp.location', | |
140 |
|
140 | |||
141 | url = 'HubFactory.url', |
|
141 | url = 'HubFactory.url', | |
142 | ip = 'HubFactory.ip', |
|
142 | ip = 'HubFactory.ip', | |
143 | transport = 'HubFactory.transport', |
|
143 | transport = 'HubFactory.transport', | |
144 | port = 'HubFactory.regport', |
|
144 | port = 'HubFactory.regport', | |
145 |
|
145 | |||
146 | ping = 'HeartMonitor.period', |
|
146 | ping = 'HeartMonitor.period', | |
147 |
|
147 | |||
148 | scheme = 'TaskScheduler.scheme_name', |
|
148 | scheme = 'TaskScheduler.scheme_name', | |
149 | hwm = 'TaskScheduler.hwm', |
|
149 | hwm = 'TaskScheduler.hwm', | |
150 | ) |
|
150 | ) | |
151 | aliases.update(base_aliases) |
|
151 | aliases.update(base_aliases) | |
152 | aliases.update(session_aliases) |
|
152 | aliases.update(session_aliases) | |
153 |
|
153 | |||
154 | class IPControllerApp(BaseParallelApplication): |
|
154 | class IPControllerApp(BaseParallelApplication): | |
155 |
|
155 | |||
156 | name = u'ipcontroller' |
|
156 | name = u'ipcontroller' | |
157 | description = _description |
|
157 | description = _description | |
158 | examples = _examples |
|
158 | examples = _examples | |
159 | config_file_name = Unicode(default_config_file_name) |
|
159 | config_file_name = Unicode(default_config_file_name) | |
160 | classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, DictDB] + real_dbs |
|
160 | classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, DictDB] + real_dbs | |
161 |
|
161 | |||
162 | # change default to True |
|
162 | # change default to True | |
163 | auto_create = Bool(True, config=True, |
|
163 | auto_create = Bool(True, config=True, | |
164 | help="""Whether to create profile dir if it doesn't exist.""") |
|
164 | help="""Whether to create profile dir if it doesn't exist.""") | |
165 |
|
165 | |||
166 | reuse_files = Bool(False, config=True, |
|
166 | reuse_files = Bool(False, config=True, | |
167 | help="""Whether to reuse existing json connection files. |
|
167 | help="""Whether to reuse existing json connection files. | |
168 | If False, connection files will be removed on a clean exit. |
|
168 | If False, connection files will be removed on a clean exit. | |
169 | """ |
|
169 | """ | |
170 | ) |
|
170 | ) | |
171 | restore_engines = Bool(False, config=True, |
|
171 | restore_engines = Bool(False, config=True, | |
172 | help="""Reload engine state from JSON file |
|
172 | help="""Reload engine state from JSON file | |
173 | """ |
|
173 | """ | |
174 | ) |
|
174 | ) | |
175 | ssh_server = Unicode(u'', config=True, |
|
175 | ssh_server = Unicode(u'', config=True, | |
176 | help="""ssh url for clients to use when connecting to the Controller |
|
176 | help="""ssh url for clients to use when connecting to the Controller | |
177 | processes. It should be of the form: [user@]server[:port]. The |
|
177 | processes. It should be of the form: [user@]server[:port]. The | |
178 | Controller's listening addresses must be accessible from the ssh server""", |
|
178 | Controller's listening addresses must be accessible from the ssh server""", | |
179 | ) |
|
179 | ) | |
180 | engine_ssh_server = Unicode(u'', config=True, |
|
180 | engine_ssh_server = Unicode(u'', config=True, | |
181 | help="""ssh url for engines to use when connecting to the Controller |
|
181 | help="""ssh url for engines to use when connecting to the Controller | |
182 | processes. It should be of the form: [user@]server[:port]. The |
|
182 | processes. It should be of the form: [user@]server[:port]. The | |
183 | Controller's listening addresses must be accessible from the ssh server""", |
|
183 | Controller's listening addresses must be accessible from the ssh server""", | |
184 | ) |
|
184 | ) | |
185 | location = Unicode(u'', config=True, |
|
185 | location = Unicode(u'', config=True, | |
186 | help="""The external IP or domain name of the Controller, used for disambiguating |
|
186 | help="""The external IP or domain name of the Controller, used for disambiguating | |
187 | engine and client connections.""", |
|
187 | engine and client connections.""", | |
188 | ) |
|
188 | ) | |
189 | import_statements = List([], config=True, |
|
189 | import_statements = List([], config=True, | |
190 | help="import statements to be run at startup. Necessary in some environments" |
|
190 | help="import statements to be run at startup. Necessary in some environments" | |
191 | ) |
|
191 | ) | |
192 |
|
192 | |||
193 | use_threads = Bool(False, config=True, |
|
193 | use_threads = Bool(False, config=True, | |
194 | help='Use threads instead of processes for the schedulers', |
|
194 | help='Use threads instead of processes for the schedulers', | |
195 | ) |
|
195 | ) | |
196 |
|
196 | |||
197 | engine_json_file = Unicode('ipcontroller-engine.json', config=True, |
|
197 | engine_json_file = Unicode('ipcontroller-engine.json', config=True, | |
198 | help="JSON filename where engine connection info will be stored.") |
|
198 | help="JSON filename where engine connection info will be stored.") | |
199 | client_json_file = Unicode('ipcontroller-client.json', config=True, |
|
199 | client_json_file = Unicode('ipcontroller-client.json', config=True, | |
200 | help="JSON filename where client connection info will be stored.") |
|
200 | help="JSON filename where client connection info will be stored.") | |
201 |
|
201 | |||
202 | def _cluster_id_changed(self, name, old, new): |
|
202 | def _cluster_id_changed(self, name, old, new): | |
203 | super(IPControllerApp, self)._cluster_id_changed(name, old, new) |
|
203 | super(IPControllerApp, self)._cluster_id_changed(name, old, new) | |
204 | self.engine_json_file = "%s-engine.json" % self.name |
|
204 | self.engine_json_file = "%s-engine.json" % self.name | |
205 | self.client_json_file = "%s-client.json" % self.name |
|
205 | self.client_json_file = "%s-client.json" % self.name | |
206 |
|
206 | |||
207 |
|
207 | |||
208 | # internal |
|
208 | # internal | |
209 | children = List() |
|
209 | children = List() | |
210 | mq_class = Unicode('zmq.devices.ProcessMonitoredQueue') |
|
210 | mq_class = Unicode('zmq.devices.ProcessMonitoredQueue') | |
211 |
|
211 | |||
212 | def _use_threads_changed(self, name, old, new): |
|
212 | def _use_threads_changed(self, name, old, new): | |
213 | self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process') |
|
213 | self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process') | |
214 |
|
214 | |||
215 | write_connection_files = Bool(True, |
|
215 | write_connection_files = Bool(True, | |
216 | help="""Whether to write connection files to disk. |
|
216 | help="""Whether to write connection files to disk. | |
217 | True in all cases other than runs with `reuse_files=True` *after the first* |
|
217 | True in all cases other than runs with `reuse_files=True` *after the first* | |
218 | """ |
|
218 | """ | |
219 | ) |
|
219 | ) | |
220 |
|
220 | |||
221 | aliases = Dict(aliases) |
|
221 | aliases = Dict(aliases) | |
222 | flags = Dict(flags) |
|
222 | flags = Dict(flags) | |
223 |
|
223 | |||
224 |
|
224 | |||
225 | def save_connection_dict(self, fname, cdict): |
|
225 | def save_connection_dict(self, fname, cdict): | |
226 | """save a connection dict to json file.""" |
|
226 | """save a connection dict to json file.""" | |
227 | c = self.config |
|
227 | c = self.config | |
228 | url = cdict['registration'] |
|
228 | url = cdict['registration'] | |
229 | location = cdict['location'] |
|
229 | location = cdict['location'] | |
230 |
|
230 | |||
231 | if not location: |
|
231 | if not location: | |
232 | if PUBLIC_IPS: |
|
232 | if PUBLIC_IPS: | |
233 | location = PUBLIC_IPS[-1] |
|
233 | location = PUBLIC_IPS[-1] | |
234 | else: |
|
234 | else: | |
235 | self.log.warn("Could not identify this machine's IP, assuming %s." |
|
235 | self.log.warn("Could not identify this machine's IP, assuming %s." | |
236 | " You may need to specify '--location=<external_ip_address>' to help" |
|
236 | " You may need to specify '--location=<external_ip_address>' to help" | |
237 | " IPython decide when to connect via loopback." % LOCALHOST) |
|
237 | " IPython decide when to connect via loopback." % LOCALHOST) | |
238 | location = LOCALHOST |
|
238 | location = LOCALHOST | |
239 | cdict['location'] = location |
|
239 | cdict['location'] = location | |
240 | fname = os.path.join(self.profile_dir.security_dir, fname) |
|
240 | fname = os.path.join(self.profile_dir.security_dir, fname) | |
241 | self.log.info("writing connection info to %s", fname) |
|
241 | self.log.info("writing connection info to %s", fname) | |
242 | with open(fname, 'w') as f: |
|
242 | with open(fname, 'w') as f: | |
243 | f.write(json.dumps(cdict, indent=2)) |
|
243 | f.write(json.dumps(cdict, indent=2)) | |
244 | os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR) |
|
244 | os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR) | |
245 |
|
245 | |||
246 | def load_config_from_json(self): |
|
246 | def load_config_from_json(self): | |
247 | """load config from existing json connector files.""" |
|
247 | """load config from existing json connector files.""" | |
248 | c = self.config |
|
248 | c = self.config | |
249 | self.log.debug("loading config from JSON") |
|
249 | self.log.debug("loading config from JSON") | |
250 |
|
250 | |||
251 | # load engine config |
|
251 | # load engine config | |
252 |
|
252 | |||
253 | fname = os.path.join(self.profile_dir.security_dir, self.engine_json_file) |
|
253 | fname = os.path.join(self.profile_dir.security_dir, self.engine_json_file) | |
254 | self.log.info("loading connection info from %s", fname) |
|
254 | self.log.info("loading connection info from %s", fname) | |
255 | with open(fname) as f: |
|
255 | with open(fname) as f: | |
256 | ecfg = json.loads(f.read()) |
|
256 | ecfg = json.loads(f.read()) | |
257 |
|
257 | |||
258 | # json gives unicode, Session.key wants bytes |
|
258 | # json gives unicode, Session.key wants bytes | |
259 | c.Session.key = ecfg['exec_key'].encode('ascii') |
|
259 | c.Session.key = ecfg['exec_key'].encode('ascii') | |
260 |
|
260 | |||
261 | xport,ip = ecfg['interface'].split('://') |
|
261 | xport,ip = ecfg['interface'].split('://') | |
262 |
|
262 | |||
263 | c.HubFactory.engine_ip = ip |
|
263 | c.HubFactory.engine_ip = ip | |
264 | c.HubFactory.engine_transport = xport |
|
264 | c.HubFactory.engine_transport = xport | |
265 |
|
265 | |||
266 | self.location = ecfg['location'] |
|
266 | self.location = ecfg['location'] | |
267 | if not self.engine_ssh_server: |
|
267 | if not self.engine_ssh_server: | |
268 | self.engine_ssh_server = ecfg['ssh'] |
|
268 | self.engine_ssh_server = ecfg['ssh'] | |
269 |
|
269 | |||
270 | # load client config |
|
270 | # load client config | |
271 |
|
271 | |||
272 | fname = os.path.join(self.profile_dir.security_dir, self.client_json_file) |
|
272 | fname = os.path.join(self.profile_dir.security_dir, self.client_json_file) | |
273 | self.log.info("loading connection info from %s", fname) |
|
273 | self.log.info("loading connection info from %s", fname) | |
274 | with open(fname) as f: |
|
274 | with open(fname) as f: | |
275 | ccfg = json.loads(f.read()) |
|
275 | ccfg = json.loads(f.read()) | |
276 |
|
276 | |||
277 | for key in ('exec_key', 'registration', 'pack', 'unpack'): |
|
277 | for key in ('exec_key', 'registration', 'pack', 'unpack'): | |
278 | assert ccfg[key] == ecfg[key], "mismatch between engine and client info: %r" % key |
|
278 | assert ccfg[key] == ecfg[key], "mismatch between engine and client info: %r" % key | |
279 |
|
279 | |||
280 | xport,addr = ccfg['interface'].split('://') |
|
280 | xport,addr = ccfg['interface'].split('://') | |
281 |
|
281 | |||
282 | c.HubFactory.client_transport = xport |
|
282 | c.HubFactory.client_transport = xport | |
283 | c.HubFactory.client_ip = ip |
|
283 | c.HubFactory.client_ip = ip | |
284 | if not self.ssh_server: |
|
284 | if not self.ssh_server: | |
285 | self.ssh_server = ccfg['ssh'] |
|
285 | self.ssh_server = ccfg['ssh'] | |
286 |
|
286 | |||
287 | # load port config: |
|
287 | # load port config: | |
288 | c.HubFactory.regport = ecfg['registration'] |
|
288 | c.HubFactory.regport = ecfg['registration'] | |
289 | c.HubFactory.hb = (ecfg['hb_ping'], ecfg['hb_pong']) |
|
289 | c.HubFactory.hb = (ecfg['hb_ping'], ecfg['hb_pong']) | |
290 | c.HubFactory.control = (ccfg['control'], ecfg['control']) |
|
290 | c.HubFactory.control = (ccfg['control'], ecfg['control']) | |
291 | c.HubFactory.mux = (ccfg['mux'], ecfg['mux']) |
|
291 | c.HubFactory.mux = (ccfg['mux'], ecfg['mux']) | |
292 | c.HubFactory.task = (ccfg['task'], ecfg['task']) |
|
292 | c.HubFactory.task = (ccfg['task'], ecfg['task']) | |
293 | c.HubFactory.iopub = (ccfg['iopub'], ecfg['iopub']) |
|
293 | c.HubFactory.iopub = (ccfg['iopub'], ecfg['iopub']) | |
294 | c.HubFactory.notifier_port = ccfg['notification'] |
|
294 | c.HubFactory.notifier_port = ccfg['notification'] | |
295 |
|
295 | |||
296 | def cleanup_connection_files(self): |
|
296 | def cleanup_connection_files(self): | |
297 | if self.reuse_files: |
|
297 | if self.reuse_files: | |
298 | self.log.debug("leaving JSON connection files for reuse") |
|
298 | self.log.debug("leaving JSON connection files for reuse") | |
299 | return |
|
299 | return | |
300 | self.log.debug("cleaning up JSON connection files") |
|
300 | self.log.debug("cleaning up JSON connection files") | |
301 | for f in (self.client_json_file, self.engine_json_file): |
|
301 | for f in (self.client_json_file, self.engine_json_file): | |
302 | f = os.path.join(self.profile_dir.security_dir, f) |
|
302 | f = os.path.join(self.profile_dir.security_dir, f) | |
303 | try: |
|
303 | try: | |
304 | os.remove(f) |
|
304 | os.remove(f) | |
305 | except Exception as e: |
|
305 | except Exception as e: | |
306 | self.log.error("Failed to cleanup connection file: %s", e) |
|
306 | self.log.error("Failed to cleanup connection file: %s", e) | |
307 | else: |
|
307 | else: | |
308 | self.log.debug(u"removed %s", f) |
|
308 | self.log.debug(u"removed %s", f) | |
309 |
|
309 | |||
310 | def load_secondary_config(self): |
|
310 | def load_secondary_config(self): | |
311 | """secondary config, loading from JSON and setting defaults""" |
|
311 | """secondary config, loading from JSON and setting defaults""" | |
312 | if self.reuse_files: |
|
312 | if self.reuse_files: | |
313 | try: |
|
313 | try: | |
314 | self.load_config_from_json() |
|
314 | self.load_config_from_json() | |
315 | except (AssertionError,IOError) as e: |
|
315 | except (AssertionError,IOError) as e: | |
316 | self.log.error("Could not load config from JSON: %s" % e) |
|
316 | self.log.error("Could not load config from JSON: %s" % e) | |
317 | else: |
|
317 | else: | |
318 | # successfully loaded config from JSON, and reuse=True |
|
318 | # successfully loaded config from JSON, and reuse=True | |
319 | # no need to wite back the same file |
|
319 | # no need to wite back the same file | |
320 | self.write_connection_files = False |
|
320 | self.write_connection_files = False | |
321 |
|
321 | |||
322 | # switch Session.key default to secure |
|
322 | # switch Session.key default to secure | |
323 | default_secure(self.config) |
|
323 | default_secure(self.config) | |
324 | self.log.debug("Config changed") |
|
324 | self.log.debug("Config changed") | |
325 | self.log.debug(repr(self.config)) |
|
325 | self.log.debug(repr(self.config)) | |
326 |
|
326 | |||
327 | def init_hub(self): |
|
327 | def init_hub(self): | |
328 | c = self.config |
|
328 | c = self.config | |
329 |
|
329 | |||
330 | self.do_import_statements() |
|
330 | self.do_import_statements() | |
331 |
|
331 | |||
332 | try: |
|
332 | try: | |
333 | self.factory = HubFactory(config=c, log=self.log) |
|
333 | self.factory = HubFactory(config=c, log=self.log) | |
334 | # self.start_logging() |
|
334 | # self.start_logging() | |
335 | self.factory.init_hub() |
|
335 | self.factory.init_hub() | |
336 | except TraitError: |
|
336 | except TraitError: | |
337 | raise |
|
337 | raise | |
338 | except Exception: |
|
338 | except Exception: | |
339 | self.log.error("Couldn't construct the Controller", exc_info=True) |
|
339 | self.log.error("Couldn't construct the Controller", exc_info=True) | |
340 | self.exit(1) |
|
340 | self.exit(1) | |
341 |
|
341 | |||
342 | if self.write_connection_files: |
|
342 | if self.write_connection_files: | |
343 | # save to new json config files |
|
343 | # save to new json config files | |
344 | f = self.factory |
|
344 | f = self.factory | |
345 | base = { |
|
345 | base = { | |
346 | 'exec_key' : f.session.key.decode('ascii'), |
|
346 | 'exec_key' : f.session.key.decode('ascii'), | |
347 | 'location' : self.location, |
|
347 | 'location' : self.location, | |
348 | 'pack' : f.session.packer, |
|
348 | 'pack' : f.session.packer, | |
349 | 'unpack' : f.session.unpacker, |
|
349 | 'unpack' : f.session.unpacker, | |
350 | } |
|
350 | } | |
351 |
|
351 | |||
352 | cdict = {'ssh' : self.ssh_server} |
|
352 | cdict = {'ssh' : self.ssh_server} | |
353 | cdict.update(f.client_info) |
|
353 | cdict.update(f.client_info) | |
354 | cdict.update(base) |
|
354 | cdict.update(base) | |
355 | self.save_connection_dict(self.client_json_file, cdict) |
|
355 | self.save_connection_dict(self.client_json_file, cdict) | |
356 |
|
356 | |||
357 | edict = {'ssh' : self.engine_ssh_server} |
|
357 | edict = {'ssh' : self.engine_ssh_server} | |
358 | edict.update(f.engine_info) |
|
358 | edict.update(f.engine_info) | |
359 | edict.update(base) |
|
359 | edict.update(base) | |
360 | self.save_connection_dict(self.engine_json_file, edict) |
|
360 | self.save_connection_dict(self.engine_json_file, edict) | |
361 |
|
361 | |||
362 | fname = "engines%s.json" % self.cluster_id |
|
362 | fname = "engines%s.json" % self.cluster_id | |
363 | self.factory.hub.engine_state_file = os.path.join(self.profile_dir.log_dir, fname) |
|
363 | self.factory.hub.engine_state_file = os.path.join(self.profile_dir.log_dir, fname) | |
364 | if self.restore_engines: |
|
364 | if self.restore_engines: | |
365 | self.factory.hub._load_engine_state() |
|
365 | self.factory.hub._load_engine_state() | |
366 |
|
366 | |||
367 | def init_schedulers(self): |
|
367 | def init_schedulers(self): | |
368 | children = self.children |
|
368 | children = self.children | |
369 | mq = import_item(str(self.mq_class)) |
|
369 | mq = import_item(str(self.mq_class)) | |
370 |
|
370 | |||
371 | f = self.factory |
|
371 | f = self.factory | |
372 | ident = f.session.bsession |
|
372 | ident = f.session.bsession | |
373 | # disambiguate url, in case of * |
|
373 | # disambiguate url, in case of * | |
374 | monitor_url = disambiguate_url(f.monitor_url) |
|
374 | monitor_url = disambiguate_url(f.monitor_url) | |
375 | # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url |
|
375 | # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url | |
376 | # IOPub relay (in a Process) |
|
376 | # IOPub relay (in a Process) | |
377 | q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub') |
|
377 | q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub') | |
378 | q.bind_in(f.client_url('iopub')) |
|
378 | q.bind_in(f.client_url('iopub')) | |
379 | q.setsockopt_in(zmq.IDENTITY, ident + b"_iopub") |
|
379 | q.setsockopt_in(zmq.IDENTITY, ident + b"_iopub") | |
380 | q.bind_out(f.engine_url('iopub')) |
|
380 | q.bind_out(f.engine_url('iopub')) | |
381 | q.setsockopt_out(zmq.SUBSCRIBE, b'') |
|
381 | q.setsockopt_out(zmq.SUBSCRIBE, b'') | |
382 | q.connect_mon(monitor_url) |
|
382 | q.connect_mon(monitor_url) | |
383 | q.daemon=True |
|
383 | q.daemon=True | |
384 | children.append(q) |
|
384 | children.append(q) | |
385 |
|
385 | |||
386 | # Multiplexer Queue (in a Process) |
|
386 | # Multiplexer Queue (in a Process) | |
387 | q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out') |
|
387 | q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out') | |
388 |
|
388 | |||
389 | q.bind_in(f.client_url('mux')) |
|
389 | q.bind_in(f.client_url('mux')) | |
390 | q.setsockopt_in(zmq.IDENTITY, b'mux_in') |
|
390 | q.setsockopt_in(zmq.IDENTITY, b'mux_in') | |
391 | q.bind_out(f.engine_url('mux')) |
|
391 | q.bind_out(f.engine_url('mux')) | |
392 | q.setsockopt_out(zmq.IDENTITY, b'mux_out') |
|
392 | q.setsockopt_out(zmq.IDENTITY, b'mux_out') | |
393 | q.connect_mon(monitor_url) |
|
393 | q.connect_mon(monitor_url) | |
394 | q.daemon=True |
|
394 | q.daemon=True | |
395 | children.append(q) |
|
395 | children.append(q) | |
396 |
|
396 | |||
397 | # Control Queue (in a Process) |
|
397 | # Control Queue (in a Process) | |
398 | q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol') |
|
398 | q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol') | |
399 | q.bind_in(f.client_url('control')) |
|
399 | q.bind_in(f.client_url('control')) | |
400 | q.setsockopt_in(zmq.IDENTITY, b'control_in') |
|
400 | q.setsockopt_in(zmq.IDENTITY, b'control_in') | |
401 | q.bind_out(f.engine_url('control')) |
|
401 | q.bind_out(f.engine_url('control')) | |
402 | q.setsockopt_out(zmq.IDENTITY, b'control_out') |
|
402 | q.setsockopt_out(zmq.IDENTITY, b'control_out') | |
403 | q.connect_mon(monitor_url) |
|
403 | q.connect_mon(monitor_url) | |
404 | q.daemon=True |
|
404 | q.daemon=True | |
405 | children.append(q) |
|
405 | children.append(q) | |
406 | try: |
|
406 | try: | |
407 | scheme = self.config.TaskScheduler.scheme_name |
|
407 | scheme = self.config.TaskScheduler.scheme_name | |
408 | except AttributeError: |
|
408 | except AttributeError: | |
409 | scheme = TaskScheduler.scheme_name.get_default_value() |
|
409 | scheme = TaskScheduler.scheme_name.get_default_value() | |
410 | # Task Queue (in a Process) |
|
410 | # Task Queue (in a Process) | |
411 | if scheme == 'pure': |
|
411 | if scheme == 'pure': | |
412 | self.log.warn("task::using pure DEALER Task scheduler") |
|
412 | self.log.warn("task::using pure DEALER Task scheduler") | |
413 | q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask') |
|
413 | q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask') | |
414 | # q.setsockopt_out(zmq.HWM, hub.hwm) |
|
414 | # q.setsockopt_out(zmq.HWM, hub.hwm) | |
415 | q.bind_in(f.client_url('task')) |
|
415 | q.bind_in(f.client_url('task')) | |
416 | q.setsockopt_in(zmq.IDENTITY, b'task_in') |
|
416 | q.setsockopt_in(zmq.IDENTITY, b'task_in') | |
417 | q.bind_out(f.engine_url('task')) |
|
417 | q.bind_out(f.engine_url('task')) | |
418 | q.setsockopt_out(zmq.IDENTITY, b'task_out') |
|
418 | q.setsockopt_out(zmq.IDENTITY, b'task_out') | |
419 | q.connect_mon(monitor_url) |
|
419 | q.connect_mon(monitor_url) | |
420 | q.daemon=True |
|
420 | q.daemon=True | |
421 | children.append(q) |
|
421 | children.append(q) | |
422 | elif scheme == 'none': |
|
422 | elif scheme == 'none': | |
423 | self.log.warn("task::using no Task scheduler") |
|
423 | self.log.warn("task::using no Task scheduler") | |
424 |
|
424 | |||
425 | else: |
|
425 | else: | |
426 | self.log.info("task::using Python %s Task scheduler"%scheme) |
|
426 | self.log.info("task::using Python %s Task scheduler"%scheme) | |
427 | sargs = (f.client_url('task'), f.engine_url('task'), |
|
427 | sargs = (f.client_url('task'), f.engine_url('task'), | |
428 | monitor_url, disambiguate_url(f.client_url('notification')), |
|
428 | monitor_url, disambiguate_url(f.client_url('notification')), | |
429 | disambiguate_url(f.client_url('registration')), |
|
429 | disambiguate_url(f.client_url('registration')), | |
430 | ) |
|
430 | ) | |
431 | kwargs = dict(logname='scheduler', loglevel=self.log_level, |
|
431 | kwargs = dict(logname='scheduler', loglevel=self.log_level, | |
432 | log_url = self.log_url, config=dict(self.config)) |
|
432 | log_url = self.log_url, config=dict(self.config)) | |
433 | if 'Process' in self.mq_class: |
|
433 | if 'Process' in self.mq_class: | |
434 | # run the Python scheduler in a Process |
|
434 | # run the Python scheduler in a Process | |
435 | q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs) |
|
435 | q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs) | |
436 | q.daemon=True |
|
436 | q.daemon=True | |
437 | children.append(q) |
|
437 | children.append(q) | |
438 | else: |
|
438 | else: | |
439 | # single-threaded Controller |
|
439 | # single-threaded Controller | |
440 | kwargs['in_thread'] = True |
|
440 | kwargs['in_thread'] = True | |
441 | launch_scheduler(*sargs, **kwargs) |
|
441 | launch_scheduler(*sargs, **kwargs) | |
442 |
|
442 | |||
443 | # set unlimited HWM for all relay devices |
|
443 | # set unlimited HWM for all relay devices | |
444 | if hasattr(zmq, 'SNDHWM'): |
|
444 | if hasattr(zmq, 'SNDHWM'): | |
445 | q = children[0] |
|
445 | q = children[0] | |
446 | q.setsockopt_in(zmq.RCVHWM, 0) |
|
446 | q.setsockopt_in(zmq.RCVHWM, 0) | |
447 | q.setsockopt_out(zmq.SNDHWM, 0) |
|
447 | q.setsockopt_out(zmq.SNDHWM, 0) | |
448 |
|
448 | |||
449 | for q in children[1:]: |
|
449 | for q in children[1:]: | |
450 | if not hasattr(q, 'setsockopt_in'): |
|
450 | if not hasattr(q, 'setsockopt_in'): | |
451 | continue |
|
451 | continue | |
452 | q.setsockopt_in(zmq.SNDHWM, 0) |
|
452 | q.setsockopt_in(zmq.SNDHWM, 0) | |
453 | q.setsockopt_in(zmq.RCVHWM, 0) |
|
453 | q.setsockopt_in(zmq.RCVHWM, 0) | |
454 | q.setsockopt_out(zmq.SNDHWM, 0) |
|
454 | q.setsockopt_out(zmq.SNDHWM, 0) | |
455 | q.setsockopt_out(zmq.RCVHWM, 0) |
|
455 | q.setsockopt_out(zmq.RCVHWM, 0) | |
456 | q.setsockopt_mon(zmq.SNDHWM, 0) |
|
456 | q.setsockopt_mon(zmq.SNDHWM, 0) | |
457 |
|
457 | |||
458 |
|
458 | |||
459 | def terminate_children(self): |
|
459 | def terminate_children(self): | |
460 | child_procs = [] |
|
460 | child_procs = [] | |
461 | for child in self.children: |
|
461 | for child in self.children: | |
462 | if isinstance(child, ProcessMonitoredQueue): |
|
462 | if isinstance(child, ProcessMonitoredQueue): | |
463 | child_procs.append(child.launcher) |
|
463 | child_procs.append(child.launcher) | |
464 | elif isinstance(child, Process): |
|
464 | elif isinstance(child, Process): | |
465 | child_procs.append(child) |
|
465 | child_procs.append(child) | |
466 | if child_procs: |
|
466 | if child_procs: | |
467 | self.log.critical("terminating children...") |
|
467 | self.log.critical("terminating children...") | |
468 | for child in child_procs: |
|
468 | for child in child_procs: | |
469 | try: |
|
469 | try: | |
470 | child.terminate() |
|
470 | child.terminate() | |
471 | except OSError: |
|
471 | except OSError: | |
472 | # already dead |
|
472 | # already dead | |
473 | pass |
|
473 | pass | |
474 |
|
474 | |||
475 | def handle_signal(self, sig, frame): |
|
475 | def handle_signal(self, sig, frame): | |
476 | self.log.critical("Received signal %i, shutting down", sig) |
|
476 | self.log.critical("Received signal %i, shutting down", sig) | |
477 | self.terminate_children() |
|
477 | self.terminate_children() | |
478 | self.loop.stop() |
|
478 | self.loop.stop() | |
479 |
|
479 | |||
480 | def init_signal(self): |
|
480 | def init_signal(self): | |
481 | for sig in (SIGINT, SIGABRT, SIGTERM): |
|
481 | for sig in (SIGINT, SIGABRT, SIGTERM): | |
482 | signal(sig, self.handle_signal) |
|
482 | signal(sig, self.handle_signal) | |
483 |
|
483 | |||
484 | def do_import_statements(self): |
|
484 | def do_import_statements(self): | |
485 | statements = self.import_statements |
|
485 | statements = self.import_statements | |
486 | for s in statements: |
|
486 | for s in statements: | |
487 | try: |
|
487 | try: | |
488 | self.log.msg("Executing statement: '%s'" % s) |
|
488 | self.log.msg("Executing statement: '%s'" % s) | |
489 | exec s in globals(), locals() |
|
489 | exec s in globals(), locals() | |
490 | except: |
|
490 | except: | |
491 | self.log.msg("Error running statement: %s" % s) |
|
491 | self.log.msg("Error running statement: %s" % s) | |
492 |
|
492 | |||
493 | def forward_logging(self): |
|
493 | def forward_logging(self): | |
494 | if self.log_url: |
|
494 | if self.log_url: | |
495 | self.log.info("Forwarding logging to %s"%self.log_url) |
|
495 | self.log.info("Forwarding logging to %s"%self.log_url) | |
496 | context = zmq.Context.instance() |
|
496 | context = zmq.Context.instance() | |
497 | lsock = context.socket(zmq.PUB) |
|
497 | lsock = context.socket(zmq.PUB) | |
498 | lsock.connect(self.log_url) |
|
498 | lsock.connect(self.log_url) | |
499 | handler = PUBHandler(lsock) |
|
499 | handler = PUBHandler(lsock) | |
500 | handler.root_topic = 'controller' |
|
500 | handler.root_topic = 'controller' | |
501 | handler.setLevel(self.log_level) |
|
501 | handler.setLevel(self.log_level) | |
502 | self.log.addHandler(handler) |
|
502 | self.log.addHandler(handler) | |
503 |
|
503 | |||
504 | @catch_config_error |
|
504 | @catch_config_error | |
505 | def initialize(self, argv=None): |
|
505 | def initialize(self, argv=None): | |
506 | super(IPControllerApp, self).initialize(argv) |
|
506 | super(IPControllerApp, self).initialize(argv) | |
507 | self.forward_logging() |
|
507 | self.forward_logging() | |
508 | self.load_secondary_config() |
|
508 | self.load_secondary_config() | |
509 | self.init_hub() |
|
509 | self.init_hub() | |
510 | self.init_schedulers() |
|
510 | self.init_schedulers() | |
511 |
|
511 | |||
512 | def start(self): |
|
512 | def start(self): | |
513 | # Start the subprocesses: |
|
513 | # Start the subprocesses: | |
514 | self.factory.start() |
|
514 | self.factory.start() | |
515 | # children must be started before signals are setup, |
|
515 | # children must be started before signals are setup, | |
516 | # otherwise signal-handling will fire multiple times |
|
516 | # otherwise signal-handling will fire multiple times | |
517 | for child in self.children: |
|
517 | for child in self.children: | |
518 | child.start() |
|
518 | child.start() | |
519 | self.init_signal() |
|
519 | self.init_signal() | |
520 |
|
520 | |||
521 | self.write_pid_file(overwrite=True) |
|
521 | self.write_pid_file(overwrite=True) | |
522 |
|
522 | |||
523 | try: |
|
523 | try: | |
524 | self.factory.loop.start() |
|
524 | self.factory.loop.start() | |
525 | except KeyboardInterrupt: |
|
525 | except KeyboardInterrupt: | |
526 | self.log.critical("Interrupted, Exiting...\n") |
|
526 | self.log.critical("Interrupted, Exiting...\n") | |
527 | finally: |
|
527 | finally: | |
528 | self.cleanup_connection_files() |
|
528 | self.cleanup_connection_files() | |
529 |
|
529 | |||
530 |
|
530 | |||
531 | def launch_new_instance(*args, **kwargs): |
|
531 | def launch_new_instance(*args, **kwargs): | |
532 | """Create and run the IPython controller""" |
|
532 | """Create and run the IPython controller""" | |
533 | if sys.platform == 'win32': |
|
533 | if sys.platform == 'win32': | |
534 | # make sure we don't get called from a multiprocessing subprocess |
|
534 | # make sure we don't get called from a multiprocessing subprocess | |
535 | # this can result in infinite Controllers being started on Windows |
|
535 | # this can result in infinite Controllers being started on Windows | |
536 | # which doesn't have a proper fork, so multiprocessing is wonky |
|
536 | # which doesn't have a proper fork, so multiprocessing is wonky | |
537 |
|
537 | |||
538 | # this only comes up when IPython has been installed using vanilla |
|
538 | # this only comes up when IPython has been installed using vanilla | |
539 | # setuptools, and *not* distribute. |
|
539 | # setuptools, and *not* distribute. | |
540 | import multiprocessing |
|
540 | import multiprocessing | |
541 | p = multiprocessing.current_process() |
|
541 | p = multiprocessing.current_process() | |
542 | # the main process has name 'MainProcess' |
|
542 | # the main process has name 'MainProcess' | |
543 | # subprocesses will have names like 'Process-1' |
|
543 | # subprocesses will have names like 'Process-1' | |
544 | if p.name != 'MainProcess': |
|
544 | if p.name != 'MainProcess': | |
545 | # we are a subprocess, don't start another Controller! |
|
545 | # we are a subprocess, don't start another Controller! | |
546 | return |
|
546 | return | |
547 |
return IPControllerApp.launch_ |
|
547 | return IPControllerApp.launch_instance(*args, **kwargs) | |
548 |
|
548 | |||
549 |
|
549 | |||
550 | if __name__ == '__main__': |
|
550 | if __name__ == '__main__': | |
551 | launch_new_instance() |
|
551 | launch_new_instance() |
@@ -1,396 +1,396 | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | The IPython engine application |
|
4 | The IPython engine application | |
5 |
|
5 | |||
6 | Authors: |
|
6 | Authors: | |
7 |
|
7 | |||
8 | * Brian Granger |
|
8 | * Brian Granger | |
9 | * MinRK |
|
9 | * MinRK | |
10 |
|
10 | |||
11 | """ |
|
11 | """ | |
12 |
|
12 | |||
13 | #----------------------------------------------------------------------------- |
|
13 | #----------------------------------------------------------------------------- | |
14 | # Copyright (C) 2008-2011 The IPython Development Team |
|
14 | # Copyright (C) 2008-2011 The IPython Development Team | |
15 | # |
|
15 | # | |
16 | # Distributed under the terms of the BSD License. The full license is in |
|
16 | # Distributed under the terms of the BSD License. The full license is in | |
17 | # the file COPYING, distributed as part of this software. |
|
17 | # the file COPYING, distributed as part of this software. | |
18 | #----------------------------------------------------------------------------- |
|
18 | #----------------------------------------------------------------------------- | |
19 |
|
19 | |||
20 | #----------------------------------------------------------------------------- |
|
20 | #----------------------------------------------------------------------------- | |
21 | # Imports |
|
21 | # Imports | |
22 | #----------------------------------------------------------------------------- |
|
22 | #----------------------------------------------------------------------------- | |
23 |
|
23 | |||
24 | import json |
|
24 | import json | |
25 | import os |
|
25 | import os | |
26 | import sys |
|
26 | import sys | |
27 | import time |
|
27 | import time | |
28 |
|
28 | |||
29 | import zmq |
|
29 | import zmq | |
30 | from zmq.eventloop import ioloop |
|
30 | from zmq.eventloop import ioloop | |
31 |
|
31 | |||
32 | from IPython.core.profiledir import ProfileDir |
|
32 | from IPython.core.profiledir import ProfileDir | |
33 | from IPython.parallel.apps.baseapp import ( |
|
33 | from IPython.parallel.apps.baseapp import ( | |
34 | BaseParallelApplication, |
|
34 | BaseParallelApplication, | |
35 | base_aliases, |
|
35 | base_aliases, | |
36 | base_flags, |
|
36 | base_flags, | |
37 | catch_config_error, |
|
37 | catch_config_error, | |
38 | ) |
|
38 | ) | |
39 | from IPython.kernel.zmq.log import EnginePUBHandler |
|
39 | from IPython.kernel.zmq.log import EnginePUBHandler | |
40 | from IPython.kernel.zmq.ipkernel import Kernel |
|
40 | from IPython.kernel.zmq.ipkernel import Kernel | |
41 | from IPython.kernel.zmq.kernelapp import IPKernelApp |
|
41 | from IPython.kernel.zmq.kernelapp import IPKernelApp | |
42 | from IPython.kernel.zmq.session import ( |
|
42 | from IPython.kernel.zmq.session import ( | |
43 | Session, session_aliases, session_flags |
|
43 | Session, session_aliases, session_flags | |
44 | ) |
|
44 | ) | |
45 | from IPython.kernel.zmq.zmqshell import ZMQInteractiveShell |
|
45 | from IPython.kernel.zmq.zmqshell import ZMQInteractiveShell | |
46 |
|
46 | |||
47 | from IPython.config.configurable import Configurable |
|
47 | from IPython.config.configurable import Configurable | |
48 |
|
48 | |||
49 | from IPython.parallel.engine.engine import EngineFactory |
|
49 | from IPython.parallel.engine.engine import EngineFactory | |
50 | from IPython.parallel.util import disambiguate_ip_address |
|
50 | from IPython.parallel.util import disambiguate_ip_address | |
51 |
|
51 | |||
52 | from IPython.utils.importstring import import_item |
|
52 | from IPython.utils.importstring import import_item | |
53 | from IPython.utils.py3compat import cast_bytes |
|
53 | from IPython.utils.py3compat import cast_bytes | |
54 | from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float, Instance |
|
54 | from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float, Instance | |
55 |
|
55 | |||
56 |
|
56 | |||
57 | #----------------------------------------------------------------------------- |
|
57 | #----------------------------------------------------------------------------- | |
58 | # Module level variables |
|
58 | # Module level variables | |
59 | #----------------------------------------------------------------------------- |
|
59 | #----------------------------------------------------------------------------- | |
60 |
|
60 | |||
61 | #: The default config file name for this application |
|
61 | #: The default config file name for this application | |
62 | default_config_file_name = u'ipengine_config.py' |
|
62 | default_config_file_name = u'ipengine_config.py' | |
63 |
|
63 | |||
64 | _description = """Start an IPython engine for parallel computing. |
|
64 | _description = """Start an IPython engine for parallel computing. | |
65 |
|
65 | |||
66 | IPython engines run in parallel and perform computations on behalf of a client |
|
66 | IPython engines run in parallel and perform computations on behalf of a client | |
67 | and controller. A controller needs to be started before the engines. The |
|
67 | and controller. A controller needs to be started before the engines. The | |
68 | engine can be configured using command line options or using a cluster |
|
68 | engine can be configured using command line options or using a cluster | |
69 | directory. Cluster directories contain config, log and security files and are |
|
69 | directory. Cluster directories contain config, log and security files and are | |
70 | usually located in your ipython directory and named as "profile_name". |
|
70 | usually located in your ipython directory and named as "profile_name". | |
71 | See the `profile` and `profile-dir` options for details. |
|
71 | See the `profile` and `profile-dir` options for details. | |
72 | """ |
|
72 | """ | |
73 |
|
73 | |||
74 | _examples = """ |
|
74 | _examples = """ | |
75 | ipengine --ip=192.168.0.1 --port=1000 # connect to hub at ip and port |
|
75 | ipengine --ip=192.168.0.1 --port=1000 # connect to hub at ip and port | |
76 | ipengine --log-to-file --log-level=DEBUG # log to a file with DEBUG verbosity |
|
76 | ipengine --log-to-file --log-level=DEBUG # log to a file with DEBUG verbosity | |
77 | """ |
|
77 | """ | |
78 |
|
78 | |||
79 | #----------------------------------------------------------------------------- |
|
79 | #----------------------------------------------------------------------------- | |
80 | # MPI configuration |
|
80 | # MPI configuration | |
81 | #----------------------------------------------------------------------------- |
|
81 | #----------------------------------------------------------------------------- | |
82 |
|
82 | |||
83 | mpi4py_init = """from mpi4py import MPI as mpi |
|
83 | mpi4py_init = """from mpi4py import MPI as mpi | |
84 | mpi.size = mpi.COMM_WORLD.Get_size() |
|
84 | mpi.size = mpi.COMM_WORLD.Get_size() | |
85 | mpi.rank = mpi.COMM_WORLD.Get_rank() |
|
85 | mpi.rank = mpi.COMM_WORLD.Get_rank() | |
86 | """ |
|
86 | """ | |
87 |
|
87 | |||
88 |
|
88 | |||
89 | pytrilinos_init = """from PyTrilinos import Epetra |
|
89 | pytrilinos_init = """from PyTrilinos import Epetra | |
90 | class SimpleStruct: |
|
90 | class SimpleStruct: | |
91 | pass |
|
91 | pass | |
92 | mpi = SimpleStruct() |
|
92 | mpi = SimpleStruct() | |
93 | mpi.rank = 0 |
|
93 | mpi.rank = 0 | |
94 | mpi.size = 0 |
|
94 | mpi.size = 0 | |
95 | """ |
|
95 | """ | |
96 |
|
96 | |||
97 | class MPI(Configurable): |
|
97 | class MPI(Configurable): | |
98 | """Configurable for MPI initialization""" |
|
98 | """Configurable for MPI initialization""" | |
99 | use = Unicode('', config=True, |
|
99 | use = Unicode('', config=True, | |
100 | help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).' |
|
100 | help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).' | |
101 | ) |
|
101 | ) | |
102 |
|
102 | |||
103 | def _use_changed(self, name, old, new): |
|
103 | def _use_changed(self, name, old, new): | |
104 | # load default init script if it's not set |
|
104 | # load default init script if it's not set | |
105 | if not self.init_script: |
|
105 | if not self.init_script: | |
106 | self.init_script = self.default_inits.get(new, '') |
|
106 | self.init_script = self.default_inits.get(new, '') | |
107 |
|
107 | |||
108 | init_script = Unicode('', config=True, |
|
108 | init_script = Unicode('', config=True, | |
109 | help="Initialization code for MPI") |
|
109 | help="Initialization code for MPI") | |
110 |
|
110 | |||
111 | default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init}, |
|
111 | default_inits = Dict({'mpi4py' : mpi4py_init, 'pytrilinos':pytrilinos_init}, | |
112 | config=True) |
|
112 | config=True) | |
113 |
|
113 | |||
114 |
|
114 | |||
115 | #----------------------------------------------------------------------------- |
|
115 | #----------------------------------------------------------------------------- | |
116 | # Main application |
|
116 | # Main application | |
117 | #----------------------------------------------------------------------------- |
|
117 | #----------------------------------------------------------------------------- | |
118 | aliases = dict( |
|
118 | aliases = dict( | |
119 | file = 'IPEngineApp.url_file', |
|
119 | file = 'IPEngineApp.url_file', | |
120 | c = 'IPEngineApp.startup_command', |
|
120 | c = 'IPEngineApp.startup_command', | |
121 | s = 'IPEngineApp.startup_script', |
|
121 | s = 'IPEngineApp.startup_script', | |
122 |
|
122 | |||
123 | url = 'EngineFactory.url', |
|
123 | url = 'EngineFactory.url', | |
124 | ssh = 'EngineFactory.sshserver', |
|
124 | ssh = 'EngineFactory.sshserver', | |
125 | sshkey = 'EngineFactory.sshkey', |
|
125 | sshkey = 'EngineFactory.sshkey', | |
126 | ip = 'EngineFactory.ip', |
|
126 | ip = 'EngineFactory.ip', | |
127 | transport = 'EngineFactory.transport', |
|
127 | transport = 'EngineFactory.transport', | |
128 | port = 'EngineFactory.regport', |
|
128 | port = 'EngineFactory.regport', | |
129 | location = 'EngineFactory.location', |
|
129 | location = 'EngineFactory.location', | |
130 |
|
130 | |||
131 | timeout = 'EngineFactory.timeout', |
|
131 | timeout = 'EngineFactory.timeout', | |
132 |
|
132 | |||
133 | mpi = 'MPI.use', |
|
133 | mpi = 'MPI.use', | |
134 |
|
134 | |||
135 | ) |
|
135 | ) | |
136 | aliases.update(base_aliases) |
|
136 | aliases.update(base_aliases) | |
137 | aliases.update(session_aliases) |
|
137 | aliases.update(session_aliases) | |
138 | flags = {} |
|
138 | flags = {} | |
139 | flags.update(base_flags) |
|
139 | flags.update(base_flags) | |
140 | flags.update(session_flags) |
|
140 | flags.update(session_flags) | |
141 |
|
141 | |||
142 | class IPEngineApp(BaseParallelApplication): |
|
142 | class IPEngineApp(BaseParallelApplication): | |
143 |
|
143 | |||
144 | name = 'ipengine' |
|
144 | name = 'ipengine' | |
145 | description = _description |
|
145 | description = _description | |
146 | examples = _examples |
|
146 | examples = _examples | |
147 | config_file_name = Unicode(default_config_file_name) |
|
147 | config_file_name = Unicode(default_config_file_name) | |
148 | classes = List([ZMQInteractiveShell, ProfileDir, Session, EngineFactory, Kernel, MPI]) |
|
148 | classes = List([ZMQInteractiveShell, ProfileDir, Session, EngineFactory, Kernel, MPI]) | |
149 |
|
149 | |||
150 | startup_script = Unicode(u'', config=True, |
|
150 | startup_script = Unicode(u'', config=True, | |
151 | help='specify a script to be run at startup') |
|
151 | help='specify a script to be run at startup') | |
152 | startup_command = Unicode('', config=True, |
|
152 | startup_command = Unicode('', config=True, | |
153 | help='specify a command to be run at startup') |
|
153 | help='specify a command to be run at startup') | |
154 |
|
154 | |||
155 | url_file = Unicode(u'', config=True, |
|
155 | url_file = Unicode(u'', config=True, | |
156 | help="""The full location of the file containing the connection information for |
|
156 | help="""The full location of the file containing the connection information for | |
157 | the controller. If this is not given, the file must be in the |
|
157 | the controller. If this is not given, the file must be in the | |
158 | security directory of the cluster directory. This location is |
|
158 | security directory of the cluster directory. This location is | |
159 | resolved using the `profile` or `profile_dir` options.""", |
|
159 | resolved using the `profile` or `profile_dir` options.""", | |
160 | ) |
|
160 | ) | |
161 | wait_for_url_file = Float(5, config=True, |
|
161 | wait_for_url_file = Float(5, config=True, | |
162 | help="""The maximum number of seconds to wait for url_file to exist. |
|
162 | help="""The maximum number of seconds to wait for url_file to exist. | |
163 | This is useful for batch-systems and shared-filesystems where the |
|
163 | This is useful for batch-systems and shared-filesystems where the | |
164 | controller and engine are started at the same time and it |
|
164 | controller and engine are started at the same time and it | |
165 | may take a moment for the controller to write the connector files.""") |
|
165 | may take a moment for the controller to write the connector files.""") | |
166 |
|
166 | |||
167 | url_file_name = Unicode(u'ipcontroller-engine.json', config=True) |
|
167 | url_file_name = Unicode(u'ipcontroller-engine.json', config=True) | |
168 |
|
168 | |||
169 | def _cluster_id_changed(self, name, old, new): |
|
169 | def _cluster_id_changed(self, name, old, new): | |
170 | if new: |
|
170 | if new: | |
171 | base = 'ipcontroller-%s' % new |
|
171 | base = 'ipcontroller-%s' % new | |
172 | else: |
|
172 | else: | |
173 | base = 'ipcontroller' |
|
173 | base = 'ipcontroller' | |
174 | self.url_file_name = "%s-engine.json" % base |
|
174 | self.url_file_name = "%s-engine.json" % base | |
175 |
|
175 | |||
176 | log_url = Unicode('', config=True, |
|
176 | log_url = Unicode('', config=True, | |
177 | help="""The URL for the iploggerapp instance, for forwarding |
|
177 | help="""The URL for the iploggerapp instance, for forwarding | |
178 | logging to a central location.""") |
|
178 | logging to a central location.""") | |
179 |
|
179 | |||
180 | # an IPKernelApp instance, used to setup listening for shell frontends |
|
180 | # an IPKernelApp instance, used to setup listening for shell frontends | |
181 | kernel_app = Instance(IPKernelApp) |
|
181 | kernel_app = Instance(IPKernelApp) | |
182 |
|
182 | |||
183 | aliases = Dict(aliases) |
|
183 | aliases = Dict(aliases) | |
184 | flags = Dict(flags) |
|
184 | flags = Dict(flags) | |
185 |
|
185 | |||
186 | @property |
|
186 | @property | |
187 | def kernel(self): |
|
187 | def kernel(self): | |
188 | """allow access to the Kernel object, so I look like IPKernelApp""" |
|
188 | """allow access to the Kernel object, so I look like IPKernelApp""" | |
189 | return self.engine.kernel |
|
189 | return self.engine.kernel | |
190 |
|
190 | |||
191 | def find_url_file(self): |
|
191 | def find_url_file(self): | |
192 | """Set the url file. |
|
192 | """Set the url file. | |
193 |
|
193 | |||
194 | Here we don't try to actually see if it exists for is valid as that |
|
194 | Here we don't try to actually see if it exists for is valid as that | |
195 | is hadled by the connection logic. |
|
195 | is hadled by the connection logic. | |
196 | """ |
|
196 | """ | |
197 | config = self.config |
|
197 | config = self.config | |
198 | # Find the actual controller key file |
|
198 | # Find the actual controller key file | |
199 | if not self.url_file: |
|
199 | if not self.url_file: | |
200 | self.url_file = os.path.join( |
|
200 | self.url_file = os.path.join( | |
201 | self.profile_dir.security_dir, |
|
201 | self.profile_dir.security_dir, | |
202 | self.url_file_name |
|
202 | self.url_file_name | |
203 | ) |
|
203 | ) | |
204 |
|
204 | |||
205 | def load_connector_file(self): |
|
205 | def load_connector_file(self): | |
206 | """load config from a JSON connector file, |
|
206 | """load config from a JSON connector file, | |
207 | at a *lower* priority than command-line/config files. |
|
207 | at a *lower* priority than command-line/config files. | |
208 | """ |
|
208 | """ | |
209 |
|
209 | |||
210 | self.log.info("Loading url_file %r", self.url_file) |
|
210 | self.log.info("Loading url_file %r", self.url_file) | |
211 | config = self.config |
|
211 | config = self.config | |
212 |
|
212 | |||
213 | with open(self.url_file) as f: |
|
213 | with open(self.url_file) as f: | |
214 | d = json.loads(f.read()) |
|
214 | d = json.loads(f.read()) | |
215 |
|
215 | |||
216 | # allow hand-override of location for disambiguation |
|
216 | # allow hand-override of location for disambiguation | |
217 | # and ssh-server |
|
217 | # and ssh-server | |
218 | try: |
|
218 | try: | |
219 | config.EngineFactory.location |
|
219 | config.EngineFactory.location | |
220 | except AttributeError: |
|
220 | except AttributeError: | |
221 | config.EngineFactory.location = d['location'] |
|
221 | config.EngineFactory.location = d['location'] | |
222 |
|
222 | |||
223 | try: |
|
223 | try: | |
224 | config.EngineFactory.sshserver |
|
224 | config.EngineFactory.sshserver | |
225 | except AttributeError: |
|
225 | except AttributeError: | |
226 | config.EngineFactory.sshserver = d.get('ssh') |
|
226 | config.EngineFactory.sshserver = d.get('ssh') | |
227 |
|
227 | |||
228 | location = config.EngineFactory.location |
|
228 | location = config.EngineFactory.location | |
229 |
|
229 | |||
230 | proto, ip = d['interface'].split('://') |
|
230 | proto, ip = d['interface'].split('://') | |
231 | ip = disambiguate_ip_address(ip, location) |
|
231 | ip = disambiguate_ip_address(ip, location) | |
232 | d['interface'] = '%s://%s' % (proto, ip) |
|
232 | d['interface'] = '%s://%s' % (proto, ip) | |
233 |
|
233 | |||
234 | # DO NOT allow override of basic URLs, serialization, or exec_key |
|
234 | # DO NOT allow override of basic URLs, serialization, or exec_key | |
235 | # JSON file takes top priority there |
|
235 | # JSON file takes top priority there | |
236 | config.Session.key = cast_bytes(d['exec_key']) |
|
236 | config.Session.key = cast_bytes(d['exec_key']) | |
237 |
|
237 | |||
238 | config.EngineFactory.url = d['interface'] + ':%i' % d['registration'] |
|
238 | config.EngineFactory.url = d['interface'] + ':%i' % d['registration'] | |
239 |
|
239 | |||
240 | config.Session.packer = d['pack'] |
|
240 | config.Session.packer = d['pack'] | |
241 | config.Session.unpacker = d['unpack'] |
|
241 | config.Session.unpacker = d['unpack'] | |
242 |
|
242 | |||
243 | self.log.debug("Config changed:") |
|
243 | self.log.debug("Config changed:") | |
244 | self.log.debug("%r", config) |
|
244 | self.log.debug("%r", config) | |
245 | self.connection_info = d |
|
245 | self.connection_info = d | |
246 |
|
246 | |||
247 | def bind_kernel(self, **kwargs): |
|
247 | def bind_kernel(self, **kwargs): | |
248 | """Promote engine to listening kernel, accessible to frontends.""" |
|
248 | """Promote engine to listening kernel, accessible to frontends.""" | |
249 | if self.kernel_app is not None: |
|
249 | if self.kernel_app is not None: | |
250 | return |
|
250 | return | |
251 |
|
251 | |||
252 | self.log.info("Opening ports for direct connections as an IPython kernel") |
|
252 | self.log.info("Opening ports for direct connections as an IPython kernel") | |
253 |
|
253 | |||
254 | kernel = self.kernel |
|
254 | kernel = self.kernel | |
255 |
|
255 | |||
256 | kwargs.setdefault('config', self.config) |
|
256 | kwargs.setdefault('config', self.config) | |
257 | kwargs.setdefault('log', self.log) |
|
257 | kwargs.setdefault('log', self.log) | |
258 | kwargs.setdefault('profile_dir', self.profile_dir) |
|
258 | kwargs.setdefault('profile_dir', self.profile_dir) | |
259 | kwargs.setdefault('session', self.engine.session) |
|
259 | kwargs.setdefault('session', self.engine.session) | |
260 |
|
260 | |||
261 | app = self.kernel_app = IPKernelApp(**kwargs) |
|
261 | app = self.kernel_app = IPKernelApp(**kwargs) | |
262 |
|
262 | |||
263 | # allow IPKernelApp.instance(): |
|
263 | # allow IPKernelApp.instance(): | |
264 | IPKernelApp._instance = app |
|
264 | IPKernelApp._instance = app | |
265 |
|
265 | |||
266 | app.init_connection_file() |
|
266 | app.init_connection_file() | |
267 | # relevant contents of init_sockets: |
|
267 | # relevant contents of init_sockets: | |
268 |
|
268 | |||
269 | app.shell_port = app._bind_socket(kernel.shell_streams[0], app.shell_port) |
|
269 | app.shell_port = app._bind_socket(kernel.shell_streams[0], app.shell_port) | |
270 | app.log.debug("shell ROUTER Channel on port: %i", app.shell_port) |
|
270 | app.log.debug("shell ROUTER Channel on port: %i", app.shell_port) | |
271 |
|
271 | |||
272 | app.iopub_port = app._bind_socket(kernel.iopub_socket, app.iopub_port) |
|
272 | app.iopub_port = app._bind_socket(kernel.iopub_socket, app.iopub_port) | |
273 | app.log.debug("iopub PUB Channel on port: %i", app.iopub_port) |
|
273 | app.log.debug("iopub PUB Channel on port: %i", app.iopub_port) | |
274 |
|
274 | |||
275 | kernel.stdin_socket = self.engine.context.socket(zmq.ROUTER) |
|
275 | kernel.stdin_socket = self.engine.context.socket(zmq.ROUTER) | |
276 | app.stdin_port = app._bind_socket(kernel.stdin_socket, app.stdin_port) |
|
276 | app.stdin_port = app._bind_socket(kernel.stdin_socket, app.stdin_port) | |
277 | app.log.debug("stdin ROUTER Channel on port: %i", app.stdin_port) |
|
277 | app.log.debug("stdin ROUTER Channel on port: %i", app.stdin_port) | |
278 |
|
278 | |||
279 | # start the heartbeat, and log connection info: |
|
279 | # start the heartbeat, and log connection info: | |
280 |
|
280 | |||
281 | app.init_heartbeat() |
|
281 | app.init_heartbeat() | |
282 |
|
282 | |||
283 | app.log_connection_info() |
|
283 | app.log_connection_info() | |
284 | app.write_connection_file() |
|
284 | app.write_connection_file() | |
285 |
|
285 | |||
286 |
|
286 | |||
287 | def init_engine(self): |
|
287 | def init_engine(self): | |
288 | # This is the working dir by now. |
|
288 | # This is the working dir by now. | |
289 | sys.path.insert(0, '') |
|
289 | sys.path.insert(0, '') | |
290 | config = self.config |
|
290 | config = self.config | |
291 | # print config |
|
291 | # print config | |
292 | self.find_url_file() |
|
292 | self.find_url_file() | |
293 |
|
293 | |||
294 | # was the url manually specified? |
|
294 | # was the url manually specified? | |
295 | keys = set(self.config.EngineFactory.keys()) |
|
295 | keys = set(self.config.EngineFactory.keys()) | |
296 | keys = keys.union(set(self.config.RegistrationFactory.keys())) |
|
296 | keys = keys.union(set(self.config.RegistrationFactory.keys())) | |
297 |
|
297 | |||
298 | if keys.intersection(set(['ip', 'url', 'port'])): |
|
298 | if keys.intersection(set(['ip', 'url', 'port'])): | |
299 | # Connection info was specified, don't wait for the file |
|
299 | # Connection info was specified, don't wait for the file | |
300 | url_specified = True |
|
300 | url_specified = True | |
301 | self.wait_for_url_file = 0 |
|
301 | self.wait_for_url_file = 0 | |
302 | else: |
|
302 | else: | |
303 | url_specified = False |
|
303 | url_specified = False | |
304 |
|
304 | |||
305 | if self.wait_for_url_file and not os.path.exists(self.url_file): |
|
305 | if self.wait_for_url_file and not os.path.exists(self.url_file): | |
306 | self.log.warn("url_file %r not found", self.url_file) |
|
306 | self.log.warn("url_file %r not found", self.url_file) | |
307 | self.log.warn("Waiting up to %.1f seconds for it to arrive.", self.wait_for_url_file) |
|
307 | self.log.warn("Waiting up to %.1f seconds for it to arrive.", self.wait_for_url_file) | |
308 | tic = time.time() |
|
308 | tic = time.time() | |
309 | while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file): |
|
309 | while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file): | |
310 | # wait for url_file to exist, or until time limit |
|
310 | # wait for url_file to exist, or until time limit | |
311 | time.sleep(0.1) |
|
311 | time.sleep(0.1) | |
312 |
|
312 | |||
313 | if os.path.exists(self.url_file): |
|
313 | if os.path.exists(self.url_file): | |
314 | self.load_connector_file() |
|
314 | self.load_connector_file() | |
315 | elif not url_specified: |
|
315 | elif not url_specified: | |
316 | self.log.fatal("Fatal: url file never arrived: %s", self.url_file) |
|
316 | self.log.fatal("Fatal: url file never arrived: %s", self.url_file) | |
317 | self.exit(1) |
|
317 | self.exit(1) | |
318 |
|
318 | |||
319 |
|
319 | |||
320 | try: |
|
320 | try: | |
321 | exec_lines = config.IPKernelApp.exec_lines |
|
321 | exec_lines = config.IPKernelApp.exec_lines | |
322 | except AttributeError: |
|
322 | except AttributeError: | |
323 | try: |
|
323 | try: | |
324 | exec_lines = config.InteractiveShellApp.exec_lines |
|
324 | exec_lines = config.InteractiveShellApp.exec_lines | |
325 | except AttributeError: |
|
325 | except AttributeError: | |
326 | exec_lines = config.IPKernelApp.exec_lines = [] |
|
326 | exec_lines = config.IPKernelApp.exec_lines = [] | |
327 | try: |
|
327 | try: | |
328 | exec_files = config.IPKernelApp.exec_files |
|
328 | exec_files = config.IPKernelApp.exec_files | |
329 | except AttributeError: |
|
329 | except AttributeError: | |
330 | try: |
|
330 | try: | |
331 | exec_files = config.InteractiveShellApp.exec_files |
|
331 | exec_files = config.InteractiveShellApp.exec_files | |
332 | except AttributeError: |
|
332 | except AttributeError: | |
333 | exec_files = config.IPKernelApp.exec_files = [] |
|
333 | exec_files = config.IPKernelApp.exec_files = [] | |
334 |
|
334 | |||
335 | if self.startup_script: |
|
335 | if self.startup_script: | |
336 | exec_files.append(self.startup_script) |
|
336 | exec_files.append(self.startup_script) | |
337 | if self.startup_command: |
|
337 | if self.startup_command: | |
338 | exec_lines.append(self.startup_command) |
|
338 | exec_lines.append(self.startup_command) | |
339 |
|
339 | |||
340 | # Create the underlying shell class and Engine |
|
340 | # Create the underlying shell class and Engine | |
341 | # shell_class = import_item(self.master_config.Global.shell_class) |
|
341 | # shell_class = import_item(self.master_config.Global.shell_class) | |
342 | # print self.config |
|
342 | # print self.config | |
343 | try: |
|
343 | try: | |
344 | self.engine = EngineFactory(config=config, log=self.log, |
|
344 | self.engine = EngineFactory(config=config, log=self.log, | |
345 | connection_info=self.connection_info, |
|
345 | connection_info=self.connection_info, | |
346 | ) |
|
346 | ) | |
347 | except: |
|
347 | except: | |
348 | self.log.error("Couldn't start the Engine", exc_info=True) |
|
348 | self.log.error("Couldn't start the Engine", exc_info=True) | |
349 | self.exit(1) |
|
349 | self.exit(1) | |
350 |
|
350 | |||
351 | def forward_logging(self): |
|
351 | def forward_logging(self): | |
352 | if self.log_url: |
|
352 | if self.log_url: | |
353 | self.log.info("Forwarding logging to %s", self.log_url) |
|
353 | self.log.info("Forwarding logging to %s", self.log_url) | |
354 | context = self.engine.context |
|
354 | context = self.engine.context | |
355 | lsock = context.socket(zmq.PUB) |
|
355 | lsock = context.socket(zmq.PUB) | |
356 | lsock.connect(self.log_url) |
|
356 | lsock.connect(self.log_url) | |
357 | handler = EnginePUBHandler(self.engine, lsock) |
|
357 | handler = EnginePUBHandler(self.engine, lsock) | |
358 | handler.setLevel(self.log_level) |
|
358 | handler.setLevel(self.log_level) | |
359 | self.log.addHandler(handler) |
|
359 | self.log.addHandler(handler) | |
360 |
|
360 | |||
361 | def init_mpi(self): |
|
361 | def init_mpi(self): | |
362 | global mpi |
|
362 | global mpi | |
363 | self.mpi = MPI(parent=self) |
|
363 | self.mpi = MPI(parent=self) | |
364 |
|
364 | |||
365 | mpi_import_statement = self.mpi.init_script |
|
365 | mpi_import_statement = self.mpi.init_script | |
366 | if mpi_import_statement: |
|
366 | if mpi_import_statement: | |
367 | try: |
|
367 | try: | |
368 | self.log.info("Initializing MPI:") |
|
368 | self.log.info("Initializing MPI:") | |
369 | self.log.info(mpi_import_statement) |
|
369 | self.log.info(mpi_import_statement) | |
370 | exec mpi_import_statement in globals() |
|
370 | exec mpi_import_statement in globals() | |
371 | except: |
|
371 | except: | |
372 | mpi = None |
|
372 | mpi = None | |
373 | else: |
|
373 | else: | |
374 | mpi = None |
|
374 | mpi = None | |
375 |
|
375 | |||
376 | @catch_config_error |
|
376 | @catch_config_error | |
377 | def initialize(self, argv=None): |
|
377 | def initialize(self, argv=None): | |
378 | super(IPEngineApp, self).initialize(argv) |
|
378 | super(IPEngineApp, self).initialize(argv) | |
379 | self.init_mpi() |
|
379 | self.init_mpi() | |
380 | self.init_engine() |
|
380 | self.init_engine() | |
381 | self.forward_logging() |
|
381 | self.forward_logging() | |
382 |
|
382 | |||
383 | def start(self): |
|
383 | def start(self): | |
384 | self.engine.start() |
|
384 | self.engine.start() | |
385 | try: |
|
385 | try: | |
386 | self.engine.loop.start() |
|
386 | self.engine.loop.start() | |
387 | except KeyboardInterrupt: |
|
387 | except KeyboardInterrupt: | |
388 | self.log.critical("Engine Interrupted, shutting down...\n") |
|
388 | self.log.critical("Engine Interrupted, shutting down...\n") | |
389 |
|
389 | |||
390 |
|
390 | |||
391 |
launch_new_instance = IPEngineApp.launch_ |
|
391 | launch_new_instance = IPEngineApp.launch_instance | |
392 |
|
392 | |||
393 |
|
393 | |||
394 | if __name__ == '__main__': |
|
394 | if __name__ == '__main__': | |
395 | launch_new_instance() |
|
395 | launch_new_instance() | |
396 |
|
396 |
@@ -1,99 +1,99 | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | A simple IPython logger application |
|
4 | A simple IPython logger application | |
5 |
|
5 | |||
6 | Authors: |
|
6 | Authors: | |
7 |
|
7 | |||
8 | * MinRK |
|
8 | * MinRK | |
9 |
|
9 | |||
10 | """ |
|
10 | """ | |
11 |
|
11 | |||
12 | #----------------------------------------------------------------------------- |
|
12 | #----------------------------------------------------------------------------- | |
13 | # Copyright (C) 2011 The IPython Development Team |
|
13 | # Copyright (C) 2011 The IPython Development Team | |
14 | # |
|
14 | # | |
15 | # Distributed under the terms of the BSD License. The full license is in |
|
15 | # Distributed under the terms of the BSD License. The full license is in | |
16 | # the file COPYING, distributed as part of this software. |
|
16 | # the file COPYING, distributed as part of this software. | |
17 | #----------------------------------------------------------------------------- |
|
17 | #----------------------------------------------------------------------------- | |
18 |
|
18 | |||
19 | #----------------------------------------------------------------------------- |
|
19 | #----------------------------------------------------------------------------- | |
20 | # Imports |
|
20 | # Imports | |
21 | #----------------------------------------------------------------------------- |
|
21 | #----------------------------------------------------------------------------- | |
22 |
|
22 | |||
23 | import os |
|
23 | import os | |
24 | import sys |
|
24 | import sys | |
25 |
|
25 | |||
26 | import zmq |
|
26 | import zmq | |
27 |
|
27 | |||
28 | from IPython.core.profiledir import ProfileDir |
|
28 | from IPython.core.profiledir import ProfileDir | |
29 | from IPython.utils.traitlets import Bool, Dict, Unicode |
|
29 | from IPython.utils.traitlets import Bool, Dict, Unicode | |
30 |
|
30 | |||
31 | from IPython.parallel.apps.baseapp import ( |
|
31 | from IPython.parallel.apps.baseapp import ( | |
32 | BaseParallelApplication, |
|
32 | BaseParallelApplication, | |
33 | base_aliases, |
|
33 | base_aliases, | |
34 | catch_config_error, |
|
34 | catch_config_error, | |
35 | ) |
|
35 | ) | |
36 | from IPython.parallel.apps.logwatcher import LogWatcher |
|
36 | from IPython.parallel.apps.logwatcher import LogWatcher | |
37 |
|
37 | |||
38 | #----------------------------------------------------------------------------- |
|
38 | #----------------------------------------------------------------------------- | |
39 | # Module level variables |
|
39 | # Module level variables | |
40 | #----------------------------------------------------------------------------- |
|
40 | #----------------------------------------------------------------------------- | |
41 |
|
41 | |||
42 | #: The default config file name for this application |
|
42 | #: The default config file name for this application | |
43 | default_config_file_name = u'iplogger_config.py' |
|
43 | default_config_file_name = u'iplogger_config.py' | |
44 |
|
44 | |||
45 | _description = """Start an IPython logger for parallel computing. |
|
45 | _description = """Start an IPython logger for parallel computing. | |
46 |
|
46 | |||
47 | IPython controllers and engines (and your own processes) can broadcast log messages |
|
47 | IPython controllers and engines (and your own processes) can broadcast log messages | |
48 | by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The |
|
48 | by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The | |
49 | logger can be configured using command line options or using a cluster |
|
49 | logger can be configured using command line options or using a cluster | |
50 | directory. Cluster directories contain config, log and security files and are |
|
50 | directory. Cluster directories contain config, log and security files and are | |
51 | usually located in your ipython directory and named as "profile_name". |
|
51 | usually located in your ipython directory and named as "profile_name". | |
52 | See the `profile` and `profile-dir` options for details. |
|
52 | See the `profile` and `profile-dir` options for details. | |
53 | """ |
|
53 | """ | |
54 |
|
54 | |||
55 |
|
55 | |||
56 | #----------------------------------------------------------------------------- |
|
56 | #----------------------------------------------------------------------------- | |
57 | # Main application |
|
57 | # Main application | |
58 | #----------------------------------------------------------------------------- |
|
58 | #----------------------------------------------------------------------------- | |
59 | aliases = {} |
|
59 | aliases = {} | |
60 | aliases.update(base_aliases) |
|
60 | aliases.update(base_aliases) | |
61 | aliases.update(dict(url='LogWatcher.url', topics='LogWatcher.topics')) |
|
61 | aliases.update(dict(url='LogWatcher.url', topics='LogWatcher.topics')) | |
62 |
|
62 | |||
63 | class IPLoggerApp(BaseParallelApplication): |
|
63 | class IPLoggerApp(BaseParallelApplication): | |
64 |
|
64 | |||
65 | name = u'iplogger' |
|
65 | name = u'iplogger' | |
66 | description = _description |
|
66 | description = _description | |
67 | config_file_name = Unicode(default_config_file_name) |
|
67 | config_file_name = Unicode(default_config_file_name) | |
68 |
|
68 | |||
69 | classes = [LogWatcher, ProfileDir] |
|
69 | classes = [LogWatcher, ProfileDir] | |
70 | aliases = Dict(aliases) |
|
70 | aliases = Dict(aliases) | |
71 |
|
71 | |||
72 | @catch_config_error |
|
72 | @catch_config_error | |
73 | def initialize(self, argv=None): |
|
73 | def initialize(self, argv=None): | |
74 | super(IPLoggerApp, self).initialize(argv) |
|
74 | super(IPLoggerApp, self).initialize(argv) | |
75 | self.init_watcher() |
|
75 | self.init_watcher() | |
76 |
|
76 | |||
77 | def init_watcher(self): |
|
77 | def init_watcher(self): | |
78 | try: |
|
78 | try: | |
79 | self.watcher = LogWatcher(parent=self, log=self.log) |
|
79 | self.watcher = LogWatcher(parent=self, log=self.log) | |
80 | except: |
|
80 | except: | |
81 | self.log.error("Couldn't start the LogWatcher", exc_info=True) |
|
81 | self.log.error("Couldn't start the LogWatcher", exc_info=True) | |
82 | self.exit(1) |
|
82 | self.exit(1) | |
83 | self.log.info("Listening for log messages on %r"%self.watcher.url) |
|
83 | self.log.info("Listening for log messages on %r"%self.watcher.url) | |
84 |
|
84 | |||
85 |
|
85 | |||
86 | def start(self): |
|
86 | def start(self): | |
87 | self.watcher.start() |
|
87 | self.watcher.start() | |
88 | try: |
|
88 | try: | |
89 | self.watcher.loop.start() |
|
89 | self.watcher.loop.start() | |
90 | except KeyboardInterrupt: |
|
90 | except KeyboardInterrupt: | |
91 | self.log.critical("Logging Interrupted, shutting down...\n") |
|
91 | self.log.critical("Logging Interrupted, shutting down...\n") | |
92 |
|
92 | |||
93 |
|
93 | |||
94 |
launch_new_instance = IPLoggerApp.launch_ |
|
94 | launch_new_instance = IPLoggerApp.launch_instance | |
95 |
|
95 | |||
96 |
|
96 | |||
97 | if __name__ == '__main__': |
|
97 | if __name__ == '__main__': | |
98 | launch_new_instance() |
|
98 | launch_new_instance() | |
99 |
|
99 |
@@ -1,149 +1,149 | |||||
1 | """ A minimal application using the ZMQ-based terminal IPython frontend. |
|
1 | """ A minimal application using the ZMQ-based terminal IPython frontend. | |
2 |
|
2 | |||
3 | This is not a complete console app, as subprocess will not be able to receive |
|
3 | This is not a complete console app, as subprocess will not be able to receive | |
4 | input, there is no real readline support, among other limitations. |
|
4 | input, there is no real readline support, among other limitations. | |
5 |
|
5 | |||
6 | Authors: |
|
6 | Authors: | |
7 |
|
7 | |||
8 | * Min RK |
|
8 | * Min RK | |
9 | * Paul Ivanov |
|
9 | * Paul Ivanov | |
10 |
|
10 | |||
11 | """ |
|
11 | """ | |
12 |
|
12 | |||
13 | #----------------------------------------------------------------------------- |
|
13 | #----------------------------------------------------------------------------- | |
14 | # Imports |
|
14 | # Imports | |
15 | #----------------------------------------------------------------------------- |
|
15 | #----------------------------------------------------------------------------- | |
16 | import signal |
|
16 | import signal | |
17 |
|
17 | |||
18 | from IPython.terminal.ipapp import TerminalIPythonApp, frontend_flags as term_flags |
|
18 | from IPython.terminal.ipapp import TerminalIPythonApp, frontend_flags as term_flags | |
19 |
|
19 | |||
20 | from IPython.utils.traitlets import ( |
|
20 | from IPython.utils.traitlets import ( | |
21 | Dict, Any |
|
21 | Dict, Any | |
22 | ) |
|
22 | ) | |
23 | from IPython.utils.warn import error |
|
23 | from IPython.utils.warn import error | |
24 |
|
24 | |||
25 | from IPython.consoleapp import ( |
|
25 | from IPython.consoleapp import ( | |
26 | IPythonConsoleApp, app_aliases, app_flags, aliases, flags |
|
26 | IPythonConsoleApp, app_aliases, app_flags, aliases, flags | |
27 | ) |
|
27 | ) | |
28 |
|
28 | |||
29 | from IPython.terminal.console.interactiveshell import ZMQTerminalInteractiveShell |
|
29 | from IPython.terminal.console.interactiveshell import ZMQTerminalInteractiveShell | |
30 |
|
30 | |||
31 | #----------------------------------------------------------------------------- |
|
31 | #----------------------------------------------------------------------------- | |
32 | # Globals |
|
32 | # Globals | |
33 | #----------------------------------------------------------------------------- |
|
33 | #----------------------------------------------------------------------------- | |
34 |
|
34 | |||
35 | _examples = """ |
|
35 | _examples = """ | |
36 | ipython console # start the ZMQ-based console |
|
36 | ipython console # start the ZMQ-based console | |
37 | ipython console --existing # connect to an existing ipython session |
|
37 | ipython console --existing # connect to an existing ipython session | |
38 | """ |
|
38 | """ | |
39 |
|
39 | |||
40 | #----------------------------------------------------------------------------- |
|
40 | #----------------------------------------------------------------------------- | |
41 | # Flags and Aliases |
|
41 | # Flags and Aliases | |
42 | #----------------------------------------------------------------------------- |
|
42 | #----------------------------------------------------------------------------- | |
43 |
|
43 | |||
44 | # copy flags from mixin: |
|
44 | # copy flags from mixin: | |
45 | flags = dict(flags) |
|
45 | flags = dict(flags) | |
46 | # start with mixin frontend flags: |
|
46 | # start with mixin frontend flags: | |
47 | frontend_flags = dict(app_flags) |
|
47 | frontend_flags = dict(app_flags) | |
48 | # add TerminalIPApp flags: |
|
48 | # add TerminalIPApp flags: | |
49 | frontend_flags.update(term_flags) |
|
49 | frontend_flags.update(term_flags) | |
50 | # disable quick startup, as it won't propagate to the kernel anyway |
|
50 | # disable quick startup, as it won't propagate to the kernel anyway | |
51 | frontend_flags.pop('quick') |
|
51 | frontend_flags.pop('quick') | |
52 | # update full dict with frontend flags: |
|
52 | # update full dict with frontend flags: | |
53 | flags.update(frontend_flags) |
|
53 | flags.update(frontend_flags) | |
54 |
|
54 | |||
55 | # copy flags from mixin |
|
55 | # copy flags from mixin | |
56 | aliases = dict(aliases) |
|
56 | aliases = dict(aliases) | |
57 | # start with mixin frontend flags |
|
57 | # start with mixin frontend flags | |
58 | frontend_aliases = dict(app_aliases) |
|
58 | frontend_aliases = dict(app_aliases) | |
59 | # load updated frontend flags into full dict |
|
59 | # load updated frontend flags into full dict | |
60 | aliases.update(frontend_aliases) |
|
60 | aliases.update(frontend_aliases) | |
61 |
|
61 | |||
62 | # get flags&aliases into sets, and remove a couple that |
|
62 | # get flags&aliases into sets, and remove a couple that | |
63 | # shouldn't be scrubbed from backend flags: |
|
63 | # shouldn't be scrubbed from backend flags: | |
64 | frontend_aliases = set(frontend_aliases.keys()) |
|
64 | frontend_aliases = set(frontend_aliases.keys()) | |
65 | frontend_flags = set(frontend_flags.keys()) |
|
65 | frontend_flags = set(frontend_flags.keys()) | |
66 |
|
66 | |||
67 |
|
67 | |||
68 | #----------------------------------------------------------------------------- |
|
68 | #----------------------------------------------------------------------------- | |
69 | # Classes |
|
69 | # Classes | |
70 | #----------------------------------------------------------------------------- |
|
70 | #----------------------------------------------------------------------------- | |
71 |
|
71 | |||
72 |
|
72 | |||
73 | class ZMQTerminalIPythonApp(TerminalIPythonApp, IPythonConsoleApp): |
|
73 | class ZMQTerminalIPythonApp(TerminalIPythonApp, IPythonConsoleApp): | |
74 | name = "ipython-console" |
|
74 | name = "ipython-console" | |
75 | """Start a terminal frontend to the IPython zmq kernel.""" |
|
75 | """Start a terminal frontend to the IPython zmq kernel.""" | |
76 |
|
76 | |||
77 | description = """ |
|
77 | description = """ | |
78 | The IPython terminal-based Console. |
|
78 | The IPython terminal-based Console. | |
79 |
|
79 | |||
80 | This launches a Console application inside a terminal. |
|
80 | This launches a Console application inside a terminal. | |
81 |
|
81 | |||
82 | The Console supports various extra features beyond the traditional |
|
82 | The Console supports various extra features beyond the traditional | |
83 | single-process Terminal IPython shell, such as connecting to an |
|
83 | single-process Terminal IPython shell, such as connecting to an | |
84 | existing ipython session, via: |
|
84 | existing ipython session, via: | |
85 |
|
85 | |||
86 | ipython console --existing |
|
86 | ipython console --existing | |
87 |
|
87 | |||
88 | where the previous session could have been created by another ipython |
|
88 | where the previous session could have been created by another ipython | |
89 | console, an ipython qtconsole, or by opening an ipython notebook. |
|
89 | console, an ipython qtconsole, or by opening an ipython notebook. | |
90 |
|
90 | |||
91 | """ |
|
91 | """ | |
92 | examples = _examples |
|
92 | examples = _examples | |
93 |
|
93 | |||
94 | classes = [ZMQTerminalInteractiveShell] + IPythonConsoleApp.classes |
|
94 | classes = [ZMQTerminalInteractiveShell] + IPythonConsoleApp.classes | |
95 | flags = Dict(flags) |
|
95 | flags = Dict(flags) | |
96 | aliases = Dict(aliases) |
|
96 | aliases = Dict(aliases) | |
97 | frontend_aliases = Any(frontend_aliases) |
|
97 | frontend_aliases = Any(frontend_aliases) | |
98 | frontend_flags = Any(frontend_flags) |
|
98 | frontend_flags = Any(frontend_flags) | |
99 |
|
99 | |||
100 | subcommands = Dict() |
|
100 | subcommands = Dict() | |
101 |
|
101 | |||
102 | def parse_command_line(self, argv=None): |
|
102 | def parse_command_line(self, argv=None): | |
103 | super(ZMQTerminalIPythonApp, self).parse_command_line(argv) |
|
103 | super(ZMQTerminalIPythonApp, self).parse_command_line(argv) | |
104 | self.build_kernel_argv(argv) |
|
104 | self.build_kernel_argv(argv) | |
105 |
|
105 | |||
106 | def init_shell(self): |
|
106 | def init_shell(self): | |
107 | IPythonConsoleApp.initialize(self) |
|
107 | IPythonConsoleApp.initialize(self) | |
108 | # relay sigint to kernel |
|
108 | # relay sigint to kernel | |
109 | signal.signal(signal.SIGINT, self.handle_sigint) |
|
109 | signal.signal(signal.SIGINT, self.handle_sigint) | |
110 | self.shell = ZMQTerminalInteractiveShell.instance(parent=self, |
|
110 | self.shell = ZMQTerminalInteractiveShell.instance(parent=self, | |
111 | display_banner=False, profile_dir=self.profile_dir, |
|
111 | display_banner=False, profile_dir=self.profile_dir, | |
112 | ipython_dir=self.ipython_dir, |
|
112 | ipython_dir=self.ipython_dir, | |
113 | manager=self.kernel_manager, |
|
113 | manager=self.kernel_manager, | |
114 | client=self.kernel_client, |
|
114 | client=self.kernel_client, | |
115 | ) |
|
115 | ) | |
116 |
|
116 | |||
117 | def init_gui_pylab(self): |
|
117 | def init_gui_pylab(self): | |
118 | # no-op, because we don't want to import matplotlib in the frontend. |
|
118 | # no-op, because we don't want to import matplotlib in the frontend. | |
119 | pass |
|
119 | pass | |
120 |
|
120 | |||
121 | def handle_sigint(self, *args): |
|
121 | def handle_sigint(self, *args): | |
122 | if self.shell._executing: |
|
122 | if self.shell._executing: | |
123 | if self.kernel_manager: |
|
123 | if self.kernel_manager: | |
124 | # interrupt already gets passed to subprocess by signal handler. |
|
124 | # interrupt already gets passed to subprocess by signal handler. | |
125 | # Only if we prevent that should we need to explicitly call |
|
125 | # Only if we prevent that should we need to explicitly call | |
126 | # interrupt_kernel, until which time, this would result in a |
|
126 | # interrupt_kernel, until which time, this would result in a | |
127 | # double-interrupt: |
|
127 | # double-interrupt: | |
128 | # self.kernel_manager.interrupt_kernel() |
|
128 | # self.kernel_manager.interrupt_kernel() | |
129 | pass |
|
129 | pass | |
130 | else: |
|
130 | else: | |
131 | self.shell.write_err('\n') |
|
131 | self.shell.write_err('\n') | |
132 | error("Cannot interrupt kernels we didn't start.\n") |
|
132 | error("Cannot interrupt kernels we didn't start.\n") | |
133 | else: |
|
133 | else: | |
134 | # raise the KeyboardInterrupt if we aren't waiting for execution, |
|
134 | # raise the KeyboardInterrupt if we aren't waiting for execution, | |
135 | # so that the interact loop advances, and prompt is redrawn, etc. |
|
135 | # so that the interact loop advances, and prompt is redrawn, etc. | |
136 | raise KeyboardInterrupt |
|
136 | raise KeyboardInterrupt | |
137 |
|
137 | |||
138 |
|
138 | |||
139 | def init_code(self): |
|
139 | def init_code(self): | |
140 | # no-op in the frontend, code gets run in the backend |
|
140 | # no-op in the frontend, code gets run in the backend | |
141 | pass |
|
141 | pass | |
142 |
|
142 | |||
143 |
|
143 | |||
144 |
launch_new_instance = ZMQTerminalIPythonApp.launch_ |
|
144 | launch_new_instance = ZMQTerminalIPythonApp.launch_instance | |
145 |
|
145 | |||
146 |
|
146 | |||
147 | if __name__ == '__main__': |
|
147 | if __name__ == '__main__': | |
148 | launch_new_instance() |
|
148 | launch_new_instance() | |
149 |
|
149 |
@@ -1,396 +1,396 | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | The :class:`~IPython.core.application.Application` object for the command |
|
4 | The :class:`~IPython.core.application.Application` object for the command | |
5 | line :command:`ipython` program. |
|
5 | line :command:`ipython` program. | |
6 |
|
6 | |||
7 | Authors |
|
7 | Authors | |
8 | ------- |
|
8 | ------- | |
9 |
|
9 | |||
10 | * Brian Granger |
|
10 | * Brian Granger | |
11 | * Fernando Perez |
|
11 | * Fernando Perez | |
12 | * Min Ragan-Kelley |
|
12 | * Min Ragan-Kelley | |
13 | """ |
|
13 | """ | |
14 |
|
14 | |||
15 | #----------------------------------------------------------------------------- |
|
15 | #----------------------------------------------------------------------------- | |
16 | # Copyright (C) 2008-2011 The IPython Development Team |
|
16 | # Copyright (C) 2008-2011 The IPython Development Team | |
17 | # |
|
17 | # | |
18 | # Distributed under the terms of the BSD License. The full license is in |
|
18 | # Distributed under the terms of the BSD License. The full license is in | |
19 | # the file COPYING, distributed as part of this software. |
|
19 | # the file COPYING, distributed as part of this software. | |
20 | #----------------------------------------------------------------------------- |
|
20 | #----------------------------------------------------------------------------- | |
21 |
|
21 | |||
22 | #----------------------------------------------------------------------------- |
|
22 | #----------------------------------------------------------------------------- | |
23 | # Imports |
|
23 | # Imports | |
24 | #----------------------------------------------------------------------------- |
|
24 | #----------------------------------------------------------------------------- | |
25 |
|
25 | |||
26 | from __future__ import absolute_import |
|
26 | from __future__ import absolute_import | |
27 |
|
27 | |||
28 | import logging |
|
28 | import logging | |
29 | import os |
|
29 | import os | |
30 | import sys |
|
30 | import sys | |
31 |
|
31 | |||
32 | from IPython.config.loader import ( |
|
32 | from IPython.config.loader import ( | |
33 | Config, PyFileConfigLoader, ConfigFileNotFound |
|
33 | Config, PyFileConfigLoader, ConfigFileNotFound | |
34 | ) |
|
34 | ) | |
35 | from IPython.config.application import boolean_flag, catch_config_error |
|
35 | from IPython.config.application import boolean_flag, catch_config_error | |
36 | from IPython.core import release |
|
36 | from IPython.core import release | |
37 | from IPython.core import usage |
|
37 | from IPython.core import usage | |
38 | from IPython.core.completer import IPCompleter |
|
38 | from IPython.core.completer import IPCompleter | |
39 | from IPython.core.crashhandler import CrashHandler |
|
39 | from IPython.core.crashhandler import CrashHandler | |
40 | from IPython.core.formatters import PlainTextFormatter |
|
40 | from IPython.core.formatters import PlainTextFormatter | |
41 | from IPython.core.history import HistoryManager |
|
41 | from IPython.core.history import HistoryManager | |
42 | from IPython.core.prompts import PromptManager |
|
42 | from IPython.core.prompts import PromptManager | |
43 | from IPython.core.application import ( |
|
43 | from IPython.core.application import ( | |
44 | ProfileDir, BaseIPythonApplication, base_flags, base_aliases |
|
44 | ProfileDir, BaseIPythonApplication, base_flags, base_aliases | |
45 | ) |
|
45 | ) | |
46 | from IPython.core.magics import ScriptMagics |
|
46 | from IPython.core.magics import ScriptMagics | |
47 | from IPython.core.shellapp import ( |
|
47 | from IPython.core.shellapp import ( | |
48 | InteractiveShellApp, shell_flags, shell_aliases |
|
48 | InteractiveShellApp, shell_flags, shell_aliases | |
49 | ) |
|
49 | ) | |
50 | from IPython.terminal.interactiveshell import TerminalInteractiveShell |
|
50 | from IPython.terminal.interactiveshell import TerminalInteractiveShell | |
51 | from IPython.utils import warn |
|
51 | from IPython.utils import warn | |
52 | from IPython.utils.path import get_ipython_dir, check_for_old_config |
|
52 | from IPython.utils.path import get_ipython_dir, check_for_old_config | |
53 | from IPython.utils.traitlets import ( |
|
53 | from IPython.utils.traitlets import ( | |
54 | Bool, List, Dict, |
|
54 | Bool, List, Dict, | |
55 | ) |
|
55 | ) | |
56 |
|
56 | |||
57 | #----------------------------------------------------------------------------- |
|
57 | #----------------------------------------------------------------------------- | |
58 | # Globals, utilities and helpers |
|
58 | # Globals, utilities and helpers | |
59 | #----------------------------------------------------------------------------- |
|
59 | #----------------------------------------------------------------------------- | |
60 |
|
60 | |||
61 | #: The default config file name for this application. |
|
61 | #: The default config file name for this application. | |
62 | default_config_file_name = u'ipython_config.py' |
|
62 | default_config_file_name = u'ipython_config.py' | |
63 |
|
63 | |||
64 | _examples = """ |
|
64 | _examples = """ | |
65 | ipython --pylab # start in pylab mode |
|
65 | ipython --pylab # start in pylab mode | |
66 | ipython --pylab=qt # start in pylab mode with the qt4 backend |
|
66 | ipython --pylab=qt # start in pylab mode with the qt4 backend | |
67 | ipython --log-level=DEBUG # set logging to DEBUG |
|
67 | ipython --log-level=DEBUG # set logging to DEBUG | |
68 | ipython --profile=foo # start with profile foo |
|
68 | ipython --profile=foo # start with profile foo | |
69 |
|
69 | |||
70 | ipython qtconsole # start the qtconsole GUI application |
|
70 | ipython qtconsole # start the qtconsole GUI application | |
71 | ipython help qtconsole # show the help for the qtconsole subcmd |
|
71 | ipython help qtconsole # show the help for the qtconsole subcmd | |
72 |
|
72 | |||
73 | ipython console # start the terminal-based console application |
|
73 | ipython console # start the terminal-based console application | |
74 | ipython help console # show the help for the console subcmd |
|
74 | ipython help console # show the help for the console subcmd | |
75 |
|
75 | |||
76 | ipython notebook # start the IPython notebook |
|
76 | ipython notebook # start the IPython notebook | |
77 | ipython help notebook # show the help for the notebook subcmd |
|
77 | ipython help notebook # show the help for the notebook subcmd | |
78 |
|
78 | |||
79 | ipython profile create foo # create profile foo w/ default config files |
|
79 | ipython profile create foo # create profile foo w/ default config files | |
80 | ipython help profile # show the help for the profile subcmd |
|
80 | ipython help profile # show the help for the profile subcmd | |
81 |
|
81 | |||
82 | ipython locate # print the path to the IPython directory |
|
82 | ipython locate # print the path to the IPython directory | |
83 | ipython locate profile foo # print the path to the directory for profile `foo` |
|
83 | ipython locate profile foo # print the path to the directory for profile `foo` | |
84 |
|
84 | |||
85 | ipython nbconvert # convert notebooks to/from other formats |
|
85 | ipython nbconvert # convert notebooks to/from other formats | |
86 | """ |
|
86 | """ | |
87 |
|
87 | |||
88 | #----------------------------------------------------------------------------- |
|
88 | #----------------------------------------------------------------------------- | |
89 | # Crash handler for this application |
|
89 | # Crash handler for this application | |
90 | #----------------------------------------------------------------------------- |
|
90 | #----------------------------------------------------------------------------- | |
91 |
|
91 | |||
92 | class IPAppCrashHandler(CrashHandler): |
|
92 | class IPAppCrashHandler(CrashHandler): | |
93 | """sys.excepthook for IPython itself, leaves a detailed report on disk.""" |
|
93 | """sys.excepthook for IPython itself, leaves a detailed report on disk.""" | |
94 |
|
94 | |||
95 | def __init__(self, app): |
|
95 | def __init__(self, app): | |
96 | contact_name = release.author |
|
96 | contact_name = release.author | |
97 | contact_email = release.author_email |
|
97 | contact_email = release.author_email | |
98 | bug_tracker = 'https://github.com/ipython/ipython/issues' |
|
98 | bug_tracker = 'https://github.com/ipython/ipython/issues' | |
99 | super(IPAppCrashHandler,self).__init__( |
|
99 | super(IPAppCrashHandler,self).__init__( | |
100 | app, contact_name, contact_email, bug_tracker |
|
100 | app, contact_name, contact_email, bug_tracker | |
101 | ) |
|
101 | ) | |
102 |
|
102 | |||
103 | def make_report(self,traceback): |
|
103 | def make_report(self,traceback): | |
104 | """Return a string containing a crash report.""" |
|
104 | """Return a string containing a crash report.""" | |
105 |
|
105 | |||
106 | sec_sep = self.section_sep |
|
106 | sec_sep = self.section_sep | |
107 | # Start with parent report |
|
107 | # Start with parent report | |
108 | report = [super(IPAppCrashHandler, self).make_report(traceback)] |
|
108 | report = [super(IPAppCrashHandler, self).make_report(traceback)] | |
109 | # Add interactive-specific info we may have |
|
109 | # Add interactive-specific info we may have | |
110 | rpt_add = report.append |
|
110 | rpt_add = report.append | |
111 | try: |
|
111 | try: | |
112 | rpt_add(sec_sep+"History of session input:") |
|
112 | rpt_add(sec_sep+"History of session input:") | |
113 | for line in self.app.shell.user_ns['_ih']: |
|
113 | for line in self.app.shell.user_ns['_ih']: | |
114 | rpt_add(line) |
|
114 | rpt_add(line) | |
115 | rpt_add('\n*** Last line of input (may not be in above history):\n') |
|
115 | rpt_add('\n*** Last line of input (may not be in above history):\n') | |
116 | rpt_add(self.app.shell._last_input_line+'\n') |
|
116 | rpt_add(self.app.shell._last_input_line+'\n') | |
117 | except: |
|
117 | except: | |
118 | pass |
|
118 | pass | |
119 |
|
119 | |||
120 | return ''.join(report) |
|
120 | return ''.join(report) | |
121 |
|
121 | |||
122 | #----------------------------------------------------------------------------- |
|
122 | #----------------------------------------------------------------------------- | |
123 | # Aliases and Flags |
|
123 | # Aliases and Flags | |
124 | #----------------------------------------------------------------------------- |
|
124 | #----------------------------------------------------------------------------- | |
125 | flags = dict(base_flags) |
|
125 | flags = dict(base_flags) | |
126 | flags.update(shell_flags) |
|
126 | flags.update(shell_flags) | |
127 | frontend_flags = {} |
|
127 | frontend_flags = {} | |
128 | addflag = lambda *args: frontend_flags.update(boolean_flag(*args)) |
|
128 | addflag = lambda *args: frontend_flags.update(boolean_flag(*args)) | |
129 | addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax', |
|
129 | addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax', | |
130 | 'Turn on auto editing of files with syntax errors.', |
|
130 | 'Turn on auto editing of files with syntax errors.', | |
131 | 'Turn off auto editing of files with syntax errors.' |
|
131 | 'Turn off auto editing of files with syntax errors.' | |
132 | ) |
|
132 | ) | |
133 | addflag('banner', 'TerminalIPythonApp.display_banner', |
|
133 | addflag('banner', 'TerminalIPythonApp.display_banner', | |
134 | "Display a banner upon starting IPython.", |
|
134 | "Display a banner upon starting IPython.", | |
135 | "Don't display a banner upon starting IPython." |
|
135 | "Don't display a banner upon starting IPython." | |
136 | ) |
|
136 | ) | |
137 | addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit', |
|
137 | addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit', | |
138 | """Set to confirm when you try to exit IPython with an EOF (Control-D |
|
138 | """Set to confirm when you try to exit IPython with an EOF (Control-D | |
139 | in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit', |
|
139 | in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit', | |
140 | you can force a direct exit without any confirmation.""", |
|
140 | you can force a direct exit without any confirmation.""", | |
141 | "Don't prompt the user when exiting." |
|
141 | "Don't prompt the user when exiting." | |
142 | ) |
|
142 | ) | |
143 | addflag('term-title', 'TerminalInteractiveShell.term_title', |
|
143 | addflag('term-title', 'TerminalInteractiveShell.term_title', | |
144 | "Enable auto setting the terminal title.", |
|
144 | "Enable auto setting the terminal title.", | |
145 | "Disable auto setting the terminal title." |
|
145 | "Disable auto setting the terminal title." | |
146 | ) |
|
146 | ) | |
147 | classic_config = Config() |
|
147 | classic_config = Config() | |
148 | classic_config.InteractiveShell.cache_size = 0 |
|
148 | classic_config.InteractiveShell.cache_size = 0 | |
149 | classic_config.PlainTextFormatter.pprint = False |
|
149 | classic_config.PlainTextFormatter.pprint = False | |
150 | classic_config.PromptManager.in_template = '>>> ' |
|
150 | classic_config.PromptManager.in_template = '>>> ' | |
151 | classic_config.PromptManager.in2_template = '... ' |
|
151 | classic_config.PromptManager.in2_template = '... ' | |
152 | classic_config.PromptManager.out_template = '' |
|
152 | classic_config.PromptManager.out_template = '' | |
153 | classic_config.InteractiveShell.separate_in = '' |
|
153 | classic_config.InteractiveShell.separate_in = '' | |
154 | classic_config.InteractiveShell.separate_out = '' |
|
154 | classic_config.InteractiveShell.separate_out = '' | |
155 | classic_config.InteractiveShell.separate_out2 = '' |
|
155 | classic_config.InteractiveShell.separate_out2 = '' | |
156 | classic_config.InteractiveShell.colors = 'NoColor' |
|
156 | classic_config.InteractiveShell.colors = 'NoColor' | |
157 | classic_config.InteractiveShell.xmode = 'Plain' |
|
157 | classic_config.InteractiveShell.xmode = 'Plain' | |
158 |
|
158 | |||
159 | frontend_flags['classic']=( |
|
159 | frontend_flags['classic']=( | |
160 | classic_config, |
|
160 | classic_config, | |
161 | "Gives IPython a similar feel to the classic Python prompt." |
|
161 | "Gives IPython a similar feel to the classic Python prompt." | |
162 | ) |
|
162 | ) | |
163 | # # log doesn't make so much sense this way anymore |
|
163 | # # log doesn't make so much sense this way anymore | |
164 | # paa('--log','-l', |
|
164 | # paa('--log','-l', | |
165 | # action='store_true', dest='InteractiveShell.logstart', |
|
165 | # action='store_true', dest='InteractiveShell.logstart', | |
166 | # help="Start logging to the default log file (./ipython_log.py).") |
|
166 | # help="Start logging to the default log file (./ipython_log.py).") | |
167 | # |
|
167 | # | |
168 | # # quick is harder to implement |
|
168 | # # quick is harder to implement | |
169 | frontend_flags['quick']=( |
|
169 | frontend_flags['quick']=( | |
170 | {'TerminalIPythonApp' : {'quick' : True}}, |
|
170 | {'TerminalIPythonApp' : {'quick' : True}}, | |
171 | "Enable quick startup with no config files." |
|
171 | "Enable quick startup with no config files." | |
172 | ) |
|
172 | ) | |
173 |
|
173 | |||
174 | frontend_flags['i'] = ( |
|
174 | frontend_flags['i'] = ( | |
175 | {'TerminalIPythonApp' : {'force_interact' : True}}, |
|
175 | {'TerminalIPythonApp' : {'force_interact' : True}}, | |
176 | """If running code from the command line, become interactive afterwards. |
|
176 | """If running code from the command line, become interactive afterwards. | |
177 | Note: can also be given simply as '-i.'""" |
|
177 | Note: can also be given simply as '-i.'""" | |
178 | ) |
|
178 | ) | |
179 | flags.update(frontend_flags) |
|
179 | flags.update(frontend_flags) | |
180 |
|
180 | |||
181 | aliases = dict(base_aliases) |
|
181 | aliases = dict(base_aliases) | |
182 | aliases.update(shell_aliases) |
|
182 | aliases.update(shell_aliases) | |
183 |
|
183 | |||
184 | #----------------------------------------------------------------------------- |
|
184 | #----------------------------------------------------------------------------- | |
185 | # Main classes and functions |
|
185 | # Main classes and functions | |
186 | #----------------------------------------------------------------------------- |
|
186 | #----------------------------------------------------------------------------- | |
187 |
|
187 | |||
188 |
|
188 | |||
189 | class LocateIPythonApp(BaseIPythonApplication): |
|
189 | class LocateIPythonApp(BaseIPythonApplication): | |
190 | description = """print the path to the IPython dir""" |
|
190 | description = """print the path to the IPython dir""" | |
191 | subcommands = Dict(dict( |
|
191 | subcommands = Dict(dict( | |
192 | profile=('IPython.core.profileapp.ProfileLocate', |
|
192 | profile=('IPython.core.profileapp.ProfileLocate', | |
193 | "print the path to an IPython profile directory", |
|
193 | "print the path to an IPython profile directory", | |
194 | ), |
|
194 | ), | |
195 | )) |
|
195 | )) | |
196 | def start(self): |
|
196 | def start(self): | |
197 | if self.subapp is not None: |
|
197 | if self.subapp is not None: | |
198 | return self.subapp.start() |
|
198 | return self.subapp.start() | |
199 | else: |
|
199 | else: | |
200 | print self.ipython_dir |
|
200 | print self.ipython_dir | |
201 |
|
201 | |||
202 |
|
202 | |||
203 | class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp): |
|
203 | class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp): | |
204 | name = u'ipython' |
|
204 | name = u'ipython' | |
205 | description = usage.cl_usage |
|
205 | description = usage.cl_usage | |
206 | default_config_file_name = default_config_file_name |
|
206 | default_config_file_name = default_config_file_name | |
207 | crash_handler_class = IPAppCrashHandler |
|
207 | crash_handler_class = IPAppCrashHandler | |
208 | examples = _examples |
|
208 | examples = _examples | |
209 |
|
209 | |||
210 | flags = Dict(flags) |
|
210 | flags = Dict(flags) | |
211 | aliases = Dict(aliases) |
|
211 | aliases = Dict(aliases) | |
212 | classes = List() |
|
212 | classes = List() | |
213 | def _classes_default(self): |
|
213 | def _classes_default(self): | |
214 | """This has to be in a method, for TerminalIPythonApp to be available.""" |
|
214 | """This has to be in a method, for TerminalIPythonApp to be available.""" | |
215 | return [ |
|
215 | return [ | |
216 | InteractiveShellApp, # ShellApp comes before TerminalApp, because |
|
216 | InteractiveShellApp, # ShellApp comes before TerminalApp, because | |
217 | self.__class__, # it will also affect subclasses (e.g. QtConsole) |
|
217 | self.__class__, # it will also affect subclasses (e.g. QtConsole) | |
218 | TerminalInteractiveShell, |
|
218 | TerminalInteractiveShell, | |
219 | PromptManager, |
|
219 | PromptManager, | |
220 | HistoryManager, |
|
220 | HistoryManager, | |
221 | ProfileDir, |
|
221 | ProfileDir, | |
222 | PlainTextFormatter, |
|
222 | PlainTextFormatter, | |
223 | IPCompleter, |
|
223 | IPCompleter, | |
224 | ScriptMagics, |
|
224 | ScriptMagics, | |
225 | ] |
|
225 | ] | |
226 |
|
226 | |||
227 | subcommands = Dict(dict( |
|
227 | subcommands = Dict(dict( | |
228 | qtconsole=('IPython.qt.console.qtconsoleapp.IPythonQtConsoleApp', |
|
228 | qtconsole=('IPython.qt.console.qtconsoleapp.IPythonQtConsoleApp', | |
229 | """Launch the IPython Qt Console.""" |
|
229 | """Launch the IPython Qt Console.""" | |
230 | ), |
|
230 | ), | |
231 | notebook=('IPython.html.notebookapp.NotebookApp', |
|
231 | notebook=('IPython.html.notebookapp.NotebookApp', | |
232 | """Launch the IPython HTML Notebook Server.""" |
|
232 | """Launch the IPython HTML Notebook Server.""" | |
233 | ), |
|
233 | ), | |
234 | profile = ("IPython.core.profileapp.ProfileApp", |
|
234 | profile = ("IPython.core.profileapp.ProfileApp", | |
235 | "Create and manage IPython profiles." |
|
235 | "Create and manage IPython profiles." | |
236 | ), |
|
236 | ), | |
237 | kernel = ("IPython.kernel.zmq.kernelapp.IPKernelApp", |
|
237 | kernel = ("IPython.kernel.zmq.kernelapp.IPKernelApp", | |
238 | "Start a kernel without an attached frontend." |
|
238 | "Start a kernel without an attached frontend." | |
239 | ), |
|
239 | ), | |
240 | console=('IPython.terminal.console.app.ZMQTerminalIPythonApp', |
|
240 | console=('IPython.terminal.console.app.ZMQTerminalIPythonApp', | |
241 | """Launch the IPython terminal-based Console.""" |
|
241 | """Launch the IPython terminal-based Console.""" | |
242 | ), |
|
242 | ), | |
243 | locate=('IPython.terminal.ipapp.LocateIPythonApp', |
|
243 | locate=('IPython.terminal.ipapp.LocateIPythonApp', | |
244 | LocateIPythonApp.description |
|
244 | LocateIPythonApp.description | |
245 | ), |
|
245 | ), | |
246 | history=('IPython.core.historyapp.HistoryApp', |
|
246 | history=('IPython.core.historyapp.HistoryApp', | |
247 | "Manage the IPython history database." |
|
247 | "Manage the IPython history database." | |
248 | ), |
|
248 | ), | |
249 | nbconvert=('IPython.nbconvert.nbconvertapp.NbConvertApp', |
|
249 | nbconvert=('IPython.nbconvert.nbconvertapp.NbConvertApp', | |
250 | "Convert notebooks to/from other formats." |
|
250 | "Convert notebooks to/from other formats." | |
251 | ), |
|
251 | ), | |
252 | )) |
|
252 | )) | |
253 |
|
253 | |||
254 | # *do* autocreate requested profile, but don't create the config file. |
|
254 | # *do* autocreate requested profile, but don't create the config file. | |
255 | auto_create=Bool(True) |
|
255 | auto_create=Bool(True) | |
256 | # configurables |
|
256 | # configurables | |
257 | ignore_old_config=Bool(False, config=True, |
|
257 | ignore_old_config=Bool(False, config=True, | |
258 | help="Suppress warning messages about legacy config files" |
|
258 | help="Suppress warning messages about legacy config files" | |
259 | ) |
|
259 | ) | |
260 | quick = Bool(False, config=True, |
|
260 | quick = Bool(False, config=True, | |
261 | help="""Start IPython quickly by skipping the loading of config files.""" |
|
261 | help="""Start IPython quickly by skipping the loading of config files.""" | |
262 | ) |
|
262 | ) | |
263 | def _quick_changed(self, name, old, new): |
|
263 | def _quick_changed(self, name, old, new): | |
264 | if new: |
|
264 | if new: | |
265 | self.load_config_file = lambda *a, **kw: None |
|
265 | self.load_config_file = lambda *a, **kw: None | |
266 | self.ignore_old_config=True |
|
266 | self.ignore_old_config=True | |
267 |
|
267 | |||
268 | display_banner = Bool(True, config=True, |
|
268 | display_banner = Bool(True, config=True, | |
269 | help="Whether to display a banner upon starting IPython." |
|
269 | help="Whether to display a banner upon starting IPython." | |
270 | ) |
|
270 | ) | |
271 |
|
271 | |||
272 | # if there is code of files to run from the cmd line, don't interact |
|
272 | # if there is code of files to run from the cmd line, don't interact | |
273 | # unless the --i flag (App.force_interact) is true. |
|
273 | # unless the --i flag (App.force_interact) is true. | |
274 | force_interact = Bool(False, config=True, |
|
274 | force_interact = Bool(False, config=True, | |
275 | help="""If a command or file is given via the command-line, |
|
275 | help="""If a command or file is given via the command-line, | |
276 | e.g. 'ipython foo.py""" |
|
276 | e.g. 'ipython foo.py""" | |
277 | ) |
|
277 | ) | |
278 | def _force_interact_changed(self, name, old, new): |
|
278 | def _force_interact_changed(self, name, old, new): | |
279 | if new: |
|
279 | if new: | |
280 | self.interact = True |
|
280 | self.interact = True | |
281 |
|
281 | |||
282 | def _file_to_run_changed(self, name, old, new): |
|
282 | def _file_to_run_changed(self, name, old, new): | |
283 | if new: |
|
283 | if new: | |
284 | self.something_to_run = True |
|
284 | self.something_to_run = True | |
285 | if new and not self.force_interact: |
|
285 | if new and not self.force_interact: | |
286 | self.interact = False |
|
286 | self.interact = False | |
287 | _code_to_run_changed = _file_to_run_changed |
|
287 | _code_to_run_changed = _file_to_run_changed | |
288 | _module_to_run_changed = _file_to_run_changed |
|
288 | _module_to_run_changed = _file_to_run_changed | |
289 |
|
289 | |||
290 | # internal, not-configurable |
|
290 | # internal, not-configurable | |
291 | interact=Bool(True) |
|
291 | interact=Bool(True) | |
292 | something_to_run=Bool(False) |
|
292 | something_to_run=Bool(False) | |
293 |
|
293 | |||
294 | def parse_command_line(self, argv=None): |
|
294 | def parse_command_line(self, argv=None): | |
295 | """override to allow old '-pylab' flag with deprecation warning""" |
|
295 | """override to allow old '-pylab' flag with deprecation warning""" | |
296 |
|
296 | |||
297 | argv = sys.argv[1:] if argv is None else argv |
|
297 | argv = sys.argv[1:] if argv is None else argv | |
298 |
|
298 | |||
299 | if '-pylab' in argv: |
|
299 | if '-pylab' in argv: | |
300 | # deprecated `-pylab` given, |
|
300 | # deprecated `-pylab` given, | |
301 | # warn and transform into current syntax |
|
301 | # warn and transform into current syntax | |
302 | argv = argv[:] # copy, don't clobber |
|
302 | argv = argv[:] # copy, don't clobber | |
303 | idx = argv.index('-pylab') |
|
303 | idx = argv.index('-pylab') | |
304 | warn.warn("`-pylab` flag has been deprecated.\n" |
|
304 | warn.warn("`-pylab` flag has been deprecated.\n" | |
305 | " Use `--pylab` instead, or `--pylab=foo` to specify a backend.") |
|
305 | " Use `--pylab` instead, or `--pylab=foo` to specify a backend.") | |
306 | sub = '--pylab' |
|
306 | sub = '--pylab' | |
307 | if len(argv) > idx+1: |
|
307 | if len(argv) > idx+1: | |
308 | # check for gui arg, as in '-pylab qt' |
|
308 | # check for gui arg, as in '-pylab qt' | |
309 | gui = argv[idx+1] |
|
309 | gui = argv[idx+1] | |
310 | if gui in ('wx', 'qt', 'qt4', 'gtk', 'auto'): |
|
310 | if gui in ('wx', 'qt', 'qt4', 'gtk', 'auto'): | |
311 | sub = '--pylab='+gui |
|
311 | sub = '--pylab='+gui | |
312 | argv.pop(idx+1) |
|
312 | argv.pop(idx+1) | |
313 | argv[idx] = sub |
|
313 | argv[idx] = sub | |
314 |
|
314 | |||
315 | return super(TerminalIPythonApp, self).parse_command_line(argv) |
|
315 | return super(TerminalIPythonApp, self).parse_command_line(argv) | |
316 |
|
316 | |||
317 | @catch_config_error |
|
317 | @catch_config_error | |
318 | def initialize(self, argv=None): |
|
318 | def initialize(self, argv=None): | |
319 | """Do actions after construct, but before starting the app.""" |
|
319 | """Do actions after construct, but before starting the app.""" | |
320 | super(TerminalIPythonApp, self).initialize(argv) |
|
320 | super(TerminalIPythonApp, self).initialize(argv) | |
321 | if self.subapp is not None: |
|
321 | if self.subapp is not None: | |
322 | # don't bother initializing further, starting subapp |
|
322 | # don't bother initializing further, starting subapp | |
323 | return |
|
323 | return | |
324 | if not self.ignore_old_config: |
|
324 | if not self.ignore_old_config: | |
325 | check_for_old_config(self.ipython_dir) |
|
325 | check_for_old_config(self.ipython_dir) | |
326 | # print self.extra_args |
|
326 | # print self.extra_args | |
327 | if self.extra_args and not self.something_to_run: |
|
327 | if self.extra_args and not self.something_to_run: | |
328 | self.file_to_run = self.extra_args[0] |
|
328 | self.file_to_run = self.extra_args[0] | |
329 | self.init_path() |
|
329 | self.init_path() | |
330 | # create the shell |
|
330 | # create the shell | |
331 | self.init_shell() |
|
331 | self.init_shell() | |
332 | # and draw the banner |
|
332 | # and draw the banner | |
333 | self.init_banner() |
|
333 | self.init_banner() | |
334 | # Now a variety of things that happen after the banner is printed. |
|
334 | # Now a variety of things that happen after the banner is printed. | |
335 | self.init_gui_pylab() |
|
335 | self.init_gui_pylab() | |
336 | self.init_extensions() |
|
336 | self.init_extensions() | |
337 | self.init_code() |
|
337 | self.init_code() | |
338 |
|
338 | |||
339 | def init_shell(self): |
|
339 | def init_shell(self): | |
340 | """initialize the InteractiveShell instance""" |
|
340 | """initialize the InteractiveShell instance""" | |
341 | # Create an InteractiveShell instance. |
|
341 | # Create an InteractiveShell instance. | |
342 | # shell.display_banner should always be False for the terminal |
|
342 | # shell.display_banner should always be False for the terminal | |
343 | # based app, because we call shell.show_banner() by hand below |
|
343 | # based app, because we call shell.show_banner() by hand below | |
344 | # so the banner shows *before* all extension loading stuff. |
|
344 | # so the banner shows *before* all extension loading stuff. | |
345 | self.shell = TerminalInteractiveShell.instance(parent=self, |
|
345 | self.shell = TerminalInteractiveShell.instance(parent=self, | |
346 | display_banner=False, profile_dir=self.profile_dir, |
|
346 | display_banner=False, profile_dir=self.profile_dir, | |
347 | ipython_dir=self.ipython_dir) |
|
347 | ipython_dir=self.ipython_dir) | |
348 | self.shell.configurables.append(self) |
|
348 | self.shell.configurables.append(self) | |
349 |
|
349 | |||
350 | def init_banner(self): |
|
350 | def init_banner(self): | |
351 | """optionally display the banner""" |
|
351 | """optionally display the banner""" | |
352 | if self.display_banner and self.interact: |
|
352 | if self.display_banner and self.interact: | |
353 | self.shell.show_banner() |
|
353 | self.shell.show_banner() | |
354 | # Make sure there is a space below the banner. |
|
354 | # Make sure there is a space below the banner. | |
355 | if self.log_level <= logging.INFO: print |
|
355 | if self.log_level <= logging.INFO: print | |
356 |
|
356 | |||
357 | def _pylab_changed(self, name, old, new): |
|
357 | def _pylab_changed(self, name, old, new): | |
358 | """Replace --pylab='inline' with --pylab='auto'""" |
|
358 | """Replace --pylab='inline' with --pylab='auto'""" | |
359 | if new == 'inline': |
|
359 | if new == 'inline': | |
360 | warn.warn("'inline' not available as pylab backend, " |
|
360 | warn.warn("'inline' not available as pylab backend, " | |
361 | "using 'auto' instead.") |
|
361 | "using 'auto' instead.") | |
362 | self.pylab = 'auto' |
|
362 | self.pylab = 'auto' | |
363 |
|
363 | |||
364 | def start(self): |
|
364 | def start(self): | |
365 | if self.subapp is not None: |
|
365 | if self.subapp is not None: | |
366 | return self.subapp.start() |
|
366 | return self.subapp.start() | |
367 | # perform any prexec steps: |
|
367 | # perform any prexec steps: | |
368 | if self.interact: |
|
368 | if self.interact: | |
369 | self.log.debug("Starting IPython's mainloop...") |
|
369 | self.log.debug("Starting IPython's mainloop...") | |
370 | self.shell.mainloop() |
|
370 | self.shell.mainloop() | |
371 | else: |
|
371 | else: | |
372 | self.log.debug("IPython not interactive...") |
|
372 | self.log.debug("IPython not interactive...") | |
373 |
|
373 | |||
374 |
|
374 | |||
375 | def load_default_config(ipython_dir=None): |
|
375 | def load_default_config(ipython_dir=None): | |
376 | """Load the default config file from the default ipython_dir. |
|
376 | """Load the default config file from the default ipython_dir. | |
377 |
|
377 | |||
378 | This is useful for embedded shells. |
|
378 | This is useful for embedded shells. | |
379 | """ |
|
379 | """ | |
380 | if ipython_dir is None: |
|
380 | if ipython_dir is None: | |
381 | ipython_dir = get_ipython_dir() |
|
381 | ipython_dir = get_ipython_dir() | |
382 | profile_dir = os.path.join(ipython_dir, 'profile_default') |
|
382 | profile_dir = os.path.join(ipython_dir, 'profile_default') | |
383 | cl = PyFileConfigLoader(default_config_file_name, profile_dir) |
|
383 | cl = PyFileConfigLoader(default_config_file_name, profile_dir) | |
384 | try: |
|
384 | try: | |
385 | config = cl.load_config() |
|
385 | config = cl.load_config() | |
386 | except ConfigFileNotFound: |
|
386 | except ConfigFileNotFound: | |
387 | # no config found |
|
387 | # no config found | |
388 | config = Config() |
|
388 | config = Config() | |
389 | return config |
|
389 | return config | |
390 |
|
390 | |||
391 |
|
391 | |||
392 |
launch_new_instance = TerminalIPythonApp.launch_ |
|
392 | launch_new_instance = TerminalIPythonApp.launch_instance | |
393 |
|
393 | |||
394 |
|
394 | |||
395 | if __name__ == '__main__': |
|
395 | if __name__ == '__main__': | |
396 | launch_new_instance() |
|
396 | launch_new_instance() |
General Comments 0
You need to be logged in to leave comments.
Login now