##// END OF EJS Templates
Various fixes to ipapp/ipcluster/ipengine/ipcontroller....
Brian Granger -
Show More
@@ -1,449 +1,453 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 An application for IPython.
3 An application for IPython.
4
4
5 All top-level applications should use the classes in this module for
5 All top-level applications should use the classes in this module for
6 handling configuration and creating componenets.
6 handling configuration and creating componenets.
7
7
8 The job of an :class:`Application` is to create the master configuration
8 The job of an :class:`Application` is to create the master configuration
9 object and then create the components, passing the config to them.
9 object and then create the components, passing the config to them.
10
10
11 Authors:
11 Authors:
12
12
13 * Brian Granger
13 * Brian Granger
14 * Fernando Perez
14 * Fernando Perez
15
15
16 Notes
16 Notes
17 -----
17 -----
18 """
18 """
19
19
20 #-----------------------------------------------------------------------------
20 #-----------------------------------------------------------------------------
21 # Copyright (C) 2008-2009 The IPython Development Team
21 # Copyright (C) 2008-2009 The IPython Development Team
22 #
22 #
23 # Distributed under the terms of the BSD License. The full license is in
23 # Distributed under the terms of the BSD License. The full license is in
24 # the file COPYING, distributed as part of this software.
24 # the file COPYING, distributed as part of this software.
25 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
26
26
27 #-----------------------------------------------------------------------------
27 #-----------------------------------------------------------------------------
28 # Imports
28 # Imports
29 #-----------------------------------------------------------------------------
29 #-----------------------------------------------------------------------------
30
30
31 import logging
31 import logging
32 import os
32 import os
33 import sys
33 import sys
34
34
35 from IPython.core import release, crashhandler
35 from IPython.core import release, crashhandler
36 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
36 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
37 from IPython.config.loader import (
37 from IPython.config.loader import (
38 PyFileConfigLoader,
38 PyFileConfigLoader,
39 ArgParseConfigLoader,
39 ArgParseConfigLoader,
40 Config,
40 Config,
41 )
41 )
42
42
43 #-----------------------------------------------------------------------------
43 #-----------------------------------------------------------------------------
44 # Classes and functions
44 # Classes and functions
45 #-----------------------------------------------------------------------------
45 #-----------------------------------------------------------------------------
46
46
47 class ApplicationError(Exception):
47 class ApplicationError(Exception):
48 pass
48 pass
49
49
50
50
51 class BaseAppConfigLoader(ArgParseConfigLoader):
51 class BaseAppConfigLoader(ArgParseConfigLoader):
52 """Default command line options for IPython based applications."""
52 """Default command line options for IPython based applications."""
53
53
54 def _add_ipython_dir(self, parser):
54 def _add_ipython_dir(self, parser):
55 """Add the --ipython-dir option to the parser."""
55 """Add the --ipython-dir option to the parser."""
56 paa = parser.add_argument
56 paa = parser.add_argument
57 paa('--ipython-dir',
57 paa('--ipython-dir',
58 dest='Global.ipython_dir',type=unicode,
58 dest='Global.ipython_dir',type=unicode,
59 help=
59 help=
60 """Set to override default location of the IPython directory
60 """Set to override default location of the IPython directory
61 IPYTHON_DIR, stored as Global.ipython_dir. This can also be
61 IPYTHON_DIR, stored as Global.ipython_dir. This can also be
62 specified through the environment variable IPYTHON_DIR.""",
62 specified through the environment variable IPYTHON_DIR.""",
63 metavar='Global.ipython_dir')
63 metavar='Global.ipython_dir')
64
64
65 def _add_log_level(self, parser):
65 def _add_log_level(self, parser):
66 """Add the --log-level option to the parser."""
66 """Add the --log-level option to the parser."""
67 paa = parser.add_argument
67 paa = parser.add_argument
68 paa('--log-level',
68 paa('--log-level',
69 dest="Global.log_level",type=int,
69 dest="Global.log_level",type=int,
70 help='Set the log level (0,10,20,30,40,50). Default is 30.',
70 help='Set the log level (0,10,20,30,40,50). Default is 30.',
71 metavar='Global.log_level')
71 metavar='Global.log_level')
72
72
73 def _add_arguments(self):
73 def _add_arguments(self):
74 self._add_ipython_dir(self.parser)
74 self._add_ipython_dir(self.parser)
75 self._add_log_level(self.parser)
75 self._add_log_level(self.parser)
76
76
77
77
78 class Application(object):
78 class Application(object):
79 """Load a config, construct components and set them running.
79 """Load a config, construct components and set them running.
80
80
81 The configuration of an application can be done via three different Config
81 The configuration of an application can be done via three different Config
82 objects, which are loaded and ultimately merged into a single one used
82 objects, which are loaded and ultimately merged into a single one used
83 from that point on by the app. These are:
83 from that point on by the app. These are:
84
84
85 1. default_config: internal defaults, implemented in code.
85 1. default_config: internal defaults, implemented in code.
86 2. file_config: read from the filesystem.
86 2. file_config: read from the filesystem.
87 3. command_line_config: read from the system's command line flags.
87 3. command_line_config: read from the system's command line flags.
88
88
89 During initialization, 3 is actually read before 2, since at the
89 During initialization, 3 is actually read before 2, since at the
90 command-line one may override the location of the file to be read. But the
90 command-line one may override the location of the file to be read. But the
91 above is the order in which the merge is made.
91 above is the order in which the merge is made.
92 """
92 """
93
93
94 name = u'ipython'
94 name = u'ipython'
95 description = 'IPython: an enhanced interactive Python shell.'
95 description = 'IPython: an enhanced interactive Python shell.'
96 #: Usage message printed by argparse. If None, auto-generate
96 #: Usage message printed by argparse. If None, auto-generate
97 usage = None
97 usage = None
98 #: The command line config loader. Subclass of ArgParseConfigLoader.
98 #: The command line config loader. Subclass of ArgParseConfigLoader.
99 command_line_loader = BaseAppConfigLoader
99 command_line_loader = BaseAppConfigLoader
100 #: The name of the config file to load.
100 #: The name of the config file to load, determined at runtime
101 config_file_name = u'ipython_config.py'
101 config_file_name = None
102 #: The name of the default config file. Track separately from the actual
102 #: The name of the default config file. Track separately from the actual
103 #: name because some logic happens only if we aren't using the default.
103 #: name because some logic happens only if we aren't using the default.
104 default_config_file_name = config_file_name
104 default_config_file_name = u'ipython_config.py'
105 default_log_level = logging.WARN
105 default_log_level = logging.WARN
106 #: Set by --profile option
106 #: Set by --profile option
107 profile_name = None
107 profile_name = None
108 #: User's ipython directory, typically ~/.ipython/
108 #: User's ipython directory, typically ~/.ipython/
109 ipython_dir = None
109 ipython_dir = None
110 #: Internal defaults, implemented in code.
110 #: Internal defaults, implemented in code.
111 default_config = None
111 default_config = None
112 #: Read from the filesystem.
112 #: Read from the filesystem.
113 file_config = None
113 file_config = None
114 #: Read from the system's command line flags.
114 #: Read from the system's command line flags.
115 command_line_config = None
115 command_line_config = None
116 #: The final config that will be passed to the component.
116 #: The final config that will be passed to the component.
117 master_config = None
117 master_config = None
118 #: A reference to the argv to be used (typically ends up being sys.argv[1:])
118 #: A reference to the argv to be used (typically ends up being sys.argv[1:])
119 argv = None
119 argv = None
120 #: extra arguments computed by the command-line loader
120 #: extra arguments computed by the command-line loader
121 extra_args = None
121 extra_args = None
122 #: The class to use as the crash handler.
122 #: The class to use as the crash handler.
123 crash_handler_class = crashhandler.CrashHandler
123 crash_handler_class = crashhandler.CrashHandler
124
124
125 # Private attributes
125 # Private attributes
126 _exiting = False
126 _exiting = False
127 _initialized = False
127 _initialized = False
128
128
129 def __init__(self, argv=None):
129 def __init__(self, argv=None):
130 self.argv = sys.argv[1:] if argv is None else argv
130 self.argv = sys.argv[1:] if argv is None else argv
131 self.init_logger()
131 self.init_logger()
132
132
133 def init_logger(self):
133 def init_logger(self):
134 self.log = logging.getLogger(self.__class__.__name__)
134 self.log = logging.getLogger(self.__class__.__name__)
135 # This is used as the default until the command line arguments are read.
135 # This is used as the default until the command line arguments are read.
136 self.log.setLevel(self.default_log_level)
136 self.log.setLevel(self.default_log_level)
137 self._log_handler = logging.StreamHandler()
137 self._log_handler = logging.StreamHandler()
138 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
138 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
139 self._log_handler.setFormatter(self._log_formatter)
139 self._log_handler.setFormatter(self._log_formatter)
140 self.log.addHandler(self._log_handler)
140 self.log.addHandler(self._log_handler)
141
141
142 def _set_log_level(self, level):
142 def _set_log_level(self, level):
143 self.log.setLevel(level)
143 self.log.setLevel(level)
144
144
145 def _get_log_level(self):
145 def _get_log_level(self):
146 return self.log.level
146 return self.log.level
147
147
148 log_level = property(_get_log_level, _set_log_level)
148 log_level = property(_get_log_level, _set_log_level)
149
149
150 def initialize(self):
150 def initialize(self):
151 """Initialize the application.
151 """Initialize the application.
152
152
153 Loads all configuration information and sets all application state, but
153 Loads all configuration information and sets all application state, but
154 does not start any relevant processing (typically some kind of event
154 does not start any relevant processing (typically some kind of event
155 loop).
155 loop).
156
156
157 Once this method has been called, the application is flagged as
157 Once this method has been called, the application is flagged as
158 initialized and the method becomes a no-op."""
158 initialized and the method becomes a no-op."""
159
159
160 if self._initialized:
160 if self._initialized:
161 return
161 return
162
162
163 # The first part is protected with an 'attempt' wrapper, that will log
163 # The first part is protected with an 'attempt' wrapper, that will log
164 # failures with the basic system traceback machinery. Once our crash
164 # failures with the basic system traceback machinery. Once our crash
165 # handler is in place, we can let any subsequent exception propagate,
165 # handler is in place, we can let any subsequent exception propagate,
166 # as our handler will log it with much better detail than the default.
166 # as our handler will log it with much better detail than the default.
167 self.attempt(self.create_crash_handler)
167 self.attempt(self.create_crash_handler)
168
168
169 # Configuration phase
169 # Configuration phase
170 # Default config (internally hardwired in application code)
170 # Default config (internally hardwired in application code)
171 self.create_default_config()
171 self.create_default_config()
172 self.log_default_config()
172 self.log_default_config()
173 self.set_default_config_log_level()
173 self.set_default_config_log_level()
174
174
175 # Command-line config
175 # Command-line config
176 self.pre_load_command_line_config()
176 self.pre_load_command_line_config()
177 self.load_command_line_config()
177 self.load_command_line_config()
178 self.set_command_line_config_log_level()
178 self.set_command_line_config_log_level()
179 self.post_load_command_line_config()
179 self.post_load_command_line_config()
180 self.log_command_line_config()
180 self.log_command_line_config()
181
181
182 # Find resources needed for filesystem access, using information from
182 # Find resources needed for filesystem access, using information from
183 # the above two
183 # the above two
184 self.find_ipython_dir()
184 self.find_ipython_dir()
185 self.find_resources()
185 self.find_resources()
186 self.find_config_file_name()
186 self.find_config_file_name()
187 self.find_config_file_paths()
187 self.find_config_file_paths()
188
188
189 # File-based config
189 # File-based config
190 self.pre_load_file_config()
190 self.pre_load_file_config()
191 self.load_file_config()
191 self.load_file_config()
192 self.set_file_config_log_level()
192 self.set_file_config_log_level()
193 self.post_load_file_config()
193 self.post_load_file_config()
194 self.log_file_config()
194 self.log_file_config()
195
195
196 # Merge all config objects into a single one the app can then use
196 # Merge all config objects into a single one the app can then use
197 self.merge_configs()
197 self.merge_configs()
198 self.log_master_config()
198 self.log_master_config()
199
199
200 # Construction phase
200 # Construction phase
201 self.pre_construct()
201 self.pre_construct()
202 self.construct()
202 self.construct()
203 self.post_construct()
203 self.post_construct()
204
204
205 # Done, flag as such and
205 # Done, flag as such and
206 self._initialized = True
206 self._initialized = True
207
207
208 def start(self):
208 def start(self):
209 """Start the application."""
209 """Start the application."""
210 self.initialize()
210 self.initialize()
211 self.start_app()
211 self.start_app()
212
212
213 #-------------------------------------------------------------------------
213 #-------------------------------------------------------------------------
214 # Various stages of Application creation
214 # Various stages of Application creation
215 #-------------------------------------------------------------------------
215 #-------------------------------------------------------------------------
216
216
217 def create_crash_handler(self):
217 def create_crash_handler(self):
218 """Create a crash handler, typically setting sys.excepthook to it."""
218 """Create a crash handler, typically setting sys.excepthook to it."""
219 self.crash_handler = self.crash_handler_class(self)
219 self.crash_handler = self.crash_handler_class(self)
220 sys.excepthook = self.crash_handler
220 sys.excepthook = self.crash_handler
221
221
222 def create_default_config(self):
222 def create_default_config(self):
223 """Create defaults that can't be set elsewhere.
223 """Create defaults that can't be set elsewhere.
224
224
225 For the most part, we try to set default in the class attributes
225 For the most part, we try to set default in the class attributes
226 of Components. But, defaults the top-level Application (which is
226 of Components. But, defaults the top-level Application (which is
227 not a HasTraitlets or Component) are not set in this way. Instead
227 not a HasTraitlets or Component) are not set in this way. Instead
228 we set them here. The Global section is for variables like this that
228 we set them here. The Global section is for variables like this that
229 don't belong to a particular component.
229 don't belong to a particular component.
230 """
230 """
231 c = Config()
231 c = Config()
232 c.Global.ipython_dir = get_ipython_dir()
232 c.Global.ipython_dir = get_ipython_dir()
233 c.Global.log_level = self.log_level
233 c.Global.log_level = self.log_level
234 self.default_config = c
234 self.default_config = c
235
235
236 def log_default_config(self):
236 def log_default_config(self):
237 self.log.debug('Default config loaded:')
237 self.log.debug('Default config loaded:')
238 self.log.debug(repr(self.default_config))
238 self.log.debug(repr(self.default_config))
239
239
240 def set_default_config_log_level(self):
240 def set_default_config_log_level(self):
241 try:
241 try:
242 self.log_level = self.default_config.Global.log_level
242 self.log_level = self.default_config.Global.log_level
243 except AttributeError:
243 except AttributeError:
244 # Fallback to the default_log_level class attribute
244 # Fallback to the default_log_level class attribute
245 pass
245 pass
246
246
247 def create_command_line_config(self):
247 def create_command_line_config(self):
248 """Create and return a command line config loader."""
248 """Create and return a command line config loader."""
249 return self.command_line_loader(
249 return self.command_line_loader(
250 self.argv,
250 self.argv,
251 description=self.description,
251 description=self.description,
252 version=release.version,
252 version=release.version,
253 usage=self.usage
253 usage=self.usage
254 )
254 )
255
255
256 def pre_load_command_line_config(self):
256 def pre_load_command_line_config(self):
257 """Do actions just before loading the command line config."""
257 """Do actions just before loading the command line config."""
258 pass
258 pass
259
259
260 def load_command_line_config(self):
260 def load_command_line_config(self):
261 """Load the command line config."""
261 """Load the command line config."""
262 loader = self.create_command_line_config()
262 loader = self.create_command_line_config()
263 self.command_line_config = loader.load_config()
263 self.command_line_config = loader.load_config()
264 self.extra_args = loader.get_extra_args()
264 self.extra_args = loader.get_extra_args()
265
265
266 def set_command_line_config_log_level(self):
266 def set_command_line_config_log_level(self):
267 try:
267 try:
268 self.log_level = self.command_line_config.Global.log_level
268 self.log_level = self.command_line_config.Global.log_level
269 except AttributeError:
269 except AttributeError:
270 pass
270 pass
271
271
272 def post_load_command_line_config(self):
272 def post_load_command_line_config(self):
273 """Do actions just after loading the command line config."""
273 """Do actions just after loading the command line config."""
274 pass
274 pass
275
275
276 def log_command_line_config(self):
276 def log_command_line_config(self):
277 self.log.debug("Command line config loaded:")
277 self.log.debug("Command line config loaded:")
278 self.log.debug(repr(self.command_line_config))
278 self.log.debug(repr(self.command_line_config))
279
279
280 def find_ipython_dir(self):
280 def find_ipython_dir(self):
281 """Set the IPython directory.
281 """Set the IPython directory.
282
282
283 This sets ``self.ipython_dir``, but the actual value that is passed to
283 This sets ``self.ipython_dir``, but the actual value that is passed to
284 the application is kept in either ``self.default_config`` or
284 the application is kept in either ``self.default_config`` or
285 ``self.command_line_config``. This also adds ``self.ipython_dir`` to
285 ``self.command_line_config``. This also adds ``self.ipython_dir`` to
286 ``sys.path`` so config files there can be referenced by other config
286 ``sys.path`` so config files there can be referenced by other config
287 files.
287 files.
288 """
288 """
289
289
290 try:
290 try:
291 self.ipython_dir = self.command_line_config.Global.ipython_dir
291 self.ipython_dir = self.command_line_config.Global.ipython_dir
292 except AttributeError:
292 except AttributeError:
293 self.ipython_dir = self.default_config.Global.ipython_dir
293 self.ipython_dir = self.default_config.Global.ipython_dir
294 sys.path.append(os.path.abspath(self.ipython_dir))
294 sys.path.append(os.path.abspath(self.ipython_dir))
295 if not os.path.isdir(self.ipython_dir):
295 if not os.path.isdir(self.ipython_dir):
296 os.makedirs(self.ipython_dir, mode=0777)
296 os.makedirs(self.ipython_dir, mode=0777)
297 self.log.debug("IPYTHON_DIR set to: %s" % self.ipython_dir)
297 self.log.debug("IPYTHON_DIR set to: %s" % self.ipython_dir)
298
298
299 def find_resources(self):
299 def find_resources(self):
300 """Find other resources that need to be in place.
300 """Find other resources that need to be in place.
301
301
302 Things like cluster directories need to be in place to find the
302 Things like cluster directories need to be in place to find the
303 config file. These happen right after the IPython directory has
303 config file. These happen right after the IPython directory has
304 been set.
304 been set.
305 """
305 """
306 pass
306 pass
307
307
308 def find_config_file_name(self):
308 def find_config_file_name(self):
309 """Find the config file name for this application.
309 """Find the config file name for this application.
310
310
311 This must set ``self.config_file_name`` to the filename of the
311 This must set ``self.config_file_name`` to the filename of the
312 config file to use (just the filename). The search paths for the
312 config file to use (just the filename). The search paths for the
313 config file are set in :meth:`find_config_file_paths` and then passed
313 config file are set in :meth:`find_config_file_paths` and then passed
314 to the config file loader where they are resolved to an absolute path.
314 to the config file loader where they are resolved to an absolute path.
315
315
316 If a profile has been set at the command line, this will resolve it.
316 If a profile has been set at the command line, this will resolve it.
317 """
317 """
318
319 try:
318 try:
320 self.config_file_name = self.command_line_config.Global.config_file
319 self.config_file_name = self.command_line_config.Global.config_file
321 except AttributeError:
320 except AttributeError:
322 pass
321 pass
322 else:
323 return
323
324
324 try:
325 try:
325 self.profile_name = self.command_line_config.Global.profile
326 self.profile_name = self.command_line_config.Global.profile
326 except AttributeError:
327 except AttributeError:
327 pass
328 # Just use the default as there is no profile
329 self.config_file_name = self.default_config_file_name
328 else:
330 else:
329 name_parts = self.config_file_name.split('.')
331 # Use the default config file name and profile name if set
332 # to determine the used config file name.
333 name_parts = self.default_config_file_name.split('.')
330 name_parts.insert(1, u'_' + self.profile_name + u'.')
334 name_parts.insert(1, u'_' + self.profile_name + u'.')
331 self.config_file_name = ''.join(name_parts)
335 self.config_file_name = ''.join(name_parts)
332
336
333 def find_config_file_paths(self):
337 def find_config_file_paths(self):
334 """Set the search paths for resolving the config file.
338 """Set the search paths for resolving the config file.
335
339
336 This must set ``self.config_file_paths`` to a sequence of search
340 This must set ``self.config_file_paths`` to a sequence of search
337 paths to pass to the config file loader.
341 paths to pass to the config file loader.
338 """
342 """
339 # Include our own profiles directory last, so that users can still find
343 # Include our own profiles directory last, so that users can still find
340 # our shipped copies of builtin profiles even if they don't have them
344 # our shipped copies of builtin profiles even if they don't have them
341 # in their local ipython directory.
345 # in their local ipython directory.
342 prof_dir = os.path.join(get_ipython_package_dir(), 'config', 'profile')
346 prof_dir = os.path.join(get_ipython_package_dir(), 'config', 'profile')
343 self.config_file_paths = (os.getcwd(), self.ipython_dir, prof_dir)
347 self.config_file_paths = (os.getcwd(), self.ipython_dir, prof_dir)
344
348
345 def pre_load_file_config(self):
349 def pre_load_file_config(self):
346 """Do actions before the config file is loaded."""
350 """Do actions before the config file is loaded."""
347 pass
351 pass
348
352
349 def load_file_config(self):
353 def load_file_config(self):
350 """Load the config file.
354 """Load the config file.
351
355
352 This tries to load the config file from disk. If successful, the
356 This tries to load the config file from disk. If successful, the
353 ``CONFIG_FILE`` config variable is set to the resolved config file
357 ``CONFIG_FILE`` config variable is set to the resolved config file
354 location. If not successful, an empty config is used.
358 location. If not successful, an empty config is used.
355 """
359 """
356 self.log.debug("Attempting to load config file: %s" %
360 self.log.debug("Attempting to load config file: %s" %
357 self.config_file_name)
361 self.config_file_name)
358 loader = PyFileConfigLoader(self.config_file_name,
362 loader = PyFileConfigLoader(self.config_file_name,
359 path=self.config_file_paths)
363 path=self.config_file_paths)
360 try:
364 try:
361 self.file_config = loader.load_config()
365 self.file_config = loader.load_config()
362 self.file_config.Global.config_file = loader.full_filename
366 self.file_config.Global.config_file = loader.full_filename
363 except IOError:
367 except IOError:
364 # Only warn if the default config file was NOT being used.
368 # Only warn if the default config file was NOT being used.
365 if not self.config_file_name==self.default_config_file_name:
369 if not self.config_file_name==self.default_config_file_name:
366 self.log.warn("Config file not found, skipping: %s" %
370 self.log.warn("Config file not found, skipping: %s" %
367 self.config_file_name, exc_info=True)
371 self.config_file_name, exc_info=True)
368 self.file_config = Config()
372 self.file_config = Config()
369 except:
373 except:
370 self.log.warn("Error loading config file: %s" %
374 self.log.warn("Error loading config file: %s" %
371 self.config_file_name, exc_info=True)
375 self.config_file_name, exc_info=True)
372 self.file_config = Config()
376 self.file_config = Config()
373
377
374 def set_file_config_log_level(self):
378 def set_file_config_log_level(self):
375 # We need to keeep self.log_level updated. But we only use the value
379 # We need to keeep self.log_level updated. But we only use the value
376 # of the file_config if a value was not specified at the command
380 # of the file_config if a value was not specified at the command
377 # line, because the command line overrides everything.
381 # line, because the command line overrides everything.
378 if not hasattr(self.command_line_config.Global, 'log_level'):
382 if not hasattr(self.command_line_config.Global, 'log_level'):
379 try:
383 try:
380 self.log_level = self.file_config.Global.log_level
384 self.log_level = self.file_config.Global.log_level
381 except AttributeError:
385 except AttributeError:
382 pass # Use existing value
386 pass # Use existing value
383
387
384 def post_load_file_config(self):
388 def post_load_file_config(self):
385 """Do actions after the config file is loaded."""
389 """Do actions after the config file is loaded."""
386 pass
390 pass
387
391
388 def log_file_config(self):
392 def log_file_config(self):
389 if hasattr(self.file_config.Global, 'config_file'):
393 if hasattr(self.file_config.Global, 'config_file'):
390 self.log.debug("Config file loaded: %s" %
394 self.log.debug("Config file loaded: %s" %
391 self.file_config.Global.config_file)
395 self.file_config.Global.config_file)
392 self.log.debug(repr(self.file_config))
396 self.log.debug(repr(self.file_config))
393
397
394 def merge_configs(self):
398 def merge_configs(self):
395 """Merge the default, command line and file config objects."""
399 """Merge the default, command line and file config objects."""
396 config = Config()
400 config = Config()
397 config._merge(self.default_config)
401 config._merge(self.default_config)
398 config._merge(self.file_config)
402 config._merge(self.file_config)
399 config._merge(self.command_line_config)
403 config._merge(self.command_line_config)
400
404
401 # XXX fperez - propose to Brian we rename master_config to simply
405 # XXX fperez - propose to Brian we rename master_config to simply
402 # config, I think this is going to be heavily used in examples and
406 # config, I think this is going to be heavily used in examples and
403 # application code and the name is shorter/easier to find/remember.
407 # application code and the name is shorter/easier to find/remember.
404 # For now, just alias it...
408 # For now, just alias it...
405 self.master_config = config
409 self.master_config = config
406 self.config = config
410 self.config = config
407
411
408 def log_master_config(self):
412 def log_master_config(self):
409 self.log.debug("Master config created:")
413 self.log.debug("Master config created:")
410 self.log.debug(repr(self.master_config))
414 self.log.debug(repr(self.master_config))
411
415
412 def pre_construct(self):
416 def pre_construct(self):
413 """Do actions after the config has been built, but before construct."""
417 """Do actions after the config has been built, but before construct."""
414 pass
418 pass
415
419
416 def construct(self):
420 def construct(self):
417 """Construct the main components that make up this app."""
421 """Construct the main components that make up this app."""
418 self.log.debug("Constructing components for application")
422 self.log.debug("Constructing components for application")
419
423
420 def post_construct(self):
424 def post_construct(self):
421 """Do actions after construct, but before starting the app."""
425 """Do actions after construct, but before starting the app."""
422 pass
426 pass
423
427
424 def start_app(self):
428 def start_app(self):
425 """Actually start the app."""
429 """Actually start the app."""
426 self.log.debug("Starting application")
430 self.log.debug("Starting application")
427
431
428 #-------------------------------------------------------------------------
432 #-------------------------------------------------------------------------
429 # Utility methods
433 # Utility methods
430 #-------------------------------------------------------------------------
434 #-------------------------------------------------------------------------
431
435
432 def exit(self, exit_status=0):
436 def exit(self, exit_status=0):
433 if self._exiting:
437 if self._exiting:
434 pass
438 pass
435 else:
439 else:
436 self.log.debug("Exiting application: %s" % self.name)
440 self.log.debug("Exiting application: %s" % self.name)
437 self._exiting = True
441 self._exiting = True
438 sys.exit(exit_status)
442 sys.exit(exit_status)
439
443
440 def attempt(self, func):
444 def attempt(self, func):
441 try:
445 try:
442 func()
446 func()
443 except SystemExit:
447 except SystemExit:
444 raise
448 raise
445 except:
449 except:
446 self.log.critical("Aborting application: %s" % self.name,
450 self.log.critical("Aborting application: %s" % self.name,
447 exc_info=True)
451 exc_info=True)
448 self.exit(0)
452 self.exit(0)
449
453
@@ -1,654 +1,653 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The :class:`~IPython.core.application.Application` object for the command
4 The :class:`~IPython.core.application.Application` object for the command
5 line :command:`ipython` program.
5 line :command:`ipython` program.
6
6
7 Authors
7 Authors
8 -------
8 -------
9
9
10 * Brian Granger
10 * Brian Granger
11 * Fernando Perez
11 * Fernando Perez
12 """
12 """
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Copyright (C) 2008-2010 The IPython Development Team
15 # Copyright (C) 2008-2010 The IPython Development Team
16 #
16 #
17 # Distributed under the terms of the BSD License. The full license is in
17 # Distributed under the terms of the BSD License. The full license is in
18 # the file COPYING, distributed as part of this software.
18 # the file COPYING, distributed as part of this software.
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20
20
21 #-----------------------------------------------------------------------------
21 #-----------------------------------------------------------------------------
22 # Imports
22 # Imports
23 #-----------------------------------------------------------------------------
23 #-----------------------------------------------------------------------------
24
24
25 from __future__ import absolute_import
25 from __future__ import absolute_import
26
26
27 import logging
27 import logging
28 import os
28 import os
29 import sys
29 import sys
30
30
31 from IPython.core import release
31 from IPython.core import release
32 from IPython.core.crashhandler import CrashHandler
32 from IPython.core.crashhandler import CrashHandler
33 from IPython.core.application import Application, BaseAppConfigLoader
33 from IPython.core.application import Application, BaseAppConfigLoader
34 from IPython.core.iplib import InteractiveShell
34 from IPython.core.iplib import InteractiveShell
35 from IPython.config.loader import (
35 from IPython.config.loader import (
36 Config,
36 Config,
37 PyFileConfigLoader
37 PyFileConfigLoader
38 )
38 )
39 from IPython.lib import inputhook
39 from IPython.lib import inputhook
40 from IPython.utils.path import filefind, get_ipython_dir
40 from IPython.utils.path import filefind, get_ipython_dir
41 from . import usage
41 from . import usage
42
42
43 #-----------------------------------------------------------------------------
43 #-----------------------------------------------------------------------------
44 # Globals, utilities and helpers
44 # Globals, utilities and helpers
45 #-----------------------------------------------------------------------------
45 #-----------------------------------------------------------------------------
46
46
47 #: The default config file name for this application.
47 #: The default config file name for this application.
48 default_config_file_name = u'ipython_config.py'
48 default_config_file_name = u'ipython_config.py'
49
49
50
50
51 class IPAppConfigLoader(BaseAppConfigLoader):
51 class IPAppConfigLoader(BaseAppConfigLoader):
52
52
53 def _add_arguments(self):
53 def _add_arguments(self):
54 super(IPAppConfigLoader, self)._add_arguments()
54 super(IPAppConfigLoader, self)._add_arguments()
55 paa = self.parser.add_argument
55 paa = self.parser.add_argument
56 paa('-p',
56 paa('-p',
57 '--profile', dest='Global.profile', type=unicode,
57 '--profile', dest='Global.profile', type=unicode,
58 help=
58 help=
59 """The string name of the ipython profile to be used. Assume that your
59 """The string name of the ipython profile to be used. Assume that your
60 config file is ipython_config-<name>.py (looks in current dir first,
60 config file is ipython_config-<name>.py (looks in current dir first,
61 then in IPYTHON_DIR). This is a quick way to keep and load multiple
61 then in IPYTHON_DIR). This is a quick way to keep and load multiple
62 config files for different tasks, especially if include your basic one
62 config files for different tasks, especially if include your basic one
63 in your more specialized ones. You can keep a basic
63 in your more specialized ones. You can keep a basic
64 IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which
64 IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which
65 include this one and load extra things for particular tasks.""",
65 include this one and load extra things for particular tasks.""",
66 metavar='Global.profile')
66 metavar='Global.profile')
67 paa('--config-file',
67 paa('--config-file',
68 dest='Global.config_file', type=unicode,
68 dest='Global.config_file', type=unicode,
69 help=
69 help=
70 """Set the config file name to override default. Normally IPython
70 """Set the config file name to override default. Normally IPython
71 loads ipython_config.py (from current directory) or
71 loads ipython_config.py (from current directory) or
72 IPYTHON_DIR/ipython_config.py. If the loading of your config file
72 IPYTHON_DIR/ipython_config.py. If the loading of your config file
73 fails, IPython starts with a bare bones configuration (no modules
73 fails, IPython starts with a bare bones configuration (no modules
74 loaded at all).""",
74 loaded at all).""",
75 metavar='Global.config_file')
75 metavar='Global.config_file')
76 paa('--autocall',
76 paa('--autocall',
77 dest='InteractiveShell.autocall', type=int,
77 dest='InteractiveShell.autocall', type=int,
78 help=
78 help=
79 """Make IPython automatically call any callable object even if you
79 """Make IPython automatically call any callable object even if you
80 didn't type explicit parentheses. For example, 'str 43' becomes
80 didn't type explicit parentheses. For example, 'str 43' becomes
81 'str(43)' automatically. The value can be '0' to disable the feature,
81 'str(43)' automatically. The value can be '0' to disable the feature,
82 '1' for 'smart' autocall, where it is not applied if there are no more
82 '1' for 'smart' autocall, where it is not applied if there are no more
83 arguments on the line, and '2' for 'full' autocall, where all callable
83 arguments on the line, and '2' for 'full' autocall, where all callable
84 objects are automatically called (even if no arguments are present).
84 objects are automatically called (even if no arguments are present).
85 The default is '1'.""",
85 The default is '1'.""",
86 metavar='InteractiveShell.autocall')
86 metavar='InteractiveShell.autocall')
87 paa('--autoindent',
87 paa('--autoindent',
88 action='store_true', dest='InteractiveShell.autoindent',
88 action='store_true', dest='InteractiveShell.autoindent',
89 help='Turn on autoindenting.')
89 help='Turn on autoindenting.')
90 paa('--no-autoindent',
90 paa('--no-autoindent',
91 action='store_false', dest='InteractiveShell.autoindent',
91 action='store_false', dest='InteractiveShell.autoindent',
92 help='Turn off autoindenting.')
92 help='Turn off autoindenting.')
93 paa('--automagic',
93 paa('--automagic',
94 action='store_true', dest='InteractiveShell.automagic',
94 action='store_true', dest='InteractiveShell.automagic',
95 help=
95 help=
96 """Turn on the auto calling of magic commands. Type %%magic at the
96 """Turn on the auto calling of magic commands. Type %%magic at the
97 IPython prompt for more information.""")
97 IPython prompt for more information.""")
98 paa('--no-automagic',
98 paa('--no-automagic',
99 action='store_false', dest='InteractiveShell.automagic',
99 action='store_false', dest='InteractiveShell.automagic',
100 help='Turn off the auto calling of magic commands.')
100 help='Turn off the auto calling of magic commands.')
101 paa('--autoedit-syntax',
101 paa('--autoedit-syntax',
102 action='store_true', dest='InteractiveShell.autoedit_syntax',
102 action='store_true', dest='InteractiveShell.autoedit_syntax',
103 help='Turn on auto editing of files with syntax errors.')
103 help='Turn on auto editing of files with syntax errors.')
104 paa('--no-autoedit-syntax',
104 paa('--no-autoedit-syntax',
105 action='store_false', dest='InteractiveShell.autoedit_syntax',
105 action='store_false', dest='InteractiveShell.autoedit_syntax',
106 help='Turn off auto editing of files with syntax errors.')
106 help='Turn off auto editing of files with syntax errors.')
107 paa('--banner',
107 paa('--banner',
108 action='store_true', dest='Global.display_banner',
108 action='store_true', dest='Global.display_banner',
109 help='Display a banner upon starting IPython.')
109 help='Display a banner upon starting IPython.')
110 paa('--no-banner',
110 paa('--no-banner',
111 action='store_false', dest='Global.display_banner',
111 action='store_false', dest='Global.display_banner',
112 help="Don't display a banner upon starting IPython.")
112 help="Don't display a banner upon starting IPython.")
113 paa('--cache-size',
113 paa('--cache-size',
114 type=int, dest='InteractiveShell.cache_size',
114 type=int, dest='InteractiveShell.cache_size',
115 help=
115 help=
116 """Set the size of the output cache. The default is 1000, you can
116 """Set the size of the output cache. The default is 1000, you can
117 change it permanently in your config file. Setting it to 0 completely
117 change it permanently in your config file. Setting it to 0 completely
118 disables the caching system, and the minimum value accepted is 20 (if
118 disables the caching system, and the minimum value accepted is 20 (if
119 you provide a value less than 20, it is reset to 0 and a warning is
119 you provide a value less than 20, it is reset to 0 and a warning is
120 issued). This limit is defined because otherwise you'll spend more
120 issued). This limit is defined because otherwise you'll spend more
121 time re-flushing a too small cache than working""",
121 time re-flushing a too small cache than working""",
122 metavar='InteractiveShell.cache_size')
122 metavar='InteractiveShell.cache_size')
123 paa('--classic',
123 paa('--classic',
124 action='store_true', dest='Global.classic',
124 action='store_true', dest='Global.classic',
125 help="Gives IPython a similar feel to the classic Python prompt.")
125 help="Gives IPython a similar feel to the classic Python prompt.")
126 paa('--colors',
126 paa('--colors',
127 type=str, dest='InteractiveShell.colors',
127 type=str, dest='InteractiveShell.colors',
128 help="Set the color scheme (NoColor, Linux, and LightBG).",
128 help="Set the color scheme (NoColor, Linux, and LightBG).",
129 metavar='InteractiveShell.colors')
129 metavar='InteractiveShell.colors')
130 paa('--color-info',
130 paa('--color-info',
131 action='store_true', dest='InteractiveShell.color_info',
131 action='store_true', dest='InteractiveShell.color_info',
132 help=
132 help=
133 """IPython can display information about objects via a set of func-
133 """IPython can display information about objects via a set of func-
134 tions, and optionally can use colors for this, syntax highlighting
134 tions, and optionally can use colors for this, syntax highlighting
135 source code and various other elements. However, because this
135 source code and various other elements. However, because this
136 information is passed through a pager (like 'less') and many pagers get
136 information is passed through a pager (like 'less') and many pagers get
137 confused with color codes, this option is off by default. You can test
137 confused with color codes, this option is off by default. You can test
138 it and turn it on permanently in your ipython_config.py file if it
138 it and turn it on permanently in your ipython_config.py file if it
139 works for you. Test it and turn it on permanently if it works with
139 works for you. Test it and turn it on permanently if it works with
140 your system. The magic function %%color_info allows you to toggle this
140 your system. The magic function %%color_info allows you to toggle this
141 inter- actively for testing.""")
141 inter- actively for testing.""")
142 paa('--no-color-info',
142 paa('--no-color-info',
143 action='store_false', dest='InteractiveShell.color_info',
143 action='store_false', dest='InteractiveShell.color_info',
144 help="Disable using colors for info related things.")
144 help="Disable using colors for info related things.")
145 paa('--confirm-exit',
145 paa('--confirm-exit',
146 action='store_true', dest='InteractiveShell.confirm_exit',
146 action='store_true', dest='InteractiveShell.confirm_exit',
147 help=
147 help=
148 """Set to confirm when you try to exit IPython with an EOF (Control-D
148 """Set to confirm when you try to exit IPython with an EOF (Control-D
149 in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or
149 in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or
150 '%%Exit', you can force a direct exit without any confirmation.""")
150 '%%Exit', you can force a direct exit without any confirmation.""")
151 paa('--no-confirm-exit',
151 paa('--no-confirm-exit',
152 action='store_false', dest='InteractiveShell.confirm_exit',
152 action='store_false', dest='InteractiveShell.confirm_exit',
153 help="Don't prompt the user when exiting.")
153 help="Don't prompt the user when exiting.")
154 paa('--deep-reload',
154 paa('--deep-reload',
155 action='store_true', dest='InteractiveShell.deep_reload',
155 action='store_true', dest='InteractiveShell.deep_reload',
156 help=
156 help=
157 """Enable deep (recursive) reloading by default. IPython can use the
157 """Enable deep (recursive) reloading by default. IPython can use the
158 deep_reload module which reloads changes in modules recursively (it
158 deep_reload module which reloads changes in modules recursively (it
159 replaces the reload() function, so you don't need to change anything to
159 replaces the reload() function, so you don't need to change anything to
160 use it). deep_reload() forces a full reload of modules whose code may
160 use it). deep_reload() forces a full reload of modules whose code may
161 have changed, which the default reload() function does not. When
161 have changed, which the default reload() function does not. When
162 deep_reload is off, IPython will use the normal reload(), but
162 deep_reload is off, IPython will use the normal reload(), but
163 deep_reload will still be available as dreload(). This fea- ture is off
163 deep_reload will still be available as dreload(). This fea- ture is off
164 by default [which means that you have both normal reload() and
164 by default [which means that you have both normal reload() and
165 dreload()].""")
165 dreload()].""")
166 paa('--no-deep-reload',
166 paa('--no-deep-reload',
167 action='store_false', dest='InteractiveShell.deep_reload',
167 action='store_false', dest='InteractiveShell.deep_reload',
168 help="Disable deep (recursive) reloading by default.")
168 help="Disable deep (recursive) reloading by default.")
169 paa('--editor',
169 paa('--editor',
170 type=str, dest='InteractiveShell.editor',
170 type=str, dest='InteractiveShell.editor',
171 help="Set the editor used by IPython (default to $EDITOR/vi/notepad).",
171 help="Set the editor used by IPython (default to $EDITOR/vi/notepad).",
172 metavar='InteractiveShell.editor')
172 metavar='InteractiveShell.editor')
173 paa('--log','-l',
173 paa('--log','-l',
174 action='store_true', dest='InteractiveShell.logstart',
174 action='store_true', dest='InteractiveShell.logstart',
175 help="Start logging to the default log file (./ipython_log.py).")
175 help="Start logging to the default log file (./ipython_log.py).")
176 paa('--logfile','-lf',
176 paa('--logfile','-lf',
177 type=unicode, dest='InteractiveShell.logfile',
177 type=unicode, dest='InteractiveShell.logfile',
178 help="Start logging to logfile with this name.",
178 help="Start logging to logfile with this name.",
179 metavar='InteractiveShell.logfile')
179 metavar='InteractiveShell.logfile')
180 paa('--log-append','-la',
180 paa('--log-append','-la',
181 type=unicode, dest='InteractiveShell.logappend',
181 type=unicode, dest='InteractiveShell.logappend',
182 help="Start logging to the given file in append mode.",
182 help="Start logging to the given file in append mode.",
183 metavar='InteractiveShell.logfile')
183 metavar='InteractiveShell.logfile')
184 paa('--pdb',
184 paa('--pdb',
185 action='store_true', dest='InteractiveShell.pdb',
185 action='store_true', dest='InteractiveShell.pdb',
186 help="Enable auto calling the pdb debugger after every exception.")
186 help="Enable auto calling the pdb debugger after every exception.")
187 paa('--no-pdb',
187 paa('--no-pdb',
188 action='store_false', dest='InteractiveShell.pdb',
188 action='store_false', dest='InteractiveShell.pdb',
189 help="Disable auto calling the pdb debugger after every exception.")
189 help="Disable auto calling the pdb debugger after every exception.")
190 paa('--pprint',
190 paa('--pprint',
191 action='store_true', dest='InteractiveShell.pprint',
191 action='store_true', dest='InteractiveShell.pprint',
192 help="Enable auto pretty printing of results.")
192 help="Enable auto pretty printing of results.")
193 paa('--no-pprint',
193 paa('--no-pprint',
194 action='store_false', dest='InteractiveShell.pprint',
194 action='store_false', dest='InteractiveShell.pprint',
195 help="Disable auto auto pretty printing of results.")
195 help="Disable auto auto pretty printing of results.")
196 paa('--prompt-in1','-pi1',
196 paa('--prompt-in1','-pi1',
197 type=str, dest='InteractiveShell.prompt_in1',
197 type=str, dest='InteractiveShell.prompt_in1',
198 help=
198 help=
199 """Set the main input prompt ('In [\#]: '). Note that if you are using
199 """Set the main input prompt ('In [\#]: '). Note that if you are using
200 numbered prompts, the number is represented with a '\#' in the string.
200 numbered prompts, the number is represented with a '\#' in the string.
201 Don't forget to quote strings with spaces embedded in them. Most
201 Don't forget to quote strings with spaces embedded in them. Most
202 bash-like escapes can be used to customize IPython's prompts, as well
202 bash-like escapes can be used to customize IPython's prompts, as well
203 as a few additional ones which are IPython-spe- cific. All valid
203 as a few additional ones which are IPython-spe- cific. All valid
204 prompt escapes are described in detail in the Customization section of
204 prompt escapes are described in detail in the Customization section of
205 the IPython manual.""",
205 the IPython manual.""",
206 metavar='InteractiveShell.prompt_in1')
206 metavar='InteractiveShell.prompt_in1')
207 paa('--prompt-in2','-pi2',
207 paa('--prompt-in2','-pi2',
208 type=str, dest='InteractiveShell.prompt_in2',
208 type=str, dest='InteractiveShell.prompt_in2',
209 help=
209 help=
210 """Set the secondary input prompt (' .\D.: '). Similar to the previous
210 """Set the secondary input prompt (' .\D.: '). Similar to the previous
211 option, but used for the continuation prompts. The special sequence
211 option, but used for the continuation prompts. The special sequence
212 '\D' is similar to '\#', but with all digits replaced by dots (so you
212 '\D' is similar to '\#', but with all digits replaced by dots (so you
213 can have your continuation prompt aligned with your input prompt).
213 can have your continuation prompt aligned with your input prompt).
214 Default: ' .\D.: ' (note three spaces at the start for alignment with
214 Default: ' .\D.: ' (note three spaces at the start for alignment with
215 'In [\#]')""",
215 'In [\#]')""",
216 metavar='InteractiveShell.prompt_in2')
216 metavar='InteractiveShell.prompt_in2')
217 paa('--prompt-out','-po',
217 paa('--prompt-out','-po',
218 type=str, dest='InteractiveShell.prompt_out',
218 type=str, dest='InteractiveShell.prompt_out',
219 help="Set the output prompt ('Out[\#]:')",
219 help="Set the output prompt ('Out[\#]:')",
220 metavar='InteractiveShell.prompt_out')
220 metavar='InteractiveShell.prompt_out')
221 paa('--quick',
221 paa('--quick',
222 action='store_true', dest='Global.quick',
222 action='store_true', dest='Global.quick',
223 help="Enable quick startup with no config files.")
223 help="Enable quick startup with no config files.")
224 paa('--readline',
224 paa('--readline',
225 action='store_true', dest='InteractiveShell.readline_use',
225 action='store_true', dest='InteractiveShell.readline_use',
226 help="Enable readline for command line usage.")
226 help="Enable readline for command line usage.")
227 paa('--no-readline',
227 paa('--no-readline',
228 action='store_false', dest='InteractiveShell.readline_use',
228 action='store_false', dest='InteractiveShell.readline_use',
229 help="Disable readline for command line usage.")
229 help="Disable readline for command line usage.")
230 paa('--screen-length','-sl',
230 paa('--screen-length','-sl',
231 type=int, dest='InteractiveShell.screen_length',
231 type=int, dest='InteractiveShell.screen_length',
232 help=
232 help=
233 """Number of lines of your screen, used to control printing of very
233 """Number of lines of your screen, used to control printing of very
234 long strings. Strings longer than this number of lines will be sent
234 long strings. Strings longer than this number of lines will be sent
235 through a pager instead of directly printed. The default value for
235 through a pager instead of directly printed. The default value for
236 this is 0, which means IPython will auto-detect your screen size every
236 this is 0, which means IPython will auto-detect your screen size every
237 time it needs to print certain potentially long strings (this doesn't
237 time it needs to print certain potentially long strings (this doesn't
238 change the behavior of the 'print' keyword, it's only triggered
238 change the behavior of the 'print' keyword, it's only triggered
239 internally). If for some reason this isn't working well (it needs
239 internally). If for some reason this isn't working well (it needs
240 curses support), specify it yourself. Otherwise don't change the
240 curses support), specify it yourself. Otherwise don't change the
241 default.""",
241 default.""",
242 metavar='InteractiveShell.screen_length')
242 metavar='InteractiveShell.screen_length')
243 paa('--separate-in','-si',
243 paa('--separate-in','-si',
244 type=str, dest='InteractiveShell.separate_in',
244 type=str, dest='InteractiveShell.separate_in',
245 help="Separator before input prompts. Default '\\n'.",
245 help="Separator before input prompts. Default '\\n'.",
246 metavar='InteractiveShell.separate_in')
246 metavar='InteractiveShell.separate_in')
247 paa('--separate-out','-so',
247 paa('--separate-out','-so',
248 type=str, dest='InteractiveShell.separate_out',
248 type=str, dest='InteractiveShell.separate_out',
249 help="Separator before output prompts. Default 0 (nothing).",
249 help="Separator before output prompts. Default 0 (nothing).",
250 metavar='InteractiveShell.separate_out')
250 metavar='InteractiveShell.separate_out')
251 paa('--separate-out2','-so2',
251 paa('--separate-out2','-so2',
252 type=str, dest='InteractiveShell.separate_out2',
252 type=str, dest='InteractiveShell.separate_out2',
253 help="Separator after output prompts. Default 0 (nonight).",
253 help="Separator after output prompts. Default 0 (nonight).",
254 metavar='InteractiveShell.separate_out2')
254 metavar='InteractiveShell.separate_out2')
255 paa('-no-sep',
255 paa('--no-sep',
256 action='store_true', dest='Global.nosep',
256 action='store_true', dest='Global.nosep',
257 help="Eliminate all spacing between prompts.")
257 help="Eliminate all spacing between prompts.")
258 paa('--term-title',
258 paa('--term-title',
259 action='store_true', dest='InteractiveShell.term_title',
259 action='store_true', dest='InteractiveShell.term_title',
260 help="Enable auto setting the terminal title.")
260 help="Enable auto setting the terminal title.")
261 paa('--no-term-title',
261 paa('--no-term-title',
262 action='store_false', dest='InteractiveShell.term_title',
262 action='store_false', dest='InteractiveShell.term_title',
263 help="Disable auto setting the terminal title.")
263 help="Disable auto setting the terminal title.")
264 paa('--xmode',
264 paa('--xmode',
265 type=str, dest='InteractiveShell.xmode',
265 type=str, dest='InteractiveShell.xmode',
266 help=
266 help=
267 """Exception reporting mode ('Plain','Context','Verbose'). Plain:
267 """Exception reporting mode ('Plain','Context','Verbose'). Plain:
268 similar to python's normal traceback printing. Context: prints 5 lines
268 similar to python's normal traceback printing. Context: prints 5 lines
269 of context source code around each line in the traceback. Verbose:
269 of context source code around each line in the traceback. Verbose:
270 similar to Context, but additionally prints the variables currently
270 similar to Context, but additionally prints the variables currently
271 visible where the exception happened (shortening their strings if too
271 visible where the exception happened (shortening their strings if too
272 long). This can potentially be very slow, if you happen to have a huge
272 long). This can potentially be very slow, if you happen to have a huge
273 data structure whose string representation is complex to compute.
273 data structure whose string representation is complex to compute.
274 Your computer may appear to freeze for a while with cpu usage at 100%%.
274 Your computer may appear to freeze for a while with cpu usage at 100%%.
275 If this occurs, you can cancel the traceback with Ctrl-C (maybe hitting
275 If this occurs, you can cancel the traceback with Ctrl-C (maybe hitting
276 it more than once).
276 it more than once).
277 """,
277 """,
278 metavar='InteractiveShell.xmode')
278 metavar='InteractiveShell.xmode')
279 paa('--ext',
279 paa('--ext',
280 type=str, dest='Global.extra_extension',
280 type=str, dest='Global.extra_extension',
281 help="The dotted module name of an IPython extension to load.",
281 help="The dotted module name of an IPython extension to load.",
282 metavar='Global.extra_extension')
282 metavar='Global.extra_extension')
283 paa('-c',
283 paa('-c',
284 type=str, dest='Global.code_to_run',
284 type=str, dest='Global.code_to_run',
285 help="Execute the given command string.",
285 help="Execute the given command string.",
286 metavar='Global.code_to_run')
286 metavar='Global.code_to_run')
287 paa('-i',
287 paa('-i',
288 action='store_true', dest='Global.force_interact',
288 action='store_true', dest='Global.force_interact',
289 help=
289 help=
290 "If running code from the command line, become interactive afterwards.")
290 "If running code from the command line, become interactive afterwards.")
291
291
292 # Options to start with GUI control enabled from the beginning
292 # Options to start with GUI control enabled from the beginning
293 paa('--gui',
293 paa('--gui',
294 type=str, dest='Global.gui',
294 type=str, dest='Global.gui',
295 help="Enable GUI event loop integration ('qt', 'wx', 'gtk').",
295 help="Enable GUI event loop integration ('qt', 'wx', 'gtk').",
296 metavar='gui-mode')
296 metavar='gui-mode')
297 paa('--pylab','-pylab',
297 paa('--pylab','-pylab',
298 type=str, dest='Global.pylab',
298 type=str, dest='Global.pylab',
299 nargs='?', const='auto', metavar='gui-mode',
299 nargs='?', const='auto', metavar='gui-mode',
300 help="Pre-load matplotlib and numpy for interactive use. "+
300 help="Pre-load matplotlib and numpy for interactive use. "+
301 "If no value is given, the gui backend is matplotlib's, else use "+
301 "If no value is given, the gui backend is matplotlib's, else use "+
302 "one of: ['tk', 'qt', 'wx', 'gtk'].")
302 "one of: ['tk', 'qt', 'wx', 'gtk'].")
303
303
304 # Legacy GUI options. Leave them in for backwards compatibility, but the
304 # Legacy GUI options. Leave them in for backwards compatibility, but the
305 # 'thread' names are really a misnomer now.
305 # 'thread' names are really a misnomer now.
306 paa('--wthread', '-wthread',
306 paa('--wthread', '-wthread',
307 action='store_true', dest='Global.wthread',
307 action='store_true', dest='Global.wthread',
308 help=
308 help=
309 """Enable wxPython event loop integration. (DEPRECATED, use --gui wx)""")
309 """Enable wxPython event loop integration. (DEPRECATED, use --gui wx)""")
310 paa('--q4thread', '--qthread', '-q4thread', '-qthread',
310 paa('--q4thread', '--qthread', '-q4thread', '-qthread',
311 action='store_true', dest='Global.q4thread',
311 action='store_true', dest='Global.q4thread',
312 help=
312 help=
313 """Enable Qt4 event loop integration. Qt3 is no longer supported.
313 """Enable Qt4 event loop integration. Qt3 is no longer supported.
314 (DEPRECATED, use --gui qt)""")
314 (DEPRECATED, use --gui qt)""")
315 paa('--gthread', '-gthread',
315 paa('--gthread', '-gthread',
316 action='store_true', dest='Global.gthread',
316 action='store_true', dest='Global.gthread',
317 help=
317 help=
318 """Enable GTK event loop integration. (DEPRECATED, use --gui gtk)""")
318 """Enable GTK event loop integration. (DEPRECATED, use --gui gtk)""")
319
319
320
320
321 #-----------------------------------------------------------------------------
321 #-----------------------------------------------------------------------------
322 # Crash handler for this application
322 # Crash handler for this application
323 #-----------------------------------------------------------------------------
323 #-----------------------------------------------------------------------------
324
324
325
325
326 _message_template = """\
326 _message_template = """\
327 Oops, $self.app_name crashed. We do our best to make it stable, but...
327 Oops, $self.app_name crashed. We do our best to make it stable, but...
328
328
329 A crash report was automatically generated with the following information:
329 A crash report was automatically generated with the following information:
330 - A verbatim copy of the crash traceback.
330 - A verbatim copy of the crash traceback.
331 - A copy of your input history during this session.
331 - A copy of your input history during this session.
332 - Data on your current $self.app_name configuration.
332 - Data on your current $self.app_name configuration.
333
333
334 It was left in the file named:
334 It was left in the file named:
335 \t'$self.crash_report_fname'
335 \t'$self.crash_report_fname'
336 If you can email this file to the developers, the information in it will help
336 If you can email this file to the developers, the information in it will help
337 them in understanding and correcting the problem.
337 them in understanding and correcting the problem.
338
338
339 You can mail it to: $self.contact_name at $self.contact_email
339 You can mail it to: $self.contact_name at $self.contact_email
340 with the subject '$self.app_name Crash Report'.
340 with the subject '$self.app_name Crash Report'.
341
341
342 If you want to do it now, the following command will work (under Unix):
342 If you want to do it now, the following command will work (under Unix):
343 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
343 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
344
344
345 To ensure accurate tracking of this issue, please file a report about it at:
345 To ensure accurate tracking of this issue, please file a report about it at:
346 $self.bug_tracker
346 $self.bug_tracker
347 """
347 """
348
348
349 class IPAppCrashHandler(CrashHandler):
349 class IPAppCrashHandler(CrashHandler):
350 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
350 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
351
351
352 message_template = _message_template
352 message_template = _message_template
353
353
354 def __init__(self, app):
354 def __init__(self, app):
355 contact_name = release.authors['Fernando'][0]
355 contact_name = release.authors['Fernando'][0]
356 contact_email = release.authors['Fernando'][1]
356 contact_email = release.authors['Fernando'][1]
357 bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug'
357 bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug'
358 super(IPAppCrashHandler,self).__init__(
358 super(IPAppCrashHandler,self).__init__(
359 app, contact_name, contact_email, bug_tracker
359 app, contact_name, contact_email, bug_tracker
360 )
360 )
361
361
362 def make_report(self,traceback):
362 def make_report(self,traceback):
363 """Return a string containing a crash report."""
363 """Return a string containing a crash report."""
364
364
365 sec_sep = self.section_sep
365 sec_sep = self.section_sep
366 # Start with parent report
366 # Start with parent report
367 report = [super(IPAppCrashHandler, self).make_report(traceback)]
367 report = [super(IPAppCrashHandler, self).make_report(traceback)]
368 # Add interactive-specific info we may have
368 # Add interactive-specific info we may have
369 rpt_add = report.append
369 rpt_add = report.append
370 try:
370 try:
371 rpt_add(sec_sep+"History of session input:")
371 rpt_add(sec_sep+"History of session input:")
372 for line in self.app.shell.user_ns['_ih']:
372 for line in self.app.shell.user_ns['_ih']:
373 rpt_add(line)
373 rpt_add(line)
374 rpt_add('\n*** Last line of input (may not be in above history):\n')
374 rpt_add('\n*** Last line of input (may not be in above history):\n')
375 rpt_add(self.app.shell._last_input_line+'\n')
375 rpt_add(self.app.shell._last_input_line+'\n')
376 except:
376 except:
377 pass
377 pass
378
378
379 return ''.join(report)
379 return ''.join(report)
380
380
381
381
382 #-----------------------------------------------------------------------------
382 #-----------------------------------------------------------------------------
383 # Main classes and functions
383 # Main classes and functions
384 #-----------------------------------------------------------------------------
384 #-----------------------------------------------------------------------------
385
385
386 class IPythonApp(Application):
386 class IPythonApp(Application):
387 name = u'ipython'
387 name = u'ipython'
388 #: argparse formats better the 'usage' than the 'description' field
388 #: argparse formats better the 'usage' than the 'description' field
389 description = None
389 description = None
390 usage = usage.cl_usage
390 usage = usage.cl_usage
391 command_line_loader = IPAppConfigLoader
391 command_line_loader = IPAppConfigLoader
392 config_file_name = default_config_file_name
392 default_config_file_name = default_config_file_name
393 crash_handler_class = IPAppCrashHandler
393 crash_handler_class = IPAppCrashHandler
394
394
395 def create_default_config(self):
395 def create_default_config(self):
396 super(IPythonApp, self).create_default_config()
396 super(IPythonApp, self).create_default_config()
397 # Eliminate multiple lookups
397 # Eliminate multiple lookups
398 Global = self.default_config.Global
398 Global = self.default_config.Global
399
399
400 # Set all default values
400 # Set all default values
401 Global.display_banner = True
401 Global.display_banner = True
402
402
403 # If the -c flag is given or a file is given to run at the cmd line
403 # If the -c flag is given or a file is given to run at the cmd line
404 # like "ipython foo.py", normally we exit without starting the main
404 # like "ipython foo.py", normally we exit without starting the main
405 # loop. The force_interact config variable allows a user to override
405 # loop. The force_interact config variable allows a user to override
406 # this and interact. It is also set by the -i cmd line flag, just
406 # this and interact. It is also set by the -i cmd line flag, just
407 # like Python.
407 # like Python.
408 Global.force_interact = False
408 Global.force_interact = False
409
409
410 # By default always interact by starting the IPython mainloop.
410 # By default always interact by starting the IPython mainloop.
411 Global.interact = True
411 Global.interact = True
412
412
413 # No GUI integration by default
413 # No GUI integration by default
414 Global.gui = False
414 Global.gui = False
415 # Pylab off by default
415 # Pylab off by default
416 Global.pylab = False
416 Global.pylab = False
417
417
418 # Deprecated versions of gui support that used threading, we support
418 # Deprecated versions of gui support that used threading, we support
419 # them just for bacwards compatibility as an alternate spelling for
419 # them just for bacwards compatibility as an alternate spelling for
420 # '--gui X'
420 # '--gui X'
421 Global.qthread = False
421 Global.qthread = False
422 Global.q4thread = False
422 Global.q4thread = False
423 Global.wthread = False
423 Global.wthread = False
424 Global.gthread = False
424 Global.gthread = False
425
425
426 def load_file_config(self):
426 def load_file_config(self):
427 if hasattr(self.command_line_config.Global, 'quick'):
427 if hasattr(self.command_line_config.Global, 'quick'):
428 if self.command_line_config.Global.quick:
428 if self.command_line_config.Global.quick:
429 self.file_config = Config()
429 self.file_config = Config()
430 return
430 return
431 super(IPythonApp, self).load_file_config()
431 super(IPythonApp, self).load_file_config()
432
432
433 def post_load_file_config(self):
433 def post_load_file_config(self):
434 if hasattr(self.command_line_config.Global, 'extra_extension'):
434 if hasattr(self.command_line_config.Global, 'extra_extension'):
435 if not hasattr(self.file_config.Global, 'extensions'):
435 if not hasattr(self.file_config.Global, 'extensions'):
436 self.file_config.Global.extensions = []
436 self.file_config.Global.extensions = []
437 self.file_config.Global.extensions.append(
437 self.file_config.Global.extensions.append(
438 self.command_line_config.Global.extra_extension)
438 self.command_line_config.Global.extra_extension)
439 del self.command_line_config.Global.extra_extension
439 del self.command_line_config.Global.extra_extension
440
440
441 def pre_construct(self):
441 def pre_construct(self):
442 config = self.master_config
442 config = self.master_config
443
443
444 if hasattr(config.Global, 'classic'):
444 if hasattr(config.Global, 'classic'):
445 if config.Global.classic:
445 if config.Global.classic:
446 config.InteractiveShell.cache_size = 0
446 config.InteractiveShell.cache_size = 0
447 config.InteractiveShell.pprint = 0
447 config.InteractiveShell.pprint = 0
448 config.InteractiveShell.prompt_in1 = '>>> '
448 config.InteractiveShell.prompt_in1 = '>>> '
449 config.InteractiveShell.prompt_in2 = '... '
449 config.InteractiveShell.prompt_in2 = '... '
450 config.InteractiveShell.prompt_out = ''
450 config.InteractiveShell.prompt_out = ''
451 config.InteractiveShell.separate_in = \
451 config.InteractiveShell.separate_in = \
452 config.InteractiveShell.separate_out = \
452 config.InteractiveShell.separate_out = \
453 config.InteractiveShell.separate_out2 = ''
453 config.InteractiveShell.separate_out2 = ''
454 config.InteractiveShell.colors = 'NoColor'
454 config.InteractiveShell.colors = 'NoColor'
455 config.InteractiveShell.xmode = 'Plain'
455 config.InteractiveShell.xmode = 'Plain'
456
456
457 if hasattr(config.Global, 'nosep'):
457 if hasattr(config.Global, 'nosep'):
458 if config.Global.nosep:
458 if config.Global.nosep:
459 config.InteractiveShell.separate_in = \
459 config.InteractiveShell.separate_in = \
460 config.InteractiveShell.separate_out = \
460 config.InteractiveShell.separate_out = \
461 config.InteractiveShell.separate_out2 = ''
461 config.InteractiveShell.separate_out2 = ''
462
462
463 # if there is code of files to run from the cmd line, don't interact
463 # if there is code of files to run from the cmd line, don't interact
464 # unless the -i flag (Global.force_interact) is true.
464 # unless the -i flag (Global.force_interact) is true.
465 code_to_run = config.Global.get('code_to_run','')
465 code_to_run = config.Global.get('code_to_run','')
466 file_to_run = False
466 file_to_run = False
467 if self.extra_args and self.extra_args[0]:
467 if self.extra_args and self.extra_args[0]:
468 file_to_run = True
468 file_to_run = True
469 if file_to_run or code_to_run:
469 if file_to_run or code_to_run:
470 if not config.Global.force_interact:
470 if not config.Global.force_interact:
471 config.Global.interact = False
471 config.Global.interact = False
472
472
473 def construct(self):
473 def construct(self):
474 # I am a little hesitant to put these into InteractiveShell itself.
474 # I am a little hesitant to put these into InteractiveShell itself.
475 # But that might be the place for them
475 # But that might be the place for them
476 sys.path.insert(0, '')
476 sys.path.insert(0, '')
477
477
478 # Create an InteractiveShell instance
478 # Create an InteractiveShell instance
479 self.shell = InteractiveShell(None, self.master_config)
479 self.shell = InteractiveShell(None, self.master_config)
480
480
481 def post_construct(self):
481 def post_construct(self):
482 """Do actions after construct, but before starting the app."""
482 """Do actions after construct, but before starting the app."""
483 config = self.master_config
483 config = self.master_config
484
484
485 # shell.display_banner should always be False for the terminal
485 # shell.display_banner should always be False for the terminal
486 # based app, because we call shell.show_banner() by hand below
486 # based app, because we call shell.show_banner() by hand below
487 # so the banner shows *before* all extension loading stuff.
487 # so the banner shows *before* all extension loading stuff.
488 self.shell.display_banner = False
488 self.shell.display_banner = False
489
490 if config.Global.display_banner and \
489 if config.Global.display_banner and \
491 config.Global.interact:
490 config.Global.interact:
492 self.shell.show_banner()
491 self.shell.show_banner()
493
492
494 # Make sure there is a space below the banner.
493 # Make sure there is a space below the banner.
495 if self.log_level <= logging.INFO: print
494 if self.log_level <= logging.INFO: print
496
495
497 # Now a variety of things that happen after the banner is printed.
496 # Now a variety of things that happen after the banner is printed.
498 self._enable_gui_pylab()
497 self._enable_gui_pylab()
499 self._load_extensions()
498 self._load_extensions()
500 self._run_exec_lines()
499 self._run_exec_lines()
501 self._run_exec_files()
500 self._run_exec_files()
502 self._run_cmd_line_code()
501 self._run_cmd_line_code()
503
502
504 def _enable_gui_pylab(self):
503 def _enable_gui_pylab(self):
505 """Enable GUI event loop integration, taking pylab into account."""
504 """Enable GUI event loop integration, taking pylab into account."""
506 Global = self.master_config.Global
505 Global = self.master_config.Global
507
506
508 # Select which gui to use
507 # Select which gui to use
509 if Global.gui:
508 if Global.gui:
510 gui = Global.gui
509 gui = Global.gui
511 # The following are deprecated, but there's likely to be a lot of use
510 # The following are deprecated, but there's likely to be a lot of use
512 # of this form out there, so we might as well support it for now. But
511 # of this form out there, so we might as well support it for now. But
513 # the --gui option above takes precedence.
512 # the --gui option above takes precedence.
514 elif Global.wthread:
513 elif Global.wthread:
515 gui = inputhook.GUI_WX
514 gui = inputhook.GUI_WX
516 elif Global.qthread:
515 elif Global.qthread:
517 gui = inputhook.GUI_QT
516 gui = inputhook.GUI_QT
518 elif Global.gthread:
517 elif Global.gthread:
519 gui = inputhook.GUI_GTK
518 gui = inputhook.GUI_GTK
520 else:
519 else:
521 gui = None
520 gui = None
522
521
523 # Using --pylab will also require gui activation, though which toolkit
522 # Using --pylab will also require gui activation, though which toolkit
524 # to use may be chosen automatically based on mpl configuration.
523 # to use may be chosen automatically based on mpl configuration.
525 if Global.pylab:
524 if Global.pylab:
526 activate = self.shell.enable_pylab
525 activate = self.shell.enable_pylab
527 if Global.pylab == 'auto':
526 if Global.pylab == 'auto':
528 gui = None
527 gui = None
529 else:
528 else:
530 gui = Global.pylab
529 gui = Global.pylab
531 else:
530 else:
532 # Enable only GUI integration, no pylab
531 # Enable only GUI integration, no pylab
533 activate = inputhook.enable_gui
532 activate = inputhook.enable_gui
534
533
535 if gui or Global.pylab:
534 if gui or Global.pylab:
536 try:
535 try:
537 self.log.info("Enabling GUI event loop integration, "
536 self.log.info("Enabling GUI event loop integration, "
538 "toolkit=%s, pylab=%s" % (gui, Global.pylab) )
537 "toolkit=%s, pylab=%s" % (gui, Global.pylab) )
539 activate(gui)
538 activate(gui)
540 except:
539 except:
541 self.log.warn("Error in enabling GUI event loop integration:")
540 self.log.warn("Error in enabling GUI event loop integration:")
542 self.shell.showtraceback()
541 self.shell.showtraceback()
543
542
544 def _load_extensions(self):
543 def _load_extensions(self):
545 """Load all IPython extensions in Global.extensions.
544 """Load all IPython extensions in Global.extensions.
546
545
547 This uses the :meth:`InteractiveShell.load_extensions` to load all
546 This uses the :meth:`InteractiveShell.load_extensions` to load all
548 the extensions listed in ``self.master_config.Global.extensions``.
547 the extensions listed in ``self.master_config.Global.extensions``.
549 """
548 """
550 try:
549 try:
551 if hasattr(self.master_config.Global, 'extensions'):
550 if hasattr(self.master_config.Global, 'extensions'):
552 self.log.debug("Loading IPython extensions...")
551 self.log.debug("Loading IPython extensions...")
553 extensions = self.master_config.Global.extensions
552 extensions = self.master_config.Global.extensions
554 for ext in extensions:
553 for ext in extensions:
555 try:
554 try:
556 self.log.info("Loading IPython extension: %s" % ext)
555 self.log.info("Loading IPython extension: %s" % ext)
557 self.shell.load_extension(ext)
556 self.shell.load_extension(ext)
558 except:
557 except:
559 self.log.warn("Error in loading extension: %s" % ext)
558 self.log.warn("Error in loading extension: %s" % ext)
560 self.shell.showtraceback()
559 self.shell.showtraceback()
561 except:
560 except:
562 self.log.warn("Unknown error in loading extensions:")
561 self.log.warn("Unknown error in loading extensions:")
563 self.shell.showtraceback()
562 self.shell.showtraceback()
564
563
565 def _run_exec_lines(self):
564 def _run_exec_lines(self):
566 """Run lines of code in Global.exec_lines in the user's namespace."""
565 """Run lines of code in Global.exec_lines in the user's namespace."""
567 try:
566 try:
568 if hasattr(self.master_config.Global, 'exec_lines'):
567 if hasattr(self.master_config.Global, 'exec_lines'):
569 self.log.debug("Running code from Global.exec_lines...")
568 self.log.debug("Running code from Global.exec_lines...")
570 exec_lines = self.master_config.Global.exec_lines
569 exec_lines = self.master_config.Global.exec_lines
571 for line in exec_lines:
570 for line in exec_lines:
572 try:
571 try:
573 self.log.info("Running code in user namespace: %s" % line)
572 self.log.info("Running code in user namespace: %s" % line)
574 self.shell.runlines(line)
573 self.shell.runlines(line)
575 except:
574 except:
576 self.log.warn("Error in executing line in user namespace: %s" % line)
575 self.log.warn("Error in executing line in user namespace: %s" % line)
577 self.shell.showtraceback()
576 self.shell.showtraceback()
578 except:
577 except:
579 self.log.warn("Unknown error in handling Global.exec_lines:")
578 self.log.warn("Unknown error in handling Global.exec_lines:")
580 self.shell.showtraceback()
579 self.shell.showtraceback()
581
580
582 def _exec_file(self, fname):
581 def _exec_file(self, fname):
583 full_filename = filefind(fname, [u'.', self.ipython_dir])
582 full_filename = filefind(fname, [u'.', self.ipython_dir])
584 if os.path.isfile(full_filename):
583 if os.path.isfile(full_filename):
585 if full_filename.endswith(u'.py'):
584 if full_filename.endswith(u'.py'):
586 self.log.info("Running file in user namespace: %s" % full_filename)
585 self.log.info("Running file in user namespace: %s" % full_filename)
587 self.shell.safe_execfile(full_filename, self.shell.user_ns)
586 self.shell.safe_execfile(full_filename, self.shell.user_ns)
588 elif full_filename.endswith('.ipy'):
587 elif full_filename.endswith('.ipy'):
589 self.log.info("Running file in user namespace: %s" % full_filename)
588 self.log.info("Running file in user namespace: %s" % full_filename)
590 self.shell.safe_execfile_ipy(full_filename)
589 self.shell.safe_execfile_ipy(full_filename)
591 else:
590 else:
592 self.log.warn("File does not have a .py or .ipy extension: <%s>" % full_filename)
591 self.log.warn("File does not have a .py or .ipy extension: <%s>" % full_filename)
593
592
594 def _run_exec_files(self):
593 def _run_exec_files(self):
595 try:
594 try:
596 if hasattr(self.master_config.Global, 'exec_files'):
595 if hasattr(self.master_config.Global, 'exec_files'):
597 self.log.debug("Running files in Global.exec_files...")
596 self.log.debug("Running files in Global.exec_files...")
598 exec_files = self.master_config.Global.exec_files
597 exec_files = self.master_config.Global.exec_files
599 for fname in exec_files:
598 for fname in exec_files:
600 self._exec_file(fname)
599 self._exec_file(fname)
601 except:
600 except:
602 self.log.warn("Unknown error in handling Global.exec_files:")
601 self.log.warn("Unknown error in handling Global.exec_files:")
603 self.shell.showtraceback()
602 self.shell.showtraceback()
604
603
605 def _run_cmd_line_code(self):
604 def _run_cmd_line_code(self):
606 if hasattr(self.master_config.Global, 'code_to_run'):
605 if hasattr(self.master_config.Global, 'code_to_run'):
607 line = self.master_config.Global.code_to_run
606 line = self.master_config.Global.code_to_run
608 try:
607 try:
609 self.log.info("Running code given at command line (-c): %s" % line)
608 self.log.info("Running code given at command line (-c): %s" % line)
610 self.shell.runlines(line)
609 self.shell.runlines(line)
611 except:
610 except:
612 self.log.warn("Error in executing line in user namespace: %s" % line)
611 self.log.warn("Error in executing line in user namespace: %s" % line)
613 self.shell.showtraceback()
612 self.shell.showtraceback()
614 return
613 return
615 # Like Python itself, ignore the second if the first of these is present
614 # Like Python itself, ignore the second if the first of these is present
616 try:
615 try:
617 fname = self.extra_args[0]
616 fname = self.extra_args[0]
618 except:
617 except:
619 pass
618 pass
620 else:
619 else:
621 try:
620 try:
622 self._exec_file(fname)
621 self._exec_file(fname)
623 except:
622 except:
624 self.log.warn("Error in executing file in user namespace: %s" % fname)
623 self.log.warn("Error in executing file in user namespace: %s" % fname)
625 self.shell.showtraceback()
624 self.shell.showtraceback()
626
625
627 def start_app(self):
626 def start_app(self):
628 if self.master_config.Global.interact:
627 if self.master_config.Global.interact:
629 self.log.debug("Starting IPython's mainloop...")
628 self.log.debug("Starting IPython's mainloop...")
630 self.shell.mainloop()
629 self.shell.mainloop()
631 else:
630 else:
632 self.log.debug("IPython not interactive, start_app is no-op...")
631 self.log.debug("IPython not interactive, start_app is no-op...")
633
632
634
633
635 def load_default_config(ipython_dir=None):
634 def load_default_config(ipython_dir=None):
636 """Load the default config file from the default ipython_dir.
635 """Load the default config file from the default ipython_dir.
637
636
638 This is useful for embedded shells.
637 This is useful for embedded shells.
639 """
638 """
640 if ipython_dir is None:
639 if ipython_dir is None:
641 ipython_dir = get_ipython_dir()
640 ipython_dir = get_ipython_dir()
642 cl = PyFileConfigLoader(default_config_file_name, ipython_dir)
641 cl = PyFileConfigLoader(default_config_file_name, ipython_dir)
643 config = cl.load_config()
642 config = cl.load_config()
644 return config
643 return config
645
644
646
645
647 def launch_new_instance():
646 def launch_new_instance():
648 """Create and run a full blown IPython instance"""
647 """Create and run a full blown IPython instance"""
649 app = IPythonApp()
648 app = IPythonApp()
650 app.start()
649 app.start()
651
650
652
651
653 if __name__ == '__main__':
652 if __name__ == '__main__':
654 launch_new_instance()
653 launch_new_instance()
@@ -1,537 +1,539 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The IPython cluster directory
4 The IPython cluster directory
5 """
5 """
6
6
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Copyright (C) 2008-2009 The IPython Development Team
8 # Copyright (C) 2008-2009 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 from __future__ import with_statement
18 from __future__ import with_statement
19
19
20 import os
20 import os
21 import shutil
21 import shutil
22 import sys
22 import sys
23 import warnings
23 import warnings
24
24
25 from twisted.python import log
25 from twisted.python import log
26
26
27 from IPython.config.loader import PyFileConfigLoader
27 from IPython.config.loader import PyFileConfigLoader
28 from IPython.core.application import Application, BaseAppConfigLoader
28 from IPython.core.application import Application, BaseAppConfigLoader
29 from IPython.core.component import Component
29 from IPython.core.component import Component
30 from IPython.core.crashhandler import CrashHandler
30 from IPython.core.crashhandler import CrashHandler
31 from IPython.core import release
31 from IPython.core import release
32 from IPython.utils.path import (
32 from IPython.utils.path import (
33 get_ipython_package_dir,
33 get_ipython_package_dir,
34 expand_path
34 expand_path
35 )
35 )
36 from IPython.utils.traitlets import Unicode
36 from IPython.utils.traitlets import Unicode
37
37
38 #-----------------------------------------------------------------------------
38 #-----------------------------------------------------------------------------
39 # Warnings control
39 # Warnings control
40 #-----------------------------------------------------------------------------
40 #-----------------------------------------------------------------------------
41 # Twisted generates annoying warnings with Python 2.6, as will do other code
41 # Twisted generates annoying warnings with Python 2.6, as will do other code
42 # that imports 'sets' as of today
42 # that imports 'sets' as of today
43 warnings.filterwarnings('ignore', 'the sets module is deprecated',
43 warnings.filterwarnings('ignore', 'the sets module is deprecated',
44 DeprecationWarning )
44 DeprecationWarning )
45
45
46 # This one also comes from Twisted
46 # This one also comes from Twisted
47 warnings.filterwarnings('ignore', 'the sha module is deprecated',
47 warnings.filterwarnings('ignore', 'the sha module is deprecated',
48 DeprecationWarning)
48 DeprecationWarning)
49
49
50 #-----------------------------------------------------------------------------
50 #-----------------------------------------------------------------------------
51 # Module errors
51 # Module errors
52 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
53
53
54 class ClusterDirError(Exception):
54 class ClusterDirError(Exception):
55 pass
55 pass
56
56
57
57
58 class PIDFileError(Exception):
58 class PIDFileError(Exception):
59 pass
59 pass
60
60
61
61
62 #-----------------------------------------------------------------------------
62 #-----------------------------------------------------------------------------
63 # Class for managing cluster directories
63 # Class for managing cluster directories
64 #-----------------------------------------------------------------------------
64 #-----------------------------------------------------------------------------
65
65
66 class ClusterDir(Component):
66 class ClusterDir(Component):
67 """An object to manage the cluster directory and its resources.
67 """An object to manage the cluster directory and its resources.
68
68
69 The cluster directory is used by :command:`ipcontroller`,
69 The cluster directory is used by :command:`ipcontroller`,
70 :command:`ipcontroller` and :command:`ipcontroller` to manage the
70 :command:`ipcontroller` and :command:`ipcontroller` to manage the
71 configuration, logging and security of these applications.
71 configuration, logging and security of these applications.
72
72
73 This object knows how to find, create and manage these directories. This
73 This object knows how to find, create and manage these directories. This
74 should be used by any code that want's to handle cluster directories.
74 should be used by any code that want's to handle cluster directories.
75 """
75 """
76
76
77 security_dir_name = Unicode('security')
77 security_dir_name = Unicode('security')
78 log_dir_name = Unicode('log')
78 log_dir_name = Unicode('log')
79 pid_dir_name = Unicode('pid')
79 pid_dir_name = Unicode('pid')
80 security_dir = Unicode(u'')
80 security_dir = Unicode(u'')
81 log_dir = Unicode(u'')
81 log_dir = Unicode(u'')
82 pid_dir = Unicode(u'')
82 pid_dir = Unicode(u'')
83 location = Unicode(u'')
83 location = Unicode(u'')
84
84
85 def __init__(self, location):
85 def __init__(self, location):
86 super(ClusterDir, self).__init__(None)
86 super(ClusterDir, self).__init__(None)
87 self.location = location
87 self.location = location
88
88
89 def _location_changed(self, name, old, new):
89 def _location_changed(self, name, old, new):
90 if not os.path.isdir(new):
90 if not os.path.isdir(new):
91 os.makedirs(new)
91 os.makedirs(new)
92 self.security_dir = os.path.join(new, self.security_dir_name)
92 self.security_dir = os.path.join(new, self.security_dir_name)
93 self.log_dir = os.path.join(new, self.log_dir_name)
93 self.log_dir = os.path.join(new, self.log_dir_name)
94 self.pid_dir = os.path.join(new, self.pid_dir_name)
94 self.pid_dir = os.path.join(new, self.pid_dir_name)
95 self.check_dirs()
95 self.check_dirs()
96
96
97 def _log_dir_changed(self, name, old, new):
97 def _log_dir_changed(self, name, old, new):
98 self.check_log_dir()
98 self.check_log_dir()
99
99
100 def check_log_dir(self):
100 def check_log_dir(self):
101 if not os.path.isdir(self.log_dir):
101 if not os.path.isdir(self.log_dir):
102 os.mkdir(self.log_dir)
102 os.mkdir(self.log_dir)
103
103
104 def _security_dir_changed(self, name, old, new):
104 def _security_dir_changed(self, name, old, new):
105 self.check_security_dir()
105 self.check_security_dir()
106
106
107 def check_security_dir(self):
107 def check_security_dir(self):
108 if not os.path.isdir(self.security_dir):
108 if not os.path.isdir(self.security_dir):
109 os.mkdir(self.security_dir, 0700)
109 os.mkdir(self.security_dir, 0700)
110 os.chmod(self.security_dir, 0700)
110 os.chmod(self.security_dir, 0700)
111
111
112 def _pid_dir_changed(self, name, old, new):
112 def _pid_dir_changed(self, name, old, new):
113 self.check_pid_dir()
113 self.check_pid_dir()
114
114
115 def check_pid_dir(self):
115 def check_pid_dir(self):
116 if not os.path.isdir(self.pid_dir):
116 if not os.path.isdir(self.pid_dir):
117 os.mkdir(self.pid_dir, 0700)
117 os.mkdir(self.pid_dir, 0700)
118 os.chmod(self.pid_dir, 0700)
118 os.chmod(self.pid_dir, 0700)
119
119
120 def check_dirs(self):
120 def check_dirs(self):
121 self.check_security_dir()
121 self.check_security_dir()
122 self.check_log_dir()
122 self.check_log_dir()
123 self.check_pid_dir()
123 self.check_pid_dir()
124
124
125 def load_config_file(self, filename):
125 def load_config_file(self, filename):
126 """Load a config file from the top level of the cluster dir.
126 """Load a config file from the top level of the cluster dir.
127
127
128 Parameters
128 Parameters
129 ----------
129 ----------
130 filename : unicode or str
130 filename : unicode or str
131 The filename only of the config file that must be located in
131 The filename only of the config file that must be located in
132 the top-level of the cluster directory.
132 the top-level of the cluster directory.
133 """
133 """
134 loader = PyFileConfigLoader(filename, self.location)
134 loader = PyFileConfigLoader(filename, self.location)
135 return loader.load_config()
135 return loader.load_config()
136
136
137 def copy_config_file(self, config_file, path=None, overwrite=False):
137 def copy_config_file(self, config_file, path=None, overwrite=False):
138 """Copy a default config file into the active cluster directory.
138 """Copy a default config file into the active cluster directory.
139
139
140 Default configuration files are kept in :mod:`IPython.config.default`.
140 Default configuration files are kept in :mod:`IPython.config.default`.
141 This function moves these from that location to the working cluster
141 This function moves these from that location to the working cluster
142 directory.
142 directory.
143 """
143 """
144 if path is None:
144 if path is None:
145 import IPython.config.default
145 import IPython.config.default
146 path = IPython.config.default.__file__.split(os.path.sep)[:-1]
146 path = IPython.config.default.__file__.split(os.path.sep)[:-1]
147 path = os.path.sep.join(path)
147 path = os.path.sep.join(path)
148 src = os.path.join(path, config_file)
148 src = os.path.join(path, config_file)
149 dst = os.path.join(self.location, config_file)
149 dst = os.path.join(self.location, config_file)
150 if not os.path.isfile(dst) or overwrite:
150 if not os.path.isfile(dst) or overwrite:
151 shutil.copy(src, dst)
151 shutil.copy(src, dst)
152
152
153 def copy_all_config_files(self, path=None, overwrite=False):
153 def copy_all_config_files(self, path=None, overwrite=False):
154 """Copy all config files into the active cluster directory."""
154 """Copy all config files into the active cluster directory."""
155 for f in [u'ipcontroller_config.py', u'ipengine_config.py',
155 for f in [u'ipcontroller_config.py', u'ipengine_config.py',
156 u'ipcluster_config.py']:
156 u'ipcluster_config.py']:
157 self.copy_config_file(f, path=path, overwrite=overwrite)
157 self.copy_config_file(f, path=path, overwrite=overwrite)
158
158
159 @classmethod
159 @classmethod
160 def create_cluster_dir(csl, cluster_dir):
160 def create_cluster_dir(csl, cluster_dir):
161 """Create a new cluster directory given a full path.
161 """Create a new cluster directory given a full path.
162
162
163 Parameters
163 Parameters
164 ----------
164 ----------
165 cluster_dir : str
165 cluster_dir : str
166 The full path to the cluster directory. If it does exist, it will
166 The full path to the cluster directory. If it does exist, it will
167 be used. If not, it will be created.
167 be used. If not, it will be created.
168 """
168 """
169 return ClusterDir(cluster_dir)
169 return ClusterDir(cluster_dir)
170
170
171 @classmethod
171 @classmethod
172 def create_cluster_dir_by_profile(cls, path, profile=u'default'):
172 def create_cluster_dir_by_profile(cls, path, profile=u'default'):
173 """Create a cluster dir by profile name and path.
173 """Create a cluster dir by profile name and path.
174
174
175 Parameters
175 Parameters
176 ----------
176 ----------
177 path : str
177 path : str
178 The path (directory) to put the cluster directory in.
178 The path (directory) to put the cluster directory in.
179 profile : str
179 profile : str
180 The name of the profile. The name of the cluster directory will
180 The name of the profile. The name of the cluster directory will
181 be "cluster_<profile>".
181 be "cluster_<profile>".
182 """
182 """
183 if not os.path.isdir(path):
183 if not os.path.isdir(path):
184 raise ClusterDirError('Directory not found: %s' % path)
184 raise ClusterDirError('Directory not found: %s' % path)
185 cluster_dir = os.path.join(path, u'cluster_' + profile)
185 cluster_dir = os.path.join(path, u'cluster_' + profile)
186 return ClusterDir(cluster_dir)
186 return ClusterDir(cluster_dir)
187
187
188 @classmethod
188 @classmethod
189 def find_cluster_dir_by_profile(cls, ipython_dir, profile=u'default'):
189 def find_cluster_dir_by_profile(cls, ipython_dir, profile=u'default'):
190 """Find an existing cluster dir by profile name, return its ClusterDir.
190 """Find an existing cluster dir by profile name, return its ClusterDir.
191
191
192 This searches through a sequence of paths for a cluster dir. If it
192 This searches through a sequence of paths for a cluster dir. If it
193 is not found, a :class:`ClusterDirError` exception will be raised.
193 is not found, a :class:`ClusterDirError` exception will be raised.
194
194
195 The search path algorithm is:
195 The search path algorithm is:
196 1. ``os.getcwd()``
196 1. ``os.getcwd()``
197 2. ``ipython_dir``
197 2. ``ipython_dir``
198 3. The directories found in the ":" separated
198 3. The directories found in the ":" separated
199 :env:`IPCLUSTER_DIR_PATH` environment variable.
199 :env:`IPCLUSTER_DIR_PATH` environment variable.
200
200
201 Parameters
201 Parameters
202 ----------
202 ----------
203 ipython_dir : unicode or str
203 ipython_dir : unicode or str
204 The IPython directory to use.
204 The IPython directory to use.
205 profile : unicode or str
205 profile : unicode or str
206 The name of the profile. The name of the cluster directory
206 The name of the profile. The name of the cluster directory
207 will be "cluster_<profile>".
207 will be "cluster_<profile>".
208 """
208 """
209 dirname = u'cluster_' + profile
209 dirname = u'cluster_' + profile
210 cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
210 cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
211 if cluster_dir_paths:
211 if cluster_dir_paths:
212 cluster_dir_paths = cluster_dir_paths.split(':')
212 cluster_dir_paths = cluster_dir_paths.split(':')
213 else:
213 else:
214 cluster_dir_paths = []
214 cluster_dir_paths = []
215 paths = [os.getcwd(), ipython_dir] + cluster_dir_paths
215 paths = [os.getcwd(), ipython_dir] + cluster_dir_paths
216 for p in paths:
216 for p in paths:
217 cluster_dir = os.path.join(p, dirname)
217 cluster_dir = os.path.join(p, dirname)
218 if os.path.isdir(cluster_dir):
218 if os.path.isdir(cluster_dir):
219 return ClusterDir(cluster_dir)
219 return ClusterDir(cluster_dir)
220 else:
220 else:
221 raise ClusterDirError('Cluster directory not found in paths: %s' % dirname)
221 raise ClusterDirError('Cluster directory not found in paths: %s' % dirname)
222
222
223 @classmethod
223 @classmethod
224 def find_cluster_dir(cls, cluster_dir):
224 def find_cluster_dir(cls, cluster_dir):
225 """Find/create a cluster dir and return its ClusterDir.
225 """Find/create a cluster dir and return its ClusterDir.
226
226
227 This will create the cluster directory if it doesn't exist.
227 This will create the cluster directory if it doesn't exist.
228
228
229 Parameters
229 Parameters
230 ----------
230 ----------
231 cluster_dir : unicode or str
231 cluster_dir : unicode or str
232 The path of the cluster directory. This is expanded using
232 The path of the cluster directory. This is expanded using
233 :func:`IPython.utils.genutils.expand_path`.
233 :func:`IPython.utils.genutils.expand_path`.
234 """
234 """
235 cluster_dir = expand_path(cluster_dir)
235 cluster_dir = expand_path(cluster_dir)
236 if not os.path.isdir(cluster_dir):
236 if not os.path.isdir(cluster_dir):
237 raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
237 raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
238 return ClusterDir(cluster_dir)
238 return ClusterDir(cluster_dir)
239
239
240
240
241 #-----------------------------------------------------------------------------
241 #-----------------------------------------------------------------------------
242 # Command line options
242 # Command line options
243 #-----------------------------------------------------------------------------
243 #-----------------------------------------------------------------------------
244
244
245 class ClusterDirConfigLoader(BaseAppConfigLoader):
245 class ClusterDirConfigLoader(BaseAppConfigLoader):
246
246
247 def _add_cluster_profile(self, parser):
247 def _add_cluster_profile(self, parser):
248 paa = parser.add_argument
248 paa = parser.add_argument
249 paa('-p', '--profile',
249 paa('-p', '--profile',
250 dest='Global.profile',type=unicode,
250 dest='Global.profile',type=unicode,
251 help=
251 help=
252 """The string name of the profile to be used. This determines the name
252 """The string name of the profile to be used. This determines the name
253 of the cluster dir as: cluster_<profile>. The default profile is named
253 of the cluster dir as: cluster_<profile>. The default profile is named
254 'default'. The cluster directory is resolve this way if the
254 'default'. The cluster directory is resolve this way if the
255 --cluster-dir option is not used.""",
255 --cluster-dir option is not used.""",
256 metavar='Global.profile')
256 metavar='Global.profile')
257
257
258 def _add_cluster_dir(self, parser):
258 def _add_cluster_dir(self, parser):
259 paa = parser.add_argument
259 paa = parser.add_argument
260 paa('--cluster-dir',
260 paa('--cluster-dir',
261 dest='Global.cluster_dir',type=unicode,
261 dest='Global.cluster_dir',type=unicode,
262 help="""Set the cluster dir. This overrides the logic used by the
262 help="""Set the cluster dir. This overrides the logic used by the
263 --profile option.""",
263 --profile option.""",
264 metavar='Global.cluster_dir')
264 metavar='Global.cluster_dir')
265
265
266 def _add_work_dir(self, parser):
266 def _add_work_dir(self, parser):
267 paa = parser.add_argument
267 paa = parser.add_argument
268 paa('--work-dir',
268 paa('--work-dir',
269 dest='Global.work_dir',type=unicode,
269 dest='Global.work_dir',type=unicode,
270 help='Set the working dir for the process.',
270 help='Set the working dir for the process.',
271 metavar='Global.work_dir')
271 metavar='Global.work_dir')
272
272
273 def _add_clean_logs(self, parser):
273 def _add_clean_logs(self, parser):
274 paa = parser.add_argument
274 paa = parser.add_argument
275 paa('--clean-logs',
275 paa('--clean-logs',
276 dest='Global.clean_logs', action='store_true',
276 dest='Global.clean_logs', action='store_true',
277 help='Delete old log flies before starting.')
277 help='Delete old log flies before starting.')
278
278
279 def _add_no_clean_logs(self, parser):
279 def _add_no_clean_logs(self, parser):
280 paa = parser.add_argument
280 paa = parser.add_argument
281 paa('--no-clean-logs',
281 paa('--no-clean-logs',
282 dest='Global.clean_logs', action='store_false',
282 dest='Global.clean_logs', action='store_false',
283 help="Don't Delete old log flies before starting.")
283 help="Don't Delete old log flies before starting.")
284
284
285 def _add_arguments(self):
285 def _add_arguments(self):
286 super(ClusterDirConfigLoader, self)._add_arguments()
286 super(ClusterDirConfigLoader, self)._add_arguments()
287 self._add_cluster_profile(self.parser)
287 self._add_cluster_profile(self.parser)
288 self._add_cluster_dir(self.parser)
288 self._add_cluster_dir(self.parser)
289 self._add_work_dir(self.parser)
289 self._add_work_dir(self.parser)
290 self._add_clean_logs(self.parser)
290 self._add_clean_logs(self.parser)
291 self._add_no_clean_logs(self.parser)
291 self._add_no_clean_logs(self.parser)
292
292
293
293
294 #-----------------------------------------------------------------------------
294 #-----------------------------------------------------------------------------
295 # Crash handler for this application
295 # Crash handler for this application
296 #-----------------------------------------------------------------------------
296 #-----------------------------------------------------------------------------
297
297
298
298
299 _message_template = """\
299 _message_template = """\
300 Oops, $self.app_name crashed. We do our best to make it stable, but...
300 Oops, $self.app_name crashed. We do our best to make it stable, but...
301
301
302 A crash report was automatically generated with the following information:
302 A crash report was automatically generated with the following information:
303 - A verbatim copy of the crash traceback.
303 - A verbatim copy of the crash traceback.
304 - Data on your current $self.app_name configuration.
304 - Data on your current $self.app_name configuration.
305
305
306 It was left in the file named:
306 It was left in the file named:
307 \t'$self.crash_report_fname'
307 \t'$self.crash_report_fname'
308 If you can email this file to the developers, the information in it will help
308 If you can email this file to the developers, the information in it will help
309 them in understanding and correcting the problem.
309 them in understanding and correcting the problem.
310
310
311 You can mail it to: $self.contact_name at $self.contact_email
311 You can mail it to: $self.contact_name at $self.contact_email
312 with the subject '$self.app_name Crash Report'.
312 with the subject '$self.app_name Crash Report'.
313
313
314 If you want to do it now, the following command will work (under Unix):
314 If you want to do it now, the following command will work (under Unix):
315 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
315 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
316
316
317 To ensure accurate tracking of this issue, please file a report about it at:
317 To ensure accurate tracking of this issue, please file a report about it at:
318 $self.bug_tracker
318 $self.bug_tracker
319 """
319 """
320
320
321 class ClusterDirCrashHandler(CrashHandler):
321 class ClusterDirCrashHandler(CrashHandler):
322 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
322 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
323
323
324 message_template = _message_template
324 message_template = _message_template
325
325
326 def __init__(self, app):
326 def __init__(self, app):
327 contact_name = release.authors['Brian'][0]
327 contact_name = release.authors['Brian'][0]
328 contact_email = release.authors['Brian'][1]
328 contact_email = release.authors['Brian'][1]
329 bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug'
329 bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug'
330 super(ClusterDirCrashHandler,self).__init__(
330 super(ClusterDirCrashHandler,self).__init__(
331 app, contact_name, contact_email, bug_tracker
331 app, contact_name, contact_email, bug_tracker
332 )
332 )
333
333
334
334
335 #-----------------------------------------------------------------------------
335 #-----------------------------------------------------------------------------
336 # Main application
336 # Main application
337 #-----------------------------------------------------------------------------
337 #-----------------------------------------------------------------------------
338
338
339 class ApplicationWithClusterDir(Application):
339 class ApplicationWithClusterDir(Application):
340 """An application that puts everything into a cluster directory.
340 """An application that puts everything into a cluster directory.
341
341
342 Instead of looking for things in the ipython_dir, this type of application
342 Instead of looking for things in the ipython_dir, this type of application
343 will use its own private directory called the "cluster directory"
343 will use its own private directory called the "cluster directory"
344 for things like config files, log files, etc.
344 for things like config files, log files, etc.
345
345
346 The cluster directory is resolved as follows:
346 The cluster directory is resolved as follows:
347
347
348 * If the ``--cluster-dir`` option is given, it is used.
348 * If the ``--cluster-dir`` option is given, it is used.
349 * If ``--cluster-dir`` is not given, the application directory is
349 * If ``--cluster-dir`` is not given, the application directory is
350 resolve using the profile name as ``cluster_<profile>``. The search
350 resolve using the profile name as ``cluster_<profile>``. The search
351 path for this directory is then i) cwd if it is found there
351 path for this directory is then i) cwd if it is found there
352 and ii) in ipython_dir otherwise.
352 and ii) in ipython_dir otherwise.
353
353
354 The config file for the application is to be put in the cluster
354 The config file for the application is to be put in the cluster
355 dir and named the value of the ``config_file_name`` class attribute.
355 dir and named the value of the ``config_file_name`` class attribute.
356 """
356 """
357
357
358 command_line_loader = ClusterDirConfigLoader
358 command_line_loader = ClusterDirConfigLoader
359 crash_handler_class = ClusterDirCrashHandler
359 crash_handler_class = ClusterDirCrashHandler
360 auto_create_cluster_dir = True
360 auto_create_cluster_dir = True
361
361
362 def create_default_config(self):
362 def create_default_config(self):
363 super(ApplicationWithClusterDir, self).create_default_config()
363 super(ApplicationWithClusterDir, self).create_default_config()
364 self.default_config.Global.profile = u'default'
364 self.default_config.Global.profile = u'default'
365 self.default_config.Global.cluster_dir = u''
365 self.default_config.Global.cluster_dir = u''
366 self.default_config.Global.work_dir = os.getcwd()
366 self.default_config.Global.work_dir = os.getcwd()
367 self.default_config.Global.log_to_file = False
367 self.default_config.Global.log_to_file = False
368 self.default_config.Global.clean_logs = False
368 self.default_config.Global.clean_logs = False
369
369
370 def find_resources(self):
370 def find_resources(self):
371 """This resolves the cluster directory.
371 """This resolves the cluster directory.
372
372
373 This tries to find the cluster directory and if successful, it will
373 This tries to find the cluster directory and if successful, it will
374 have done:
374 have done:
375 * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for
375 * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for
376 the application.
376 the application.
377 * Sets ``self.cluster_dir`` attribute of the application and config
377 * Sets ``self.cluster_dir`` attribute of the application and config
378 objects.
378 objects.
379
379
380 The algorithm used for this is as follows:
380 The algorithm used for this is as follows:
381 1. Try ``Global.cluster_dir``.
381 1. Try ``Global.cluster_dir``.
382 2. Try using ``Global.profile``.
382 2. Try using ``Global.profile``.
383 3. If both of these fail and ``self.auto_create_cluster_dir`` is
383 3. If both of these fail and ``self.auto_create_cluster_dir`` is
384 ``True``, then create the new cluster dir in the IPython directory.
384 ``True``, then create the new cluster dir in the IPython directory.
385 4. If all fails, then raise :class:`ClusterDirError`.
385 4. If all fails, then raise :class:`ClusterDirError`.
386 """
386 """
387
387
388 try:
388 try:
389 cluster_dir = self.command_line_config.Global.cluster_dir
389 cluster_dir = self.command_line_config.Global.cluster_dir
390 except AttributeError:
390 except AttributeError:
391 cluster_dir = self.default_config.Global.cluster_dir
391 cluster_dir = self.default_config.Global.cluster_dir
392 cluster_dir = expand_path(cluster_dir)
392 cluster_dir = expand_path(cluster_dir)
393 try:
393 try:
394 self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
394 self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
395 except ClusterDirError:
395 except ClusterDirError:
396 pass
396 pass
397 else:
397 else:
398 self.log.info('Using existing cluster dir: %s' % \
398 self.log.info('Using existing cluster dir: %s' % \
399 self.cluster_dir_obj.location
399 self.cluster_dir_obj.location
400 )
400 )
401 self.finish_cluster_dir()
401 self.finish_cluster_dir()
402 return
402 return
403
403
404 try:
404 try:
405 self.profile = self.command_line_config.Global.profile
405 self.profile = self.command_line_config.Global.profile
406 except AttributeError:
406 except AttributeError:
407 self.profile = self.default_config.Global.profile
407 self.profile = self.default_config.Global.profile
408 try:
408 try:
409 self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
409 self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
410 self.ipython_dir, self.profile)
410 self.ipython_dir, self.profile)
411 except ClusterDirError:
411 except ClusterDirError:
412 pass
412 pass
413 else:
413 else:
414 self.log.info('Using existing cluster dir: %s' % \
414 self.log.info('Using existing cluster dir: %s' % \
415 self.cluster_dir_obj.location
415 self.cluster_dir_obj.location
416 )
416 )
417 self.finish_cluster_dir()
417 self.finish_cluster_dir()
418 return
418 return
419
419
420 if self.auto_create_cluster_dir:
420 if self.auto_create_cluster_dir:
421 self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
421 self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
422 self.ipython_dir, self.profile
422 self.ipython_dir, self.profile
423 )
423 )
424 self.log.info('Creating new cluster dir: %s' % \
424 self.log.info('Creating new cluster dir: %s' % \
425 self.cluster_dir_obj.location
425 self.cluster_dir_obj.location
426 )
426 )
427 self.finish_cluster_dir()
427 self.finish_cluster_dir()
428 else:
428 else:
429 raise ClusterDirError('Could not find a valid cluster directory.')
429 raise ClusterDirError('Could not find a valid cluster directory.')
430
430
431 def finish_cluster_dir(self):
431 def finish_cluster_dir(self):
432 # Set the cluster directory
432 # Set the cluster directory
433 self.cluster_dir = self.cluster_dir_obj.location
433 self.cluster_dir = self.cluster_dir_obj.location
434
434
435 # These have to be set because they could be different from the one
435 # These have to be set because they could be different from the one
436 # that we just computed. Because command line has the highest
436 # that we just computed. Because command line has the highest
437 # priority, this will always end up in the master_config.
437 # priority, this will always end up in the master_config.
438 self.default_config.Global.cluster_dir = self.cluster_dir
438 self.default_config.Global.cluster_dir = self.cluster_dir
439 self.command_line_config.Global.cluster_dir = self.cluster_dir
439 self.command_line_config.Global.cluster_dir = self.cluster_dir
440
440
441 def find_config_file_name(self):
441 def find_config_file_name(self):
442 """Find the config file name for this application."""
442 """Find the config file name for this application."""
443 # For this type of Application it should be set as a class attribute.
443 # For this type of Application it should be set as a class attribute.
444 if not hasattr(self, 'config_file_name'):
444 if not hasattr(self, 'default_config_file_name'):
445 self.log.critical("No config filename found")
445 self.log.critical("No config filename found")
446 else:
447 self.config_file_name = self.default_config_file_name
446
448
447 def find_config_file_paths(self):
449 def find_config_file_paths(self):
448 # Set the search path to to the cluster directory. We should NOT
450 # Set the search path to to the cluster directory. We should NOT
449 # include IPython.config.default here as the default config files
451 # include IPython.config.default here as the default config files
450 # are ALWAYS automatically moved to the cluster directory.
452 # are ALWAYS automatically moved to the cluster directory.
451 conf_dir = os.path.join(get_ipython_package_dir(), 'config', 'default')
453 conf_dir = os.path.join(get_ipython_package_dir(), 'config', 'default')
452 self.config_file_paths = (self.cluster_dir,)
454 self.config_file_paths = (self.cluster_dir,)
453
455
454 def pre_construct(self):
456 def pre_construct(self):
455 # The log and security dirs were set earlier, but here we put them
457 # The log and security dirs were set earlier, but here we put them
456 # into the config and log them.
458 # into the config and log them.
457 config = self.master_config
459 config = self.master_config
458 sdir = self.cluster_dir_obj.security_dir
460 sdir = self.cluster_dir_obj.security_dir
459 self.security_dir = config.Global.security_dir = sdir
461 self.security_dir = config.Global.security_dir = sdir
460 ldir = self.cluster_dir_obj.log_dir
462 ldir = self.cluster_dir_obj.log_dir
461 self.log_dir = config.Global.log_dir = ldir
463 self.log_dir = config.Global.log_dir = ldir
462 pdir = self.cluster_dir_obj.pid_dir
464 pdir = self.cluster_dir_obj.pid_dir
463 self.pid_dir = config.Global.pid_dir = pdir
465 self.pid_dir = config.Global.pid_dir = pdir
464 self.log.info("Cluster directory set to: %s" % self.cluster_dir)
466 self.log.info("Cluster directory set to: %s" % self.cluster_dir)
465 config.Global.work_dir = unicode(expand_path(config.Global.work_dir))
467 config.Global.work_dir = unicode(expand_path(config.Global.work_dir))
466 # Change to the working directory. We do this just before construct
468 # Change to the working directory. We do this just before construct
467 # is called so all the components there have the right working dir.
469 # is called so all the components there have the right working dir.
468 self.to_work_dir()
470 self.to_work_dir()
469
471
470 def to_work_dir(self):
472 def to_work_dir(self):
471 wd = self.master_config.Global.work_dir
473 wd = self.master_config.Global.work_dir
472 if unicode(wd) != unicode(os.getcwd()):
474 if unicode(wd) != unicode(os.getcwd()):
473 os.chdir(wd)
475 os.chdir(wd)
474 self.log.info("Changing to working dir: %s" % wd)
476 self.log.info("Changing to working dir: %s" % wd)
475
477
476 def start_logging(self):
478 def start_logging(self):
477 # Remove old log files
479 # Remove old log files
478 if self.master_config.Global.clean_logs:
480 if self.master_config.Global.clean_logs:
479 log_dir = self.master_config.Global.log_dir
481 log_dir = self.master_config.Global.log_dir
480 for f in os.listdir(log_dir):
482 for f in os.listdir(log_dir):
481 if f.startswith(self.name + u'-') and f.endswith('.log'):
483 if f.startswith(self.name + u'-') and f.endswith('.log'):
482 os.remove(os.path.join(log_dir, f))
484 os.remove(os.path.join(log_dir, f))
483 # Start logging to the new log file
485 # Start logging to the new log file
484 if self.master_config.Global.log_to_file:
486 if self.master_config.Global.log_to_file:
485 log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
487 log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
486 logfile = os.path.join(self.log_dir, log_filename)
488 logfile = os.path.join(self.log_dir, log_filename)
487 open_log_file = open(logfile, 'w')
489 open_log_file = open(logfile, 'w')
488 else:
490 else:
489 open_log_file = sys.stdout
491 open_log_file = sys.stdout
490 log.startLogging(open_log_file)
492 log.startLogging(open_log_file)
491
493
492 def write_pid_file(self, overwrite=False):
494 def write_pid_file(self, overwrite=False):
493 """Create a .pid file in the pid_dir with my pid.
495 """Create a .pid file in the pid_dir with my pid.
494
496
495 This must be called after pre_construct, which sets `self.pid_dir`.
497 This must be called after pre_construct, which sets `self.pid_dir`.
496 This raises :exc:`PIDFileError` if the pid file exists already.
498 This raises :exc:`PIDFileError` if the pid file exists already.
497 """
499 """
498 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
500 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
499 if os.path.isfile(pid_file):
501 if os.path.isfile(pid_file):
500 pid = self.get_pid_from_file()
502 pid = self.get_pid_from_file()
501 if not overwrite:
503 if not overwrite:
502 raise PIDFileError(
504 raise PIDFileError(
503 'The pid file [%s] already exists. \nThis could mean that this '
505 'The pid file [%s] already exists. \nThis could mean that this '
504 'server is already running with [pid=%s].' % (pid_file, pid)
506 'server is already running with [pid=%s].' % (pid_file, pid)
505 )
507 )
506 with open(pid_file, 'w') as f:
508 with open(pid_file, 'w') as f:
507 self.log.info("Creating pid file: %s" % pid_file)
509 self.log.info("Creating pid file: %s" % pid_file)
508 f.write(repr(os.getpid())+'\n')
510 f.write(repr(os.getpid())+'\n')
509
511
510 def remove_pid_file(self):
512 def remove_pid_file(self):
511 """Remove the pid file.
513 """Remove the pid file.
512
514
513 This should be called at shutdown by registering a callback with
515 This should be called at shutdown by registering a callback with
514 :func:`reactor.addSystemEventTrigger`. This needs to return
516 :func:`reactor.addSystemEventTrigger`. This needs to return
515 ``None``.
517 ``None``.
516 """
518 """
517 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
519 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
518 if os.path.isfile(pid_file):
520 if os.path.isfile(pid_file):
519 try:
521 try:
520 self.log.info("Removing pid file: %s" % pid_file)
522 self.log.info("Removing pid file: %s" % pid_file)
521 os.remove(pid_file)
523 os.remove(pid_file)
522 except:
524 except:
523 self.log.warn("Error removing the pid file: %s" % pid_file)
525 self.log.warn("Error removing the pid file: %s" % pid_file)
524
526
525 def get_pid_from_file(self):
527 def get_pid_from_file(self):
526 """Get the pid from the pid file.
528 """Get the pid from the pid file.
527
529
528 If the pid file doesn't exist a :exc:`PIDFileError` is raised.
530 If the pid file doesn't exist a :exc:`PIDFileError` is raised.
529 """
531 """
530 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
532 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
531 if os.path.isfile(pid_file):
533 if os.path.isfile(pid_file):
532 with open(pid_file, 'r') as f:
534 with open(pid_file, 'r') as f:
533 pid = int(f.read().strip())
535 pid = int(f.read().strip())
534 return pid
536 return pid
535 else:
537 else:
536 raise PIDFileError('pid file not found: %s' % pid_file)
538 raise PIDFileError('pid file not found: %s' % pid_file)
537
539
@@ -1,273 +1,277 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 Foolscap related utilities.
4 Foolscap related utilities.
5 """
5 """
6
6
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Copyright (C) 2008-2009 The IPython Development Team
8 # Copyright (C) 2008-2009 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 from __future__ import with_statement
18 from __future__ import with_statement
19
19
20 import os
20 import os
21 import tempfile
21 import tempfile
22
22
23 from twisted.internet import reactor, defer
23 from twisted.internet import reactor, defer
24 from twisted.python import log
24 from twisted.python import log
25
25
26 from foolscap import Tub, UnauthenticatedTub
26 from foolscap import Tub, UnauthenticatedTub
27
27
28 from IPython.config.loader import Config
28 from IPython.config.loader import Config
29
29
30 from IPython.kernel.configobjfactory import AdaptedConfiguredObjectFactory
30 from IPython.kernel.configobjfactory import AdaptedConfiguredObjectFactory
31
31
32 from IPython.kernel.error import SecurityError
32 from IPython.kernel.error import SecurityError
33
33
34 from IPython.utils.traitlets import Int, Str, Bool, Instance
34 from IPython.utils.traitlets import Int, Str, Bool, Instance
35 from IPython.utils.importstring import import_item
35 from IPython.utils.importstring import import_item
36
36
37 #-----------------------------------------------------------------------------
37 #-----------------------------------------------------------------------------
38 # Code
38 # Code
39 #-----------------------------------------------------------------------------
39 #-----------------------------------------------------------------------------
40
40
41
41
42 # We do this so if a user doesn't have OpenSSL installed, it will try to use
42 # We do this so if a user doesn't have OpenSSL installed, it will try to use
43 # an UnauthenticatedTub. But, they will still run into problems if they
43 # an UnauthenticatedTub. But, they will still run into problems if they
44 # try to use encrypted furls.
44 # try to use encrypted furls.
45 try:
45 try:
46 import OpenSSL
46 import OpenSSL
47 except:
47 except:
48 Tub = UnauthenticatedTub
48 Tub = UnauthenticatedTub
49 have_crypto = False
49 have_crypto = False
50 else:
50 else:
51 have_crypto = True
51 have_crypto = True
52
52
53
53
54 class FURLError(Exception):
54 class FURLError(Exception):
55 pass
55 pass
56
56
57
57
58 def check_furl_file_security(furl_file, secure):
58 def check_furl_file_security(furl_file, secure):
59 """Remove the old furl_file if changing security modes."""
59 """Remove the old furl_file if changing security modes."""
60 if os.path.isfile(furl_file):
60 if os.path.isfile(furl_file):
61 f = open(furl_file, 'r')
61 f = open(furl_file, 'r')
62 oldfurl = f.read().strip()
62 oldfurl = f.read().strip()
63 f.close()
63 f.close()
64 if (oldfurl.startswith('pb://') and not secure) or (oldfurl.startswith('pbu://') and secure):
64 if (oldfurl.startswith('pb://') and not secure) or (oldfurl.startswith('pbu://') and secure):
65 os.remove(furl_file)
65 os.remove(furl_file)
66
66
67
67
68 def is_secure(furl):
68 def is_secure(furl):
69 """Is the given FURL secure or not."""
69 """Is the given FURL secure or not."""
70 if is_valid(furl):
70 if is_valid(furl):
71 if furl.startswith("pb://"):
71 if furl.startswith("pb://"):
72 return True
72 return True
73 elif furl.startswith("pbu://"):
73 elif furl.startswith("pbu://"):
74 return False
74 return False
75 else:
75 else:
76 raise FURLError("invalid FURL: %s" % furl)
76 raise FURLError("invalid FURL: %s" % furl)
77
77
78
78
79 def is_valid(furl):
79 def is_valid(furl):
80 """Is the str a valid FURL or not."""
80 """Is the str a valid FURL or not."""
81 if isinstance(furl, str):
81 if isinstance(furl, str):
82 if furl.startswith("pb://") or furl.startswith("pbu://"):
82 if furl.startswith("pb://") or furl.startswith("pbu://"):
83 return True
83 return True
84 else:
84 else:
85 return False
85 return False
86
86
87
87
88 def find_furl(furl_or_file):
88 def find_furl(furl_or_file):
89 """Find, validate and return a FURL in a string or file."""
89 """Find, validate and return a FURL in a string or file."""
90 if isinstance(furl_or_file, str):
90 if isinstance(furl_or_file, str):
91 if is_valid(furl_or_file):
91 if is_valid(furl_or_file):
92 return furl_or_file
92 return furl_or_file
93 if os.path.isfile(furl_or_file):
93 if os.path.isfile(furl_or_file):
94 with open(furl_or_file, 'r') as f:
94 with open(furl_or_file, 'r') as f:
95 furl = f.read().strip()
95 furl = f.read().strip()
96 if is_valid(furl):
96 if is_valid(furl):
97 return furl
97 return furl
98 raise FURLError("Not a valid FURL or FURL file: %s" % furl_or_file)
98 raise FURLError("Not a valid FURL or FURL file: %s" % furl_or_file)
99
99
100
100
101 def is_valid_furl_or_file(furl_or_file):
101 def is_valid_furl_or_file(furl_or_file):
102 """Validate a FURL or a FURL file.
102 """Validate a FURL or a FURL file.
103
103
104 If ``furl_or_file`` looks like a file, we simply make sure its directory
104 If ``furl_or_file`` looks like a file, we simply make sure its directory
105 exists and that it has a ``.furl`` file extension. We don't try to see
105 exists and that it has a ``.furl`` file extension. We don't try to see
106 if the FURL file exists or to read its contents. This is useful for
106 if the FURL file exists or to read its contents. This is useful for
107 cases where auto re-connection is being used.
107 cases where auto re-connection is being used.
108 """
108 """
109 if isinstance(furl_or_file, str):
109 if isinstance(furl_or_file, str):
110 if is_valid(furl_or_file):
110 if is_valid(furl_or_file):
111 return True
111 return True
112 if isinstance(furl_or_file, (str, unicode)):
112 if isinstance(furl_or_file, (str, unicode)):
113 path, furl_filename = os.path.split(furl_or_file)
113 path, furl_filename = os.path.split(furl_or_file)
114 if os.path.isdir(path) and furl_filename.endswith('.furl'):
114 if os.path.isdir(path) and furl_filename.endswith('.furl'):
115 return True
115 return True
116 return False
116 return False
117
117
118
118
119 def validate_furl_or_file(furl_or_file):
119 def validate_furl_or_file(furl_or_file):
120 if not is_valid_furl_or_file(furl_or_file):
120 if not is_valid_furl_or_file(furl_or_file):
121 raise FURLError('Not a valid FURL or FURL file: %r' % furl_or_file)
121 raise FURLError('Not a valid FURL or FURL file: %r' % furl_or_file)
122
122
123
123
124 def get_temp_furlfile(filename):
124 def get_temp_furlfile(filename):
125 """Return a temporary FURL file."""
125 """Return a temporary FURL file."""
126 return tempfile.mktemp(dir=os.path.dirname(filename),
126 return tempfile.mktemp(dir=os.path.dirname(filename),
127 prefix=os.path.basename(filename))
127 prefix=os.path.basename(filename))
128
128
129
129
130 def make_tub(ip, port, secure, cert_file):
130 def make_tub(ip, port, secure, cert_file):
131 """Create a listening tub given an ip, port, and cert_file location.
131 """Create a listening tub given an ip, port, and cert_file location.
132
132
133 Parameters
133 Parameters
134 ----------
134 ----------
135 ip : str
135 ip : str
136 The ip address or hostname that the tub should listen on.
136 The ip address or hostname that the tub should listen on.
137 Empty means all interfaces.
137 Empty means all interfaces.
138 port : int
138 port : int
139 The port that the tub should listen on. A value of 0 means
139 The port that the tub should listen on. A value of 0 means
140 pick a random port
140 pick a random port
141 secure: bool
141 secure: bool
142 Will the connection be secure (in the Foolscap sense).
142 Will the connection be secure (in the Foolscap sense).
143 cert_file: str
143 cert_file: str
144 A filename of a file to be used for theSSL certificate.
144 A filename of a file to be used for theSSL certificate.
145
145
146 Returns
146 Returns
147 -------
147 -------
148 A tub, listener tuple.
148 A tub, listener tuple.
149 """
149 """
150 if secure:
150 if secure:
151 if have_crypto:
151 if have_crypto:
152 tub = Tub(certFile=cert_file)
152 tub = Tub(certFile=cert_file)
153 else:
153 else:
154 raise SecurityError("OpenSSL/pyOpenSSL is not available, so we "
154 raise SecurityError("OpenSSL/pyOpenSSL is not available, so we "
155 "can't run in secure mode. Try running without "
155 "can't run in secure mode. Try running without "
156 "security using 'ipcontroller -xy'.")
156 "security using 'ipcontroller -xy'.")
157 else:
157 else:
158 tub = UnauthenticatedTub()
158 tub = UnauthenticatedTub()
159
159
160 # Set the strport based on the ip and port and start listening
160 # Set the strport based on the ip and port and start listening
161 if ip == '':
161 if ip == '':
162 strport = "tcp:%i" % port
162 strport = "tcp:%i" % port
163 else:
163 else:
164 strport = "tcp:%i:interface=%s" % (port, ip)
164 strport = "tcp:%i:interface=%s" % (port, ip)
165 log.msg("Starting listener with [secure=%r] on: %s" % (secure, strport))
165 log.msg("Starting listener with [secure=%r] on: %s" % (secure, strport))
166 listener = tub.listenOn(strport)
166 listener = tub.listenOn(strport)
167
167
168 return tub, listener
168 return tub, listener
169
169
170
170
171 class FCServiceFactory(AdaptedConfiguredObjectFactory):
171 class FCServiceFactory(AdaptedConfiguredObjectFactory):
172 """This class creates a tub with various services running in it.
172 """This class creates a tub with various services running in it.
173
173
174 The basic idea is that :meth:`create` returns a running :class:`Tub`
174 The basic idea is that :meth:`create` returns a running :class:`Tub`
175 instance that has a number of Foolscap references registered in it.
175 instance that has a number of Foolscap references registered in it.
176 This class is a subclass of :class:`IPython.core.component.Component`
176 This class is a subclass of :class:`IPython.core.component.Component`
177 so the IPython configuration and component system are used.
177 so the IPython configuration and component system are used.
178
178
179 Attributes
179 Attributes
180 ----------
180 ----------
181 interfaces : Config
181 interfaces : Config
182 A Config instance whose values are sub-Config objects having two
182 A Config instance whose values are sub-Config objects having two
183 keys: furl_file and interface_chain.
183 keys: furl_file and interface_chain.
184
184
185 The other attributes are the standard ones for Foolscap.
185 The other attributes are the standard ones for Foolscap.
186 """
186 """
187
187
188 ip = Str('', config=True)
188 ip = Str('', config=True)
189 port = Int(0, config=True)
189 port = Int(0, config=True)
190 secure = Bool(True, config=True)
190 secure = Bool(True, config=True)
191 cert_file = Str('', config=True)
191 cert_file = Str('', config=True)
192 location = Str('', config=True)
192 location = Str('', config=True)
193 reuse_furls = Bool(False, config=True)
193 reuse_furls = Bool(False, config=True)
194 interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True)
194 interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True)
195
195
196 def __init__(self, config, adaptee):
196 def __init__(self, config, adaptee):
197 super(FCServiceFactory, self).__init__(config, adaptee)
197 super(FCServiceFactory, self).__init__(config, adaptee)
198 self._check_reuse_furls()
198 self._check_reuse_furls()
199
199
200 def _ip_changed(self, name, old, new):
200 def _ip_changed(self, name, old, new):
201 if new == 'localhost' or new == '127.0.0.1':
201 if new == 'localhost' or new == '127.0.0.1':
202 self.location = '127.0.0.1'
202 self.location = '127.0.0.1'
203
203
204 def _check_reuse_furls(self):
204 def _check_reuse_furls(self):
205 furl_files = [i.furl_file for i in self.interfaces.values()]
205 furl_files = [i.furl_file for i in self.interfaces.values()]
206 for ff in furl_files:
206 for ff in furl_files:
207 fullfile = self._get_security_file(ff)
207 fullfile = self._get_security_file(ff)
208 if self.reuse_furls:
208 if self.reuse_furls:
209 if self.port==0:
209 if self.port==0:
210 raise FURLError("You are trying to reuse the FURL file "
210 raise FURLError("You are trying to reuse the FURL file "
211 "for this connection, but the port for this connection "
211 "for this connection, but the port for this connection "
212 "is set to 0 (autoselect). To reuse the FURL file "
212 "is set to 0 (autoselect). To reuse the FURL file "
213 "you need to specify specific port to listen on."
213 "you need to specify specific port to listen on."
214 )
214 )
215 else:
215 else:
216 log.msg("Reusing FURL file: %s" % fullfile)
216 log.msg("Reusing FURL file: %s" % fullfile)
217 else:
217 else:
218 if os.path.isfile(fullfile):
218 if os.path.isfile(fullfile):
219 log.msg("Removing old FURL file: %s" % fullfile)
219 log.msg("Removing old FURL file: %s" % fullfile)
220 os.remove(fullfile)
220 os.remove(fullfile)
221
221
222 def _get_security_file(self, filename):
222 def _get_security_file(self, filename):
223 return os.path.join(self.config.Global.security_dir, filename)
223 return os.path.join(self.config.Global.security_dir, filename)
224
224
225 def create(self):
225 def create(self):
226 """Create and return the Foolscap tub with everything running."""
226 """Create and return the Foolscap tub with everything running."""
227
227
228 self.tub, self.listener = make_tub(
228 self.tub, self.listener = make_tub(
229 self.ip, self.port, self.secure,
229 self.ip, self.port, self.secure,
230 self._get_security_file(self.cert_file)
230 self._get_security_file(self.cert_file)
231 )
231 )
232 # log.msg("Interfaces to register [%r]: %r" % \
232 # log.msg("Interfaces to register [%r]: %r" % \
233 # (self.__class__, self.interfaces))
233 # (self.__class__, self.interfaces))
234 if not self.secure:
234 if not self.secure:
235 log.msg("WARNING: running with no security: %s" % \
235 log.msg("WARNING: running with no security: %s" % \
236 self.__class__.__name__)
236 self.__class__.__name__)
237 reactor.callWhenRunning(self.set_location_and_register)
237 reactor.callWhenRunning(self.set_location_and_register)
238 return self.tub
238 return self.tub
239
239
240 def set_location_and_register(self):
240 def set_location_and_register(self):
241 """Set the location for the tub and return a deferred."""
241 """Set the location for the tub and return a deferred."""
242
242
243 if self.location == '':
243 if self.location == '':
244 d = self.tub.setLocationAutomatically()
244 d = self.tub.setLocationAutomatically()
245 else:
245 else:
246 d = defer.maybeDeferred(self.tub.setLocation,
246 d = defer.maybeDeferred(self.tub.setLocation,
247 "%s:%i" % (self.location, self.listener.getPortnum()))
247 "%s:%i" % (self.location, self.listener.getPortnum()))
248 self.adapt_to_interfaces(d)
248 self.adapt_to_interfaces(d)
249
249
250 def adapt_to_interfaces(self, d):
250 def adapt_to_interfaces(self, d):
251 """Run through the interfaces, adapt and register."""
251 """Run through the interfaces, adapt and register."""
252
252
253 for ifname, ifconfig in self.interfaces.iteritems():
253 for ifname, ifconfig in self.interfaces.iteritems():
254 ff = self._get_security_file(ifconfig.furl_file)
254 ff = self._get_security_file(ifconfig.furl_file)
255 log.msg("Adapting [%s] to interface: %s" % \
255 log.msg("Adapting [%s] to interface: %s" % \
256 (self.adaptee.__class__.__name__, ifname))
256 (self.adaptee.__class__.__name__, ifname))
257 log.msg("Saving FURL for interface [%s] to file: %s" % (ifname, ff))
257 log.msg("Saving FURL for interface [%s] to file: %s" % (ifname, ff))
258 check_furl_file_security(ff, self.secure)
258 check_furl_file_security(ff, self.secure)
259 adaptee = self.adaptee
259 adaptee = self.adaptee
260 for i in ifconfig.interface_chain:
260 for i in ifconfig.interface_chain:
261 adaptee = import_item(i)(adaptee)
261 adaptee = import_item(i)(adaptee)
262 d.addCallback(self.register, adaptee, furl_file=ff)
262 d.addCallback(self.register, adaptee, furl_file=ff)
263
263
264 def register(self, empty, ref, furl_file):
264 def register(self, empty, ref, furl_file):
265 """Register the reference with the FURL file.
265 """Register the reference with the FURL file.
266
266
267 The FURL file is created and then moved to make sure that when the
267 The FURL file is created and then moved to make sure that when the
268 file appears, the buffer has been flushed and the file closed.
268 file appears, the buffer has been flushed and the file closed. This
269 is not done if we are re-using FURLS however.
269 """
270 """
270 temp_furl_file = get_temp_furlfile(furl_file)
271 if self.reuse_furls:
271 self.tub.registerReference(ref, furlFile=temp_furl_file)
272 self.tub.registerReference(ref, furlFile=furl_file)
272 os.rename(temp_furl_file, furl_file)
273 else:
274 temp_furl_file = get_temp_furlfile(furl_file)
275 self.tub.registerReference(ref, furlFile=temp_furl_file)
276 os.rename(temp_furl_file, furl_file)
273
277
@@ -1,499 +1,499 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The ipcluster application.
4 The ipcluster application.
5 """
5 """
6
6
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Copyright (C) 2008-2009 The IPython Development Team
8 # Copyright (C) 2008-2009 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 import logging
18 import logging
19 import os
19 import os
20 import signal
20 import signal
21
21
22 if os.name=='posix':
22 if os.name=='posix':
23 from twisted.scripts._twistd_unix import daemonize
23 from twisted.scripts._twistd_unix import daemonize
24
24
25 from twisted.internet import reactor, defer
25 from twisted.internet import reactor, defer
26 from twisted.python import log, failure
26 from twisted.python import log, failure
27
27
28
28
29 from IPython.external.argparse import ArgumentParser, SUPPRESS
29 from IPython.external.argparse import ArgumentParser, SUPPRESS
30 from IPython.utils.importstring import import_item
30 from IPython.utils.importstring import import_item
31 from IPython.kernel.clusterdir import (
31 from IPython.kernel.clusterdir import (
32 ApplicationWithClusterDir, ClusterDirConfigLoader,
32 ApplicationWithClusterDir, ClusterDirConfigLoader,
33 ClusterDirError, PIDFileError
33 ClusterDirError, PIDFileError
34 )
34 )
35
35
36
36
37 #-----------------------------------------------------------------------------
37 #-----------------------------------------------------------------------------
38 # Module level variables
38 # Module level variables
39 #-----------------------------------------------------------------------------
39 #-----------------------------------------------------------------------------
40
40
41
41
42 default_config_file_name = u'ipcluster_config.py'
42 default_config_file_name = u'ipcluster_config.py'
43
43
44
44
45 _description = """\
45 _description = """\
46 Start an IPython cluster for parallel computing.\n\n
46 Start an IPython cluster for parallel computing.\n\n
47
47
48 An IPython cluster consists of 1 controller and 1 or more engines.
48 An IPython cluster consists of 1 controller and 1 or more engines.
49 This command automates the startup of these processes using a wide
49 This command automates the startup of these processes using a wide
50 range of startup methods (SSH, local processes, PBS, mpiexec,
50 range of startup methods (SSH, local processes, PBS, mpiexec,
51 Windows HPC Server 2008). To start a cluster with 4 engines on your
51 Windows HPC Server 2008). To start a cluster with 4 engines on your
52 local host simply do 'ipcluster start -n 4'. For more complex usage
52 local host simply do 'ipcluster start -n 4'. For more complex usage
53 you will typically do 'ipcluster create -p mycluster', then edit
53 you will typically do 'ipcluster create -p mycluster', then edit
54 configuration files, followed by 'ipcluster start -p mycluster -n 4'.
54 configuration files, followed by 'ipcluster start -p mycluster -n 4'.
55 """
55 """
56
56
57
57
58 # Exit codes for ipcluster
58 # Exit codes for ipcluster
59
59
60 # This will be the exit code if the ipcluster appears to be running because
60 # This will be the exit code if the ipcluster appears to be running because
61 # a .pid file exists
61 # a .pid file exists
62 ALREADY_STARTED = 10
62 ALREADY_STARTED = 10
63
63
64
64
65 # This will be the exit code if ipcluster stop is run, but there is not .pid
65 # This will be the exit code if ipcluster stop is run, but there is not .pid
66 # file to be found.
66 # file to be found.
67 ALREADY_STOPPED = 11
67 ALREADY_STOPPED = 11
68
68
69
69
70 #-----------------------------------------------------------------------------
70 #-----------------------------------------------------------------------------
71 # Command line options
71 # Command line options
72 #-----------------------------------------------------------------------------
72 #-----------------------------------------------------------------------------
73
73
74
74
75 class IPClusterAppConfigLoader(ClusterDirConfigLoader):
75 class IPClusterAppConfigLoader(ClusterDirConfigLoader):
76
76
77 def _add_arguments(self):
77 def _add_arguments(self):
78 # Don't call ClusterDirConfigLoader._add_arguments as we don't want
78 # Don't call ClusterDirConfigLoader._add_arguments as we don't want
79 # its defaults on self.parser. Instead, we will put those on
79 # its defaults on self.parser. Instead, we will put those on
80 # default options on our subparsers.
80 # default options on our subparsers.
81
81
82 # This has all the common options that all subcommands use
82 # This has all the common options that all subcommands use
83 parent_parser1 = ArgumentParser(
83 parent_parser1 = ArgumentParser(
84 add_help=False,
84 add_help=False,
85 argument_default=SUPPRESS
85 argument_default=SUPPRESS
86 )
86 )
87 self._add_ipython_dir(parent_parser1)
87 self._add_ipython_dir(parent_parser1)
88 self._add_log_level(parent_parser1)
88 self._add_log_level(parent_parser1)
89
89
90 # This has all the common options that other subcommands use
90 # This has all the common options that other subcommands use
91 parent_parser2 = ArgumentParser(
91 parent_parser2 = ArgumentParser(
92 add_help=False,
92 add_help=False,
93 argument_default=SUPPRESS
93 argument_default=SUPPRESS
94 )
94 )
95 self._add_cluster_profile(parent_parser2)
95 self._add_cluster_profile(parent_parser2)
96 self._add_cluster_dir(parent_parser2)
96 self._add_cluster_dir(parent_parser2)
97 self._add_work_dir(parent_parser2)
97 self._add_work_dir(parent_parser2)
98 paa = parent_parser2.add_argument
98 paa = parent_parser2.add_argument
99 paa('--log-to-file',
99 paa('--log-to-file',
100 action='store_true', dest='Global.log_to_file',
100 action='store_true', dest='Global.log_to_file',
101 help='Log to a file in the log directory (default is stdout)')
101 help='Log to a file in the log directory (default is stdout)')
102
102
103 # Create the object used to create the subparsers.
103 # Create the object used to create the subparsers.
104 subparsers = self.parser.add_subparsers(
104 subparsers = self.parser.add_subparsers(
105 dest='Global.subcommand',
105 dest='Global.subcommand',
106 title='ipcluster subcommands',
106 title='ipcluster subcommands',
107 description=
107 description=
108 """ipcluster has a variety of subcommands. The general way of
108 """ipcluster has a variety of subcommands. The general way of
109 running ipcluster is 'ipcluster <cmd> [options]'. To get help
109 running ipcluster is 'ipcluster <cmd> [options]'. To get help
110 on a particular subcommand do 'ipcluster <cmd> -h'."""
110 on a particular subcommand do 'ipcluster <cmd> -h'."""
111 # help="For more help, type 'ipcluster <cmd> -h'",
111 # help="For more help, type 'ipcluster <cmd> -h'",
112 )
112 )
113
113
114 # The "list" subcommand parser
114 # The "list" subcommand parser
115 parser_list = subparsers.add_parser(
115 parser_list = subparsers.add_parser(
116 'list',
116 'list',
117 parents=[parent_parser1],
117 parents=[parent_parser1],
118 argument_default=SUPPRESS,
118 argument_default=SUPPRESS,
119 help="List all clusters in cwd and ipython_dir.",
119 help="List all clusters in cwd and ipython_dir.",
120 description=
120 description=
121 """List all available clusters, by cluster directory, that can
121 """List all available clusters, by cluster directory, that can
122 be found in the current working directly or in the ipython
122 be found in the current working directly or in the ipython
123 directory. Cluster directories are named using the convention
123 directory. Cluster directories are named using the convention
124 'cluster_<profile>'."""
124 'cluster_<profile>'."""
125 )
125 )
126
126
127 # The "create" subcommand parser
127 # The "create" subcommand parser
128 parser_create = subparsers.add_parser(
128 parser_create = subparsers.add_parser(
129 'create',
129 'create',
130 parents=[parent_parser1, parent_parser2],
130 parents=[parent_parser1, parent_parser2],
131 argument_default=SUPPRESS,
131 argument_default=SUPPRESS,
132 help="Create a new cluster directory.",
132 help="Create a new cluster directory.",
133 description=
133 description=
134 """Create an ipython cluster directory by its profile name or
134 """Create an ipython cluster directory by its profile name or
135 cluster directory path. Cluster directories contain
135 cluster directory path. Cluster directories contain
136 configuration, log and security related files and are named
136 configuration, log and security related files and are named
137 using the convention 'cluster_<profile>'. By default they are
137 using the convention 'cluster_<profile>'. By default they are
138 located in your ipython directory. Once created, you will
138 located in your ipython directory. Once created, you will
139 probably need to edit the configuration files in the cluster
139 probably need to edit the configuration files in the cluster
140 directory to configure your cluster. Most users will create a
140 directory to configure your cluster. Most users will create a
141 cluster directory by profile name,
141 cluster directory by profile name,
142 'ipcluster create -p mycluster', which will put the directory
142 'ipcluster create -p mycluster', which will put the directory
143 in '<ipython_dir>/cluster_mycluster'.
143 in '<ipython_dir>/cluster_mycluster'.
144 """
144 """
145 )
145 )
146 paa = parser_create.add_argument
146 paa = parser_create.add_argument
147 paa('--reset-config',
147 paa('--reset-config',
148 dest='Global.reset_config', action='store_true',
148 dest='Global.reset_config', action='store_true',
149 help=
149 help=
150 """Recopy the default config files to the cluster directory.
150 """Recopy the default config files to the cluster directory.
151 You will loose any modifications you have made to these files.""")
151 You will loose any modifications you have made to these files.""")
152
152
153 # The "start" subcommand parser
153 # The "start" subcommand parser
154 parser_start = subparsers.add_parser(
154 parser_start = subparsers.add_parser(
155 'start',
155 'start',
156 parents=[parent_parser1, parent_parser2],
156 parents=[parent_parser1, parent_parser2],
157 argument_default=SUPPRESS,
157 argument_default=SUPPRESS,
158 help="Start a cluster.",
158 help="Start a cluster.",
159 description=
159 description=
160 """Start an ipython cluster by its profile name or cluster
160 """Start an ipython cluster by its profile name or cluster
161 directory. Cluster directories contain configuration, log and
161 directory. Cluster directories contain configuration, log and
162 security related files and are named using the convention
162 security related files and are named using the convention
163 'cluster_<profile>' and should be creating using the 'start'
163 'cluster_<profile>' and should be creating using the 'start'
164 subcommand of 'ipcluster'. If your cluster directory is in
164 subcommand of 'ipcluster'. If your cluster directory is in
165 the cwd or the ipython directory, you can simply refer to it
165 the cwd or the ipython directory, you can simply refer to it
166 using its profile name, 'ipcluster start -n 4 -p <profile>`,
166 using its profile name, 'ipcluster start -n 4 -p <profile>`,
167 otherwise use the '--cluster-dir' option.
167 otherwise use the '--cluster-dir' option.
168 """
168 """
169 )
169 )
170 paa = parser_start.add_argument
170 paa = parser_start.add_argument
171 paa('-n', '--number',
171 paa('-n', '--number',
172 type=int, dest='Global.n',
172 type=int, dest='Global.n',
173 help='The number of engines to start.',
173 help='The number of engines to start.',
174 metavar='Global.n')
174 metavar='Global.n')
175 paa('--clean-logs',
175 paa('--clean-logs',
176 dest='Global.clean_logs', action='store_true',
176 dest='Global.clean_logs', action='store_true',
177 help='Delete old log flies before starting.')
177 help='Delete old log flies before starting.')
178 paa('--no-clean-logs',
178 paa('--no-clean-logs',
179 dest='Global.clean_logs', action='store_false',
179 dest='Global.clean_logs', action='store_false',
180 help="Don't delete old log flies before starting.")
180 help="Don't delete old log flies before starting.")
181 paa('--daemon',
181 paa('--daemon',
182 dest='Global.daemonize', action='store_true',
182 dest='Global.daemonize', action='store_true',
183 help='Daemonize the ipcluster program. This implies --log-to-file')
183 help='Daemonize the ipcluster program. This implies --log-to-file')
184 paa('--no-daemon',
184 paa('--no-daemon',
185 dest='Global.daemonize', action='store_false',
185 dest='Global.daemonize', action='store_false',
186 help="Dont't daemonize the ipcluster program.")
186 help="Dont't daemonize the ipcluster program.")
187
187
188 # The "stop" subcommand parser
188 # The "stop" subcommand parser
189 parser_stop = subparsers.add_parser(
189 parser_stop = subparsers.add_parser(
190 'stop',
190 'stop',
191 parents=[parent_parser1, parent_parser2],
191 parents=[parent_parser1, parent_parser2],
192 argument_default=SUPPRESS,
192 argument_default=SUPPRESS,
193 help="Stop a running cluster.",
193 help="Stop a running cluster.",
194 description=
194 description=
195 """Stop a running ipython cluster by its profile name or cluster
195 """Stop a running ipython cluster by its profile name or cluster
196 directory. Cluster directories are named using the convention
196 directory. Cluster directories are named using the convention
197 'cluster_<profile>'. If your cluster directory is in
197 'cluster_<profile>'. If your cluster directory is in
198 the cwd or the ipython directory, you can simply refer to it
198 the cwd or the ipython directory, you can simply refer to it
199 using its profile name, 'ipcluster stop -p <profile>`, otherwise
199 using its profile name, 'ipcluster stop -p <profile>`, otherwise
200 use the '--cluster-dir' option.
200 use the '--cluster-dir' option.
201 """
201 """
202 )
202 )
203 paa = parser_stop.add_argument
203 paa = parser_stop.add_argument
204 paa('--signal',
204 paa('--signal',
205 dest='Global.signal', type=int,
205 dest='Global.signal', type=int,
206 help="The signal number to use in stopping the cluster (default=2).",
206 help="The signal number to use in stopping the cluster (default=2).",
207 metavar="Global.signal")
207 metavar="Global.signal")
208
208
209
209
210 #-----------------------------------------------------------------------------
210 #-----------------------------------------------------------------------------
211 # Main application
211 # Main application
212 #-----------------------------------------------------------------------------
212 #-----------------------------------------------------------------------------
213
213
214
214
215 class IPClusterApp(ApplicationWithClusterDir):
215 class IPClusterApp(ApplicationWithClusterDir):
216
216
217 name = u'ipcluster'
217 name = u'ipcluster'
218 description = _description
218 description = _description
219 usage = None
219 usage = None
220 command_line_loader = IPClusterAppConfigLoader
220 command_line_loader = IPClusterAppConfigLoader
221 config_file_name = default_config_file_name
221 default_config_file_name = default_config_file_name
222 default_log_level = logging.INFO
222 default_log_level = logging.INFO
223 auto_create_cluster_dir = False
223 auto_create_cluster_dir = False
224
224
225 def create_default_config(self):
225 def create_default_config(self):
226 super(IPClusterApp, self).create_default_config()
226 super(IPClusterApp, self).create_default_config()
227 self.default_config.Global.controller_launcher = \
227 self.default_config.Global.controller_launcher = \
228 'IPython.kernel.launcher.LocalControllerLauncher'
228 'IPython.kernel.launcher.LocalControllerLauncher'
229 self.default_config.Global.engine_launcher = \
229 self.default_config.Global.engine_launcher = \
230 'IPython.kernel.launcher.LocalEngineSetLauncher'
230 'IPython.kernel.launcher.LocalEngineSetLauncher'
231 self.default_config.Global.n = 2
231 self.default_config.Global.n = 2
232 self.default_config.Global.reset_config = False
232 self.default_config.Global.reset_config = False
233 self.default_config.Global.clean_logs = True
233 self.default_config.Global.clean_logs = True
234 self.default_config.Global.signal = 2
234 self.default_config.Global.signal = 2
235 self.default_config.Global.daemonize = False
235 self.default_config.Global.daemonize = False
236
236
237 def find_resources(self):
237 def find_resources(self):
238 subcommand = self.command_line_config.Global.subcommand
238 subcommand = self.command_line_config.Global.subcommand
239 if subcommand=='list':
239 if subcommand=='list':
240 self.list_cluster_dirs()
240 self.list_cluster_dirs()
241 # Exit immediately because there is nothing left to do.
241 # Exit immediately because there is nothing left to do.
242 self.exit()
242 self.exit()
243 elif subcommand=='create':
243 elif subcommand=='create':
244 self.auto_create_cluster_dir = True
244 self.auto_create_cluster_dir = True
245 super(IPClusterApp, self).find_resources()
245 super(IPClusterApp, self).find_resources()
246 elif subcommand=='start' or subcommand=='stop':
246 elif subcommand=='start' or subcommand=='stop':
247 self.auto_create_cluster_dir = True
247 self.auto_create_cluster_dir = True
248 try:
248 try:
249 super(IPClusterApp, self).find_resources()
249 super(IPClusterApp, self).find_resources()
250 except ClusterDirError:
250 except ClusterDirError:
251 raise ClusterDirError(
251 raise ClusterDirError(
252 "Could not find a cluster directory. A cluster dir must "
252 "Could not find a cluster directory. A cluster dir must "
253 "be created before running 'ipcluster start'. Do "
253 "be created before running 'ipcluster start'. Do "
254 "'ipcluster create -h' or 'ipcluster list -h' for more "
254 "'ipcluster create -h' or 'ipcluster list -h' for more "
255 "information about creating and listing cluster dirs."
255 "information about creating and listing cluster dirs."
256 )
256 )
257
257
258 def list_cluster_dirs(self):
258 def list_cluster_dirs(self):
259 # Find the search paths
259 # Find the search paths
260 cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
260 cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
261 if cluster_dir_paths:
261 if cluster_dir_paths:
262 cluster_dir_paths = cluster_dir_paths.split(':')
262 cluster_dir_paths = cluster_dir_paths.split(':')
263 else:
263 else:
264 cluster_dir_paths = []
264 cluster_dir_paths = []
265 try:
265 try:
266 ipython_dir = self.command_line_config.Global.ipython_dir
266 ipython_dir = self.command_line_config.Global.ipython_dir
267 except AttributeError:
267 except AttributeError:
268 ipython_dir = self.default_config.Global.ipython_dir
268 ipython_dir = self.default_config.Global.ipython_dir
269 paths = [os.getcwd(), ipython_dir] + \
269 paths = [os.getcwd(), ipython_dir] + \
270 cluster_dir_paths
270 cluster_dir_paths
271 paths = list(set(paths))
271 paths = list(set(paths))
272
272
273 self.log.info('Searching for cluster dirs in paths: %r' % paths)
273 self.log.info('Searching for cluster dirs in paths: %r' % paths)
274 for path in paths:
274 for path in paths:
275 files = os.listdir(path)
275 files = os.listdir(path)
276 for f in files:
276 for f in files:
277 full_path = os.path.join(path, f)
277 full_path = os.path.join(path, f)
278 if os.path.isdir(full_path) and f.startswith('cluster_'):
278 if os.path.isdir(full_path) and f.startswith('cluster_'):
279 profile = full_path.split('_')[-1]
279 profile = full_path.split('_')[-1]
280 start_cmd = 'ipcluster start -p %s -n 4' % profile
280 start_cmd = 'ipcluster start -p %s -n 4' % profile
281 print start_cmd + " ==> " + full_path
281 print start_cmd + " ==> " + full_path
282
282
283 def pre_construct(self):
283 def pre_construct(self):
284 # IPClusterApp.pre_construct() is where we cd to the working directory.
284 # IPClusterApp.pre_construct() is where we cd to the working directory.
285 super(IPClusterApp, self).pre_construct()
285 super(IPClusterApp, self).pre_construct()
286 config = self.master_config
286 config = self.master_config
287 try:
287 try:
288 daemon = config.Global.daemonize
288 daemon = config.Global.daemonize
289 if daemon:
289 if daemon:
290 config.Global.log_to_file = True
290 config.Global.log_to_file = True
291 except AttributeError:
291 except AttributeError:
292 pass
292 pass
293
293
294 def construct(self):
294 def construct(self):
295 config = self.master_config
295 config = self.master_config
296 subcmd = config.Global.subcommand
296 subcmd = config.Global.subcommand
297 reset = config.Global.reset_config
297 reset = config.Global.reset_config
298 if subcmd == 'list':
298 if subcmd == 'list':
299 return
299 return
300 if subcmd == 'create':
300 if subcmd == 'create':
301 self.log.info('Copying default config files to cluster directory '
301 self.log.info('Copying default config files to cluster directory '
302 '[overwrite=%r]' % (reset,))
302 '[overwrite=%r]' % (reset,))
303 self.cluster_dir_obj.copy_all_config_files(overwrite=reset)
303 self.cluster_dir_obj.copy_all_config_files(overwrite=reset)
304 if subcmd =='start':
304 if subcmd =='start':
305 self.cluster_dir_obj.copy_all_config_files(overwrite=False)
305 self.cluster_dir_obj.copy_all_config_files(overwrite=False)
306 self.start_logging()
306 self.start_logging()
307 reactor.callWhenRunning(self.start_launchers)
307 reactor.callWhenRunning(self.start_launchers)
308
308
309 def start_launchers(self):
309 def start_launchers(self):
310 config = self.master_config
310 config = self.master_config
311
311
312 # Create the launchers. In both bases, we set the work_dir of
312 # Create the launchers. In both bases, we set the work_dir of
313 # the launcher to the cluster_dir. This is where the launcher's
313 # the launcher to the cluster_dir. This is where the launcher's
314 # subprocesses will be launched. It is not where the controller
314 # subprocesses will be launched. It is not where the controller
315 # and engine will be launched.
315 # and engine will be launched.
316 el_class = import_item(config.Global.engine_launcher)
316 el_class = import_item(config.Global.engine_launcher)
317 self.engine_launcher = el_class(
317 self.engine_launcher = el_class(
318 work_dir=self.cluster_dir, config=config
318 work_dir=self.cluster_dir, config=config
319 )
319 )
320 cl_class = import_item(config.Global.controller_launcher)
320 cl_class = import_item(config.Global.controller_launcher)
321 self.controller_launcher = cl_class(
321 self.controller_launcher = cl_class(
322 work_dir=self.cluster_dir, config=config
322 work_dir=self.cluster_dir, config=config
323 )
323 )
324
324
325 # Setup signals
325 # Setup signals
326 signal.signal(signal.SIGINT, self.sigint_handler)
326 signal.signal(signal.SIGINT, self.sigint_handler)
327
327
328 # Setup the observing of stopping. If the controller dies, shut
328 # Setup the observing of stopping. If the controller dies, shut
329 # everything down as that will be completely fatal for the engines.
329 # everything down as that will be completely fatal for the engines.
330 d1 = self.controller_launcher.observe_stop()
330 d1 = self.controller_launcher.observe_stop()
331 d1.addCallback(self.stop_launchers)
331 d1.addCallback(self.stop_launchers)
332 # But, we don't monitor the stopping of engines. An engine dying
332 # But, we don't monitor the stopping of engines. An engine dying
333 # is just fine and in principle a user could start a new engine.
333 # is just fine and in principle a user could start a new engine.
334 # Also, if we did monitor engine stopping, it is difficult to
334 # Also, if we did monitor engine stopping, it is difficult to
335 # know what to do when only some engines die. Currently, the
335 # know what to do when only some engines die. Currently, the
336 # observing of engine stopping is inconsistent. Some launchers
336 # observing of engine stopping is inconsistent. Some launchers
337 # might trigger on a single engine stopping, other wait until
337 # might trigger on a single engine stopping, other wait until
338 # all stop. TODO: think more about how to handle this.
338 # all stop. TODO: think more about how to handle this.
339
339
340 # Start the controller and engines
340 # Start the controller and engines
341 self._stopping = False # Make sure stop_launchers is not called 2x.
341 self._stopping = False # Make sure stop_launchers is not called 2x.
342 d = self.start_controller()
342 d = self.start_controller()
343 d.addCallback(self.start_engines)
343 d.addCallback(self.start_engines)
344 d.addCallback(self.startup_message)
344 d.addCallback(self.startup_message)
345 # If the controller or engines fail to start, stop everything
345 # If the controller or engines fail to start, stop everything
346 d.addErrback(self.stop_launchers)
346 d.addErrback(self.stop_launchers)
347 return d
347 return d
348
348
349 def startup_message(self, r=None):
349 def startup_message(self, r=None):
350 log.msg("IPython cluster: started")
350 log.msg("IPython cluster: started")
351 return r
351 return r
352
352
353 def start_controller(self, r=None):
353 def start_controller(self, r=None):
354 # log.msg("In start_controller")
354 # log.msg("In start_controller")
355 config = self.master_config
355 config = self.master_config
356 d = self.controller_launcher.start(
356 d = self.controller_launcher.start(
357 cluster_dir=config.Global.cluster_dir
357 cluster_dir=config.Global.cluster_dir
358 )
358 )
359 return d
359 return d
360
360
361 def start_engines(self, r=None):
361 def start_engines(self, r=None):
362 # log.msg("In start_engines")
362 # log.msg("In start_engines")
363 config = self.master_config
363 config = self.master_config
364 d = self.engine_launcher.start(
364 d = self.engine_launcher.start(
365 config.Global.n,
365 config.Global.n,
366 cluster_dir=config.Global.cluster_dir
366 cluster_dir=config.Global.cluster_dir
367 )
367 )
368 return d
368 return d
369
369
370 def stop_controller(self, r=None):
370 def stop_controller(self, r=None):
371 # log.msg("In stop_controller")
371 # log.msg("In stop_controller")
372 if self.controller_launcher.running:
372 if self.controller_launcher.running:
373 d = self.controller_launcher.stop()
373 d = self.controller_launcher.stop()
374 d.addErrback(self.log_err)
374 d.addErrback(self.log_err)
375 return d
375 return d
376 else:
376 else:
377 return defer.succeed(None)
377 return defer.succeed(None)
378
378
379 def stop_engines(self, r=None):
379 def stop_engines(self, r=None):
380 # log.msg("In stop_engines")
380 # log.msg("In stop_engines")
381 if self.engine_launcher.running:
381 if self.engine_launcher.running:
382 d = self.engine_launcher.stop()
382 d = self.engine_launcher.stop()
383 d.addErrback(self.log_err)
383 d.addErrback(self.log_err)
384 return d
384 return d
385 else:
385 else:
386 return defer.succeed(None)
386 return defer.succeed(None)
387
387
388 def log_err(self, f):
388 def log_err(self, f):
389 log.msg(f.getTraceback())
389 log.msg(f.getTraceback())
390 return None
390 return None
391
391
392 def stop_launchers(self, r=None):
392 def stop_launchers(self, r=None):
393 if not self._stopping:
393 if not self._stopping:
394 self._stopping = True
394 self._stopping = True
395 if isinstance(r, failure.Failure):
395 if isinstance(r, failure.Failure):
396 log.msg('Unexpected error in ipcluster:')
396 log.msg('Unexpected error in ipcluster:')
397 log.msg(r.getTraceback())
397 log.msg(r.getTraceback())
398 log.msg("IPython cluster: stopping")
398 log.msg("IPython cluster: stopping")
399 # These return deferreds. We are not doing anything with them
399 # These return deferreds. We are not doing anything with them
400 # but we are holding refs to them as a reminder that they
400 # but we are holding refs to them as a reminder that they
401 # do return deferreds.
401 # do return deferreds.
402 d1 = self.stop_engines()
402 d1 = self.stop_engines()
403 d2 = self.stop_controller()
403 d2 = self.stop_controller()
404 # Wait a few seconds to let things shut down.
404 # Wait a few seconds to let things shut down.
405 reactor.callLater(4.0, reactor.stop)
405 reactor.callLater(4.0, reactor.stop)
406
406
407 def sigint_handler(self, signum, frame):
407 def sigint_handler(self, signum, frame):
408 self.stop_launchers()
408 self.stop_launchers()
409
409
410 def start_logging(self):
410 def start_logging(self):
411 # Remove old log files of the controller and engine
411 # Remove old log files of the controller and engine
412 if self.master_config.Global.clean_logs:
412 if self.master_config.Global.clean_logs:
413 log_dir = self.master_config.Global.log_dir
413 log_dir = self.master_config.Global.log_dir
414 for f in os.listdir(log_dir):
414 for f in os.listdir(log_dir):
415 if f.startswith('ipengine' + '-'):
415 if f.startswith('ipengine' + '-'):
416 if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
416 if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
417 os.remove(os.path.join(log_dir, f))
417 os.remove(os.path.join(log_dir, f))
418 if f.startswith('ipcontroller' + '-'):
418 if f.startswith('ipcontroller' + '-'):
419 if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
419 if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
420 os.remove(os.path.join(log_dir, f))
420 os.remove(os.path.join(log_dir, f))
421 # This will remote old log files for ipcluster itself
421 # This will remote old log files for ipcluster itself
422 super(IPClusterApp, self).start_logging()
422 super(IPClusterApp, self).start_logging()
423
423
424 def start_app(self):
424 def start_app(self):
425 """Start the application, depending on what subcommand is used."""
425 """Start the application, depending on what subcommand is used."""
426 subcmd = self.master_config.Global.subcommand
426 subcmd = self.master_config.Global.subcommand
427 if subcmd=='create' or subcmd=='list':
427 if subcmd=='create' or subcmd=='list':
428 return
428 return
429 elif subcmd=='start':
429 elif subcmd=='start':
430 self.start_app_start()
430 self.start_app_start()
431 elif subcmd=='stop':
431 elif subcmd=='stop':
432 self.start_app_stop()
432 self.start_app_stop()
433
433
434 def start_app_start(self):
434 def start_app_start(self):
435 """Start the app for the start subcommand."""
435 """Start the app for the start subcommand."""
436 config = self.master_config
436 config = self.master_config
437 # First see if the cluster is already running
437 # First see if the cluster is already running
438 try:
438 try:
439 pid = self.get_pid_from_file()
439 pid = self.get_pid_from_file()
440 except PIDFileError:
440 except PIDFileError:
441 pass
441 pass
442 else:
442 else:
443 self.log.critical(
443 self.log.critical(
444 'Cluster is already running with [pid=%s]. '
444 'Cluster is already running with [pid=%s]. '
445 'use "ipcluster stop" to stop the cluster.' % pid
445 'use "ipcluster stop" to stop the cluster.' % pid
446 )
446 )
447 # Here I exit with a unusual exit status that other processes
447 # Here I exit with a unusual exit status that other processes
448 # can watch for to learn how I existed.
448 # can watch for to learn how I existed.
449 self.exit(ALREADY_STARTED)
449 self.exit(ALREADY_STARTED)
450
450
451 # Now log and daemonize
451 # Now log and daemonize
452 self.log.info(
452 self.log.info(
453 'Starting ipcluster with [daemon=%r]' % config.Global.daemonize
453 'Starting ipcluster with [daemon=%r]' % config.Global.daemonize
454 )
454 )
455 # TODO: Get daemonize working on Windows or as a Windows Server.
455 # TODO: Get daemonize working on Windows or as a Windows Server.
456 if config.Global.daemonize:
456 if config.Global.daemonize:
457 if os.name=='posix':
457 if os.name=='posix':
458 daemonize()
458 daemonize()
459
459
460 # Now write the new pid file AFTER our new forked pid is active.
460 # Now write the new pid file AFTER our new forked pid is active.
461 self.write_pid_file()
461 self.write_pid_file()
462 reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
462 reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
463 reactor.run()
463 reactor.run()
464
464
465 def start_app_stop(self):
465 def start_app_stop(self):
466 """Start the app for the stop subcommand."""
466 """Start the app for the stop subcommand."""
467 config = self.master_config
467 config = self.master_config
468 try:
468 try:
469 pid = self.get_pid_from_file()
469 pid = self.get_pid_from_file()
470 except PIDFileError:
470 except PIDFileError:
471 self.log.critical(
471 self.log.critical(
472 'Problem reading pid file, cluster is probably not running.'
472 'Problem reading pid file, cluster is probably not running.'
473 )
473 )
474 # Here I exit with a unusual exit status that other processes
474 # Here I exit with a unusual exit status that other processes
475 # can watch for to learn how I existed.
475 # can watch for to learn how I existed.
476 self.exit(ALREADY_STOPPED)
476 self.exit(ALREADY_STOPPED)
477 else:
477 else:
478 if os.name=='posix':
478 if os.name=='posix':
479 sig = config.Global.signal
479 sig = config.Global.signal
480 self.log.info(
480 self.log.info(
481 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
481 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
482 )
482 )
483 os.kill(pid, sig)
483 os.kill(pid, sig)
484 elif os.name=='nt':
484 elif os.name=='nt':
485 # As of right now, we don't support daemonize on Windows, so
485 # As of right now, we don't support daemonize on Windows, so
486 # stop will not do anything. Minimally, it should clean up the
486 # stop will not do anything. Minimally, it should clean up the
487 # old .pid files.
487 # old .pid files.
488 self.remove_pid_file()
488 self.remove_pid_file()
489
489
490
490
491 def launch_new_instance():
491 def launch_new_instance():
492 """Create and run the IPython cluster."""
492 """Create and run the IPython cluster."""
493 app = IPClusterApp()
493 app = IPClusterApp()
494 app.start()
494 app.start()
495
495
496
496
497 if __name__ == '__main__':
497 if __name__ == '__main__':
498 launch_new_instance()
498 launch_new_instance()
499
499
@@ -1,260 +1,271 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The IPython controller application.
4 The IPython controller application.
5 """
5 """
6
6
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Copyright (C) 2008-2009 The IPython Development Team
8 # Copyright (C) 2008-2009 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 from __future__ import with_statement
18 from __future__ import with_statement
19
19
20 import copy
20 import copy
21 import sys
21 import sys
22
22
23 from twisted.application import service
23 from twisted.application import service
24 from twisted.internet import reactor
24 from twisted.internet import reactor
25 from twisted.python import log
25 from twisted.python import log
26
26
27 from IPython.config.loader import Config
27 from IPython.config.loader import Config
28 from IPython.kernel import controllerservice
28 from IPython.kernel import controllerservice
29 from IPython.kernel.clusterdir import (
29 from IPython.kernel.clusterdir import (
30 ApplicationWithClusterDir,
30 ApplicationWithClusterDir,
31 ClusterDirConfigLoader
31 ClusterDirConfigLoader
32 )
32 )
33 from IPython.kernel.fcutil import FCServiceFactory
33 from IPython.kernel.fcutil import FCServiceFactory, FURLError
34 from IPython.utils.traitlets import Instance, Unicode
34 from IPython.utils.traitlets import Instance, Unicode
35
35
36
36
37 #-----------------------------------------------------------------------------
37 #-----------------------------------------------------------------------------
38 # Module level variables
38 # Module level variables
39 #-----------------------------------------------------------------------------
39 #-----------------------------------------------------------------------------
40
40
41
41
42 #: The default config file name for this application
42 #: The default config file name for this application
43 default_config_file_name = u'ipcontroller_config.py'
43 default_config_file_name = u'ipcontroller_config.py'
44
44
45
45
46 _description = """Start the IPython controller for parallel computing.
46 _description = """Start the IPython controller for parallel computing.
47
47
48 The IPython controller provides a gateway between the IPython engines and
48 The IPython controller provides a gateway between the IPython engines and
49 clients. The controller needs to be started before the engines and can be
49 clients. The controller needs to be started before the engines and can be
50 configured using command line options or using a cluster directory. Cluster
50 configured using command line options or using a cluster directory. Cluster
51 directories contain config, log and security files and are usually located in
51 directories contain config, log and security files and are usually located in
52 your .ipython directory and named as "cluster_<profile>". See the --profile
52 your .ipython directory and named as "cluster_<profile>". See the --profile
53 and --cluster-dir options for details.
53 and --cluster-dir options for details.
54 """
54 """
55
55
56 #-----------------------------------------------------------------------------
56 #-----------------------------------------------------------------------------
57 # Default interfaces
57 # Default interfaces
58 #-----------------------------------------------------------------------------
58 #-----------------------------------------------------------------------------
59
59
60 # The default client interfaces for FCClientServiceFactory.interfaces
60 # The default client interfaces for FCClientServiceFactory.interfaces
61 default_client_interfaces = Config()
61 default_client_interfaces = Config()
62 default_client_interfaces.Task.interface_chain = [
62 default_client_interfaces.Task.interface_chain = [
63 'IPython.kernel.task.ITaskController',
63 'IPython.kernel.task.ITaskController',
64 'IPython.kernel.taskfc.IFCTaskController'
64 'IPython.kernel.taskfc.IFCTaskController'
65 ]
65 ]
66
66
67 default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
67 default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
68
68
69 default_client_interfaces.MultiEngine.interface_chain = [
69 default_client_interfaces.MultiEngine.interface_chain = [
70 'IPython.kernel.multiengine.IMultiEngine',
70 'IPython.kernel.multiengine.IMultiEngine',
71 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
71 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
72 ]
72 ]
73
73
74 default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
74 default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
75
75
76 # Make this a dict we can pass to Config.__init__ for the default
76 # Make this a dict we can pass to Config.__init__ for the default
77 default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
77 default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
78
78
79
79
80
80
81 # The default engine interfaces for FCEngineServiceFactory.interfaces
81 # The default engine interfaces for FCEngineServiceFactory.interfaces
82 default_engine_interfaces = Config()
82 default_engine_interfaces = Config()
83 default_engine_interfaces.Default.interface_chain = [
83 default_engine_interfaces.Default.interface_chain = [
84 'IPython.kernel.enginefc.IFCControllerBase'
84 'IPython.kernel.enginefc.IFCControllerBase'
85 ]
85 ]
86
86
87 default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
87 default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
88
88
89 # Make this a dict we can pass to Config.__init__ for the default
89 # Make this a dict we can pass to Config.__init__ for the default
90 default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
90 default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
91
91
92
92
93 #-----------------------------------------------------------------------------
93 #-----------------------------------------------------------------------------
94 # Service factories
94 # Service factories
95 #-----------------------------------------------------------------------------
95 #-----------------------------------------------------------------------------
96
96
97
97
98 class FCClientServiceFactory(FCServiceFactory):
98 class FCClientServiceFactory(FCServiceFactory):
99 """A Foolscap implementation of the client services."""
99 """A Foolscap implementation of the client services."""
100
100
101 cert_file = Unicode(u'ipcontroller-client.pem', config=True)
101 cert_file = Unicode(u'ipcontroller-client.pem', config=True)
102 interfaces = Instance(klass=Config, kw=default_client_interfaces,
102 interfaces = Instance(klass=Config, kw=default_client_interfaces,
103 allow_none=False, config=True)
103 allow_none=False, config=True)
104
104
105
105
106 class FCEngineServiceFactory(FCServiceFactory):
106 class FCEngineServiceFactory(FCServiceFactory):
107 """A Foolscap implementation of the engine services."""
107 """A Foolscap implementation of the engine services."""
108
108
109 cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
109 cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
110 interfaces = Instance(klass=dict, kw=default_engine_interfaces,
110 interfaces = Instance(klass=dict, kw=default_engine_interfaces,
111 allow_none=False, config=True)
111 allow_none=False, config=True)
112
112
113
113
114 #-----------------------------------------------------------------------------
114 #-----------------------------------------------------------------------------
115 # Command line options
115 # Command line options
116 #-----------------------------------------------------------------------------
116 #-----------------------------------------------------------------------------
117
117
118
118
119 class IPControllerAppConfigLoader(ClusterDirConfigLoader):
119 class IPControllerAppConfigLoader(ClusterDirConfigLoader):
120
120
121 def _add_arguments(self):
121 def _add_arguments(self):
122 super(IPControllerAppConfigLoader, self)._add_arguments()
122 super(IPControllerAppConfigLoader, self)._add_arguments()
123 paa = self.parser.add_argument
123 paa = self.parser.add_argument
124 # Client config
124 # Client config
125 paa('--client-ip',
125 paa('--client-ip',
126 type=str, dest='FCClientServiceFactory.ip',
126 type=str, dest='FCClientServiceFactory.ip',
127 help='The IP address or hostname the controller will listen on for '
127 help='The IP address or hostname the controller will listen on for '
128 'client connections.',
128 'client connections.',
129 metavar='FCClientServiceFactory.ip')
129 metavar='FCClientServiceFactory.ip')
130 paa('--client-port',
130 paa('--client-port',
131 type=int, dest='FCClientServiceFactory.port',
131 type=int, dest='FCClientServiceFactory.port',
132 help='The port the controller will listen on for client connections. '
132 help='The port the controller will listen on for client connections. '
133 'The default is to use 0, which will autoselect an open port.',
133 'The default is to use 0, which will autoselect an open port.',
134 metavar='FCClientServiceFactory.port')
134 metavar='FCClientServiceFactory.port')
135 paa('--client-location',), dict(
135 paa('--client-location',), dict(
136 type=str, dest='FCClientServiceFactory.location',
136 type=str, dest='FCClientServiceFactory.location',
137 help='The hostname or IP that clients should connect to. This does '
137 help='The hostname or IP that clients should connect to. This does '
138 'not control which interface the controller listens on. Instead, this '
138 'not control which interface the controller listens on. Instead, this '
139 'determines the hostname/IP that is listed in the FURL, which is how '
139 'determines the hostname/IP that is listed in the FURL, which is how '
140 'clients know where to connect. Useful if the controller is listening '
140 'clients know where to connect. Useful if the controller is listening '
141 'on multiple interfaces.',
141 'on multiple interfaces.',
142 metavar='FCClientServiceFactory.location')
142 metavar='FCClientServiceFactory.location')
143 # Engine config
143 # Engine config
144 paa('--engine-ip',
144 paa('--engine-ip',
145 type=str, dest='FCEngineServiceFactory.ip',
145 type=str, dest='FCEngineServiceFactory.ip',
146 help='The IP address or hostname the controller will listen on for '
146 help='The IP address or hostname the controller will listen on for '
147 'engine connections.',
147 'engine connections.',
148 metavar='FCEngineServiceFactory.ip')
148 metavar='FCEngineServiceFactory.ip')
149 paa('--engine-port',
149 paa('--engine-port',
150 type=int, dest='FCEngineServiceFactory.port',
150 type=int, dest='FCEngineServiceFactory.port',
151 help='The port the controller will listen on for engine connections. '
151 help='The port the controller will listen on for engine connections. '
152 'The default is to use 0, which will autoselect an open port.',
152 'The default is to use 0, which will autoselect an open port.',
153 metavar='FCEngineServiceFactory.port')
153 metavar='FCEngineServiceFactory.port')
154 paa('--engine-location',
154 paa('--engine-location',
155 type=str, dest='FCEngineServiceFactory.location',
155 type=str, dest='FCEngineServiceFactory.location',
156 help='The hostname or IP that engines should connect to. This does '
156 help='The hostname or IP that engines should connect to. This does '
157 'not control which interface the controller listens on. Instead, this '
157 'not control which interface the controller listens on. Instead, this '
158 'determines the hostname/IP that is listed in the FURL, which is how '
158 'determines the hostname/IP that is listed in the FURL, which is how '
159 'engines know where to connect. Useful if the controller is listening '
159 'engines know where to connect. Useful if the controller is listening '
160 'on multiple interfaces.',
160 'on multiple interfaces.',
161 metavar='FCEngineServiceFactory.location')
161 metavar='FCEngineServiceFactory.location')
162 # Global config
162 # Global config
163 paa('--log-to-file',
163 paa('--log-to-file',
164 action='store_true', dest='Global.log_to_file',
164 action='store_true', dest='Global.log_to_file',
165 help='Log to a file in the log directory (default is stdout)')
165 help='Log to a file in the log directory (default is stdout)')
166 paa('-r','--reuse-furls',
166 paa('-r','--reuse-furls',
167 action='store_true', dest='Global.reuse_furls',
167 action='store_true', dest='Global.reuse_furls',
168 help='Try to reuse all FURL files. If this is not set all FURL files '
168 help='Try to reuse all FURL files. If this is not set all FURL files '
169 'are deleted before the controller starts. This must be set if '
169 'are deleted before the controller starts. This must be set if '
170 'specific ports are specified by --engine-port or --client-port.')
170 'specific ports are specified by --engine-port or --client-port.')
171 paa('--no-secure',
171 paa('--no-secure',
172 action='store_false', dest='Global.secure',
172 action='store_false', dest='Global.secure',
173 help='Turn off SSL encryption for all connections.')
173 help='Turn off SSL encryption for all connections.')
174 paa('--secure',
174 paa('--secure',
175 action='store_true', dest='Global.secure',
175 action='store_true', dest='Global.secure',
176 help='Turn off SSL encryption for all connections.')
176 help='Turn off SSL encryption for all connections.')
177
177
178
178
179 #-----------------------------------------------------------------------------
179 #-----------------------------------------------------------------------------
180 # The main application
180 # The main application
181 #-----------------------------------------------------------------------------
181 #-----------------------------------------------------------------------------
182
182
183
183
184 class IPControllerApp(ApplicationWithClusterDir):
184 class IPControllerApp(ApplicationWithClusterDir):
185
185
186 name = u'ipcontroller'
186 name = u'ipcontroller'
187 description = _description
187 description = _description
188 command_line_loader = IPControllerAppConfigLoader
188 command_line_loader = IPControllerAppConfigLoader
189 config_file_name = default_config_file_name
189 default_config_file_name = default_config_file_name
190 auto_create_cluster_dir = True
190 auto_create_cluster_dir = True
191
191
192 def create_default_config(self):
192 def create_default_config(self):
193 super(IPControllerApp, self).create_default_config()
193 super(IPControllerApp, self).create_default_config()
194 self.default_config.Global.reuse_furls = False
194 # Don't set defaults for Global.secure or Global.reuse_furls
195 self.default_config.Global.secure = True
195 # as those are set in a component.
196 self.default_config.Global.import_statements = []
196 self.default_config.Global.import_statements = []
197 self.default_config.Global.clean_logs = True
197 self.default_config.Global.clean_logs = True
198
198
199 def post_load_command_line_config(self):
199 def pre_construct(self):
200 # Now setup reuse_furls
200 super(IPControllerApp, self).pre_construct()
201 c = self.command_line_config
201 c = self.master_config
202 # The defaults for these are set in FCClientServiceFactory and
203 # FCEngineServiceFactory, so we only set them here if the global
204 # options have be set to override the class level defaults.
202 if hasattr(c.Global, 'reuse_furls'):
205 if hasattr(c.Global, 'reuse_furls'):
203 c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
206 c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
204 c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
207 c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
205 del c.Global.reuse_furls
208 del c.Global.reuse_furls
206 if hasattr(c.Global, 'secure'):
209 if hasattr(c.Global, 'secure'):
207 c.FCClientServiceFactory.secure = c.Global.secure
210 c.FCClientServiceFactory.secure = c.Global.secure
208 c.FCEngineServiceFactory.secure = c.Global.secure
211 c.FCEngineServiceFactory.secure = c.Global.secure
209 del c.Global.secure
212 del c.Global.secure
210
213
211 def construct(self):
214 def construct(self):
212 # This is the working dir by now.
215 # This is the working dir by now.
213 sys.path.insert(0, '')
216 sys.path.insert(0, '')
214
217
215 self.start_logging()
218 self.start_logging()
216 self.import_statements()
219 self.import_statements()
217
220
218 # Create the service hierarchy
221 # Create the service hierarchy
219 self.main_service = service.MultiService()
222 self.main_service = service.MultiService()
220 # The controller service
223 # The controller service
221 controller_service = controllerservice.ControllerService()
224 controller_service = controllerservice.ControllerService()
222 controller_service.setServiceParent(self.main_service)
225 controller_service.setServiceParent(self.main_service)
223 # The client tub and all its refereceables
226 # The client tub and all its refereceables
224 csfactory = FCClientServiceFactory(self.master_config, controller_service)
227 try:
228 csfactory = FCClientServiceFactory(self.master_config, controller_service)
229 except FURLError, e:
230 log.err(e)
231 self.exit(0)
225 client_service = csfactory.create()
232 client_service = csfactory.create()
226 client_service.setServiceParent(self.main_service)
233 client_service.setServiceParent(self.main_service)
227 # The engine tub
234 # The engine tub
228 esfactory = FCEngineServiceFactory(self.master_config, controller_service)
235 try:
236 esfactory = FCEngineServiceFactory(self.master_config, controller_service)
237 except FURLError, e:
238 log.err(e)
239 self.exit(0)
229 engine_service = esfactory.create()
240 engine_service = esfactory.create()
230 engine_service.setServiceParent(self.main_service)
241 engine_service.setServiceParent(self.main_service)
231
242
232 def import_statements(self):
243 def import_statements(self):
233 statements = self.master_config.Global.import_statements
244 statements = self.master_config.Global.import_statements
234 for s in statements:
245 for s in statements:
235 try:
246 try:
236 log.msg("Executing statement: '%s'" % s)
247 log.msg("Executing statement: '%s'" % s)
237 exec s in globals(), locals()
248 exec s in globals(), locals()
238 except:
249 except:
239 log.msg("Error running statement: %s" % s)
250 log.msg("Error running statement: %s" % s)
240
251
241 def start_app(self):
252 def start_app(self):
242 # Start the controller service.
253 # Start the controller service.
243 self.main_service.startService()
254 self.main_service.startService()
244 # Write the .pid file overwriting old ones. This allow multiple
255 # Write the .pid file overwriting old ones. This allow multiple
245 # controllers to clober each other. But Windows is not cleaning
256 # controllers to clober each other. But Windows is not cleaning
246 # these up properly.
257 # these up properly.
247 self.write_pid_file(overwrite=True)
258 self.write_pid_file(overwrite=True)
248 # Add a trigger to delete the .pid file upon shutting down.
259 # Add a trigger to delete the .pid file upon shutting down.
249 reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
260 reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
250 reactor.run()
261 reactor.run()
251
262
252
263
253 def launch_new_instance():
264 def launch_new_instance():
254 """Create and run the IPython controller"""
265 """Create and run the IPython controller"""
255 app = IPControllerApp()
266 app = IPControllerApp()
256 app.start()
267 app.start()
257
268
258
269
259 if __name__ == '__main__':
270 if __name__ == '__main__':
260 launch_new_instance()
271 launch_new_instance()
@@ -1,242 +1,242 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The IPython controller application
4 The IPython controller application
5 """
5 """
6
6
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Copyright (C) 2008-2009 The IPython Development Team
8 # Copyright (C) 2008-2009 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 import os
18 import os
19 import sys
19 import sys
20
20
21 from twisted.application import service
21 from twisted.application import service
22 from twisted.internet import reactor
22 from twisted.internet import reactor
23 from twisted.python import log
23 from twisted.python import log
24
24
25 from IPython.kernel.clusterdir import (
25 from IPython.kernel.clusterdir import (
26 ApplicationWithClusterDir,
26 ApplicationWithClusterDir,
27 ClusterDirConfigLoader
27 ClusterDirConfigLoader
28 )
28 )
29 from IPython.kernel.engineconnector import EngineConnector
29 from IPython.kernel.engineconnector import EngineConnector
30 from IPython.kernel.engineservice import EngineService
30 from IPython.kernel.engineservice import EngineService
31 from IPython.kernel.fcutil import Tub
31 from IPython.kernel.fcutil import Tub
32 from IPython.utils.importstring import import_item
32 from IPython.utils.importstring import import_item
33
33
34 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
35 # Module level variables
35 # Module level variables
36 #-----------------------------------------------------------------------------
36 #-----------------------------------------------------------------------------
37
37
38 #: The default config file name for this application
38 #: The default config file name for this application
39 default_config_file_name = u'ipengine_config.py'
39 default_config_file_name = u'ipengine_config.py'
40
40
41
41
42 mpi4py_init = """from mpi4py import MPI as mpi
42 mpi4py_init = """from mpi4py import MPI as mpi
43 mpi.size = mpi.COMM_WORLD.Get_size()
43 mpi.size = mpi.COMM_WORLD.Get_size()
44 mpi.rank = mpi.COMM_WORLD.Get_rank()
44 mpi.rank = mpi.COMM_WORLD.Get_rank()
45 """
45 """
46
46
47
47
48 pytrilinos_init = """from PyTrilinos import Epetra
48 pytrilinos_init = """from PyTrilinos import Epetra
49 class SimpleStruct:
49 class SimpleStruct:
50 pass
50 pass
51 mpi = SimpleStruct()
51 mpi = SimpleStruct()
52 mpi.rank = 0
52 mpi.rank = 0
53 mpi.size = 0
53 mpi.size = 0
54 """
54 """
55
55
56
56
57 _description = """Start an IPython engine for parallel computing.\n\n
57 _description = """Start an IPython engine for parallel computing.\n\n
58
58
59 IPython engines run in parallel and perform computations on behalf of a client
59 IPython engines run in parallel and perform computations on behalf of a client
60 and controller. A controller needs to be started before the engines. The
60 and controller. A controller needs to be started before the engines. The
61 engine can be configured using command line options or using a cluster
61 engine can be configured using command line options or using a cluster
62 directory. Cluster directories contain config, log and security files and are
62 directory. Cluster directories contain config, log and security files and are
63 usually located in your .ipython directory and named as "cluster_<profile>".
63 usually located in your .ipython directory and named as "cluster_<profile>".
64 See the --profile and --cluster-dir options for details.
64 See the --profile and --cluster-dir options for details.
65 """
65 """
66
66
67 #-----------------------------------------------------------------------------
67 #-----------------------------------------------------------------------------
68 # Command line options
68 # Command line options
69 #-----------------------------------------------------------------------------
69 #-----------------------------------------------------------------------------
70
70
71
71
72 class IPEngineAppConfigLoader(ClusterDirConfigLoader):
72 class IPEngineAppConfigLoader(ClusterDirConfigLoader):
73
73
74 def _add_arguments(self):
74 def _add_arguments(self):
75 super(IPEngineAppConfigLoader, self)._add_arguments()
75 super(IPEngineAppConfigLoader, self)._add_arguments()
76 paa = self.parser.add_argument
76 paa = self.parser.add_argument
77 # Controller config
77 # Controller config
78 paa('--furl-file',
78 paa('--furl-file',
79 type=unicode, dest='Global.furl_file',
79 type=unicode, dest='Global.furl_file',
80 help='The full location of the file containing the FURL of the '
80 help='The full location of the file containing the FURL of the '
81 'controller. If this is not given, the FURL file must be in the '
81 'controller. If this is not given, the FURL file must be in the '
82 'security directory of the cluster directory. This location is '
82 'security directory of the cluster directory. This location is '
83 'resolved using the --profile and --app-dir options.',
83 'resolved using the --profile and --app-dir options.',
84 metavar='Global.furl_file')
84 metavar='Global.furl_file')
85 # MPI
85 # MPI
86 paa('--mpi',
86 paa('--mpi',
87 type=str, dest='MPI.use',
87 type=str, dest='MPI.use',
88 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
88 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
89 metavar='MPI.use')
89 metavar='MPI.use')
90 # Global config
90 # Global config
91 paa('--log-to-file',
91 paa('--log-to-file',
92 action='store_true', dest='Global.log_to_file',
92 action='store_true', dest='Global.log_to_file',
93 help='Log to a file in the log directory (default is stdout)')
93 help='Log to a file in the log directory (default is stdout)')
94
94
95
95
96 #-----------------------------------------------------------------------------
96 #-----------------------------------------------------------------------------
97 # Main application
97 # Main application
98 #-----------------------------------------------------------------------------
98 #-----------------------------------------------------------------------------
99
99
100
100
101 class IPEngineApp(ApplicationWithClusterDir):
101 class IPEngineApp(ApplicationWithClusterDir):
102
102
103 name = u'ipengine'
103 name = u'ipengine'
104 description = _description
104 description = _description
105 command_line_loader = IPEngineAppConfigLoader
105 command_line_loader = IPEngineAppConfigLoader
106 config_file_name = default_config_file_name
106 default_config_file_name = default_config_file_name
107 auto_create_cluster_dir = True
107 auto_create_cluster_dir = True
108
108
109 def create_default_config(self):
109 def create_default_config(self):
110 super(IPEngineApp, self).create_default_config()
110 super(IPEngineApp, self).create_default_config()
111
111
112 # The engine should not clean logs as we don't want to remove the
112 # The engine should not clean logs as we don't want to remove the
113 # active log files of other running engines.
113 # active log files of other running engines.
114 self.default_config.Global.clean_logs = False
114 self.default_config.Global.clean_logs = False
115
115
116 # Global config attributes
116 # Global config attributes
117 self.default_config.Global.exec_lines = []
117 self.default_config.Global.exec_lines = []
118 self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
118 self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
119
119
120 # Configuration related to the controller
120 # Configuration related to the controller
121 # This must match the filename (path not included) that the controller
121 # This must match the filename (path not included) that the controller
122 # used for the FURL file.
122 # used for the FURL file.
123 self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl'
123 self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl'
124 # If given, this is the actual location of the controller's FURL file.
124 # If given, this is the actual location of the controller's FURL file.
125 # If not, this is computed using the profile, app_dir and furl_file_name
125 # If not, this is computed using the profile, app_dir and furl_file_name
126 self.default_config.Global.furl_file = u''
126 self.default_config.Global.furl_file = u''
127
127
128 # The max number of connection attemps and the initial delay between
128 # The max number of connection attemps and the initial delay between
129 # those attemps.
129 # those attemps.
130 self.default_config.Global.connect_delay = 0.1
130 self.default_config.Global.connect_delay = 0.1
131 self.default_config.Global.connect_max_tries = 15
131 self.default_config.Global.connect_max_tries = 15
132
132
133 # MPI related config attributes
133 # MPI related config attributes
134 self.default_config.MPI.use = ''
134 self.default_config.MPI.use = ''
135 self.default_config.MPI.mpi4py = mpi4py_init
135 self.default_config.MPI.mpi4py = mpi4py_init
136 self.default_config.MPI.pytrilinos = pytrilinos_init
136 self.default_config.MPI.pytrilinos = pytrilinos_init
137
137
138 def post_load_command_line_config(self):
138 def post_load_command_line_config(self):
139 pass
139 pass
140
140
141 def pre_construct(self):
141 def pre_construct(self):
142 super(IPEngineApp, self).pre_construct()
142 super(IPEngineApp, self).pre_construct()
143 self.find_cont_furl_file()
143 self.find_cont_furl_file()
144
144
145 def find_cont_furl_file(self):
145 def find_cont_furl_file(self):
146 """Set the furl file.
146 """Set the furl file.
147
147
148 Here we don't try to actually see if it exists for is valid as that
148 Here we don't try to actually see if it exists for is valid as that
149 is hadled by the connection logic.
149 is hadled by the connection logic.
150 """
150 """
151 config = self.master_config
151 config = self.master_config
152 # Find the actual controller FURL file
152 # Find the actual controller FURL file
153 if not config.Global.furl_file:
153 if not config.Global.furl_file:
154 try_this = os.path.join(
154 try_this = os.path.join(
155 config.Global.cluster_dir,
155 config.Global.cluster_dir,
156 config.Global.security_dir,
156 config.Global.security_dir,
157 config.Global.furl_file_name
157 config.Global.furl_file_name
158 )
158 )
159 config.Global.furl_file = try_this
159 config.Global.furl_file = try_this
160
160
161 def construct(self):
161 def construct(self):
162 # This is the working dir by now.
162 # This is the working dir by now.
163 sys.path.insert(0, '')
163 sys.path.insert(0, '')
164
164
165 self.start_mpi()
165 self.start_mpi()
166 self.start_logging()
166 self.start_logging()
167
167
168 # Create the underlying shell class and EngineService
168 # Create the underlying shell class and EngineService
169 shell_class = import_item(self.master_config.Global.shell_class)
169 shell_class = import_item(self.master_config.Global.shell_class)
170 self.engine_service = EngineService(shell_class, mpi=mpi)
170 self.engine_service = EngineService(shell_class, mpi=mpi)
171
171
172 self.exec_lines()
172 self.exec_lines()
173
173
174 # Create the service hierarchy
174 # Create the service hierarchy
175 self.main_service = service.MultiService()
175 self.main_service = service.MultiService()
176 self.engine_service.setServiceParent(self.main_service)
176 self.engine_service.setServiceParent(self.main_service)
177 self.tub_service = Tub()
177 self.tub_service = Tub()
178 self.tub_service.setServiceParent(self.main_service)
178 self.tub_service.setServiceParent(self.main_service)
179 # This needs to be called before the connection is initiated
179 # This needs to be called before the connection is initiated
180 self.main_service.startService()
180 self.main_service.startService()
181
181
182 # This initiates the connection to the controller and calls
182 # This initiates the connection to the controller and calls
183 # register_engine to tell the controller we are ready to do work
183 # register_engine to tell the controller we are ready to do work
184 self.engine_connector = EngineConnector(self.tub_service)
184 self.engine_connector = EngineConnector(self.tub_service)
185
185
186 log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
186 log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
187
187
188 reactor.callWhenRunning(self.call_connect)
188 reactor.callWhenRunning(self.call_connect)
189
189
190 def call_connect(self):
190 def call_connect(self):
191 d = self.engine_connector.connect_to_controller(
191 d = self.engine_connector.connect_to_controller(
192 self.engine_service,
192 self.engine_service,
193 self.master_config.Global.furl_file,
193 self.master_config.Global.furl_file,
194 self.master_config.Global.connect_delay,
194 self.master_config.Global.connect_delay,
195 self.master_config.Global.connect_max_tries
195 self.master_config.Global.connect_max_tries
196 )
196 )
197
197
198 def handle_error(f):
198 def handle_error(f):
199 log.msg('Error connecting to controller. This usually means that '
199 log.msg('Error connecting to controller. This usually means that '
200 'i) the controller was not started, ii) a firewall was blocking '
200 'i) the controller was not started, ii) a firewall was blocking '
201 'the engine from connecting to the controller or iii) the engine '
201 'the engine from connecting to the controller or iii) the engine '
202 ' was not pointed at the right FURL file:')
202 ' was not pointed at the right FURL file:')
203 log.msg(f.getErrorMessage())
203 log.msg(f.getErrorMessage())
204 reactor.callLater(0.1, reactor.stop)
204 reactor.callLater(0.1, reactor.stop)
205
205
206 d.addErrback(handle_error)
206 d.addErrback(handle_error)
207
207
208 def start_mpi(self):
208 def start_mpi(self):
209 global mpi
209 global mpi
210 mpikey = self.master_config.MPI.use
210 mpikey = self.master_config.MPI.use
211 mpi_import_statement = self.master_config.MPI.get(mpikey, None)
211 mpi_import_statement = self.master_config.MPI.get(mpikey, None)
212 if mpi_import_statement is not None:
212 if mpi_import_statement is not None:
213 try:
213 try:
214 self.log.info("Initializing MPI:")
214 self.log.info("Initializing MPI:")
215 self.log.info(mpi_import_statement)
215 self.log.info(mpi_import_statement)
216 exec mpi_import_statement in globals()
216 exec mpi_import_statement in globals()
217 except:
217 except:
218 mpi = None
218 mpi = None
219 else:
219 else:
220 mpi = None
220 mpi = None
221
221
222 def exec_lines(self):
222 def exec_lines(self):
223 for line in self.master_config.Global.exec_lines:
223 for line in self.master_config.Global.exec_lines:
224 try:
224 try:
225 log.msg("Executing statement: '%s'" % line)
225 log.msg("Executing statement: '%s'" % line)
226 self.engine_service.execute(line)
226 self.engine_service.execute(line)
227 except:
227 except:
228 log.msg("Error executing statement: %s" % line)
228 log.msg("Error executing statement: %s" % line)
229
229
230 def start_app(self):
230 def start_app(self):
231 reactor.run()
231 reactor.run()
232
232
233
233
234 def launch_new_instance():
234 def launch_new_instance():
235 """Create and run the IPython controller"""
235 """Create and run the IPython controller"""
236 app = IPEngineApp()
236 app = IPEngineApp()
237 app.start()
237 app.start()
238
238
239
239
240 if __name__ == '__main__':
240 if __name__ == '__main__':
241 launch_new_instance()
241 launch_new_instance()
242
242
General Comments 0
You need to be logged in to leave comments. Login now