##// END OF EJS Templates
Various fixes to ipapp/ipcluster/ipengine/ipcontroller....
Brian Granger -
Show More
@@ -1,449 +1,453 b''
1 1 # encoding: utf-8
2 2 """
3 3 An application for IPython.
4 4
5 5 All top-level applications should use the classes in this module for
6 6 handling configuration and creating componenets.
7 7
8 8 The job of an :class:`Application` is to create the master configuration
9 9 object and then create the components, passing the config to them.
10 10
11 11 Authors:
12 12
13 13 * Brian Granger
14 14 * Fernando Perez
15 15
16 16 Notes
17 17 -----
18 18 """
19 19
20 20 #-----------------------------------------------------------------------------
21 21 # Copyright (C) 2008-2009 The IPython Development Team
22 22 #
23 23 # Distributed under the terms of the BSD License. The full license is in
24 24 # the file COPYING, distributed as part of this software.
25 25 #-----------------------------------------------------------------------------
26 26
27 27 #-----------------------------------------------------------------------------
28 28 # Imports
29 29 #-----------------------------------------------------------------------------
30 30
31 31 import logging
32 32 import os
33 33 import sys
34 34
35 35 from IPython.core import release, crashhandler
36 36 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
37 37 from IPython.config.loader import (
38 38 PyFileConfigLoader,
39 39 ArgParseConfigLoader,
40 40 Config,
41 41 )
42 42
43 43 #-----------------------------------------------------------------------------
44 44 # Classes and functions
45 45 #-----------------------------------------------------------------------------
46 46
47 47 class ApplicationError(Exception):
48 48 pass
49 49
50 50
51 51 class BaseAppConfigLoader(ArgParseConfigLoader):
52 52 """Default command line options for IPython based applications."""
53 53
54 54 def _add_ipython_dir(self, parser):
55 55 """Add the --ipython-dir option to the parser."""
56 56 paa = parser.add_argument
57 57 paa('--ipython-dir',
58 58 dest='Global.ipython_dir',type=unicode,
59 59 help=
60 60 """Set to override default location of the IPython directory
61 61 IPYTHON_DIR, stored as Global.ipython_dir. This can also be
62 62 specified through the environment variable IPYTHON_DIR.""",
63 63 metavar='Global.ipython_dir')
64 64
65 65 def _add_log_level(self, parser):
66 66 """Add the --log-level option to the parser."""
67 67 paa = parser.add_argument
68 68 paa('--log-level',
69 69 dest="Global.log_level",type=int,
70 70 help='Set the log level (0,10,20,30,40,50). Default is 30.',
71 71 metavar='Global.log_level')
72 72
73 73 def _add_arguments(self):
74 74 self._add_ipython_dir(self.parser)
75 75 self._add_log_level(self.parser)
76 76
77 77
78 78 class Application(object):
79 79 """Load a config, construct components and set them running.
80 80
81 81 The configuration of an application can be done via three different Config
82 82 objects, which are loaded and ultimately merged into a single one used
83 83 from that point on by the app. These are:
84 84
85 85 1. default_config: internal defaults, implemented in code.
86 86 2. file_config: read from the filesystem.
87 87 3. command_line_config: read from the system's command line flags.
88 88
89 89 During initialization, 3 is actually read before 2, since at the
90 90 command-line one may override the location of the file to be read. But the
91 91 above is the order in which the merge is made.
92 92 """
93 93
94 94 name = u'ipython'
95 95 description = 'IPython: an enhanced interactive Python shell.'
96 96 #: Usage message printed by argparse. If None, auto-generate
97 97 usage = None
98 98 #: The command line config loader. Subclass of ArgParseConfigLoader.
99 99 command_line_loader = BaseAppConfigLoader
100 #: The name of the config file to load.
101 config_file_name = u'ipython_config.py'
100 #: The name of the config file to load, determined at runtime
101 config_file_name = None
102 102 #: The name of the default config file. Track separately from the actual
103 103 #: name because some logic happens only if we aren't using the default.
104 default_config_file_name = config_file_name
104 default_config_file_name = u'ipython_config.py'
105 105 default_log_level = logging.WARN
106 106 #: Set by --profile option
107 107 profile_name = None
108 108 #: User's ipython directory, typically ~/.ipython/
109 109 ipython_dir = None
110 110 #: Internal defaults, implemented in code.
111 111 default_config = None
112 112 #: Read from the filesystem.
113 113 file_config = None
114 114 #: Read from the system's command line flags.
115 115 command_line_config = None
116 116 #: The final config that will be passed to the component.
117 117 master_config = None
118 118 #: A reference to the argv to be used (typically ends up being sys.argv[1:])
119 119 argv = None
120 120 #: extra arguments computed by the command-line loader
121 121 extra_args = None
122 122 #: The class to use as the crash handler.
123 123 crash_handler_class = crashhandler.CrashHandler
124 124
125 125 # Private attributes
126 126 _exiting = False
127 127 _initialized = False
128 128
129 129 def __init__(self, argv=None):
130 130 self.argv = sys.argv[1:] if argv is None else argv
131 131 self.init_logger()
132 132
133 133 def init_logger(self):
134 134 self.log = logging.getLogger(self.__class__.__name__)
135 135 # This is used as the default until the command line arguments are read.
136 136 self.log.setLevel(self.default_log_level)
137 137 self._log_handler = logging.StreamHandler()
138 138 self._log_formatter = logging.Formatter("[%(name)s] %(message)s")
139 139 self._log_handler.setFormatter(self._log_formatter)
140 140 self.log.addHandler(self._log_handler)
141 141
142 142 def _set_log_level(self, level):
143 143 self.log.setLevel(level)
144 144
145 145 def _get_log_level(self):
146 146 return self.log.level
147 147
148 148 log_level = property(_get_log_level, _set_log_level)
149 149
150 150 def initialize(self):
151 151 """Initialize the application.
152 152
153 153 Loads all configuration information and sets all application state, but
154 154 does not start any relevant processing (typically some kind of event
155 155 loop).
156 156
157 157 Once this method has been called, the application is flagged as
158 158 initialized and the method becomes a no-op."""
159 159
160 160 if self._initialized:
161 161 return
162 162
163 163 # The first part is protected with an 'attempt' wrapper, that will log
164 164 # failures with the basic system traceback machinery. Once our crash
165 165 # handler is in place, we can let any subsequent exception propagate,
166 166 # as our handler will log it with much better detail than the default.
167 167 self.attempt(self.create_crash_handler)
168 168
169 169 # Configuration phase
170 170 # Default config (internally hardwired in application code)
171 171 self.create_default_config()
172 172 self.log_default_config()
173 173 self.set_default_config_log_level()
174 174
175 175 # Command-line config
176 176 self.pre_load_command_line_config()
177 177 self.load_command_line_config()
178 178 self.set_command_line_config_log_level()
179 179 self.post_load_command_line_config()
180 180 self.log_command_line_config()
181 181
182 182 # Find resources needed for filesystem access, using information from
183 183 # the above two
184 184 self.find_ipython_dir()
185 185 self.find_resources()
186 186 self.find_config_file_name()
187 187 self.find_config_file_paths()
188 188
189 189 # File-based config
190 190 self.pre_load_file_config()
191 191 self.load_file_config()
192 192 self.set_file_config_log_level()
193 193 self.post_load_file_config()
194 194 self.log_file_config()
195 195
196 196 # Merge all config objects into a single one the app can then use
197 197 self.merge_configs()
198 198 self.log_master_config()
199 199
200 200 # Construction phase
201 201 self.pre_construct()
202 202 self.construct()
203 203 self.post_construct()
204 204
205 205 # Done, flag as such and
206 206 self._initialized = True
207 207
208 208 def start(self):
209 209 """Start the application."""
210 210 self.initialize()
211 211 self.start_app()
212 212
213 213 #-------------------------------------------------------------------------
214 214 # Various stages of Application creation
215 215 #-------------------------------------------------------------------------
216 216
217 217 def create_crash_handler(self):
218 218 """Create a crash handler, typically setting sys.excepthook to it."""
219 219 self.crash_handler = self.crash_handler_class(self)
220 220 sys.excepthook = self.crash_handler
221 221
222 222 def create_default_config(self):
223 223 """Create defaults that can't be set elsewhere.
224 224
225 225 For the most part, we try to set default in the class attributes
226 226 of Components. But, defaults the top-level Application (which is
227 227 not a HasTraitlets or Component) are not set in this way. Instead
228 228 we set them here. The Global section is for variables like this that
229 229 don't belong to a particular component.
230 230 """
231 231 c = Config()
232 232 c.Global.ipython_dir = get_ipython_dir()
233 233 c.Global.log_level = self.log_level
234 234 self.default_config = c
235 235
236 236 def log_default_config(self):
237 237 self.log.debug('Default config loaded:')
238 238 self.log.debug(repr(self.default_config))
239 239
240 240 def set_default_config_log_level(self):
241 241 try:
242 242 self.log_level = self.default_config.Global.log_level
243 243 except AttributeError:
244 244 # Fallback to the default_log_level class attribute
245 245 pass
246 246
247 247 def create_command_line_config(self):
248 248 """Create and return a command line config loader."""
249 249 return self.command_line_loader(
250 250 self.argv,
251 251 description=self.description,
252 252 version=release.version,
253 253 usage=self.usage
254 254 )
255 255
256 256 def pre_load_command_line_config(self):
257 257 """Do actions just before loading the command line config."""
258 258 pass
259 259
260 260 def load_command_line_config(self):
261 261 """Load the command line config."""
262 262 loader = self.create_command_line_config()
263 263 self.command_line_config = loader.load_config()
264 264 self.extra_args = loader.get_extra_args()
265 265
266 266 def set_command_line_config_log_level(self):
267 267 try:
268 268 self.log_level = self.command_line_config.Global.log_level
269 269 except AttributeError:
270 270 pass
271 271
272 272 def post_load_command_line_config(self):
273 273 """Do actions just after loading the command line config."""
274 274 pass
275 275
276 276 def log_command_line_config(self):
277 277 self.log.debug("Command line config loaded:")
278 278 self.log.debug(repr(self.command_line_config))
279 279
280 280 def find_ipython_dir(self):
281 281 """Set the IPython directory.
282 282
283 283 This sets ``self.ipython_dir``, but the actual value that is passed to
284 284 the application is kept in either ``self.default_config`` or
285 285 ``self.command_line_config``. This also adds ``self.ipython_dir`` to
286 286 ``sys.path`` so config files there can be referenced by other config
287 287 files.
288 288 """
289 289
290 290 try:
291 291 self.ipython_dir = self.command_line_config.Global.ipython_dir
292 292 except AttributeError:
293 293 self.ipython_dir = self.default_config.Global.ipython_dir
294 294 sys.path.append(os.path.abspath(self.ipython_dir))
295 295 if not os.path.isdir(self.ipython_dir):
296 296 os.makedirs(self.ipython_dir, mode=0777)
297 297 self.log.debug("IPYTHON_DIR set to: %s" % self.ipython_dir)
298 298
299 299 def find_resources(self):
300 300 """Find other resources that need to be in place.
301 301
302 302 Things like cluster directories need to be in place to find the
303 303 config file. These happen right after the IPython directory has
304 304 been set.
305 305 """
306 306 pass
307 307
308 308 def find_config_file_name(self):
309 309 """Find the config file name for this application.
310 310
311 311 This must set ``self.config_file_name`` to the filename of the
312 312 config file to use (just the filename). The search paths for the
313 313 config file are set in :meth:`find_config_file_paths` and then passed
314 314 to the config file loader where they are resolved to an absolute path.
315 315
316 316 If a profile has been set at the command line, this will resolve it.
317 317 """
318
319 318 try:
320 319 self.config_file_name = self.command_line_config.Global.config_file
321 320 except AttributeError:
322 321 pass
322 else:
323 return
323 324
324 325 try:
325 326 self.profile_name = self.command_line_config.Global.profile
326 327 except AttributeError:
327 pass
328 # Just use the default as there is no profile
329 self.config_file_name = self.default_config_file_name
328 330 else:
329 name_parts = self.config_file_name.split('.')
331 # Use the default config file name and profile name if set
332 # to determine the used config file name.
333 name_parts = self.default_config_file_name.split('.')
330 334 name_parts.insert(1, u'_' + self.profile_name + u'.')
331 335 self.config_file_name = ''.join(name_parts)
332 336
333 337 def find_config_file_paths(self):
334 338 """Set the search paths for resolving the config file.
335 339
336 340 This must set ``self.config_file_paths`` to a sequence of search
337 341 paths to pass to the config file loader.
338 342 """
339 343 # Include our own profiles directory last, so that users can still find
340 344 # our shipped copies of builtin profiles even if they don't have them
341 345 # in their local ipython directory.
342 346 prof_dir = os.path.join(get_ipython_package_dir(), 'config', 'profile')
343 347 self.config_file_paths = (os.getcwd(), self.ipython_dir, prof_dir)
344 348
345 349 def pre_load_file_config(self):
346 350 """Do actions before the config file is loaded."""
347 351 pass
348 352
349 353 def load_file_config(self):
350 354 """Load the config file.
351 355
352 356 This tries to load the config file from disk. If successful, the
353 357 ``CONFIG_FILE`` config variable is set to the resolved config file
354 358 location. If not successful, an empty config is used.
355 359 """
356 360 self.log.debug("Attempting to load config file: %s" %
357 361 self.config_file_name)
358 362 loader = PyFileConfigLoader(self.config_file_name,
359 363 path=self.config_file_paths)
360 364 try:
361 365 self.file_config = loader.load_config()
362 366 self.file_config.Global.config_file = loader.full_filename
363 367 except IOError:
364 368 # Only warn if the default config file was NOT being used.
365 369 if not self.config_file_name==self.default_config_file_name:
366 370 self.log.warn("Config file not found, skipping: %s" %
367 371 self.config_file_name, exc_info=True)
368 372 self.file_config = Config()
369 373 except:
370 374 self.log.warn("Error loading config file: %s" %
371 375 self.config_file_name, exc_info=True)
372 376 self.file_config = Config()
373 377
374 378 def set_file_config_log_level(self):
375 379 # We need to keeep self.log_level updated. But we only use the value
376 380 # of the file_config if a value was not specified at the command
377 381 # line, because the command line overrides everything.
378 382 if not hasattr(self.command_line_config.Global, 'log_level'):
379 383 try:
380 384 self.log_level = self.file_config.Global.log_level
381 385 except AttributeError:
382 386 pass # Use existing value
383 387
384 388 def post_load_file_config(self):
385 389 """Do actions after the config file is loaded."""
386 390 pass
387 391
388 392 def log_file_config(self):
389 393 if hasattr(self.file_config.Global, 'config_file'):
390 394 self.log.debug("Config file loaded: %s" %
391 395 self.file_config.Global.config_file)
392 396 self.log.debug(repr(self.file_config))
393 397
394 398 def merge_configs(self):
395 399 """Merge the default, command line and file config objects."""
396 400 config = Config()
397 401 config._merge(self.default_config)
398 402 config._merge(self.file_config)
399 403 config._merge(self.command_line_config)
400 404
401 405 # XXX fperez - propose to Brian we rename master_config to simply
402 406 # config, I think this is going to be heavily used in examples and
403 407 # application code and the name is shorter/easier to find/remember.
404 408 # For now, just alias it...
405 409 self.master_config = config
406 410 self.config = config
407 411
408 412 def log_master_config(self):
409 413 self.log.debug("Master config created:")
410 414 self.log.debug(repr(self.master_config))
411 415
412 416 def pre_construct(self):
413 417 """Do actions after the config has been built, but before construct."""
414 418 pass
415 419
416 420 def construct(self):
417 421 """Construct the main components that make up this app."""
418 422 self.log.debug("Constructing components for application")
419 423
420 424 def post_construct(self):
421 425 """Do actions after construct, but before starting the app."""
422 426 pass
423 427
424 428 def start_app(self):
425 429 """Actually start the app."""
426 430 self.log.debug("Starting application")
427 431
428 432 #-------------------------------------------------------------------------
429 433 # Utility methods
430 434 #-------------------------------------------------------------------------
431 435
432 436 def exit(self, exit_status=0):
433 437 if self._exiting:
434 438 pass
435 439 else:
436 440 self.log.debug("Exiting application: %s" % self.name)
437 441 self._exiting = True
438 442 sys.exit(exit_status)
439 443
440 444 def attempt(self, func):
441 445 try:
442 446 func()
443 447 except SystemExit:
444 448 raise
445 449 except:
446 450 self.log.critical("Aborting application: %s" % self.name,
447 451 exc_info=True)
448 452 self.exit(0)
449 453
@@ -1,654 +1,653 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The :class:`~IPython.core.application.Application` object for the command
5 5 line :command:`ipython` program.
6 6
7 7 Authors
8 8 -------
9 9
10 10 * Brian Granger
11 11 * Fernando Perez
12 12 """
13 13
14 14 #-----------------------------------------------------------------------------
15 15 # Copyright (C) 2008-2010 The IPython Development Team
16 16 #
17 17 # Distributed under the terms of the BSD License. The full license is in
18 18 # the file COPYING, distributed as part of this software.
19 19 #-----------------------------------------------------------------------------
20 20
21 21 #-----------------------------------------------------------------------------
22 22 # Imports
23 23 #-----------------------------------------------------------------------------
24 24
25 25 from __future__ import absolute_import
26 26
27 27 import logging
28 28 import os
29 29 import sys
30 30
31 31 from IPython.core import release
32 32 from IPython.core.crashhandler import CrashHandler
33 33 from IPython.core.application import Application, BaseAppConfigLoader
34 34 from IPython.core.iplib import InteractiveShell
35 35 from IPython.config.loader import (
36 36 Config,
37 37 PyFileConfigLoader
38 38 )
39 39 from IPython.lib import inputhook
40 40 from IPython.utils.path import filefind, get_ipython_dir
41 41 from . import usage
42 42
43 43 #-----------------------------------------------------------------------------
44 44 # Globals, utilities and helpers
45 45 #-----------------------------------------------------------------------------
46 46
47 47 #: The default config file name for this application.
48 48 default_config_file_name = u'ipython_config.py'
49 49
50 50
51 51 class IPAppConfigLoader(BaseAppConfigLoader):
52 52
53 53 def _add_arguments(self):
54 54 super(IPAppConfigLoader, self)._add_arguments()
55 55 paa = self.parser.add_argument
56 56 paa('-p',
57 57 '--profile', dest='Global.profile', type=unicode,
58 58 help=
59 59 """The string name of the ipython profile to be used. Assume that your
60 60 config file is ipython_config-<name>.py (looks in current dir first,
61 61 then in IPYTHON_DIR). This is a quick way to keep and load multiple
62 62 config files for different tasks, especially if include your basic one
63 63 in your more specialized ones. You can keep a basic
64 64 IPYTHON_DIR/ipython_config.py file and then have other 'profiles' which
65 65 include this one and load extra things for particular tasks.""",
66 66 metavar='Global.profile')
67 67 paa('--config-file',
68 68 dest='Global.config_file', type=unicode,
69 69 help=
70 70 """Set the config file name to override default. Normally IPython
71 71 loads ipython_config.py (from current directory) or
72 72 IPYTHON_DIR/ipython_config.py. If the loading of your config file
73 73 fails, IPython starts with a bare bones configuration (no modules
74 74 loaded at all).""",
75 75 metavar='Global.config_file')
76 76 paa('--autocall',
77 77 dest='InteractiveShell.autocall', type=int,
78 78 help=
79 79 """Make IPython automatically call any callable object even if you
80 80 didn't type explicit parentheses. For example, 'str 43' becomes
81 81 'str(43)' automatically. The value can be '0' to disable the feature,
82 82 '1' for 'smart' autocall, where it is not applied if there are no more
83 83 arguments on the line, and '2' for 'full' autocall, where all callable
84 84 objects are automatically called (even if no arguments are present).
85 85 The default is '1'.""",
86 86 metavar='InteractiveShell.autocall')
87 87 paa('--autoindent',
88 88 action='store_true', dest='InteractiveShell.autoindent',
89 89 help='Turn on autoindenting.')
90 90 paa('--no-autoindent',
91 91 action='store_false', dest='InteractiveShell.autoindent',
92 92 help='Turn off autoindenting.')
93 93 paa('--automagic',
94 94 action='store_true', dest='InteractiveShell.automagic',
95 95 help=
96 96 """Turn on the auto calling of magic commands. Type %%magic at the
97 97 IPython prompt for more information.""")
98 98 paa('--no-automagic',
99 99 action='store_false', dest='InteractiveShell.automagic',
100 100 help='Turn off the auto calling of magic commands.')
101 101 paa('--autoedit-syntax',
102 102 action='store_true', dest='InteractiveShell.autoedit_syntax',
103 103 help='Turn on auto editing of files with syntax errors.')
104 104 paa('--no-autoedit-syntax',
105 105 action='store_false', dest='InteractiveShell.autoedit_syntax',
106 106 help='Turn off auto editing of files with syntax errors.')
107 107 paa('--banner',
108 108 action='store_true', dest='Global.display_banner',
109 109 help='Display a banner upon starting IPython.')
110 110 paa('--no-banner',
111 111 action='store_false', dest='Global.display_banner',
112 112 help="Don't display a banner upon starting IPython.")
113 113 paa('--cache-size',
114 114 type=int, dest='InteractiveShell.cache_size',
115 115 help=
116 116 """Set the size of the output cache. The default is 1000, you can
117 117 change it permanently in your config file. Setting it to 0 completely
118 118 disables the caching system, and the minimum value accepted is 20 (if
119 119 you provide a value less than 20, it is reset to 0 and a warning is
120 120 issued). This limit is defined because otherwise you'll spend more
121 121 time re-flushing a too small cache than working""",
122 122 metavar='InteractiveShell.cache_size')
123 123 paa('--classic',
124 124 action='store_true', dest='Global.classic',
125 125 help="Gives IPython a similar feel to the classic Python prompt.")
126 126 paa('--colors',
127 127 type=str, dest='InteractiveShell.colors',
128 128 help="Set the color scheme (NoColor, Linux, and LightBG).",
129 129 metavar='InteractiveShell.colors')
130 130 paa('--color-info',
131 131 action='store_true', dest='InteractiveShell.color_info',
132 132 help=
133 133 """IPython can display information about objects via a set of func-
134 134 tions, and optionally can use colors for this, syntax highlighting
135 135 source code and various other elements. However, because this
136 136 information is passed through a pager (like 'less') and many pagers get
137 137 confused with color codes, this option is off by default. You can test
138 138 it and turn it on permanently in your ipython_config.py file if it
139 139 works for you. Test it and turn it on permanently if it works with
140 140 your system. The magic function %%color_info allows you to toggle this
141 141 inter- actively for testing.""")
142 142 paa('--no-color-info',
143 143 action='store_false', dest='InteractiveShell.color_info',
144 144 help="Disable using colors for info related things.")
145 145 paa('--confirm-exit',
146 146 action='store_true', dest='InteractiveShell.confirm_exit',
147 147 help=
148 148 """Set to confirm when you try to exit IPython with an EOF (Control-D
149 149 in Unix, Control-Z/Enter in Windows). By typing 'exit', 'quit' or
150 150 '%%Exit', you can force a direct exit without any confirmation.""")
151 151 paa('--no-confirm-exit',
152 152 action='store_false', dest='InteractiveShell.confirm_exit',
153 153 help="Don't prompt the user when exiting.")
154 154 paa('--deep-reload',
155 155 action='store_true', dest='InteractiveShell.deep_reload',
156 156 help=
157 157 """Enable deep (recursive) reloading by default. IPython can use the
158 158 deep_reload module which reloads changes in modules recursively (it
159 159 replaces the reload() function, so you don't need to change anything to
160 160 use it). deep_reload() forces a full reload of modules whose code may
161 161 have changed, which the default reload() function does not. When
162 162 deep_reload is off, IPython will use the normal reload(), but
163 163 deep_reload will still be available as dreload(). This fea- ture is off
164 164 by default [which means that you have both normal reload() and
165 165 dreload()].""")
166 166 paa('--no-deep-reload',
167 167 action='store_false', dest='InteractiveShell.deep_reload',
168 168 help="Disable deep (recursive) reloading by default.")
169 169 paa('--editor',
170 170 type=str, dest='InteractiveShell.editor',
171 171 help="Set the editor used by IPython (default to $EDITOR/vi/notepad).",
172 172 metavar='InteractiveShell.editor')
173 173 paa('--log','-l',
174 174 action='store_true', dest='InteractiveShell.logstart',
175 175 help="Start logging to the default log file (./ipython_log.py).")
176 176 paa('--logfile','-lf',
177 177 type=unicode, dest='InteractiveShell.logfile',
178 178 help="Start logging to logfile with this name.",
179 179 metavar='InteractiveShell.logfile')
180 180 paa('--log-append','-la',
181 181 type=unicode, dest='InteractiveShell.logappend',
182 182 help="Start logging to the given file in append mode.",
183 183 metavar='InteractiveShell.logfile')
184 184 paa('--pdb',
185 185 action='store_true', dest='InteractiveShell.pdb',
186 186 help="Enable auto calling the pdb debugger after every exception.")
187 187 paa('--no-pdb',
188 188 action='store_false', dest='InteractiveShell.pdb',
189 189 help="Disable auto calling the pdb debugger after every exception.")
190 190 paa('--pprint',
191 191 action='store_true', dest='InteractiveShell.pprint',
192 192 help="Enable auto pretty printing of results.")
193 193 paa('--no-pprint',
194 194 action='store_false', dest='InteractiveShell.pprint',
195 195 help="Disable auto auto pretty printing of results.")
196 196 paa('--prompt-in1','-pi1',
197 197 type=str, dest='InteractiveShell.prompt_in1',
198 198 help=
199 199 """Set the main input prompt ('In [\#]: '). Note that if you are using
200 200 numbered prompts, the number is represented with a '\#' in the string.
201 201 Don't forget to quote strings with spaces embedded in them. Most
202 202 bash-like escapes can be used to customize IPython's prompts, as well
203 203 as a few additional ones which are IPython-spe- cific. All valid
204 204 prompt escapes are described in detail in the Customization section of
205 205 the IPython manual.""",
206 206 metavar='InteractiveShell.prompt_in1')
207 207 paa('--prompt-in2','-pi2',
208 208 type=str, dest='InteractiveShell.prompt_in2',
209 209 help=
210 210 """Set the secondary input prompt (' .\D.: '). Similar to the previous
211 211 option, but used for the continuation prompts. The special sequence
212 212 '\D' is similar to '\#', but with all digits replaced by dots (so you
213 213 can have your continuation prompt aligned with your input prompt).
214 214 Default: ' .\D.: ' (note three spaces at the start for alignment with
215 215 'In [\#]')""",
216 216 metavar='InteractiveShell.prompt_in2')
217 217 paa('--prompt-out','-po',
218 218 type=str, dest='InteractiveShell.prompt_out',
219 219 help="Set the output prompt ('Out[\#]:')",
220 220 metavar='InteractiveShell.prompt_out')
221 221 paa('--quick',
222 222 action='store_true', dest='Global.quick',
223 223 help="Enable quick startup with no config files.")
224 224 paa('--readline',
225 225 action='store_true', dest='InteractiveShell.readline_use',
226 226 help="Enable readline for command line usage.")
227 227 paa('--no-readline',
228 228 action='store_false', dest='InteractiveShell.readline_use',
229 229 help="Disable readline for command line usage.")
230 230 paa('--screen-length','-sl',
231 231 type=int, dest='InteractiveShell.screen_length',
232 232 help=
233 233 """Number of lines of your screen, used to control printing of very
234 234 long strings. Strings longer than this number of lines will be sent
235 235 through a pager instead of directly printed. The default value for
236 236 this is 0, which means IPython will auto-detect your screen size every
237 237 time it needs to print certain potentially long strings (this doesn't
238 238 change the behavior of the 'print' keyword, it's only triggered
239 239 internally). If for some reason this isn't working well (it needs
240 240 curses support), specify it yourself. Otherwise don't change the
241 241 default.""",
242 242 metavar='InteractiveShell.screen_length')
243 243 paa('--separate-in','-si',
244 244 type=str, dest='InteractiveShell.separate_in',
245 245 help="Separator before input prompts. Default '\\n'.",
246 246 metavar='InteractiveShell.separate_in')
247 247 paa('--separate-out','-so',
248 248 type=str, dest='InteractiveShell.separate_out',
249 249 help="Separator before output prompts. Default 0 (nothing).",
250 250 metavar='InteractiveShell.separate_out')
251 251 paa('--separate-out2','-so2',
252 252 type=str, dest='InteractiveShell.separate_out2',
253 253 help="Separator after output prompts. Default 0 (nonight).",
254 254 metavar='InteractiveShell.separate_out2')
255 paa('-no-sep',
255 paa('--no-sep',
256 256 action='store_true', dest='Global.nosep',
257 257 help="Eliminate all spacing between prompts.")
258 258 paa('--term-title',
259 259 action='store_true', dest='InteractiveShell.term_title',
260 260 help="Enable auto setting the terminal title.")
261 261 paa('--no-term-title',
262 262 action='store_false', dest='InteractiveShell.term_title',
263 263 help="Disable auto setting the terminal title.")
264 264 paa('--xmode',
265 265 type=str, dest='InteractiveShell.xmode',
266 266 help=
267 267 """Exception reporting mode ('Plain','Context','Verbose'). Plain:
268 268 similar to python's normal traceback printing. Context: prints 5 lines
269 269 of context source code around each line in the traceback. Verbose:
270 270 similar to Context, but additionally prints the variables currently
271 271 visible where the exception happened (shortening their strings if too
272 272 long). This can potentially be very slow, if you happen to have a huge
273 273 data structure whose string representation is complex to compute.
274 274 Your computer may appear to freeze for a while with cpu usage at 100%%.
275 275 If this occurs, you can cancel the traceback with Ctrl-C (maybe hitting
276 276 it more than once).
277 277 """,
278 278 metavar='InteractiveShell.xmode')
279 279 paa('--ext',
280 280 type=str, dest='Global.extra_extension',
281 281 help="The dotted module name of an IPython extension to load.",
282 282 metavar='Global.extra_extension')
283 283 paa('-c',
284 284 type=str, dest='Global.code_to_run',
285 285 help="Execute the given command string.",
286 286 metavar='Global.code_to_run')
287 287 paa('-i',
288 288 action='store_true', dest='Global.force_interact',
289 289 help=
290 290 "If running code from the command line, become interactive afterwards.")
291 291
292 292 # Options to start with GUI control enabled from the beginning
293 293 paa('--gui',
294 294 type=str, dest='Global.gui',
295 295 help="Enable GUI event loop integration ('qt', 'wx', 'gtk').",
296 296 metavar='gui-mode')
297 297 paa('--pylab','-pylab',
298 298 type=str, dest='Global.pylab',
299 299 nargs='?', const='auto', metavar='gui-mode',
300 300 help="Pre-load matplotlib and numpy for interactive use. "+
301 301 "If no value is given, the gui backend is matplotlib's, else use "+
302 302 "one of: ['tk', 'qt', 'wx', 'gtk'].")
303 303
304 304 # Legacy GUI options. Leave them in for backwards compatibility, but the
305 305 # 'thread' names are really a misnomer now.
306 306 paa('--wthread', '-wthread',
307 307 action='store_true', dest='Global.wthread',
308 308 help=
309 309 """Enable wxPython event loop integration. (DEPRECATED, use --gui wx)""")
310 310 paa('--q4thread', '--qthread', '-q4thread', '-qthread',
311 311 action='store_true', dest='Global.q4thread',
312 312 help=
313 313 """Enable Qt4 event loop integration. Qt3 is no longer supported.
314 314 (DEPRECATED, use --gui qt)""")
315 315 paa('--gthread', '-gthread',
316 316 action='store_true', dest='Global.gthread',
317 317 help=
318 318 """Enable GTK event loop integration. (DEPRECATED, use --gui gtk)""")
319 319
320 320
321 321 #-----------------------------------------------------------------------------
322 322 # Crash handler for this application
323 323 #-----------------------------------------------------------------------------
324 324
325 325
326 326 _message_template = """\
327 327 Oops, $self.app_name crashed. We do our best to make it stable, but...
328 328
329 329 A crash report was automatically generated with the following information:
330 330 - A verbatim copy of the crash traceback.
331 331 - A copy of your input history during this session.
332 332 - Data on your current $self.app_name configuration.
333 333
334 334 It was left in the file named:
335 335 \t'$self.crash_report_fname'
336 336 If you can email this file to the developers, the information in it will help
337 337 them in understanding and correcting the problem.
338 338
339 339 You can mail it to: $self.contact_name at $self.contact_email
340 340 with the subject '$self.app_name Crash Report'.
341 341
342 342 If you want to do it now, the following command will work (under Unix):
343 343 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
344 344
345 345 To ensure accurate tracking of this issue, please file a report about it at:
346 346 $self.bug_tracker
347 347 """
348 348
349 349 class IPAppCrashHandler(CrashHandler):
350 350 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
351 351
352 352 message_template = _message_template
353 353
354 354 def __init__(self, app):
355 355 contact_name = release.authors['Fernando'][0]
356 356 contact_email = release.authors['Fernando'][1]
357 357 bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug'
358 358 super(IPAppCrashHandler,self).__init__(
359 359 app, contact_name, contact_email, bug_tracker
360 360 )
361 361
362 362 def make_report(self,traceback):
363 363 """Return a string containing a crash report."""
364 364
365 365 sec_sep = self.section_sep
366 366 # Start with parent report
367 367 report = [super(IPAppCrashHandler, self).make_report(traceback)]
368 368 # Add interactive-specific info we may have
369 369 rpt_add = report.append
370 370 try:
371 371 rpt_add(sec_sep+"History of session input:")
372 372 for line in self.app.shell.user_ns['_ih']:
373 373 rpt_add(line)
374 374 rpt_add('\n*** Last line of input (may not be in above history):\n')
375 375 rpt_add(self.app.shell._last_input_line+'\n')
376 376 except:
377 377 pass
378 378
379 379 return ''.join(report)
380 380
381 381
382 382 #-----------------------------------------------------------------------------
383 383 # Main classes and functions
384 384 #-----------------------------------------------------------------------------
385 385
386 386 class IPythonApp(Application):
387 387 name = u'ipython'
388 388 #: argparse formats better the 'usage' than the 'description' field
389 389 description = None
390 390 usage = usage.cl_usage
391 391 command_line_loader = IPAppConfigLoader
392 config_file_name = default_config_file_name
392 default_config_file_name = default_config_file_name
393 393 crash_handler_class = IPAppCrashHandler
394 394
395 395 def create_default_config(self):
396 396 super(IPythonApp, self).create_default_config()
397 397 # Eliminate multiple lookups
398 398 Global = self.default_config.Global
399 399
400 400 # Set all default values
401 401 Global.display_banner = True
402 402
403 403 # If the -c flag is given or a file is given to run at the cmd line
404 404 # like "ipython foo.py", normally we exit without starting the main
405 405 # loop. The force_interact config variable allows a user to override
406 406 # this and interact. It is also set by the -i cmd line flag, just
407 407 # like Python.
408 408 Global.force_interact = False
409 409
410 410 # By default always interact by starting the IPython mainloop.
411 411 Global.interact = True
412 412
413 413 # No GUI integration by default
414 414 Global.gui = False
415 415 # Pylab off by default
416 416 Global.pylab = False
417 417
418 418 # Deprecated versions of gui support that used threading, we support
419 419 # them just for bacwards compatibility as an alternate spelling for
420 420 # '--gui X'
421 421 Global.qthread = False
422 422 Global.q4thread = False
423 423 Global.wthread = False
424 424 Global.gthread = False
425 425
426 426 def load_file_config(self):
427 427 if hasattr(self.command_line_config.Global, 'quick'):
428 428 if self.command_line_config.Global.quick:
429 429 self.file_config = Config()
430 430 return
431 431 super(IPythonApp, self).load_file_config()
432 432
433 433 def post_load_file_config(self):
434 434 if hasattr(self.command_line_config.Global, 'extra_extension'):
435 435 if not hasattr(self.file_config.Global, 'extensions'):
436 436 self.file_config.Global.extensions = []
437 437 self.file_config.Global.extensions.append(
438 438 self.command_line_config.Global.extra_extension)
439 439 del self.command_line_config.Global.extra_extension
440 440
441 441 def pre_construct(self):
442 442 config = self.master_config
443 443
444 444 if hasattr(config.Global, 'classic'):
445 445 if config.Global.classic:
446 446 config.InteractiveShell.cache_size = 0
447 447 config.InteractiveShell.pprint = 0
448 448 config.InteractiveShell.prompt_in1 = '>>> '
449 449 config.InteractiveShell.prompt_in2 = '... '
450 450 config.InteractiveShell.prompt_out = ''
451 451 config.InteractiveShell.separate_in = \
452 452 config.InteractiveShell.separate_out = \
453 453 config.InteractiveShell.separate_out2 = ''
454 454 config.InteractiveShell.colors = 'NoColor'
455 455 config.InteractiveShell.xmode = 'Plain'
456 456
457 457 if hasattr(config.Global, 'nosep'):
458 458 if config.Global.nosep:
459 459 config.InteractiveShell.separate_in = \
460 460 config.InteractiveShell.separate_out = \
461 461 config.InteractiveShell.separate_out2 = ''
462 462
463 463 # if there is code of files to run from the cmd line, don't interact
464 464 # unless the -i flag (Global.force_interact) is true.
465 465 code_to_run = config.Global.get('code_to_run','')
466 466 file_to_run = False
467 467 if self.extra_args and self.extra_args[0]:
468 468 file_to_run = True
469 469 if file_to_run or code_to_run:
470 470 if not config.Global.force_interact:
471 471 config.Global.interact = False
472 472
473 473 def construct(self):
474 474 # I am a little hesitant to put these into InteractiveShell itself.
475 475 # But that might be the place for them
476 476 sys.path.insert(0, '')
477 477
478 478 # Create an InteractiveShell instance
479 479 self.shell = InteractiveShell(None, self.master_config)
480 480
481 481 def post_construct(self):
482 482 """Do actions after construct, but before starting the app."""
483 483 config = self.master_config
484 484
485 485 # shell.display_banner should always be False for the terminal
486 486 # based app, because we call shell.show_banner() by hand below
487 487 # so the banner shows *before* all extension loading stuff.
488 488 self.shell.display_banner = False
489
490 489 if config.Global.display_banner and \
491 490 config.Global.interact:
492 491 self.shell.show_banner()
493 492
494 493 # Make sure there is a space below the banner.
495 494 if self.log_level <= logging.INFO: print
496 495
497 496 # Now a variety of things that happen after the banner is printed.
498 497 self._enable_gui_pylab()
499 498 self._load_extensions()
500 499 self._run_exec_lines()
501 500 self._run_exec_files()
502 501 self._run_cmd_line_code()
503 502
504 503 def _enable_gui_pylab(self):
505 504 """Enable GUI event loop integration, taking pylab into account."""
506 505 Global = self.master_config.Global
507 506
508 507 # Select which gui to use
509 508 if Global.gui:
510 509 gui = Global.gui
511 510 # The following are deprecated, but there's likely to be a lot of use
512 511 # of this form out there, so we might as well support it for now. But
513 512 # the --gui option above takes precedence.
514 513 elif Global.wthread:
515 514 gui = inputhook.GUI_WX
516 515 elif Global.qthread:
517 516 gui = inputhook.GUI_QT
518 517 elif Global.gthread:
519 518 gui = inputhook.GUI_GTK
520 519 else:
521 520 gui = None
522 521
523 522 # Using --pylab will also require gui activation, though which toolkit
524 523 # to use may be chosen automatically based on mpl configuration.
525 524 if Global.pylab:
526 525 activate = self.shell.enable_pylab
527 526 if Global.pylab == 'auto':
528 527 gui = None
529 528 else:
530 529 gui = Global.pylab
531 530 else:
532 531 # Enable only GUI integration, no pylab
533 532 activate = inputhook.enable_gui
534 533
535 534 if gui or Global.pylab:
536 535 try:
537 536 self.log.info("Enabling GUI event loop integration, "
538 537 "toolkit=%s, pylab=%s" % (gui, Global.pylab) )
539 538 activate(gui)
540 539 except:
541 540 self.log.warn("Error in enabling GUI event loop integration:")
542 541 self.shell.showtraceback()
543 542
544 543 def _load_extensions(self):
545 544 """Load all IPython extensions in Global.extensions.
546 545
547 546 This uses the :meth:`InteractiveShell.load_extensions` to load all
548 547 the extensions listed in ``self.master_config.Global.extensions``.
549 548 """
550 549 try:
551 550 if hasattr(self.master_config.Global, 'extensions'):
552 551 self.log.debug("Loading IPython extensions...")
553 552 extensions = self.master_config.Global.extensions
554 553 for ext in extensions:
555 554 try:
556 555 self.log.info("Loading IPython extension: %s" % ext)
557 556 self.shell.load_extension(ext)
558 557 except:
559 558 self.log.warn("Error in loading extension: %s" % ext)
560 559 self.shell.showtraceback()
561 560 except:
562 561 self.log.warn("Unknown error in loading extensions:")
563 562 self.shell.showtraceback()
564 563
565 564 def _run_exec_lines(self):
566 565 """Run lines of code in Global.exec_lines in the user's namespace."""
567 566 try:
568 567 if hasattr(self.master_config.Global, 'exec_lines'):
569 568 self.log.debug("Running code from Global.exec_lines...")
570 569 exec_lines = self.master_config.Global.exec_lines
571 570 for line in exec_lines:
572 571 try:
573 572 self.log.info("Running code in user namespace: %s" % line)
574 573 self.shell.runlines(line)
575 574 except:
576 575 self.log.warn("Error in executing line in user namespace: %s" % line)
577 576 self.shell.showtraceback()
578 577 except:
579 578 self.log.warn("Unknown error in handling Global.exec_lines:")
580 579 self.shell.showtraceback()
581 580
582 581 def _exec_file(self, fname):
583 582 full_filename = filefind(fname, [u'.', self.ipython_dir])
584 583 if os.path.isfile(full_filename):
585 584 if full_filename.endswith(u'.py'):
586 585 self.log.info("Running file in user namespace: %s" % full_filename)
587 586 self.shell.safe_execfile(full_filename, self.shell.user_ns)
588 587 elif full_filename.endswith('.ipy'):
589 588 self.log.info("Running file in user namespace: %s" % full_filename)
590 589 self.shell.safe_execfile_ipy(full_filename)
591 590 else:
592 591 self.log.warn("File does not have a .py or .ipy extension: <%s>" % full_filename)
593 592
594 593 def _run_exec_files(self):
595 594 try:
596 595 if hasattr(self.master_config.Global, 'exec_files'):
597 596 self.log.debug("Running files in Global.exec_files...")
598 597 exec_files = self.master_config.Global.exec_files
599 598 for fname in exec_files:
600 599 self._exec_file(fname)
601 600 except:
602 601 self.log.warn("Unknown error in handling Global.exec_files:")
603 602 self.shell.showtraceback()
604 603
605 604 def _run_cmd_line_code(self):
606 605 if hasattr(self.master_config.Global, 'code_to_run'):
607 606 line = self.master_config.Global.code_to_run
608 607 try:
609 608 self.log.info("Running code given at command line (-c): %s" % line)
610 609 self.shell.runlines(line)
611 610 except:
612 611 self.log.warn("Error in executing line in user namespace: %s" % line)
613 612 self.shell.showtraceback()
614 613 return
615 614 # Like Python itself, ignore the second if the first of these is present
616 615 try:
617 616 fname = self.extra_args[0]
618 617 except:
619 618 pass
620 619 else:
621 620 try:
622 621 self._exec_file(fname)
623 622 except:
624 623 self.log.warn("Error in executing file in user namespace: %s" % fname)
625 624 self.shell.showtraceback()
626 625
627 626 def start_app(self):
628 627 if self.master_config.Global.interact:
629 628 self.log.debug("Starting IPython's mainloop...")
630 629 self.shell.mainloop()
631 630 else:
632 631 self.log.debug("IPython not interactive, start_app is no-op...")
633 632
634 633
635 634 def load_default_config(ipython_dir=None):
636 635 """Load the default config file from the default ipython_dir.
637 636
638 637 This is useful for embedded shells.
639 638 """
640 639 if ipython_dir is None:
641 640 ipython_dir = get_ipython_dir()
642 641 cl = PyFileConfigLoader(default_config_file_name, ipython_dir)
643 642 config = cl.load_config()
644 643 return config
645 644
646 645
647 646 def launch_new_instance():
648 647 """Create and run a full blown IPython instance"""
649 648 app = IPythonApp()
650 649 app.start()
651 650
652 651
653 652 if __name__ == '__main__':
654 653 launch_new_instance()
@@ -1,537 +1,539 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The IPython cluster directory
5 5 """
6 6
7 7 #-----------------------------------------------------------------------------
8 8 # Copyright (C) 2008-2009 The IPython Development Team
9 9 #
10 10 # Distributed under the terms of the BSD License. The full license is in
11 11 # the file COPYING, distributed as part of this software.
12 12 #-----------------------------------------------------------------------------
13 13
14 14 #-----------------------------------------------------------------------------
15 15 # Imports
16 16 #-----------------------------------------------------------------------------
17 17
18 18 from __future__ import with_statement
19 19
20 20 import os
21 21 import shutil
22 22 import sys
23 23 import warnings
24 24
25 25 from twisted.python import log
26 26
27 27 from IPython.config.loader import PyFileConfigLoader
28 28 from IPython.core.application import Application, BaseAppConfigLoader
29 29 from IPython.core.component import Component
30 30 from IPython.core.crashhandler import CrashHandler
31 31 from IPython.core import release
32 32 from IPython.utils.path import (
33 33 get_ipython_package_dir,
34 34 expand_path
35 35 )
36 36 from IPython.utils.traitlets import Unicode
37 37
38 38 #-----------------------------------------------------------------------------
39 39 # Warnings control
40 40 #-----------------------------------------------------------------------------
41 41 # Twisted generates annoying warnings with Python 2.6, as will do other code
42 42 # that imports 'sets' as of today
43 43 warnings.filterwarnings('ignore', 'the sets module is deprecated',
44 44 DeprecationWarning )
45 45
46 46 # This one also comes from Twisted
47 47 warnings.filterwarnings('ignore', 'the sha module is deprecated',
48 48 DeprecationWarning)
49 49
50 50 #-----------------------------------------------------------------------------
51 51 # Module errors
52 52 #-----------------------------------------------------------------------------
53 53
54 54 class ClusterDirError(Exception):
55 55 pass
56 56
57 57
58 58 class PIDFileError(Exception):
59 59 pass
60 60
61 61
62 62 #-----------------------------------------------------------------------------
63 63 # Class for managing cluster directories
64 64 #-----------------------------------------------------------------------------
65 65
66 66 class ClusterDir(Component):
67 67 """An object to manage the cluster directory and its resources.
68 68
69 69 The cluster directory is used by :command:`ipcontroller`,
70 70 :command:`ipcontroller` and :command:`ipcontroller` to manage the
71 71 configuration, logging and security of these applications.
72 72
73 73 This object knows how to find, create and manage these directories. This
74 74 should be used by any code that want's to handle cluster directories.
75 75 """
76 76
77 77 security_dir_name = Unicode('security')
78 78 log_dir_name = Unicode('log')
79 79 pid_dir_name = Unicode('pid')
80 80 security_dir = Unicode(u'')
81 81 log_dir = Unicode(u'')
82 82 pid_dir = Unicode(u'')
83 83 location = Unicode(u'')
84 84
85 85 def __init__(self, location):
86 86 super(ClusterDir, self).__init__(None)
87 87 self.location = location
88 88
89 89 def _location_changed(self, name, old, new):
90 90 if not os.path.isdir(new):
91 91 os.makedirs(new)
92 92 self.security_dir = os.path.join(new, self.security_dir_name)
93 93 self.log_dir = os.path.join(new, self.log_dir_name)
94 94 self.pid_dir = os.path.join(new, self.pid_dir_name)
95 95 self.check_dirs()
96 96
97 97 def _log_dir_changed(self, name, old, new):
98 98 self.check_log_dir()
99 99
100 100 def check_log_dir(self):
101 101 if not os.path.isdir(self.log_dir):
102 102 os.mkdir(self.log_dir)
103 103
104 104 def _security_dir_changed(self, name, old, new):
105 105 self.check_security_dir()
106 106
107 107 def check_security_dir(self):
108 108 if not os.path.isdir(self.security_dir):
109 109 os.mkdir(self.security_dir, 0700)
110 110 os.chmod(self.security_dir, 0700)
111 111
112 112 def _pid_dir_changed(self, name, old, new):
113 113 self.check_pid_dir()
114 114
115 115 def check_pid_dir(self):
116 116 if not os.path.isdir(self.pid_dir):
117 117 os.mkdir(self.pid_dir, 0700)
118 118 os.chmod(self.pid_dir, 0700)
119 119
120 120 def check_dirs(self):
121 121 self.check_security_dir()
122 122 self.check_log_dir()
123 123 self.check_pid_dir()
124 124
125 125 def load_config_file(self, filename):
126 126 """Load a config file from the top level of the cluster dir.
127 127
128 128 Parameters
129 129 ----------
130 130 filename : unicode or str
131 131 The filename only of the config file that must be located in
132 132 the top-level of the cluster directory.
133 133 """
134 134 loader = PyFileConfigLoader(filename, self.location)
135 135 return loader.load_config()
136 136
137 137 def copy_config_file(self, config_file, path=None, overwrite=False):
138 138 """Copy a default config file into the active cluster directory.
139 139
140 140 Default configuration files are kept in :mod:`IPython.config.default`.
141 141 This function moves these from that location to the working cluster
142 142 directory.
143 143 """
144 144 if path is None:
145 145 import IPython.config.default
146 146 path = IPython.config.default.__file__.split(os.path.sep)[:-1]
147 147 path = os.path.sep.join(path)
148 148 src = os.path.join(path, config_file)
149 149 dst = os.path.join(self.location, config_file)
150 150 if not os.path.isfile(dst) or overwrite:
151 151 shutil.copy(src, dst)
152 152
153 153 def copy_all_config_files(self, path=None, overwrite=False):
154 154 """Copy all config files into the active cluster directory."""
155 155 for f in [u'ipcontroller_config.py', u'ipengine_config.py',
156 156 u'ipcluster_config.py']:
157 157 self.copy_config_file(f, path=path, overwrite=overwrite)
158 158
159 159 @classmethod
160 160 def create_cluster_dir(csl, cluster_dir):
161 161 """Create a new cluster directory given a full path.
162 162
163 163 Parameters
164 164 ----------
165 165 cluster_dir : str
166 166 The full path to the cluster directory. If it does exist, it will
167 167 be used. If not, it will be created.
168 168 """
169 169 return ClusterDir(cluster_dir)
170 170
171 171 @classmethod
172 172 def create_cluster_dir_by_profile(cls, path, profile=u'default'):
173 173 """Create a cluster dir by profile name and path.
174 174
175 175 Parameters
176 176 ----------
177 177 path : str
178 178 The path (directory) to put the cluster directory in.
179 179 profile : str
180 180 The name of the profile. The name of the cluster directory will
181 181 be "cluster_<profile>".
182 182 """
183 183 if not os.path.isdir(path):
184 184 raise ClusterDirError('Directory not found: %s' % path)
185 185 cluster_dir = os.path.join(path, u'cluster_' + profile)
186 186 return ClusterDir(cluster_dir)
187 187
188 188 @classmethod
189 189 def find_cluster_dir_by_profile(cls, ipython_dir, profile=u'default'):
190 190 """Find an existing cluster dir by profile name, return its ClusterDir.
191 191
192 192 This searches through a sequence of paths for a cluster dir. If it
193 193 is not found, a :class:`ClusterDirError` exception will be raised.
194 194
195 195 The search path algorithm is:
196 196 1. ``os.getcwd()``
197 197 2. ``ipython_dir``
198 198 3. The directories found in the ":" separated
199 199 :env:`IPCLUSTER_DIR_PATH` environment variable.
200 200
201 201 Parameters
202 202 ----------
203 203 ipython_dir : unicode or str
204 204 The IPython directory to use.
205 205 profile : unicode or str
206 206 The name of the profile. The name of the cluster directory
207 207 will be "cluster_<profile>".
208 208 """
209 209 dirname = u'cluster_' + profile
210 210 cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
211 211 if cluster_dir_paths:
212 212 cluster_dir_paths = cluster_dir_paths.split(':')
213 213 else:
214 214 cluster_dir_paths = []
215 215 paths = [os.getcwd(), ipython_dir] + cluster_dir_paths
216 216 for p in paths:
217 217 cluster_dir = os.path.join(p, dirname)
218 218 if os.path.isdir(cluster_dir):
219 219 return ClusterDir(cluster_dir)
220 220 else:
221 221 raise ClusterDirError('Cluster directory not found in paths: %s' % dirname)
222 222
223 223 @classmethod
224 224 def find_cluster_dir(cls, cluster_dir):
225 225 """Find/create a cluster dir and return its ClusterDir.
226 226
227 227 This will create the cluster directory if it doesn't exist.
228 228
229 229 Parameters
230 230 ----------
231 231 cluster_dir : unicode or str
232 232 The path of the cluster directory. This is expanded using
233 233 :func:`IPython.utils.genutils.expand_path`.
234 234 """
235 235 cluster_dir = expand_path(cluster_dir)
236 236 if not os.path.isdir(cluster_dir):
237 237 raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
238 238 return ClusterDir(cluster_dir)
239 239
240 240
241 241 #-----------------------------------------------------------------------------
242 242 # Command line options
243 243 #-----------------------------------------------------------------------------
244 244
245 245 class ClusterDirConfigLoader(BaseAppConfigLoader):
246 246
247 247 def _add_cluster_profile(self, parser):
248 248 paa = parser.add_argument
249 249 paa('-p', '--profile',
250 250 dest='Global.profile',type=unicode,
251 251 help=
252 252 """The string name of the profile to be used. This determines the name
253 253 of the cluster dir as: cluster_<profile>. The default profile is named
254 254 'default'. The cluster directory is resolve this way if the
255 255 --cluster-dir option is not used.""",
256 256 metavar='Global.profile')
257 257
258 258 def _add_cluster_dir(self, parser):
259 259 paa = parser.add_argument
260 260 paa('--cluster-dir',
261 261 dest='Global.cluster_dir',type=unicode,
262 262 help="""Set the cluster dir. This overrides the logic used by the
263 263 --profile option.""",
264 264 metavar='Global.cluster_dir')
265 265
266 266 def _add_work_dir(self, parser):
267 267 paa = parser.add_argument
268 268 paa('--work-dir',
269 269 dest='Global.work_dir',type=unicode,
270 270 help='Set the working dir for the process.',
271 271 metavar='Global.work_dir')
272 272
273 273 def _add_clean_logs(self, parser):
274 274 paa = parser.add_argument
275 275 paa('--clean-logs',
276 276 dest='Global.clean_logs', action='store_true',
277 277 help='Delete old log flies before starting.')
278 278
279 279 def _add_no_clean_logs(self, parser):
280 280 paa = parser.add_argument
281 281 paa('--no-clean-logs',
282 282 dest='Global.clean_logs', action='store_false',
283 283 help="Don't Delete old log flies before starting.")
284 284
285 285 def _add_arguments(self):
286 286 super(ClusterDirConfigLoader, self)._add_arguments()
287 287 self._add_cluster_profile(self.parser)
288 288 self._add_cluster_dir(self.parser)
289 289 self._add_work_dir(self.parser)
290 290 self._add_clean_logs(self.parser)
291 291 self._add_no_clean_logs(self.parser)
292 292
293 293
294 294 #-----------------------------------------------------------------------------
295 295 # Crash handler for this application
296 296 #-----------------------------------------------------------------------------
297 297
298 298
299 299 _message_template = """\
300 300 Oops, $self.app_name crashed. We do our best to make it stable, but...
301 301
302 302 A crash report was automatically generated with the following information:
303 303 - A verbatim copy of the crash traceback.
304 304 - Data on your current $self.app_name configuration.
305 305
306 306 It was left in the file named:
307 307 \t'$self.crash_report_fname'
308 308 If you can email this file to the developers, the information in it will help
309 309 them in understanding and correcting the problem.
310 310
311 311 You can mail it to: $self.contact_name at $self.contact_email
312 312 with the subject '$self.app_name Crash Report'.
313 313
314 314 If you want to do it now, the following command will work (under Unix):
315 315 mail -s '$self.app_name Crash Report' $self.contact_email < $self.crash_report_fname
316 316
317 317 To ensure accurate tracking of this issue, please file a report about it at:
318 318 $self.bug_tracker
319 319 """
320 320
321 321 class ClusterDirCrashHandler(CrashHandler):
322 322 """sys.excepthook for IPython itself, leaves a detailed report on disk."""
323 323
324 324 message_template = _message_template
325 325
326 326 def __init__(self, app):
327 327 contact_name = release.authors['Brian'][0]
328 328 contact_email = release.authors['Brian'][1]
329 329 bug_tracker = 'https://bugs.launchpad.net/ipython/+filebug'
330 330 super(ClusterDirCrashHandler,self).__init__(
331 331 app, contact_name, contact_email, bug_tracker
332 332 )
333 333
334 334
335 335 #-----------------------------------------------------------------------------
336 336 # Main application
337 337 #-----------------------------------------------------------------------------
338 338
339 339 class ApplicationWithClusterDir(Application):
340 340 """An application that puts everything into a cluster directory.
341 341
342 342 Instead of looking for things in the ipython_dir, this type of application
343 343 will use its own private directory called the "cluster directory"
344 344 for things like config files, log files, etc.
345 345
346 346 The cluster directory is resolved as follows:
347 347
348 348 * If the ``--cluster-dir`` option is given, it is used.
349 349 * If ``--cluster-dir`` is not given, the application directory is
350 350 resolve using the profile name as ``cluster_<profile>``. The search
351 351 path for this directory is then i) cwd if it is found there
352 352 and ii) in ipython_dir otherwise.
353 353
354 354 The config file for the application is to be put in the cluster
355 355 dir and named the value of the ``config_file_name`` class attribute.
356 356 """
357 357
358 358 command_line_loader = ClusterDirConfigLoader
359 359 crash_handler_class = ClusterDirCrashHandler
360 360 auto_create_cluster_dir = True
361 361
362 362 def create_default_config(self):
363 363 super(ApplicationWithClusterDir, self).create_default_config()
364 364 self.default_config.Global.profile = u'default'
365 365 self.default_config.Global.cluster_dir = u''
366 366 self.default_config.Global.work_dir = os.getcwd()
367 367 self.default_config.Global.log_to_file = False
368 368 self.default_config.Global.clean_logs = False
369 369
370 370 def find_resources(self):
371 371 """This resolves the cluster directory.
372 372
373 373 This tries to find the cluster directory and if successful, it will
374 374 have done:
375 375 * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for
376 376 the application.
377 377 * Sets ``self.cluster_dir`` attribute of the application and config
378 378 objects.
379 379
380 380 The algorithm used for this is as follows:
381 381 1. Try ``Global.cluster_dir``.
382 382 2. Try using ``Global.profile``.
383 383 3. If both of these fail and ``self.auto_create_cluster_dir`` is
384 384 ``True``, then create the new cluster dir in the IPython directory.
385 385 4. If all fails, then raise :class:`ClusterDirError`.
386 386 """
387 387
388 388 try:
389 389 cluster_dir = self.command_line_config.Global.cluster_dir
390 390 except AttributeError:
391 391 cluster_dir = self.default_config.Global.cluster_dir
392 392 cluster_dir = expand_path(cluster_dir)
393 393 try:
394 394 self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
395 395 except ClusterDirError:
396 396 pass
397 397 else:
398 398 self.log.info('Using existing cluster dir: %s' % \
399 399 self.cluster_dir_obj.location
400 400 )
401 401 self.finish_cluster_dir()
402 402 return
403 403
404 404 try:
405 405 self.profile = self.command_line_config.Global.profile
406 406 except AttributeError:
407 407 self.profile = self.default_config.Global.profile
408 408 try:
409 409 self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
410 410 self.ipython_dir, self.profile)
411 411 except ClusterDirError:
412 412 pass
413 413 else:
414 414 self.log.info('Using existing cluster dir: %s' % \
415 415 self.cluster_dir_obj.location
416 416 )
417 417 self.finish_cluster_dir()
418 418 return
419 419
420 420 if self.auto_create_cluster_dir:
421 421 self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
422 422 self.ipython_dir, self.profile
423 423 )
424 424 self.log.info('Creating new cluster dir: %s' % \
425 425 self.cluster_dir_obj.location
426 426 )
427 427 self.finish_cluster_dir()
428 428 else:
429 429 raise ClusterDirError('Could not find a valid cluster directory.')
430 430
431 431 def finish_cluster_dir(self):
432 432 # Set the cluster directory
433 433 self.cluster_dir = self.cluster_dir_obj.location
434 434
435 435 # These have to be set because they could be different from the one
436 436 # that we just computed. Because command line has the highest
437 437 # priority, this will always end up in the master_config.
438 438 self.default_config.Global.cluster_dir = self.cluster_dir
439 439 self.command_line_config.Global.cluster_dir = self.cluster_dir
440 440
441 441 def find_config_file_name(self):
442 442 """Find the config file name for this application."""
443 443 # For this type of Application it should be set as a class attribute.
444 if not hasattr(self, 'config_file_name'):
444 if not hasattr(self, 'default_config_file_name'):
445 445 self.log.critical("No config filename found")
446 else:
447 self.config_file_name = self.default_config_file_name
446 448
447 449 def find_config_file_paths(self):
448 450 # Set the search path to to the cluster directory. We should NOT
449 451 # include IPython.config.default here as the default config files
450 452 # are ALWAYS automatically moved to the cluster directory.
451 453 conf_dir = os.path.join(get_ipython_package_dir(), 'config', 'default')
452 454 self.config_file_paths = (self.cluster_dir,)
453 455
454 456 def pre_construct(self):
455 457 # The log and security dirs were set earlier, but here we put them
456 458 # into the config and log them.
457 459 config = self.master_config
458 460 sdir = self.cluster_dir_obj.security_dir
459 461 self.security_dir = config.Global.security_dir = sdir
460 462 ldir = self.cluster_dir_obj.log_dir
461 463 self.log_dir = config.Global.log_dir = ldir
462 464 pdir = self.cluster_dir_obj.pid_dir
463 465 self.pid_dir = config.Global.pid_dir = pdir
464 466 self.log.info("Cluster directory set to: %s" % self.cluster_dir)
465 467 config.Global.work_dir = unicode(expand_path(config.Global.work_dir))
466 468 # Change to the working directory. We do this just before construct
467 469 # is called so all the components there have the right working dir.
468 470 self.to_work_dir()
469 471
470 472 def to_work_dir(self):
471 473 wd = self.master_config.Global.work_dir
472 474 if unicode(wd) != unicode(os.getcwd()):
473 475 os.chdir(wd)
474 476 self.log.info("Changing to working dir: %s" % wd)
475 477
476 478 def start_logging(self):
477 479 # Remove old log files
478 480 if self.master_config.Global.clean_logs:
479 481 log_dir = self.master_config.Global.log_dir
480 482 for f in os.listdir(log_dir):
481 483 if f.startswith(self.name + u'-') and f.endswith('.log'):
482 484 os.remove(os.path.join(log_dir, f))
483 485 # Start logging to the new log file
484 486 if self.master_config.Global.log_to_file:
485 487 log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
486 488 logfile = os.path.join(self.log_dir, log_filename)
487 489 open_log_file = open(logfile, 'w')
488 490 else:
489 491 open_log_file = sys.stdout
490 492 log.startLogging(open_log_file)
491 493
492 494 def write_pid_file(self, overwrite=False):
493 495 """Create a .pid file in the pid_dir with my pid.
494 496
495 497 This must be called after pre_construct, which sets `self.pid_dir`.
496 498 This raises :exc:`PIDFileError` if the pid file exists already.
497 499 """
498 500 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
499 501 if os.path.isfile(pid_file):
500 502 pid = self.get_pid_from_file()
501 503 if not overwrite:
502 504 raise PIDFileError(
503 505 'The pid file [%s] already exists. \nThis could mean that this '
504 506 'server is already running with [pid=%s].' % (pid_file, pid)
505 507 )
506 508 with open(pid_file, 'w') as f:
507 509 self.log.info("Creating pid file: %s" % pid_file)
508 510 f.write(repr(os.getpid())+'\n')
509 511
510 512 def remove_pid_file(self):
511 513 """Remove the pid file.
512 514
513 515 This should be called at shutdown by registering a callback with
514 516 :func:`reactor.addSystemEventTrigger`. This needs to return
515 517 ``None``.
516 518 """
517 519 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
518 520 if os.path.isfile(pid_file):
519 521 try:
520 522 self.log.info("Removing pid file: %s" % pid_file)
521 523 os.remove(pid_file)
522 524 except:
523 525 self.log.warn("Error removing the pid file: %s" % pid_file)
524 526
525 527 def get_pid_from_file(self):
526 528 """Get the pid from the pid file.
527 529
528 530 If the pid file doesn't exist a :exc:`PIDFileError` is raised.
529 531 """
530 532 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
531 533 if os.path.isfile(pid_file):
532 534 with open(pid_file, 'r') as f:
533 535 pid = int(f.read().strip())
534 536 return pid
535 537 else:
536 538 raise PIDFileError('pid file not found: %s' % pid_file)
537 539
@@ -1,273 +1,277 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 Foolscap related utilities.
5 5 """
6 6
7 7 #-----------------------------------------------------------------------------
8 8 # Copyright (C) 2008-2009 The IPython Development Team
9 9 #
10 10 # Distributed under the terms of the BSD License. The full license is in
11 11 # the file COPYING, distributed as part of this software.
12 12 #-----------------------------------------------------------------------------
13 13
14 14 #-----------------------------------------------------------------------------
15 15 # Imports
16 16 #-----------------------------------------------------------------------------
17 17
18 18 from __future__ import with_statement
19 19
20 20 import os
21 21 import tempfile
22 22
23 23 from twisted.internet import reactor, defer
24 24 from twisted.python import log
25 25
26 26 from foolscap import Tub, UnauthenticatedTub
27 27
28 28 from IPython.config.loader import Config
29 29
30 30 from IPython.kernel.configobjfactory import AdaptedConfiguredObjectFactory
31 31
32 32 from IPython.kernel.error import SecurityError
33 33
34 34 from IPython.utils.traitlets import Int, Str, Bool, Instance
35 35 from IPython.utils.importstring import import_item
36 36
37 37 #-----------------------------------------------------------------------------
38 38 # Code
39 39 #-----------------------------------------------------------------------------
40 40
41 41
42 42 # We do this so if a user doesn't have OpenSSL installed, it will try to use
43 43 # an UnauthenticatedTub. But, they will still run into problems if they
44 44 # try to use encrypted furls.
45 45 try:
46 46 import OpenSSL
47 47 except:
48 48 Tub = UnauthenticatedTub
49 49 have_crypto = False
50 50 else:
51 51 have_crypto = True
52 52
53 53
54 54 class FURLError(Exception):
55 55 pass
56 56
57 57
58 58 def check_furl_file_security(furl_file, secure):
59 59 """Remove the old furl_file if changing security modes."""
60 60 if os.path.isfile(furl_file):
61 61 f = open(furl_file, 'r')
62 62 oldfurl = f.read().strip()
63 63 f.close()
64 64 if (oldfurl.startswith('pb://') and not secure) or (oldfurl.startswith('pbu://') and secure):
65 65 os.remove(furl_file)
66 66
67 67
68 68 def is_secure(furl):
69 69 """Is the given FURL secure or not."""
70 70 if is_valid(furl):
71 71 if furl.startswith("pb://"):
72 72 return True
73 73 elif furl.startswith("pbu://"):
74 74 return False
75 75 else:
76 76 raise FURLError("invalid FURL: %s" % furl)
77 77
78 78
79 79 def is_valid(furl):
80 80 """Is the str a valid FURL or not."""
81 81 if isinstance(furl, str):
82 82 if furl.startswith("pb://") or furl.startswith("pbu://"):
83 83 return True
84 84 else:
85 85 return False
86 86
87 87
88 88 def find_furl(furl_or_file):
89 89 """Find, validate and return a FURL in a string or file."""
90 90 if isinstance(furl_or_file, str):
91 91 if is_valid(furl_or_file):
92 92 return furl_or_file
93 93 if os.path.isfile(furl_or_file):
94 94 with open(furl_or_file, 'r') as f:
95 95 furl = f.read().strip()
96 96 if is_valid(furl):
97 97 return furl
98 98 raise FURLError("Not a valid FURL or FURL file: %s" % furl_or_file)
99 99
100 100
101 101 def is_valid_furl_or_file(furl_or_file):
102 102 """Validate a FURL or a FURL file.
103 103
104 104 If ``furl_or_file`` looks like a file, we simply make sure its directory
105 105 exists and that it has a ``.furl`` file extension. We don't try to see
106 106 if the FURL file exists or to read its contents. This is useful for
107 107 cases where auto re-connection is being used.
108 108 """
109 109 if isinstance(furl_or_file, str):
110 110 if is_valid(furl_or_file):
111 111 return True
112 112 if isinstance(furl_or_file, (str, unicode)):
113 113 path, furl_filename = os.path.split(furl_or_file)
114 114 if os.path.isdir(path) and furl_filename.endswith('.furl'):
115 115 return True
116 116 return False
117 117
118 118
119 119 def validate_furl_or_file(furl_or_file):
120 120 if not is_valid_furl_or_file(furl_or_file):
121 121 raise FURLError('Not a valid FURL or FURL file: %r' % furl_or_file)
122 122
123 123
124 124 def get_temp_furlfile(filename):
125 125 """Return a temporary FURL file."""
126 126 return tempfile.mktemp(dir=os.path.dirname(filename),
127 127 prefix=os.path.basename(filename))
128 128
129 129
130 130 def make_tub(ip, port, secure, cert_file):
131 131 """Create a listening tub given an ip, port, and cert_file location.
132 132
133 133 Parameters
134 134 ----------
135 135 ip : str
136 136 The ip address or hostname that the tub should listen on.
137 137 Empty means all interfaces.
138 138 port : int
139 139 The port that the tub should listen on. A value of 0 means
140 140 pick a random port
141 141 secure: bool
142 142 Will the connection be secure (in the Foolscap sense).
143 143 cert_file: str
144 144 A filename of a file to be used for theSSL certificate.
145 145
146 146 Returns
147 147 -------
148 148 A tub, listener tuple.
149 149 """
150 150 if secure:
151 151 if have_crypto:
152 152 tub = Tub(certFile=cert_file)
153 153 else:
154 154 raise SecurityError("OpenSSL/pyOpenSSL is not available, so we "
155 155 "can't run in secure mode. Try running without "
156 156 "security using 'ipcontroller -xy'.")
157 157 else:
158 158 tub = UnauthenticatedTub()
159 159
160 160 # Set the strport based on the ip and port and start listening
161 161 if ip == '':
162 162 strport = "tcp:%i" % port
163 163 else:
164 164 strport = "tcp:%i:interface=%s" % (port, ip)
165 165 log.msg("Starting listener with [secure=%r] on: %s" % (secure, strport))
166 166 listener = tub.listenOn(strport)
167 167
168 168 return tub, listener
169 169
170 170
171 171 class FCServiceFactory(AdaptedConfiguredObjectFactory):
172 172 """This class creates a tub with various services running in it.
173 173
174 174 The basic idea is that :meth:`create` returns a running :class:`Tub`
175 175 instance that has a number of Foolscap references registered in it.
176 176 This class is a subclass of :class:`IPython.core.component.Component`
177 177 so the IPython configuration and component system are used.
178 178
179 179 Attributes
180 180 ----------
181 181 interfaces : Config
182 182 A Config instance whose values are sub-Config objects having two
183 183 keys: furl_file and interface_chain.
184 184
185 185 The other attributes are the standard ones for Foolscap.
186 186 """
187 187
188 188 ip = Str('', config=True)
189 189 port = Int(0, config=True)
190 190 secure = Bool(True, config=True)
191 191 cert_file = Str('', config=True)
192 192 location = Str('', config=True)
193 193 reuse_furls = Bool(False, config=True)
194 194 interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True)
195 195
196 196 def __init__(self, config, adaptee):
197 197 super(FCServiceFactory, self).__init__(config, adaptee)
198 198 self._check_reuse_furls()
199 199
200 200 def _ip_changed(self, name, old, new):
201 201 if new == 'localhost' or new == '127.0.0.1':
202 202 self.location = '127.0.0.1'
203 203
204 204 def _check_reuse_furls(self):
205 205 furl_files = [i.furl_file for i in self.interfaces.values()]
206 206 for ff in furl_files:
207 207 fullfile = self._get_security_file(ff)
208 208 if self.reuse_furls:
209 209 if self.port==0:
210 210 raise FURLError("You are trying to reuse the FURL file "
211 211 "for this connection, but the port for this connection "
212 212 "is set to 0 (autoselect). To reuse the FURL file "
213 213 "you need to specify specific port to listen on."
214 214 )
215 215 else:
216 216 log.msg("Reusing FURL file: %s" % fullfile)
217 217 else:
218 218 if os.path.isfile(fullfile):
219 219 log.msg("Removing old FURL file: %s" % fullfile)
220 220 os.remove(fullfile)
221 221
222 222 def _get_security_file(self, filename):
223 223 return os.path.join(self.config.Global.security_dir, filename)
224 224
225 225 def create(self):
226 226 """Create and return the Foolscap tub with everything running."""
227 227
228 228 self.tub, self.listener = make_tub(
229 229 self.ip, self.port, self.secure,
230 230 self._get_security_file(self.cert_file)
231 231 )
232 232 # log.msg("Interfaces to register [%r]: %r" % \
233 233 # (self.__class__, self.interfaces))
234 234 if not self.secure:
235 235 log.msg("WARNING: running with no security: %s" % \
236 236 self.__class__.__name__)
237 237 reactor.callWhenRunning(self.set_location_and_register)
238 238 return self.tub
239 239
240 240 def set_location_and_register(self):
241 241 """Set the location for the tub and return a deferred."""
242 242
243 243 if self.location == '':
244 244 d = self.tub.setLocationAutomatically()
245 245 else:
246 246 d = defer.maybeDeferred(self.tub.setLocation,
247 247 "%s:%i" % (self.location, self.listener.getPortnum()))
248 248 self.adapt_to_interfaces(d)
249 249
250 250 def adapt_to_interfaces(self, d):
251 251 """Run through the interfaces, adapt and register."""
252 252
253 253 for ifname, ifconfig in self.interfaces.iteritems():
254 254 ff = self._get_security_file(ifconfig.furl_file)
255 255 log.msg("Adapting [%s] to interface: %s" % \
256 256 (self.adaptee.__class__.__name__, ifname))
257 257 log.msg("Saving FURL for interface [%s] to file: %s" % (ifname, ff))
258 258 check_furl_file_security(ff, self.secure)
259 259 adaptee = self.adaptee
260 260 for i in ifconfig.interface_chain:
261 261 adaptee = import_item(i)(adaptee)
262 262 d.addCallback(self.register, adaptee, furl_file=ff)
263 263
264 264 def register(self, empty, ref, furl_file):
265 265 """Register the reference with the FURL file.
266 266
267 267 The FURL file is created and then moved to make sure that when the
268 file appears, the buffer has been flushed and the file closed.
268 file appears, the buffer has been flushed and the file closed. This
269 is not done if we are re-using FURLS however.
269 270 """
271 if self.reuse_furls:
272 self.tub.registerReference(ref, furlFile=furl_file)
273 else:
270 274 temp_furl_file = get_temp_furlfile(furl_file)
271 275 self.tub.registerReference(ref, furlFile=temp_furl_file)
272 276 os.rename(temp_furl_file, furl_file)
273 277
@@ -1,499 +1,499 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The ipcluster application.
5 5 """
6 6
7 7 #-----------------------------------------------------------------------------
8 8 # Copyright (C) 2008-2009 The IPython Development Team
9 9 #
10 10 # Distributed under the terms of the BSD License. The full license is in
11 11 # the file COPYING, distributed as part of this software.
12 12 #-----------------------------------------------------------------------------
13 13
14 14 #-----------------------------------------------------------------------------
15 15 # Imports
16 16 #-----------------------------------------------------------------------------
17 17
18 18 import logging
19 19 import os
20 20 import signal
21 21
22 22 if os.name=='posix':
23 23 from twisted.scripts._twistd_unix import daemonize
24 24
25 25 from twisted.internet import reactor, defer
26 26 from twisted.python import log, failure
27 27
28 28
29 29 from IPython.external.argparse import ArgumentParser, SUPPRESS
30 30 from IPython.utils.importstring import import_item
31 31 from IPython.kernel.clusterdir import (
32 32 ApplicationWithClusterDir, ClusterDirConfigLoader,
33 33 ClusterDirError, PIDFileError
34 34 )
35 35
36 36
37 37 #-----------------------------------------------------------------------------
38 38 # Module level variables
39 39 #-----------------------------------------------------------------------------
40 40
41 41
42 42 default_config_file_name = u'ipcluster_config.py'
43 43
44 44
45 45 _description = """\
46 46 Start an IPython cluster for parallel computing.\n\n
47 47
48 48 An IPython cluster consists of 1 controller and 1 or more engines.
49 49 This command automates the startup of these processes using a wide
50 50 range of startup methods (SSH, local processes, PBS, mpiexec,
51 51 Windows HPC Server 2008). To start a cluster with 4 engines on your
52 52 local host simply do 'ipcluster start -n 4'. For more complex usage
53 53 you will typically do 'ipcluster create -p mycluster', then edit
54 54 configuration files, followed by 'ipcluster start -p mycluster -n 4'.
55 55 """
56 56
57 57
58 58 # Exit codes for ipcluster
59 59
60 60 # This will be the exit code if the ipcluster appears to be running because
61 61 # a .pid file exists
62 62 ALREADY_STARTED = 10
63 63
64 64
65 65 # This will be the exit code if ipcluster stop is run, but there is not .pid
66 66 # file to be found.
67 67 ALREADY_STOPPED = 11
68 68
69 69
70 70 #-----------------------------------------------------------------------------
71 71 # Command line options
72 72 #-----------------------------------------------------------------------------
73 73
74 74
75 75 class IPClusterAppConfigLoader(ClusterDirConfigLoader):
76 76
77 77 def _add_arguments(self):
78 78 # Don't call ClusterDirConfigLoader._add_arguments as we don't want
79 79 # its defaults on self.parser. Instead, we will put those on
80 80 # default options on our subparsers.
81 81
82 82 # This has all the common options that all subcommands use
83 83 parent_parser1 = ArgumentParser(
84 84 add_help=False,
85 85 argument_default=SUPPRESS
86 86 )
87 87 self._add_ipython_dir(parent_parser1)
88 88 self._add_log_level(parent_parser1)
89 89
90 90 # This has all the common options that other subcommands use
91 91 parent_parser2 = ArgumentParser(
92 92 add_help=False,
93 93 argument_default=SUPPRESS
94 94 )
95 95 self._add_cluster_profile(parent_parser2)
96 96 self._add_cluster_dir(parent_parser2)
97 97 self._add_work_dir(parent_parser2)
98 98 paa = parent_parser2.add_argument
99 99 paa('--log-to-file',
100 100 action='store_true', dest='Global.log_to_file',
101 101 help='Log to a file in the log directory (default is stdout)')
102 102
103 103 # Create the object used to create the subparsers.
104 104 subparsers = self.parser.add_subparsers(
105 105 dest='Global.subcommand',
106 106 title='ipcluster subcommands',
107 107 description=
108 108 """ipcluster has a variety of subcommands. The general way of
109 109 running ipcluster is 'ipcluster <cmd> [options]'. To get help
110 110 on a particular subcommand do 'ipcluster <cmd> -h'."""
111 111 # help="For more help, type 'ipcluster <cmd> -h'",
112 112 )
113 113
114 114 # The "list" subcommand parser
115 115 parser_list = subparsers.add_parser(
116 116 'list',
117 117 parents=[parent_parser1],
118 118 argument_default=SUPPRESS,
119 119 help="List all clusters in cwd and ipython_dir.",
120 120 description=
121 121 """List all available clusters, by cluster directory, that can
122 122 be found in the current working directly or in the ipython
123 123 directory. Cluster directories are named using the convention
124 124 'cluster_<profile>'."""
125 125 )
126 126
127 127 # The "create" subcommand parser
128 128 parser_create = subparsers.add_parser(
129 129 'create',
130 130 parents=[parent_parser1, parent_parser2],
131 131 argument_default=SUPPRESS,
132 132 help="Create a new cluster directory.",
133 133 description=
134 134 """Create an ipython cluster directory by its profile name or
135 135 cluster directory path. Cluster directories contain
136 136 configuration, log and security related files and are named
137 137 using the convention 'cluster_<profile>'. By default they are
138 138 located in your ipython directory. Once created, you will
139 139 probably need to edit the configuration files in the cluster
140 140 directory to configure your cluster. Most users will create a
141 141 cluster directory by profile name,
142 142 'ipcluster create -p mycluster', which will put the directory
143 143 in '<ipython_dir>/cluster_mycluster'.
144 144 """
145 145 )
146 146 paa = parser_create.add_argument
147 147 paa('--reset-config',
148 148 dest='Global.reset_config', action='store_true',
149 149 help=
150 150 """Recopy the default config files to the cluster directory.
151 151 You will loose any modifications you have made to these files.""")
152 152
153 153 # The "start" subcommand parser
154 154 parser_start = subparsers.add_parser(
155 155 'start',
156 156 parents=[parent_parser1, parent_parser2],
157 157 argument_default=SUPPRESS,
158 158 help="Start a cluster.",
159 159 description=
160 160 """Start an ipython cluster by its profile name or cluster
161 161 directory. Cluster directories contain configuration, log and
162 162 security related files and are named using the convention
163 163 'cluster_<profile>' and should be creating using the 'start'
164 164 subcommand of 'ipcluster'. If your cluster directory is in
165 165 the cwd or the ipython directory, you can simply refer to it
166 166 using its profile name, 'ipcluster start -n 4 -p <profile>`,
167 167 otherwise use the '--cluster-dir' option.
168 168 """
169 169 )
170 170 paa = parser_start.add_argument
171 171 paa('-n', '--number',
172 172 type=int, dest='Global.n',
173 173 help='The number of engines to start.',
174 174 metavar='Global.n')
175 175 paa('--clean-logs',
176 176 dest='Global.clean_logs', action='store_true',
177 177 help='Delete old log flies before starting.')
178 178 paa('--no-clean-logs',
179 179 dest='Global.clean_logs', action='store_false',
180 180 help="Don't delete old log flies before starting.")
181 181 paa('--daemon',
182 182 dest='Global.daemonize', action='store_true',
183 183 help='Daemonize the ipcluster program. This implies --log-to-file')
184 184 paa('--no-daemon',
185 185 dest='Global.daemonize', action='store_false',
186 186 help="Dont't daemonize the ipcluster program.")
187 187
188 188 # The "stop" subcommand parser
189 189 parser_stop = subparsers.add_parser(
190 190 'stop',
191 191 parents=[parent_parser1, parent_parser2],
192 192 argument_default=SUPPRESS,
193 193 help="Stop a running cluster.",
194 194 description=
195 195 """Stop a running ipython cluster by its profile name or cluster
196 196 directory. Cluster directories are named using the convention
197 197 'cluster_<profile>'. If your cluster directory is in
198 198 the cwd or the ipython directory, you can simply refer to it
199 199 using its profile name, 'ipcluster stop -p <profile>`, otherwise
200 200 use the '--cluster-dir' option.
201 201 """
202 202 )
203 203 paa = parser_stop.add_argument
204 204 paa('--signal',
205 205 dest='Global.signal', type=int,
206 206 help="The signal number to use in stopping the cluster (default=2).",
207 207 metavar="Global.signal")
208 208
209 209
210 210 #-----------------------------------------------------------------------------
211 211 # Main application
212 212 #-----------------------------------------------------------------------------
213 213
214 214
215 215 class IPClusterApp(ApplicationWithClusterDir):
216 216
217 217 name = u'ipcluster'
218 218 description = _description
219 219 usage = None
220 220 command_line_loader = IPClusterAppConfigLoader
221 config_file_name = default_config_file_name
221 default_config_file_name = default_config_file_name
222 222 default_log_level = logging.INFO
223 223 auto_create_cluster_dir = False
224 224
225 225 def create_default_config(self):
226 226 super(IPClusterApp, self).create_default_config()
227 227 self.default_config.Global.controller_launcher = \
228 228 'IPython.kernel.launcher.LocalControllerLauncher'
229 229 self.default_config.Global.engine_launcher = \
230 230 'IPython.kernel.launcher.LocalEngineSetLauncher'
231 231 self.default_config.Global.n = 2
232 232 self.default_config.Global.reset_config = False
233 233 self.default_config.Global.clean_logs = True
234 234 self.default_config.Global.signal = 2
235 235 self.default_config.Global.daemonize = False
236 236
237 237 def find_resources(self):
238 238 subcommand = self.command_line_config.Global.subcommand
239 239 if subcommand=='list':
240 240 self.list_cluster_dirs()
241 241 # Exit immediately because there is nothing left to do.
242 242 self.exit()
243 243 elif subcommand=='create':
244 244 self.auto_create_cluster_dir = True
245 245 super(IPClusterApp, self).find_resources()
246 246 elif subcommand=='start' or subcommand=='stop':
247 247 self.auto_create_cluster_dir = True
248 248 try:
249 249 super(IPClusterApp, self).find_resources()
250 250 except ClusterDirError:
251 251 raise ClusterDirError(
252 252 "Could not find a cluster directory. A cluster dir must "
253 253 "be created before running 'ipcluster start'. Do "
254 254 "'ipcluster create -h' or 'ipcluster list -h' for more "
255 255 "information about creating and listing cluster dirs."
256 256 )
257 257
258 258 def list_cluster_dirs(self):
259 259 # Find the search paths
260 260 cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
261 261 if cluster_dir_paths:
262 262 cluster_dir_paths = cluster_dir_paths.split(':')
263 263 else:
264 264 cluster_dir_paths = []
265 265 try:
266 266 ipython_dir = self.command_line_config.Global.ipython_dir
267 267 except AttributeError:
268 268 ipython_dir = self.default_config.Global.ipython_dir
269 269 paths = [os.getcwd(), ipython_dir] + \
270 270 cluster_dir_paths
271 271 paths = list(set(paths))
272 272
273 273 self.log.info('Searching for cluster dirs in paths: %r' % paths)
274 274 for path in paths:
275 275 files = os.listdir(path)
276 276 for f in files:
277 277 full_path = os.path.join(path, f)
278 278 if os.path.isdir(full_path) and f.startswith('cluster_'):
279 279 profile = full_path.split('_')[-1]
280 280 start_cmd = 'ipcluster start -p %s -n 4' % profile
281 281 print start_cmd + " ==> " + full_path
282 282
283 283 def pre_construct(self):
284 284 # IPClusterApp.pre_construct() is where we cd to the working directory.
285 285 super(IPClusterApp, self).pre_construct()
286 286 config = self.master_config
287 287 try:
288 288 daemon = config.Global.daemonize
289 289 if daemon:
290 290 config.Global.log_to_file = True
291 291 except AttributeError:
292 292 pass
293 293
294 294 def construct(self):
295 295 config = self.master_config
296 296 subcmd = config.Global.subcommand
297 297 reset = config.Global.reset_config
298 298 if subcmd == 'list':
299 299 return
300 300 if subcmd == 'create':
301 301 self.log.info('Copying default config files to cluster directory '
302 302 '[overwrite=%r]' % (reset,))
303 303 self.cluster_dir_obj.copy_all_config_files(overwrite=reset)
304 304 if subcmd =='start':
305 305 self.cluster_dir_obj.copy_all_config_files(overwrite=False)
306 306 self.start_logging()
307 307 reactor.callWhenRunning(self.start_launchers)
308 308
309 309 def start_launchers(self):
310 310 config = self.master_config
311 311
312 312 # Create the launchers. In both bases, we set the work_dir of
313 313 # the launcher to the cluster_dir. This is where the launcher's
314 314 # subprocesses will be launched. It is not where the controller
315 315 # and engine will be launched.
316 316 el_class = import_item(config.Global.engine_launcher)
317 317 self.engine_launcher = el_class(
318 318 work_dir=self.cluster_dir, config=config
319 319 )
320 320 cl_class = import_item(config.Global.controller_launcher)
321 321 self.controller_launcher = cl_class(
322 322 work_dir=self.cluster_dir, config=config
323 323 )
324 324
325 325 # Setup signals
326 326 signal.signal(signal.SIGINT, self.sigint_handler)
327 327
328 328 # Setup the observing of stopping. If the controller dies, shut
329 329 # everything down as that will be completely fatal for the engines.
330 330 d1 = self.controller_launcher.observe_stop()
331 331 d1.addCallback(self.stop_launchers)
332 332 # But, we don't monitor the stopping of engines. An engine dying
333 333 # is just fine and in principle a user could start a new engine.
334 334 # Also, if we did monitor engine stopping, it is difficult to
335 335 # know what to do when only some engines die. Currently, the
336 336 # observing of engine stopping is inconsistent. Some launchers
337 337 # might trigger on a single engine stopping, other wait until
338 338 # all stop. TODO: think more about how to handle this.
339 339
340 340 # Start the controller and engines
341 341 self._stopping = False # Make sure stop_launchers is not called 2x.
342 342 d = self.start_controller()
343 343 d.addCallback(self.start_engines)
344 344 d.addCallback(self.startup_message)
345 345 # If the controller or engines fail to start, stop everything
346 346 d.addErrback(self.stop_launchers)
347 347 return d
348 348
349 349 def startup_message(self, r=None):
350 350 log.msg("IPython cluster: started")
351 351 return r
352 352
353 353 def start_controller(self, r=None):
354 354 # log.msg("In start_controller")
355 355 config = self.master_config
356 356 d = self.controller_launcher.start(
357 357 cluster_dir=config.Global.cluster_dir
358 358 )
359 359 return d
360 360
361 361 def start_engines(self, r=None):
362 362 # log.msg("In start_engines")
363 363 config = self.master_config
364 364 d = self.engine_launcher.start(
365 365 config.Global.n,
366 366 cluster_dir=config.Global.cluster_dir
367 367 )
368 368 return d
369 369
370 370 def stop_controller(self, r=None):
371 371 # log.msg("In stop_controller")
372 372 if self.controller_launcher.running:
373 373 d = self.controller_launcher.stop()
374 374 d.addErrback(self.log_err)
375 375 return d
376 376 else:
377 377 return defer.succeed(None)
378 378
379 379 def stop_engines(self, r=None):
380 380 # log.msg("In stop_engines")
381 381 if self.engine_launcher.running:
382 382 d = self.engine_launcher.stop()
383 383 d.addErrback(self.log_err)
384 384 return d
385 385 else:
386 386 return defer.succeed(None)
387 387
388 388 def log_err(self, f):
389 389 log.msg(f.getTraceback())
390 390 return None
391 391
392 392 def stop_launchers(self, r=None):
393 393 if not self._stopping:
394 394 self._stopping = True
395 395 if isinstance(r, failure.Failure):
396 396 log.msg('Unexpected error in ipcluster:')
397 397 log.msg(r.getTraceback())
398 398 log.msg("IPython cluster: stopping")
399 399 # These return deferreds. We are not doing anything with them
400 400 # but we are holding refs to them as a reminder that they
401 401 # do return deferreds.
402 402 d1 = self.stop_engines()
403 403 d2 = self.stop_controller()
404 404 # Wait a few seconds to let things shut down.
405 405 reactor.callLater(4.0, reactor.stop)
406 406
407 407 def sigint_handler(self, signum, frame):
408 408 self.stop_launchers()
409 409
410 410 def start_logging(self):
411 411 # Remove old log files of the controller and engine
412 412 if self.master_config.Global.clean_logs:
413 413 log_dir = self.master_config.Global.log_dir
414 414 for f in os.listdir(log_dir):
415 415 if f.startswith('ipengine' + '-'):
416 416 if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
417 417 os.remove(os.path.join(log_dir, f))
418 418 if f.startswith('ipcontroller' + '-'):
419 419 if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
420 420 os.remove(os.path.join(log_dir, f))
421 421 # This will remote old log files for ipcluster itself
422 422 super(IPClusterApp, self).start_logging()
423 423
424 424 def start_app(self):
425 425 """Start the application, depending on what subcommand is used."""
426 426 subcmd = self.master_config.Global.subcommand
427 427 if subcmd=='create' or subcmd=='list':
428 428 return
429 429 elif subcmd=='start':
430 430 self.start_app_start()
431 431 elif subcmd=='stop':
432 432 self.start_app_stop()
433 433
434 434 def start_app_start(self):
435 435 """Start the app for the start subcommand."""
436 436 config = self.master_config
437 437 # First see if the cluster is already running
438 438 try:
439 439 pid = self.get_pid_from_file()
440 440 except PIDFileError:
441 441 pass
442 442 else:
443 443 self.log.critical(
444 444 'Cluster is already running with [pid=%s]. '
445 445 'use "ipcluster stop" to stop the cluster.' % pid
446 446 )
447 447 # Here I exit with a unusual exit status that other processes
448 448 # can watch for to learn how I existed.
449 449 self.exit(ALREADY_STARTED)
450 450
451 451 # Now log and daemonize
452 452 self.log.info(
453 453 'Starting ipcluster with [daemon=%r]' % config.Global.daemonize
454 454 )
455 455 # TODO: Get daemonize working on Windows or as a Windows Server.
456 456 if config.Global.daemonize:
457 457 if os.name=='posix':
458 458 daemonize()
459 459
460 460 # Now write the new pid file AFTER our new forked pid is active.
461 461 self.write_pid_file()
462 462 reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
463 463 reactor.run()
464 464
465 465 def start_app_stop(self):
466 466 """Start the app for the stop subcommand."""
467 467 config = self.master_config
468 468 try:
469 469 pid = self.get_pid_from_file()
470 470 except PIDFileError:
471 471 self.log.critical(
472 472 'Problem reading pid file, cluster is probably not running.'
473 473 )
474 474 # Here I exit with a unusual exit status that other processes
475 475 # can watch for to learn how I existed.
476 476 self.exit(ALREADY_STOPPED)
477 477 else:
478 478 if os.name=='posix':
479 479 sig = config.Global.signal
480 480 self.log.info(
481 481 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
482 482 )
483 483 os.kill(pid, sig)
484 484 elif os.name=='nt':
485 485 # As of right now, we don't support daemonize on Windows, so
486 486 # stop will not do anything. Minimally, it should clean up the
487 487 # old .pid files.
488 488 self.remove_pid_file()
489 489
490 490
491 491 def launch_new_instance():
492 492 """Create and run the IPython cluster."""
493 493 app = IPClusterApp()
494 494 app.start()
495 495
496 496
497 497 if __name__ == '__main__':
498 498 launch_new_instance()
499 499
@@ -1,260 +1,271 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The IPython controller application.
5 5 """
6 6
7 7 #-----------------------------------------------------------------------------
8 8 # Copyright (C) 2008-2009 The IPython Development Team
9 9 #
10 10 # Distributed under the terms of the BSD License. The full license is in
11 11 # the file COPYING, distributed as part of this software.
12 12 #-----------------------------------------------------------------------------
13 13
14 14 #-----------------------------------------------------------------------------
15 15 # Imports
16 16 #-----------------------------------------------------------------------------
17 17
18 18 from __future__ import with_statement
19 19
20 20 import copy
21 21 import sys
22 22
23 23 from twisted.application import service
24 24 from twisted.internet import reactor
25 25 from twisted.python import log
26 26
27 27 from IPython.config.loader import Config
28 28 from IPython.kernel import controllerservice
29 29 from IPython.kernel.clusterdir import (
30 30 ApplicationWithClusterDir,
31 31 ClusterDirConfigLoader
32 32 )
33 from IPython.kernel.fcutil import FCServiceFactory
33 from IPython.kernel.fcutil import FCServiceFactory, FURLError
34 34 from IPython.utils.traitlets import Instance, Unicode
35 35
36 36
37 37 #-----------------------------------------------------------------------------
38 38 # Module level variables
39 39 #-----------------------------------------------------------------------------
40 40
41 41
42 42 #: The default config file name for this application
43 43 default_config_file_name = u'ipcontroller_config.py'
44 44
45 45
46 46 _description = """Start the IPython controller for parallel computing.
47 47
48 48 The IPython controller provides a gateway between the IPython engines and
49 49 clients. The controller needs to be started before the engines and can be
50 50 configured using command line options or using a cluster directory. Cluster
51 51 directories contain config, log and security files and are usually located in
52 52 your .ipython directory and named as "cluster_<profile>". See the --profile
53 53 and --cluster-dir options for details.
54 54 """
55 55
56 56 #-----------------------------------------------------------------------------
57 57 # Default interfaces
58 58 #-----------------------------------------------------------------------------
59 59
60 60 # The default client interfaces for FCClientServiceFactory.interfaces
61 61 default_client_interfaces = Config()
62 62 default_client_interfaces.Task.interface_chain = [
63 63 'IPython.kernel.task.ITaskController',
64 64 'IPython.kernel.taskfc.IFCTaskController'
65 65 ]
66 66
67 67 default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
68 68
69 69 default_client_interfaces.MultiEngine.interface_chain = [
70 70 'IPython.kernel.multiengine.IMultiEngine',
71 71 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
72 72 ]
73 73
74 74 default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
75 75
76 76 # Make this a dict we can pass to Config.__init__ for the default
77 77 default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
78 78
79 79
80 80
81 81 # The default engine interfaces for FCEngineServiceFactory.interfaces
82 82 default_engine_interfaces = Config()
83 83 default_engine_interfaces.Default.interface_chain = [
84 84 'IPython.kernel.enginefc.IFCControllerBase'
85 85 ]
86 86
87 87 default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
88 88
89 89 # Make this a dict we can pass to Config.__init__ for the default
90 90 default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
91 91
92 92
93 93 #-----------------------------------------------------------------------------
94 94 # Service factories
95 95 #-----------------------------------------------------------------------------
96 96
97 97
98 98 class FCClientServiceFactory(FCServiceFactory):
99 99 """A Foolscap implementation of the client services."""
100 100
101 101 cert_file = Unicode(u'ipcontroller-client.pem', config=True)
102 102 interfaces = Instance(klass=Config, kw=default_client_interfaces,
103 103 allow_none=False, config=True)
104 104
105 105
106 106 class FCEngineServiceFactory(FCServiceFactory):
107 107 """A Foolscap implementation of the engine services."""
108 108
109 109 cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
110 110 interfaces = Instance(klass=dict, kw=default_engine_interfaces,
111 111 allow_none=False, config=True)
112 112
113 113
114 114 #-----------------------------------------------------------------------------
115 115 # Command line options
116 116 #-----------------------------------------------------------------------------
117 117
118 118
119 119 class IPControllerAppConfigLoader(ClusterDirConfigLoader):
120 120
121 121 def _add_arguments(self):
122 122 super(IPControllerAppConfigLoader, self)._add_arguments()
123 123 paa = self.parser.add_argument
124 124 # Client config
125 125 paa('--client-ip',
126 126 type=str, dest='FCClientServiceFactory.ip',
127 127 help='The IP address or hostname the controller will listen on for '
128 128 'client connections.',
129 129 metavar='FCClientServiceFactory.ip')
130 130 paa('--client-port',
131 131 type=int, dest='FCClientServiceFactory.port',
132 132 help='The port the controller will listen on for client connections. '
133 133 'The default is to use 0, which will autoselect an open port.',
134 134 metavar='FCClientServiceFactory.port')
135 135 paa('--client-location',), dict(
136 136 type=str, dest='FCClientServiceFactory.location',
137 137 help='The hostname or IP that clients should connect to. This does '
138 138 'not control which interface the controller listens on. Instead, this '
139 139 'determines the hostname/IP that is listed in the FURL, which is how '
140 140 'clients know where to connect. Useful if the controller is listening '
141 141 'on multiple interfaces.',
142 142 metavar='FCClientServiceFactory.location')
143 143 # Engine config
144 144 paa('--engine-ip',
145 145 type=str, dest='FCEngineServiceFactory.ip',
146 146 help='The IP address or hostname the controller will listen on for '
147 147 'engine connections.',
148 148 metavar='FCEngineServiceFactory.ip')
149 149 paa('--engine-port',
150 150 type=int, dest='FCEngineServiceFactory.port',
151 151 help='The port the controller will listen on for engine connections. '
152 152 'The default is to use 0, which will autoselect an open port.',
153 153 metavar='FCEngineServiceFactory.port')
154 154 paa('--engine-location',
155 155 type=str, dest='FCEngineServiceFactory.location',
156 156 help='The hostname or IP that engines should connect to. This does '
157 157 'not control which interface the controller listens on. Instead, this '
158 158 'determines the hostname/IP that is listed in the FURL, which is how '
159 159 'engines know where to connect. Useful if the controller is listening '
160 160 'on multiple interfaces.',
161 161 metavar='FCEngineServiceFactory.location')
162 162 # Global config
163 163 paa('--log-to-file',
164 164 action='store_true', dest='Global.log_to_file',
165 165 help='Log to a file in the log directory (default is stdout)')
166 166 paa('-r','--reuse-furls',
167 167 action='store_true', dest='Global.reuse_furls',
168 168 help='Try to reuse all FURL files. If this is not set all FURL files '
169 169 'are deleted before the controller starts. This must be set if '
170 170 'specific ports are specified by --engine-port or --client-port.')
171 171 paa('--no-secure',
172 172 action='store_false', dest='Global.secure',
173 173 help='Turn off SSL encryption for all connections.')
174 174 paa('--secure',
175 175 action='store_true', dest='Global.secure',
176 176 help='Turn off SSL encryption for all connections.')
177 177
178 178
179 179 #-----------------------------------------------------------------------------
180 180 # The main application
181 181 #-----------------------------------------------------------------------------
182 182
183 183
184 184 class IPControllerApp(ApplicationWithClusterDir):
185 185
186 186 name = u'ipcontroller'
187 187 description = _description
188 188 command_line_loader = IPControllerAppConfigLoader
189 config_file_name = default_config_file_name
189 default_config_file_name = default_config_file_name
190 190 auto_create_cluster_dir = True
191 191
192 192 def create_default_config(self):
193 193 super(IPControllerApp, self).create_default_config()
194 self.default_config.Global.reuse_furls = False
195 self.default_config.Global.secure = True
194 # Don't set defaults for Global.secure or Global.reuse_furls
195 # as those are set in a component.
196 196 self.default_config.Global.import_statements = []
197 197 self.default_config.Global.clean_logs = True
198 198
199 def post_load_command_line_config(self):
200 # Now setup reuse_furls
201 c = self.command_line_config
199 def pre_construct(self):
200 super(IPControllerApp, self).pre_construct()
201 c = self.master_config
202 # The defaults for these are set in FCClientServiceFactory and
203 # FCEngineServiceFactory, so we only set them here if the global
204 # options have be set to override the class level defaults.
202 205 if hasattr(c.Global, 'reuse_furls'):
203 206 c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
204 207 c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
205 208 del c.Global.reuse_furls
206 209 if hasattr(c.Global, 'secure'):
207 210 c.FCClientServiceFactory.secure = c.Global.secure
208 211 c.FCEngineServiceFactory.secure = c.Global.secure
209 212 del c.Global.secure
210 213
211 214 def construct(self):
212 215 # This is the working dir by now.
213 216 sys.path.insert(0, '')
214 217
215 218 self.start_logging()
216 219 self.import_statements()
217 220
218 221 # Create the service hierarchy
219 222 self.main_service = service.MultiService()
220 223 # The controller service
221 224 controller_service = controllerservice.ControllerService()
222 225 controller_service.setServiceParent(self.main_service)
223 226 # The client tub and all its refereceables
227 try:
224 228 csfactory = FCClientServiceFactory(self.master_config, controller_service)
229 except FURLError, e:
230 log.err(e)
231 self.exit(0)
225 232 client_service = csfactory.create()
226 233 client_service.setServiceParent(self.main_service)
227 234 # The engine tub
235 try:
228 236 esfactory = FCEngineServiceFactory(self.master_config, controller_service)
237 except FURLError, e:
238 log.err(e)
239 self.exit(0)
229 240 engine_service = esfactory.create()
230 241 engine_service.setServiceParent(self.main_service)
231 242
232 243 def import_statements(self):
233 244 statements = self.master_config.Global.import_statements
234 245 for s in statements:
235 246 try:
236 247 log.msg("Executing statement: '%s'" % s)
237 248 exec s in globals(), locals()
238 249 except:
239 250 log.msg("Error running statement: %s" % s)
240 251
241 252 def start_app(self):
242 253 # Start the controller service.
243 254 self.main_service.startService()
244 255 # Write the .pid file overwriting old ones. This allow multiple
245 256 # controllers to clober each other. But Windows is not cleaning
246 257 # these up properly.
247 258 self.write_pid_file(overwrite=True)
248 259 # Add a trigger to delete the .pid file upon shutting down.
249 260 reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
250 261 reactor.run()
251 262
252 263
253 264 def launch_new_instance():
254 265 """Create and run the IPython controller"""
255 266 app = IPControllerApp()
256 267 app.start()
257 268
258 269
259 270 if __name__ == '__main__':
260 271 launch_new_instance()
@@ -1,242 +1,242 b''
1 1 #!/usr/bin/env python
2 2 # encoding: utf-8
3 3 """
4 4 The IPython controller application
5 5 """
6 6
7 7 #-----------------------------------------------------------------------------
8 8 # Copyright (C) 2008-2009 The IPython Development Team
9 9 #
10 10 # Distributed under the terms of the BSD License. The full license is in
11 11 # the file COPYING, distributed as part of this software.
12 12 #-----------------------------------------------------------------------------
13 13
14 14 #-----------------------------------------------------------------------------
15 15 # Imports
16 16 #-----------------------------------------------------------------------------
17 17
18 18 import os
19 19 import sys
20 20
21 21 from twisted.application import service
22 22 from twisted.internet import reactor
23 23 from twisted.python import log
24 24
25 25 from IPython.kernel.clusterdir import (
26 26 ApplicationWithClusterDir,
27 27 ClusterDirConfigLoader
28 28 )
29 29 from IPython.kernel.engineconnector import EngineConnector
30 30 from IPython.kernel.engineservice import EngineService
31 31 from IPython.kernel.fcutil import Tub
32 32 from IPython.utils.importstring import import_item
33 33
34 34 #-----------------------------------------------------------------------------
35 35 # Module level variables
36 36 #-----------------------------------------------------------------------------
37 37
38 38 #: The default config file name for this application
39 39 default_config_file_name = u'ipengine_config.py'
40 40
41 41
42 42 mpi4py_init = """from mpi4py import MPI as mpi
43 43 mpi.size = mpi.COMM_WORLD.Get_size()
44 44 mpi.rank = mpi.COMM_WORLD.Get_rank()
45 45 """
46 46
47 47
48 48 pytrilinos_init = """from PyTrilinos import Epetra
49 49 class SimpleStruct:
50 50 pass
51 51 mpi = SimpleStruct()
52 52 mpi.rank = 0
53 53 mpi.size = 0
54 54 """
55 55
56 56
57 57 _description = """Start an IPython engine for parallel computing.\n\n
58 58
59 59 IPython engines run in parallel and perform computations on behalf of a client
60 60 and controller. A controller needs to be started before the engines. The
61 61 engine can be configured using command line options or using a cluster
62 62 directory. Cluster directories contain config, log and security files and are
63 63 usually located in your .ipython directory and named as "cluster_<profile>".
64 64 See the --profile and --cluster-dir options for details.
65 65 """
66 66
67 67 #-----------------------------------------------------------------------------
68 68 # Command line options
69 69 #-----------------------------------------------------------------------------
70 70
71 71
72 72 class IPEngineAppConfigLoader(ClusterDirConfigLoader):
73 73
74 74 def _add_arguments(self):
75 75 super(IPEngineAppConfigLoader, self)._add_arguments()
76 76 paa = self.parser.add_argument
77 77 # Controller config
78 78 paa('--furl-file',
79 79 type=unicode, dest='Global.furl_file',
80 80 help='The full location of the file containing the FURL of the '
81 81 'controller. If this is not given, the FURL file must be in the '
82 82 'security directory of the cluster directory. This location is '
83 83 'resolved using the --profile and --app-dir options.',
84 84 metavar='Global.furl_file')
85 85 # MPI
86 86 paa('--mpi',
87 87 type=str, dest='MPI.use',
88 88 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
89 89 metavar='MPI.use')
90 90 # Global config
91 91 paa('--log-to-file',
92 92 action='store_true', dest='Global.log_to_file',
93 93 help='Log to a file in the log directory (default is stdout)')
94 94
95 95
96 96 #-----------------------------------------------------------------------------
97 97 # Main application
98 98 #-----------------------------------------------------------------------------
99 99
100 100
101 101 class IPEngineApp(ApplicationWithClusterDir):
102 102
103 103 name = u'ipengine'
104 104 description = _description
105 105 command_line_loader = IPEngineAppConfigLoader
106 config_file_name = default_config_file_name
106 default_config_file_name = default_config_file_name
107 107 auto_create_cluster_dir = True
108 108
109 109 def create_default_config(self):
110 110 super(IPEngineApp, self).create_default_config()
111 111
112 112 # The engine should not clean logs as we don't want to remove the
113 113 # active log files of other running engines.
114 114 self.default_config.Global.clean_logs = False
115 115
116 116 # Global config attributes
117 117 self.default_config.Global.exec_lines = []
118 118 self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
119 119
120 120 # Configuration related to the controller
121 121 # This must match the filename (path not included) that the controller
122 122 # used for the FURL file.
123 123 self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl'
124 124 # If given, this is the actual location of the controller's FURL file.
125 125 # If not, this is computed using the profile, app_dir and furl_file_name
126 126 self.default_config.Global.furl_file = u''
127 127
128 128 # The max number of connection attemps and the initial delay between
129 129 # those attemps.
130 130 self.default_config.Global.connect_delay = 0.1
131 131 self.default_config.Global.connect_max_tries = 15
132 132
133 133 # MPI related config attributes
134 134 self.default_config.MPI.use = ''
135 135 self.default_config.MPI.mpi4py = mpi4py_init
136 136 self.default_config.MPI.pytrilinos = pytrilinos_init
137 137
138 138 def post_load_command_line_config(self):
139 139 pass
140 140
141 141 def pre_construct(self):
142 142 super(IPEngineApp, self).pre_construct()
143 143 self.find_cont_furl_file()
144 144
145 145 def find_cont_furl_file(self):
146 146 """Set the furl file.
147 147
148 148 Here we don't try to actually see if it exists for is valid as that
149 149 is hadled by the connection logic.
150 150 """
151 151 config = self.master_config
152 152 # Find the actual controller FURL file
153 153 if not config.Global.furl_file:
154 154 try_this = os.path.join(
155 155 config.Global.cluster_dir,
156 156 config.Global.security_dir,
157 157 config.Global.furl_file_name
158 158 )
159 159 config.Global.furl_file = try_this
160 160
161 161 def construct(self):
162 162 # This is the working dir by now.
163 163 sys.path.insert(0, '')
164 164
165 165 self.start_mpi()
166 166 self.start_logging()
167 167
168 168 # Create the underlying shell class and EngineService
169 169 shell_class = import_item(self.master_config.Global.shell_class)
170 170 self.engine_service = EngineService(shell_class, mpi=mpi)
171 171
172 172 self.exec_lines()
173 173
174 174 # Create the service hierarchy
175 175 self.main_service = service.MultiService()
176 176 self.engine_service.setServiceParent(self.main_service)
177 177 self.tub_service = Tub()
178 178 self.tub_service.setServiceParent(self.main_service)
179 179 # This needs to be called before the connection is initiated
180 180 self.main_service.startService()
181 181
182 182 # This initiates the connection to the controller and calls
183 183 # register_engine to tell the controller we are ready to do work
184 184 self.engine_connector = EngineConnector(self.tub_service)
185 185
186 186 log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
187 187
188 188 reactor.callWhenRunning(self.call_connect)
189 189
190 190 def call_connect(self):
191 191 d = self.engine_connector.connect_to_controller(
192 192 self.engine_service,
193 193 self.master_config.Global.furl_file,
194 194 self.master_config.Global.connect_delay,
195 195 self.master_config.Global.connect_max_tries
196 196 )
197 197
198 198 def handle_error(f):
199 199 log.msg('Error connecting to controller. This usually means that '
200 200 'i) the controller was not started, ii) a firewall was blocking '
201 201 'the engine from connecting to the controller or iii) the engine '
202 202 ' was not pointed at the right FURL file:')
203 203 log.msg(f.getErrorMessage())
204 204 reactor.callLater(0.1, reactor.stop)
205 205
206 206 d.addErrback(handle_error)
207 207
208 208 def start_mpi(self):
209 209 global mpi
210 210 mpikey = self.master_config.MPI.use
211 211 mpi_import_statement = self.master_config.MPI.get(mpikey, None)
212 212 if mpi_import_statement is not None:
213 213 try:
214 214 self.log.info("Initializing MPI:")
215 215 self.log.info(mpi_import_statement)
216 216 exec mpi_import_statement in globals()
217 217 except:
218 218 mpi = None
219 219 else:
220 220 mpi = None
221 221
222 222 def exec_lines(self):
223 223 for line in self.master_config.Global.exec_lines:
224 224 try:
225 225 log.msg("Executing statement: '%s'" % line)
226 226 self.engine_service.execute(line)
227 227 except:
228 228 log.msg("Error executing statement: %s" % line)
229 229
230 230 def start_app(self):
231 231 reactor.run()
232 232
233 233
234 234 def launch_new_instance():
235 235 """Create and run the IPython controller"""
236 236 app = IPEngineApp()
237 237 app.start()
238 238
239 239
240 240 if __name__ == '__main__':
241 241 launch_new_instance()
242 242
General Comments 0
You need to be logged in to leave comments. Login now