##// END OF EJS Templates
Apply argparse code simplification to all kernel scripts.
Fernando Perez -
Show More
@@ -1,376 +1,377 b''
1 # coding: utf-8
1 # coding: utf-8
2 """A simple configuration system.
2 """A simple configuration system.
3
3
4 Authors
4 Authors
5 -------
5 -------
6 * Brian Granger
6 * Brian Granger
7 * Fernando Perez
7 * Fernando Perez
8 """
8 """
9
9
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11 # Copyright (C) 2008-2009 The IPython Development Team
11 # Copyright (C) 2008-2009 The IPython Development Team
12 #
12 #
13 # Distributed under the terms of the BSD License. The full license is in
13 # Distributed under the terms of the BSD License. The full license is in
14 # the file COPYING, distributed as part of this software.
14 # the file COPYING, distributed as part of this software.
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16
16
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # Imports
18 # Imports
19 #-----------------------------------------------------------------------------
19 #-----------------------------------------------------------------------------
20
20
21 import __builtin__
21 import __builtin__
22 import os
22 import os
23 import sys
23 import sys
24
24
25 from IPython.external import argparse
25 from IPython.external import argparse
26 from IPython.utils.genutils import filefind
26 from IPython.utils.genutils import filefind
27
27
28 #-----------------------------------------------------------------------------
28 #-----------------------------------------------------------------------------
29 # Exceptions
29 # Exceptions
30 #-----------------------------------------------------------------------------
30 #-----------------------------------------------------------------------------
31
31
32
32
33 class ConfigError(Exception):
33 class ConfigError(Exception):
34 pass
34 pass
35
35
36
36
37 class ConfigLoaderError(ConfigError):
37 class ConfigLoaderError(ConfigError):
38 pass
38 pass
39
39
40 #-----------------------------------------------------------------------------
40 #-----------------------------------------------------------------------------
41 # Argparse fix
41 # Argparse fix
42 #-----------------------------------------------------------------------------
42 #-----------------------------------------------------------------------------
43 # Unfortunately argparse by default prints help messages to stderr instead of
43 # Unfortunately argparse by default prints help messages to stderr instead of
44 # stdout. This makes it annoying to capture long help screens at the command
44 # stdout. This makes it annoying to capture long help screens at the command
45 # line, since one must know how to pipe stderr, which many users don't know how
45 # line, since one must know how to pipe stderr, which many users don't know how
46 # to do. So we override the print_help method with one that defaults to
46 # to do. So we override the print_help method with one that defaults to
47 # stdout and use our class instead.
47 # stdout and use our class instead.
48
48
49 class ArgumentParser(argparse.ArgumentParser):
49 class ArgumentParser(argparse.ArgumentParser):
50 """Simple argparse subclass that prints help to stdout by default."""
50 """Simple argparse subclass that prints help to stdout by default."""
51
51
52 def print_help(self, file=None):
52 def print_help(self, file=None):
53 if file is None:
53 if file is None:
54 file = sys.stdout
54 file = sys.stdout
55 return super(ArgumentParser, self).print_help(file)
55 return super(ArgumentParser, self).print_help(file)
56
56
57 print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
57 print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
58
58
59 #-----------------------------------------------------------------------------
59 #-----------------------------------------------------------------------------
60 # Config class for holding config information
60 # Config class for holding config information
61 #-----------------------------------------------------------------------------
61 #-----------------------------------------------------------------------------
62
62
63
63
64 class Config(dict):
64 class Config(dict):
65 """An attribute based dict that can do smart merges."""
65 """An attribute based dict that can do smart merges."""
66
66
67 def __init__(self, *args, **kwds):
67 def __init__(self, *args, **kwds):
68 dict.__init__(self, *args, **kwds)
68 dict.__init__(self, *args, **kwds)
69 # This sets self.__dict__ = self, but it has to be done this way
69 # This sets self.__dict__ = self, but it has to be done this way
70 # because we are also overriding __setattr__.
70 # because we are also overriding __setattr__.
71 dict.__setattr__(self, '__dict__', self)
71 dict.__setattr__(self, '__dict__', self)
72
72
73 def _merge(self, other):
73 def _merge(self, other):
74 to_update = {}
74 to_update = {}
75 for k, v in other.items():
75 for k, v in other.items():
76 if not self.has_key(k):
76 if not self.has_key(k):
77 to_update[k] = v
77 to_update[k] = v
78 else: # I have this key
78 else: # I have this key
79 if isinstance(v, Config):
79 if isinstance(v, Config):
80 # Recursively merge common sub Configs
80 # Recursively merge common sub Configs
81 self[k]._merge(v)
81 self[k]._merge(v)
82 else:
82 else:
83 # Plain updates for non-Configs
83 # Plain updates for non-Configs
84 to_update[k] = v
84 to_update[k] = v
85
85
86 self.update(to_update)
86 self.update(to_update)
87
87
88 def _is_section_key(self, key):
88 def _is_section_key(self, key):
89 if key[0].upper()==key[0] and not key.startswith('_'):
89 if key[0].upper()==key[0] and not key.startswith('_'):
90 return True
90 return True
91 else:
91 else:
92 return False
92 return False
93
93
94 def has_key(self, key):
94 def has_key(self, key):
95 if self._is_section_key(key):
95 if self._is_section_key(key):
96 return True
96 return True
97 else:
97 else:
98 return dict.has_key(self, key)
98 return dict.has_key(self, key)
99
99
100 def _has_section(self, key):
100 def _has_section(self, key):
101 if self._is_section_key(key):
101 if self._is_section_key(key):
102 if dict.has_key(self, key):
102 if dict.has_key(self, key):
103 return True
103 return True
104 return False
104 return False
105
105
106 def copy(self):
106 def copy(self):
107 return type(self)(dict.copy(self))
107 return type(self)(dict.copy(self))
108
108
109 def __copy__(self):
109 def __copy__(self):
110 return self.copy()
110 return self.copy()
111
111
112 def __deepcopy__(self, memo):
112 def __deepcopy__(self, memo):
113 import copy
113 import copy
114 return type(self)(copy.deepcopy(self.items()))
114 return type(self)(copy.deepcopy(self.items()))
115
115
116 def __getitem__(self, key):
116 def __getitem__(self, key):
117 # Because we use this for an exec namespace, we need to delegate
117 # Because we use this for an exec namespace, we need to delegate
118 # the lookup of names in __builtin__ to itself. This means
118 # the lookup of names in __builtin__ to itself. This means
119 # that you can't have section or attribute names that are
119 # that you can't have section or attribute names that are
120 # builtins.
120 # builtins.
121 try:
121 try:
122 return getattr(__builtin__, key)
122 return getattr(__builtin__, key)
123 except AttributeError:
123 except AttributeError:
124 pass
124 pass
125 if self._is_section_key(key):
125 if self._is_section_key(key):
126 try:
126 try:
127 return dict.__getitem__(self, key)
127 return dict.__getitem__(self, key)
128 except KeyError:
128 except KeyError:
129 c = Config()
129 c = Config()
130 dict.__setitem__(self, key, c)
130 dict.__setitem__(self, key, c)
131 return c
131 return c
132 else:
132 else:
133 return dict.__getitem__(self, key)
133 return dict.__getitem__(self, key)
134
134
135 def __setitem__(self, key, value):
135 def __setitem__(self, key, value):
136 # Don't allow names in __builtin__ to be modified.
136 # Don't allow names in __builtin__ to be modified.
137 if hasattr(__builtin__, key):
137 if hasattr(__builtin__, key):
138 raise ConfigError('Config variable names cannot have the same name '
138 raise ConfigError('Config variable names cannot have the same name '
139 'as a Python builtin: %s' % key)
139 'as a Python builtin: %s' % key)
140 if self._is_section_key(key):
140 if self._is_section_key(key):
141 if not isinstance(value, Config):
141 if not isinstance(value, Config):
142 raise ValueError('values whose keys begin with an uppercase '
142 raise ValueError('values whose keys begin with an uppercase '
143 'char must be Config instances: %r, %r' % (key, value))
143 'char must be Config instances: %r, %r' % (key, value))
144 else:
144 else:
145 dict.__setitem__(self, key, value)
145 dict.__setitem__(self, key, value)
146
146
147 def __getattr__(self, key):
147 def __getattr__(self, key):
148 try:
148 try:
149 return self.__getitem__(key)
149 return self.__getitem__(key)
150 except KeyError, e:
150 except KeyError, e:
151 raise AttributeError(e)
151 raise AttributeError(e)
152
152
153 def __setattr__(self, key, value):
153 def __setattr__(self, key, value):
154 try:
154 try:
155 self.__setitem__(key, value)
155 self.__setitem__(key, value)
156 except KeyError, e:
156 except KeyError, e:
157 raise AttributeError(e)
157 raise AttributeError(e)
158
158
159 def __delattr__(self, key):
159 def __delattr__(self, key):
160 try:
160 try:
161 dict.__delitem__(self, key)
161 dict.__delitem__(self, key)
162 except KeyError, e:
162 except KeyError, e:
163 raise AttributeError(e)
163 raise AttributeError(e)
164
164
165
165
166 #-----------------------------------------------------------------------------
166 #-----------------------------------------------------------------------------
167 # Config loading classes
167 # Config loading classes
168 #-----------------------------------------------------------------------------
168 #-----------------------------------------------------------------------------
169
169
170
170
171 class ConfigLoader(object):
171 class ConfigLoader(object):
172 """A object for loading configurations from just about anywhere.
172 """A object for loading configurations from just about anywhere.
173
173
174 The resulting configuration is packaged as a :class:`Struct`.
174 The resulting configuration is packaged as a :class:`Struct`.
175
175
176 Notes
176 Notes
177 -----
177 -----
178 A :class:`ConfigLoader` does one thing: load a config from a source
178 A :class:`ConfigLoader` does one thing: load a config from a source
179 (file, command line arguments) and returns the data as a :class:`Struct`.
179 (file, command line arguments) and returns the data as a :class:`Struct`.
180 There are lots of things that :class:`ConfigLoader` does not do. It does
180 There are lots of things that :class:`ConfigLoader` does not do. It does
181 not implement complex logic for finding config files. It does not handle
181 not implement complex logic for finding config files. It does not handle
182 default values or merge multiple configs. These things need to be
182 default values or merge multiple configs. These things need to be
183 handled elsewhere.
183 handled elsewhere.
184 """
184 """
185
185
186 def __init__(self):
186 def __init__(self):
187 """A base class for config loaders.
187 """A base class for config loaders.
188
188
189 Examples
189 Examples
190 --------
190 --------
191
191
192 >>> cl = ConfigLoader()
192 >>> cl = ConfigLoader()
193 >>> config = cl.load_config()
193 >>> config = cl.load_config()
194 >>> config
194 >>> config
195 {}
195 {}
196 """
196 """
197 self.clear()
197 self.clear()
198
198
199 def clear(self):
199 def clear(self):
200 self.config = Config()
200 self.config = Config()
201
201
202 def load_config(self):
202 def load_config(self):
203 """Load a config from somewhere, return a Struct.
203 """Load a config from somewhere, return a Struct.
204
204
205 Usually, this will cause self.config to be set and then returned.
205 Usually, this will cause self.config to be set and then returned.
206 """
206 """
207 return self.config
207 return self.config
208
208
209
209
210 class FileConfigLoader(ConfigLoader):
210 class FileConfigLoader(ConfigLoader):
211 """A base class for file based configurations.
211 """A base class for file based configurations.
212
212
213 As we add more file based config loaders, the common logic should go
213 As we add more file based config loaders, the common logic should go
214 here.
214 here.
215 """
215 """
216 pass
216 pass
217
217
218
218
219 class PyFileConfigLoader(FileConfigLoader):
219 class PyFileConfigLoader(FileConfigLoader):
220 """A config loader for pure python files.
220 """A config loader for pure python files.
221
221
222 This calls execfile on a plain python file and looks for attributes
222 This calls execfile on a plain python file and looks for attributes
223 that are all caps. These attribute are added to the config Struct.
223 that are all caps. These attribute are added to the config Struct.
224 """
224 """
225
225
226 def __init__(self, filename, path=None):
226 def __init__(self, filename, path=None):
227 """Build a config loader for a filename and path.
227 """Build a config loader for a filename and path.
228
228
229 Parameters
229 Parameters
230 ----------
230 ----------
231 filename : str
231 filename : str
232 The file name of the config file.
232 The file name of the config file.
233 path : str, list, tuple
233 path : str, list, tuple
234 The path to search for the config file on, or a sequence of
234 The path to search for the config file on, or a sequence of
235 paths to try in order.
235 paths to try in order.
236 """
236 """
237 super(PyFileConfigLoader, self).__init__()
237 super(PyFileConfigLoader, self).__init__()
238 self.filename = filename
238 self.filename = filename
239 self.path = path
239 self.path = path
240 self.full_filename = ''
240 self.full_filename = ''
241 self.data = None
241 self.data = None
242
242
243 def load_config(self):
243 def load_config(self):
244 """Load the config from a file and return it as a Struct."""
244 """Load the config from a file and return it as a Struct."""
245 self._find_file()
245 self._find_file()
246 self._read_file_as_dict()
246 self._read_file_as_dict()
247 self._convert_to_config()
247 self._convert_to_config()
248 return self.config
248 return self.config
249
249
250 def _find_file(self):
250 def _find_file(self):
251 """Try to find the file by searching the paths."""
251 """Try to find the file by searching the paths."""
252 self.full_filename = filefind(self.filename, self.path)
252 self.full_filename = filefind(self.filename, self.path)
253
253
254 def _read_file_as_dict(self):
254 def _read_file_as_dict(self):
255 """Load the config file into self.config, with recursive loading."""
255 """Load the config file into self.config, with recursive loading."""
256 # This closure is made available in the namespace that is used
256 # This closure is made available in the namespace that is used
257 # to exec the config file. This allows users to call
257 # to exec the config file. This allows users to call
258 # load_subconfig('myconfig.py') to load config files recursively.
258 # load_subconfig('myconfig.py') to load config files recursively.
259 # It needs to be a closure because it has references to self.path
259 # It needs to be a closure because it has references to self.path
260 # and self.config. The sub-config is loaded with the same path
260 # and self.config. The sub-config is loaded with the same path
261 # as the parent, but it uses an empty config which is then merged
261 # as the parent, but it uses an empty config which is then merged
262 # with the parents.
262 # with the parents.
263 def load_subconfig(fname):
263 def load_subconfig(fname):
264 loader = PyFileConfigLoader(fname, self.path)
264 loader = PyFileConfigLoader(fname, self.path)
265 try:
265 try:
266 sub_config = loader.load_config()
266 sub_config = loader.load_config()
267 except IOError:
267 except IOError:
268 # Pass silently if the sub config is not there. This happens
268 # Pass silently if the sub config is not there. This happens
269 # when a user us using a profile, but not the default config.
269 # when a user us using a profile, but not the default config.
270 pass
270 pass
271 else:
271 else:
272 self.config._merge(sub_config)
272 self.config._merge(sub_config)
273
273
274 # Again, this needs to be a closure and should be used in config
274 # Again, this needs to be a closure and should be used in config
275 # files to get the config being loaded.
275 # files to get the config being loaded.
276 def get_config():
276 def get_config():
277 return self.config
277 return self.config
278
278
279 namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
279 namespace = dict(load_subconfig=load_subconfig, get_config=get_config)
280 execfile(self.full_filename, namespace)
280 execfile(self.full_filename, namespace)
281
281
282 def _convert_to_config(self):
282 def _convert_to_config(self):
283 if self.data is None:
283 if self.data is None:
284 ConfigLoaderError('self.data does not exist')
284 ConfigLoaderError('self.data does not exist')
285
285
286
286
287 class CommandLineConfigLoader(ConfigLoader):
287 class CommandLineConfigLoader(ConfigLoader):
288 """A config loader for command line arguments.
288 """A config loader for command line arguments.
289
289
290 As we add more command line based loaders, the common logic should go
290 As we add more command line based loaders, the common logic should go
291 here.
291 here.
292 """
292 """
293
293
294
294
295 class __NoConfigDefault(object): pass
295 class __NoConfigDefault(object): pass
296 NoConfigDefault = __NoConfigDefault()
296 NoConfigDefault = __NoConfigDefault()
297
297
298
298
299 class ArgParseConfigLoader(CommandLineConfigLoader):
299 class ArgParseConfigLoader(CommandLineConfigLoader):
300 #: Global default for arguments (see argparse docs for details)
300 #: Global default for arguments (see argparse docs for details)
301 argument_default = NoConfigDefault
301 argument_default = NoConfigDefault
302
302
303 def __init__(self, argv=None, arguments=(), *args, **kw):
303 def __init__(self, argv=None, arguments=(), *args, **kw):
304 """Create a config loader for use with argparse.
304 """Create a config loader for use with argparse.
305
305
306 With the exception of ``argv`` and ``arguments``, other args and kwargs
306 With the exception of ``argv`` and ``arguments``, other args and kwargs
307 arguments here are passed onto the constructor of
307 arguments here are passed onto the constructor of
308 :class:`argparse.ArgumentParser`.
308 :class:`argparse.ArgumentParser`.
309
309
310 Parameters
310 Parameters
311 ----------
311 ----------
312
312
313 argv : optional, list
313 argv : optional, list
314 If given, used to read command-line arguments from, otherwise
314 If given, used to read command-line arguments from, otherwise
315 sys.argv[1:] is used.
315 sys.argv[1:] is used.
316
316
317 arguments : optional, tuple
317 arguments : optional, tuple
318 Description of valid command-line arguments, to be called in sequence
318 Description of valid command-line arguments, to be called in sequence
319 with parser.add_argument() to configure the parser.
319 with parser.add_argument() to configure the parser.
320 """
320 """
321 super(CommandLineConfigLoader, self).__init__()
321 super(CommandLineConfigLoader, self).__init__()
322 if argv == None:
322 if argv == None:
323 argv = sys.argv[1:]
323 argv = sys.argv[1:]
324 self.argv = argv
324 self.argv = argv
325 self.arguments = arguments
325 self.arguments = arguments
326 self.args = args
326 self.args = args
327 kwargs = dict(argument_default=self.argument_default)
327 kwargs = dict(argument_default=self.argument_default)
328 kwargs.update(kw)
328 kwargs.update(kw)
329 self.kw = kwargs
329 self.kw = kwargs
330
330
331 def load_config(self, args=None):
331 def load_config(self, args=None):
332 """Parse command line arguments and return as a Struct.
332 """Parse command line arguments and return as a Struct.
333
333
334 Parameters
334 Parameters
335 ----------
335 ----------
336
336
337 args : optional, list
337 args : optional, list
338 If given, a list with the structure of sys.argv[1:] to parse arguments
338 If given, a list with the structure of sys.argv[1:] to parse arguments
339 from. If not given, the instance's self.argv attribute (given at
339 from. If not given, the instance's self.argv attribute (given at
340 construction time) is used."""
340 construction time) is used."""
341
341
342 if args is None:
342 if args is None:
343 args = self.argv
343 args = self.argv
344 self._create_parser()
344 self._create_parser()
345 self._parse_args(args)
345 self._parse_args(args)
346 self._convert_to_config()
346 self._convert_to_config()
347 return self.config
347 return self.config
348
348
349 def get_extra_args(self):
349 def get_extra_args(self):
350 if hasattr(self, 'extra_args'):
350 if hasattr(self, 'extra_args'):
351 return self.extra_args
351 return self.extra_args
352 else:
352 else:
353 return []
353 return []
354
354
355 def _create_parser(self):
355 def _create_parser(self):
356 self.parser = ArgumentParser(*self.args, **self.kw)
356 self.parser = ArgumentParser(*self.args, **self.kw)
357 self._add_arguments()
357 self._add_arguments()
358 self._add_other_arguments()
358 self._add_other_arguments()
359
359
360 def _add_arguments(self):
360 def _add_arguments(self):
361 for argument in self.arguments:
361 for argument in self.arguments:
362 self.parser.add_argument(*argument[0],**argument[1])
362 self.parser.add_argument(*argument[0],**argument[1])
363
363
364 def _add_other_arguments(self):
364 def _add_other_arguments(self):
365 """Meant for subclasses to add their own arguments."""
365 pass
366 pass
366
367
367 def _parse_args(self, args):
368 def _parse_args(self, args):
368 """self.parser->self.parsed_data"""
369 """self.parser->self.parsed_data"""
369 self.parsed_data, self.extra_args = self.parser.parse_known_args(args)
370 self.parsed_data, self.extra_args = self.parser.parse_known_args(args)
370
371
371 def _convert_to_config(self):
372 def _convert_to_config(self):
372 """self.parsed_data->self.config"""
373 """self.parsed_data->self.config"""
373 for k, v in vars(self.parsed_data).items():
374 for k, v in vars(self.parsed_data).items():
374 if v is not NoConfigDefault:
375 if v is not NoConfigDefault:
375 exec_str = 'self.config.' + k + '= v'
376 exec_str = 'self.config.' + k + '= v'
376 exec exec_str in locals(), globals()
377 exec exec_str in locals(), globals()
@@ -1,475 +1,450 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The IPython cluster directory
4 The IPython cluster directory
5 """
5 """
6
6
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Copyright (C) 2008-2009 The IPython Development Team
8 # Copyright (C) 2008-2009 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 from __future__ import with_statement
18 from __future__ import with_statement
19
19
20 import os
20 import os
21 import shutil
21 import shutil
22 import sys
22 import sys
23
23
24 from twisted.python import log
24 from twisted.python import log
25
25
26 from IPython.core import release
26 from IPython.core import release
27 from IPython.config.loader import PyFileConfigLoader
27 from IPython.config.loader import PyFileConfigLoader
28 from IPython.core.application import Application
28 from IPython.core.application import Application
29 from IPython.core.component import Component
29 from IPython.core.component import Component
30 from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault
31 from IPython.utils.traitlets import Unicode, Bool
30 from IPython.utils.traitlets import Unicode, Bool
32 from IPython.utils import genutils
31 from IPython.utils import genutils
33
32
34 #-----------------------------------------------------------------------------
33 #-----------------------------------------------------------------------------
35 # Imports
34 # Imports
36 #-----------------------------------------------------------------------------
35 #-----------------------------------------------------------------------------
37
36
38
37
39 class ClusterDirError(Exception):
38 class ClusterDirError(Exception):
40 pass
39 pass
41
40
42
41
43 class PIDFileError(Exception):
42 class PIDFileError(Exception):
44 pass
43 pass
45
44
46
45
47 class ClusterDir(Component):
46 class ClusterDir(Component):
48 """An object to manage the cluster directory and its resources.
47 """An object to manage the cluster directory and its resources.
49
48
50 The cluster directory is used by :command:`ipcontroller`,
49 The cluster directory is used by :command:`ipcontroller`,
51 :command:`ipcontroller` and :command:`ipcontroller` to manage the
50 :command:`ipcontroller` and :command:`ipcontroller` to manage the
52 configuration, logging and security of these applications.
51 configuration, logging and security of these applications.
53
52
54 This object knows how to find, create and manage these directories. This
53 This object knows how to find, create and manage these directories. This
55 should be used by any code that want's to handle cluster directories.
54 should be used by any code that want's to handle cluster directories.
56 """
55 """
57
56
58 security_dir_name = Unicode('security')
57 security_dir_name = Unicode('security')
59 log_dir_name = Unicode('log')
58 log_dir_name = Unicode('log')
60 pid_dir_name = Unicode('pid')
59 pid_dir_name = Unicode('pid')
61 security_dir = Unicode(u'')
60 security_dir = Unicode(u'')
62 log_dir = Unicode(u'')
61 log_dir = Unicode(u'')
63 pid_dir = Unicode(u'')
62 pid_dir = Unicode(u'')
64 location = Unicode(u'')
63 location = Unicode(u'')
65
64
66 def __init__(self, location):
65 def __init__(self, location):
67 super(ClusterDir, self).__init__(None)
66 super(ClusterDir, self).__init__(None)
68 self.location = location
67 self.location = location
69
68
70 def _location_changed(self, name, old, new):
69 def _location_changed(self, name, old, new):
71 if not os.path.isdir(new):
70 if not os.path.isdir(new):
72 os.makedirs(new)
71 os.makedirs(new)
73 self.security_dir = os.path.join(new, self.security_dir_name)
72 self.security_dir = os.path.join(new, self.security_dir_name)
74 self.log_dir = os.path.join(new, self.log_dir_name)
73 self.log_dir = os.path.join(new, self.log_dir_name)
75 self.pid_dir = os.path.join(new, self.pid_dir_name)
74 self.pid_dir = os.path.join(new, self.pid_dir_name)
76 self.check_dirs()
75 self.check_dirs()
77
76
78 def _log_dir_changed(self, name, old, new):
77 def _log_dir_changed(self, name, old, new):
79 self.check_log_dir()
78 self.check_log_dir()
80
79
81 def check_log_dir(self):
80 def check_log_dir(self):
82 if not os.path.isdir(self.log_dir):
81 if not os.path.isdir(self.log_dir):
83 os.mkdir(self.log_dir)
82 os.mkdir(self.log_dir)
84
83
85 def _security_dir_changed(self, name, old, new):
84 def _security_dir_changed(self, name, old, new):
86 self.check_security_dir()
85 self.check_security_dir()
87
86
88 def check_security_dir(self):
87 def check_security_dir(self):
89 if not os.path.isdir(self.security_dir):
88 if not os.path.isdir(self.security_dir):
90 os.mkdir(self.security_dir, 0700)
89 os.mkdir(self.security_dir, 0700)
91 os.chmod(self.security_dir, 0700)
90 os.chmod(self.security_dir, 0700)
92
91
93 def _pid_dir_changed(self, name, old, new):
92 def _pid_dir_changed(self, name, old, new):
94 self.check_pid_dir()
93 self.check_pid_dir()
95
94
96 def check_pid_dir(self):
95 def check_pid_dir(self):
97 if not os.path.isdir(self.pid_dir):
96 if not os.path.isdir(self.pid_dir):
98 os.mkdir(self.pid_dir, 0700)
97 os.mkdir(self.pid_dir, 0700)
99 os.chmod(self.pid_dir, 0700)
98 os.chmod(self.pid_dir, 0700)
100
99
101 def check_dirs(self):
100 def check_dirs(self):
102 self.check_security_dir()
101 self.check_security_dir()
103 self.check_log_dir()
102 self.check_log_dir()
104 self.check_pid_dir()
103 self.check_pid_dir()
105
104
106 def load_config_file(self, filename):
105 def load_config_file(self, filename):
107 """Load a config file from the top level of the cluster dir.
106 """Load a config file from the top level of the cluster dir.
108
107
109 Parameters
108 Parameters
110 ----------
109 ----------
111 filename : unicode or str
110 filename : unicode or str
112 The filename only of the config file that must be located in
111 The filename only of the config file that must be located in
113 the top-level of the cluster directory.
112 the top-level of the cluster directory.
114 """
113 """
115 loader = PyFileConfigLoader(filename, self.location)
114 loader = PyFileConfigLoader(filename, self.location)
116 return loader.load_config()
115 return loader.load_config()
117
116
118 def copy_config_file(self, config_file, path=None, overwrite=False):
117 def copy_config_file(self, config_file, path=None, overwrite=False):
119 """Copy a default config file into the active cluster directory.
118 """Copy a default config file into the active cluster directory.
120
119
121 Default configuration files are kept in :mod:`IPython.config.default`.
120 Default configuration files are kept in :mod:`IPython.config.default`.
122 This function moves these from that location to the working cluster
121 This function moves these from that location to the working cluster
123 directory.
122 directory.
124 """
123 """
125 if path is None:
124 if path is None:
126 import IPython.config.default
125 import IPython.config.default
127 path = IPython.config.default.__file__.split(os.path.sep)[:-1]
126 path = IPython.config.default.__file__.split(os.path.sep)[:-1]
128 path = os.path.sep.join(path)
127 path = os.path.sep.join(path)
129 src = os.path.join(path, config_file)
128 src = os.path.join(path, config_file)
130 dst = os.path.join(self.location, config_file)
129 dst = os.path.join(self.location, config_file)
131 if not os.path.isfile(dst) or overwrite:
130 if not os.path.isfile(dst) or overwrite:
132 shutil.copy(src, dst)
131 shutil.copy(src, dst)
133
132
134 def copy_all_config_files(self, path=None, overwrite=False):
133 def copy_all_config_files(self, path=None, overwrite=False):
135 """Copy all config files into the active cluster directory."""
134 """Copy all config files into the active cluster directory."""
136 for f in [u'ipcontroller_config.py', u'ipengine_config.py',
135 for f in [u'ipcontroller_config.py', u'ipengine_config.py',
137 u'ipcluster_config.py']:
136 u'ipcluster_config.py']:
138 self.copy_config_file(f, path=path, overwrite=overwrite)
137 self.copy_config_file(f, path=path, overwrite=overwrite)
139
138
140 @classmethod
139 @classmethod
141 def create_cluster_dir(csl, cluster_dir):
140 def create_cluster_dir(csl, cluster_dir):
142 """Create a new cluster directory given a full path.
141 """Create a new cluster directory given a full path.
143
142
144 Parameters
143 Parameters
145 ----------
144 ----------
146 cluster_dir : str
145 cluster_dir : str
147 The full path to the cluster directory. If it does exist, it will
146 The full path to the cluster directory. If it does exist, it will
148 be used. If not, it will be created.
147 be used. If not, it will be created.
149 """
148 """
150 return ClusterDir(cluster_dir)
149 return ClusterDir(cluster_dir)
151
150
152 @classmethod
151 @classmethod
153 def create_cluster_dir_by_profile(cls, path, profile=u'default'):
152 def create_cluster_dir_by_profile(cls, path, profile=u'default'):
154 """Create a cluster dir by profile name and path.
153 """Create a cluster dir by profile name and path.
155
154
156 Parameters
155 Parameters
157 ----------
156 ----------
158 path : str
157 path : str
159 The path (directory) to put the cluster directory in.
158 The path (directory) to put the cluster directory in.
160 profile : str
159 profile : str
161 The name of the profile. The name of the cluster directory will
160 The name of the profile. The name of the cluster directory will
162 be "cluster_<profile>".
161 be "cluster_<profile>".
163 """
162 """
164 if not os.path.isdir(path):
163 if not os.path.isdir(path):
165 raise ClusterDirError('Directory not found: %s' % path)
164 raise ClusterDirError('Directory not found: %s' % path)
166 cluster_dir = os.path.join(path, u'cluster_' + profile)
165 cluster_dir = os.path.join(path, u'cluster_' + profile)
167 return ClusterDir(cluster_dir)
166 return ClusterDir(cluster_dir)
168
167
169 @classmethod
168 @classmethod
170 def find_cluster_dir_by_profile(cls, ipython_dir, profile=u'default'):
169 def find_cluster_dir_by_profile(cls, ipython_dir, profile=u'default'):
171 """Find an existing cluster dir by profile name, return its ClusterDir.
170 """Find an existing cluster dir by profile name, return its ClusterDir.
172
171
173 This searches through a sequence of paths for a cluster dir. If it
172 This searches through a sequence of paths for a cluster dir. If it
174 is not found, a :class:`ClusterDirError` exception will be raised.
173 is not found, a :class:`ClusterDirError` exception will be raised.
175
174
176 The search path algorithm is:
175 The search path algorithm is:
177 1. ``os.getcwd()``
176 1. ``os.getcwd()``
178 2. ``ipython_dir``
177 2. ``ipython_dir``
179 3. The directories found in the ":" separated
178 3. The directories found in the ":" separated
180 :env:`IPCLUSTER_DIR_PATH` environment variable.
179 :env:`IPCLUSTER_DIR_PATH` environment variable.
181
180
182 Parameters
181 Parameters
183 ----------
182 ----------
184 ipython_dir : unicode or str
183 ipython_dir : unicode or str
185 The IPython directory to use.
184 The IPython directory to use.
186 profile : unicode or str
185 profile : unicode or str
187 The name of the profile. The name of the cluster directory
186 The name of the profile. The name of the cluster directory
188 will be "cluster_<profile>".
187 will be "cluster_<profile>".
189 """
188 """
190 dirname = u'cluster_' + profile
189 dirname = u'cluster_' + profile
191 cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
190 cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
192 if cluster_dir_paths:
191 if cluster_dir_paths:
193 cluster_dir_paths = cluster_dir_paths.split(':')
192 cluster_dir_paths = cluster_dir_paths.split(':')
194 else:
193 else:
195 cluster_dir_paths = []
194 cluster_dir_paths = []
196 paths = [os.getcwd(), ipython_dir] + cluster_dir_paths
195 paths = [os.getcwd(), ipython_dir] + cluster_dir_paths
197 for p in paths:
196 for p in paths:
198 cluster_dir = os.path.join(p, dirname)
197 cluster_dir = os.path.join(p, dirname)
199 if os.path.isdir(cluster_dir):
198 if os.path.isdir(cluster_dir):
200 return ClusterDir(cluster_dir)
199 return ClusterDir(cluster_dir)
201 else:
200 else:
202 raise ClusterDirError('Cluster directory not found in paths: %s' % dirname)
201 raise ClusterDirError('Cluster directory not found in paths: %s' % dirname)
203
202
204 @classmethod
203 @classmethod
205 def find_cluster_dir(cls, cluster_dir):
204 def find_cluster_dir(cls, cluster_dir):
206 """Find/create a cluster dir and return its ClusterDir.
205 """Find/create a cluster dir and return its ClusterDir.
207
206
208 This will create the cluster directory if it doesn't exist.
207 This will create the cluster directory if it doesn't exist.
209
208
210 Parameters
209 Parameters
211 ----------
210 ----------
212 cluster_dir : unicode or str
211 cluster_dir : unicode or str
213 The path of the cluster directory. This is expanded using
212 The path of the cluster directory. This is expanded using
214 :func:`IPython.utils.genutils.expand_path`.
213 :func:`IPython.utils.genutils.expand_path`.
215 """
214 """
216 cluster_dir = genutils.expand_path(cluster_dir)
215 cluster_dir = genutils.expand_path(cluster_dir)
217 if not os.path.isdir(cluster_dir):
216 if not os.path.isdir(cluster_dir):
218 raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
217 raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
219 return ClusterDir(cluster_dir)
218 return ClusterDir(cluster_dir)
220
219
221
220
222 class AppWithClusterDirArgParseConfigLoader(ArgParseConfigLoader):
221 # Default command line options for IPython cluster applications.
223 """Default command line options for IPython cluster applications."""
222 cl_args = (
224
223 (('--ipython-dir',), dict(
225 def _add_other_arguments(self):
226 self.parser.add_argument('--ipython-dir',
227 dest='Global.ipython_dir',type=unicode,
224 dest='Global.ipython_dir',type=unicode,
228 help='Set to override default location of Global.ipython_dir.',
225 help='Set to override default location of Global.ipython_dir.',
229 default=NoConfigDefault,
226 metavar='Global.ipython_dir') ),
230 metavar='Global.ipython_dir'
227 (('-p', '--profile',), dict(
231 )
232 self.parser.add_argument('-p', '--profile',
233 dest='Global.profile',type=unicode,
228 dest='Global.profile',type=unicode,
234 help='The string name of the profile to be used. This determines '
229 help=
235 'the name of the cluster dir as: cluster_<profile>. The default profile '
230 """The string name of the profile to be used. This determines the name
236 'is named "default". The cluster directory is resolve this way '
231 of the cluster dir as: cluster_<profile>. The default profile is named
237 'if the --cluster-dir option is not used.',
232 'default'. The cluster directory is resolve this way if the
238 default=NoConfigDefault,
233 --cluster-dir option is not used.""",
239 metavar='Global.profile'
234 metavar='Global.profile') ),
240 )
235 (('--cluster-dir',), dict(
241 self.parser.add_argument('--log-level',
242 dest="Global.log_level",type=int,
243 help='Set the log level (0,10,20,30,40,50). Default is 30.',
244 default=NoConfigDefault,
245 metavar="Global.log_level"
246 )
247 self.parser.add_argument('--cluster-dir',
248 dest='Global.cluster_dir',type=unicode,
236 dest='Global.cluster_dir',type=unicode,
249 help='Set the cluster dir. This overrides the logic used by the '
237 help="""Set the cluster dir. This overrides the logic used by the
250 '--profile option.',
238 --profile option.""",
251 default=NoConfigDefault,
239 metavar='Global.cluster_dir') ),
252 metavar='Global.cluster_dir'
240 (('--work-dir',), dict(
253 ),
254 self.parser.add_argument('--work-dir',
255 dest='Global.work_dir',type=unicode,
241 dest='Global.work_dir',type=unicode,
256 help='Set the working dir for the process.',
242 help='Set the working dir for the process.',
257 default=NoConfigDefault,
243 metavar='Global.work_dir') ),
258 metavar='Global.work_dir'
244 (('--clean-logs',), dict(
259 )
260 self.parser.add_argument('--clean-logs',
261 dest='Global.clean_logs', action='store_true',
245 dest='Global.clean_logs', action='store_true',
262 help='Delete old log flies before starting.',
246 help='Delete old log flies before starting.') ),
263 default=NoConfigDefault
247 (('--no-clean-logs',), dict(
264 )
265 self.parser.add_argument('--no-clean-logs',
266 dest='Global.clean_logs', action='store_false',
248 dest='Global.clean_logs', action='store_false',
267 help="Don't Delete old log flies before starting.",
249 help="Don't Delete old log flies before starting.") ),
268 default=NoConfigDefault
269 )
250 )
270
251
252
271 class ApplicationWithClusterDir(Application):
253 class ApplicationWithClusterDir(Application):
272 """An application that puts everything into a cluster directory.
254 """An application that puts everything into a cluster directory.
273
255
274 Instead of looking for things in the ipython_dir, this type of application
256 Instead of looking for things in the ipython_dir, this type of application
275 will use its own private directory called the "cluster directory"
257 will use its own private directory called the "cluster directory"
276 for things like config files, log files, etc.
258 for things like config files, log files, etc.
277
259
278 The cluster directory is resolved as follows:
260 The cluster directory is resolved as follows:
279
261
280 * If the ``--cluster-dir`` option is given, it is used.
262 * If the ``--cluster-dir`` option is given, it is used.
281 * If ``--cluster-dir`` is not given, the application directory is
263 * If ``--cluster-dir`` is not given, the application directory is
282 resolve using the profile name as ``cluster_<profile>``. The search
264 resolve using the profile name as ``cluster_<profile>``. The search
283 path for this directory is then i) cwd if it is found there
265 path for this directory is then i) cwd if it is found there
284 and ii) in ipython_dir otherwise.
266 and ii) in ipython_dir otherwise.
285
267
286 The config file for the application is to be put in the cluster
268 The config file for the application is to be put in the cluster
287 dir and named the value of the ``config_file_name`` class attribute.
269 dir and named the value of the ``config_file_name`` class attribute.
288 """
270 """
289
271
290 auto_create_cluster_dir = True
272 auto_create_cluster_dir = True
291
273
274 cl_arguments = Application.cl_arguments + cl_args
275
292 def create_default_config(self):
276 def create_default_config(self):
293 super(ApplicationWithClusterDir, self).create_default_config()
277 super(ApplicationWithClusterDir, self).create_default_config()
294 self.default_config.Global.profile = u'default'
278 self.default_config.Global.profile = u'default'
295 self.default_config.Global.cluster_dir = u''
279 self.default_config.Global.cluster_dir = u''
296 self.default_config.Global.work_dir = os.getcwd()
280 self.default_config.Global.work_dir = os.getcwd()
297 self.default_config.Global.log_to_file = False
281 self.default_config.Global.log_to_file = False
298 self.default_config.Global.clean_logs = False
282 self.default_config.Global.clean_logs = False
299
283
300 def create_command_line_config(self):
301 """Create and return a command line config loader."""
302 return AppWithClusterDirArgParseConfigLoader(
303 description=self.description,
304 version=release.version
305 )
306
307 def find_resources(self):
284 def find_resources(self):
308 """This resolves the cluster directory.
285 """This resolves the cluster directory.
309
286
310 This tries to find the cluster directory and if successful, it will
287 This tries to find the cluster directory and if successful, it will
311 have done:
288 have done:
312 * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for
289 * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for
313 the application.
290 the application.
314 * Sets ``self.cluster_dir`` attribute of the application and config
291 * Sets ``self.cluster_dir`` attribute of the application and config
315 objects.
292 objects.
316
293
317 The algorithm used for this is as follows:
294 The algorithm used for this is as follows:
318 1. Try ``Global.cluster_dir``.
295 1. Try ``Global.cluster_dir``.
319 2. Try using ``Global.profile``.
296 2. Try using ``Global.profile``.
320 3. If both of these fail and ``self.auto_create_cluster_dir`` is
297 3. If both of these fail and ``self.auto_create_cluster_dir`` is
321 ``True``, then create the new cluster dir in the IPython directory.
298 ``True``, then create the new cluster dir in the IPython directory.
322 4. If all fails, then raise :class:`ClusterDirError`.
299 4. If all fails, then raise :class:`ClusterDirError`.
323 """
300 """
324
301
325 try:
302 try:
326 cluster_dir = self.command_line_config.Global.cluster_dir
303 cluster_dir = self.command_line_config.Global.cluster_dir
327 except AttributeError:
304 except AttributeError:
328 cluster_dir = self.default_config.Global.cluster_dir
305 cluster_dir = self.default_config.Global.cluster_dir
329 cluster_dir = genutils.expand_path(cluster_dir)
306 cluster_dir = genutils.expand_path(cluster_dir)
330 try:
307 try:
331 self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
308 self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
332 except ClusterDirError:
309 except ClusterDirError:
333 pass
310 pass
334 else:
311 else:
335 self.log.info('Using existing cluster dir: %s' % \
312 self.log.info('Using existing cluster dir: %s' % \
336 self.cluster_dir_obj.location
313 self.cluster_dir_obj.location
337 )
314 )
338 self.finish_cluster_dir()
315 self.finish_cluster_dir()
339 return
316 return
340
317
341 try:
318 try:
342 self.profile = self.command_line_config.Global.profile
319 self.profile = self.command_line_config.Global.profile
343 except AttributeError:
320 except AttributeError:
344 self.profile = self.default_config.Global.profile
321 self.profile = self.default_config.Global.profile
345 try:
322 try:
346 self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
323 self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
347 self.ipython_dir, self.profile)
324 self.ipython_dir, self.profile)
348 except ClusterDirError:
325 except ClusterDirError:
349 pass
326 pass
350 else:
327 else:
351 self.log.info('Using existing cluster dir: %s' % \
328 self.log.info('Using existing cluster dir: %s' % \
352 self.cluster_dir_obj.location
329 self.cluster_dir_obj.location
353 )
330 )
354 self.finish_cluster_dir()
331 self.finish_cluster_dir()
355 return
332 return
356
333
357 if self.auto_create_cluster_dir:
334 if self.auto_create_cluster_dir:
358 self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
335 self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
359 self.ipython_dir, self.profile
336 self.ipython_dir, self.profile
360 )
337 )
361 self.log.info('Creating new cluster dir: %s' % \
338 self.log.info('Creating new cluster dir: %s' % \
362 self.cluster_dir_obj.location
339 self.cluster_dir_obj.location
363 )
340 )
364 self.finish_cluster_dir()
341 self.finish_cluster_dir()
365 else:
342 else:
366 raise ClusterDirError('Could not find a valid cluster directory.')
343 raise ClusterDirError('Could not find a valid cluster directory.')
367
344
368 def finish_cluster_dir(self):
345 def finish_cluster_dir(self):
369 # Set the cluster directory
346 # Set the cluster directory
370 self.cluster_dir = self.cluster_dir_obj.location
347 self.cluster_dir = self.cluster_dir_obj.location
371
348
372 # These have to be set because they could be different from the one
349 # These have to be set because they could be different from the one
373 # that we just computed. Because command line has the highest
350 # that we just computed. Because command line has the highest
374 # priority, this will always end up in the master_config.
351 # priority, this will always end up in the master_config.
375 self.default_config.Global.cluster_dir = self.cluster_dir
352 self.default_config.Global.cluster_dir = self.cluster_dir
376 self.command_line_config.Global.cluster_dir = self.cluster_dir
353 self.command_line_config.Global.cluster_dir = self.cluster_dir
377
354
378 # Set the search path to the cluster directory
355 # Set the search path to the cluster directory
379 self.config_file_paths = (self.cluster_dir,)
356 self.config_file_paths = (self.cluster_dir,)
380
357
381 def find_config_file_name(self):
358 def find_config_file_name(self):
382 """Find the config file name for this application."""
359 """Find the config file name for this application."""
383 # For this type of Application it should be set as a class attribute.
360 # For this type of Application it should be set as a class attribute.
384 if not hasattr(self, 'config_file_name'):
361 if not hasattr(self, 'config_file_name'):
385 self.log.critical("No config filename found")
362 self.log.critical("No config filename found")
386
363
387 def find_config_file_paths(self):
364 def find_config_file_paths(self):
388 # Set the search path to the cluster directory
365 # Set the search path to the cluster directory
389 self.config_file_paths = (self.cluster_dir,)
366 self.config_file_paths = (self.cluster_dir,)
390
367
391 def pre_construct(self):
368 def pre_construct(self):
392 # The log and security dirs were set earlier, but here we put them
369 # The log and security dirs were set earlier, but here we put them
393 # into the config and log them.
370 # into the config and log them.
394 config = self.master_config
371 config = self.master_config
395 sdir = self.cluster_dir_obj.security_dir
372 sdir = self.cluster_dir_obj.security_dir
396 self.security_dir = config.Global.security_dir = sdir
373 self.security_dir = config.Global.security_dir = sdir
397 ldir = self.cluster_dir_obj.log_dir
374 ldir = self.cluster_dir_obj.log_dir
398 self.log_dir = config.Global.log_dir = ldir
375 self.log_dir = config.Global.log_dir = ldir
399 pdir = self.cluster_dir_obj.pid_dir
376 pdir = self.cluster_dir_obj.pid_dir
400 self.pid_dir = config.Global.pid_dir = pdir
377 self.pid_dir = config.Global.pid_dir = pdir
401 self.log.info("Cluster directory set to: %s" % self.cluster_dir)
378 self.log.info("Cluster directory set to: %s" % self.cluster_dir)
402 config.Global.work_dir = unicode(genutils.expand_path(config.Global.work_dir))
379 config.Global.work_dir = unicode(genutils.expand_path(config.Global.work_dir))
403 # Change to the working directory. We do this just before construct
380 # Change to the working directory. We do this just before construct
404 # is called so all the components there have the right working dir.
381 # is called so all the components there have the right working dir.
405 self.to_work_dir()
382 self.to_work_dir()
406
383
407 def to_work_dir(self):
384 def to_work_dir(self):
408 wd = self.master_config.Global.work_dir
385 wd = self.master_config.Global.work_dir
409 if unicode(wd) != unicode(os.getcwd()):
386 if unicode(wd) != unicode(os.getcwd()):
410 os.chdir(wd)
387 os.chdir(wd)
411 self.log.info("Changing to working dir: %s" % wd)
388 self.log.info("Changing to working dir: %s" % wd)
412
389
413 def start_logging(self):
390 def start_logging(self):
414 # Remove old log files
391 # Remove old log files
415 if self.master_config.Global.clean_logs:
392 if self.master_config.Global.clean_logs:
416 log_dir = self.master_config.Global.log_dir
393 log_dir = self.master_config.Global.log_dir
417 for f in os.listdir(log_dir):
394 for f in os.listdir(log_dir):
418 if f.startswith(self.name + u'-') and f.endswith('.log'):
395 if f.startswith(self.name + u'-') and f.endswith('.log'):
419 os.remove(os.path.join(log_dir, f))
396 os.remove(os.path.join(log_dir, f))
420 # Start logging to the new log file
397 # Start logging to the new log file
421 if self.master_config.Global.log_to_file:
398 if self.master_config.Global.log_to_file:
422 log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
399 log_filename = self.name + u'-' + str(os.getpid()) + u'.log'
423 logfile = os.path.join(self.log_dir, log_filename)
400 logfile = os.path.join(self.log_dir, log_filename)
424 open_log_file = open(logfile, 'w')
401 open_log_file = open(logfile, 'w')
425 else:
402 else:
426 open_log_file = sys.stdout
403 open_log_file = sys.stdout
427 log.startLogging(open_log_file)
404 log.startLogging(open_log_file)
428
405
429 def write_pid_file(self, overwrite=False):
406 def write_pid_file(self, overwrite=False):
430 """Create a .pid file in the pid_dir with my pid.
407 """Create a .pid file in the pid_dir with my pid.
431
408
432 This must be called after pre_construct, which sets `self.pid_dir`.
409 This must be called after pre_construct, which sets `self.pid_dir`.
433 This raises :exc:`PIDFileError` if the pid file exists already.
410 This raises :exc:`PIDFileError` if the pid file exists already.
434 """
411 """
435 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
412 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
436 if os.path.isfile(pid_file):
413 if os.path.isfile(pid_file):
437 pid = self.get_pid_from_file()
414 pid = self.get_pid_from_file()
438 if not overwrite:
415 if not overwrite:
439 raise PIDFileError(
416 raise PIDFileError(
440 'The pid file [%s] already exists. \nThis could mean that this '
417 'The pid file [%s] already exists. \nThis could mean that this '
441 'server is already running with [pid=%s].' % (pid_file, pid)
418 'server is already running with [pid=%s].' % (pid_file, pid)
442 )
419 )
443 with open(pid_file, 'w') as f:
420 with open(pid_file, 'w') as f:
444 self.log.info("Creating pid file: %s" % pid_file)
421 self.log.info("Creating pid file: %s" % pid_file)
445 f.write(repr(os.getpid())+'\n')
422 f.write(repr(os.getpid())+'\n')
446
423
447 def remove_pid_file(self):
424 def remove_pid_file(self):
448 """Remove the pid file.
425 """Remove the pid file.
449
426
450 This should be called at shutdown by registering a callback with
427 This should be called at shutdown by registering a callback with
451 :func:`reactor.addSystemEventTrigger`. This needs to return
428 :func:`reactor.addSystemEventTrigger`. This needs to return
452 ``None``.
429 ``None``.
453 """
430 """
454 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
431 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
455 if os.path.isfile(pid_file):
432 if os.path.isfile(pid_file):
456 try:
433 try:
457 self.log.info("Removing pid file: %s" % pid_file)
434 self.log.info("Removing pid file: %s" % pid_file)
458 os.remove(pid_file)
435 os.remove(pid_file)
459 except:
436 except:
460 self.log.warn("Error removing the pid file: %s" % pid_file)
437 self.log.warn("Error removing the pid file: %s" % pid_file)
461
438
462 def get_pid_from_file(self):
439 def get_pid_from_file(self):
463 """Get the pid from the pid file.
440 """Get the pid from the pid file.
464
441
465 If the pid file doesn't exist a :exc:`PIDFileError` is raised.
442 If the pid file doesn't exist a :exc:`PIDFileError` is raised.
466 """
443 """
467 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
444 pid_file = os.path.join(self.pid_dir, self.name + u'.pid')
468 if os.path.isfile(pid_file):
445 if os.path.isfile(pid_file):
469 with open(pid_file, 'r') as f:
446 with open(pid_file, 'r') as f:
470 pid = int(f.read().strip())
447 pid = int(f.read().strip())
471 return pid
448 return pid
472 else:
449 else:
473 raise PIDFileError('pid file not found: %s' % pid_file)
450 raise PIDFileError('pid file not found: %s' % pid_file)
474
475
@@ -1,471 +1,460 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The ipcluster application.
4 The ipcluster application.
5 """
5 """
6
6
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Copyright (C) 2008-2009 The IPython Development Team
8 # Copyright (C) 2008-2009 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 import logging
18 import logging
19 import os
19 import os
20 import signal
20 import signal
21 import sys
22
21
23 if os.name=='posix':
22 if os.name=='posix':
24 from twisted.scripts._twistd_unix import daemonize
23 from twisted.scripts._twistd_unix import daemonize
25
24
26 from IPython.core import release
25 from IPython.core import release
27 from IPython.external import argparse
26 from IPython.external.argparse import ArgumentParser
28 from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault
27 from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault
29 from IPython.utils.importstring import import_item
28 from IPython.utils.importstring import import_item
30
29
31 from IPython.kernel.clusterdir import (
30 from IPython.kernel.clusterdir import (
32 ApplicationWithClusterDir, ClusterDirError, PIDFileError
31 ApplicationWithClusterDir, ClusterDirError, PIDFileError
33 )
32 )
34
33
35 from twisted.internet import reactor, defer
34 from twisted.internet import reactor, defer
36 from twisted.python import log, failure
35 from twisted.python import log, failure
37
36
38
37
39 #-----------------------------------------------------------------------------
38 #-----------------------------------------------------------------------------
40 # The ipcluster application
39 # The ipcluster application
41 #-----------------------------------------------------------------------------
40 #-----------------------------------------------------------------------------
42
41
43
42
44 # Exit codes for ipcluster
43 # Exit codes for ipcluster
45
44
46 # This will be the exit code if the ipcluster appears to be running because
45 # This will be the exit code if the ipcluster appears to be running because
47 # a .pid file exists
46 # a .pid file exists
48 ALREADY_STARTED = 10
47 ALREADY_STARTED = 10
49
48
50 # This will be the exit code if ipcluster stop is run, but there is not .pid
49 # This will be the exit code if ipcluster stop is run, but there is not .pid
51 # file to be found.
50 # file to be found.
52 ALREADY_STOPPED = 11
51 ALREADY_STOPPED = 11
53
52
54
53
55 class IPClusterCLLoader(ArgParseConfigLoader):
54 class IPClusterCLLoader(ArgParseConfigLoader):
56
55
57 def _add_arguments(self):
56 def _add_other_arguments(self):
58 # This has all the common options that all subcommands use
57 # This has all the common options that all subcommands use
59 parent_parser1 = argparse.ArgumentParser(add_help=False)
58 parent_parser1 = ArgumentParser(add_help=False,
59 argument_default=NoConfigDefault)
60 parent_parser1.add_argument('--ipython-dir',
60 parent_parser1.add_argument('--ipython-dir',
61 dest='Global.ipython_dir',type=unicode,
61 dest='Global.ipython_dir',type=unicode,
62 help='Set to override default location of Global.ipython_dir.',
62 help='Set to override default location of Global.ipython_dir.',
63 default=NoConfigDefault,
64 metavar='Global.ipython_dir')
63 metavar='Global.ipython_dir')
65 parent_parser1.add_argument('--log-level',
64 parent_parser1.add_argument('--log-level',
66 dest="Global.log_level",type=int,
65 dest="Global.log_level",type=int,
67 help='Set the log level (0,10,20,30,40,50). Default is 30.',
66 help='Set the log level (0,10,20,30,40,50). Default is 30.',
68 default=NoConfigDefault,
69 metavar='Global.log_level')
67 metavar='Global.log_level')
70
68
71 # This has all the common options that other subcommands use
69 # This has all the common options that other subcommands use
72 parent_parser2 = argparse.ArgumentParser(add_help=False)
70 parent_parser2 = ArgumentParser(add_help=False,
71 argument_default=NoConfigDefault)
73 parent_parser2.add_argument('-p','--profile',
72 parent_parser2.add_argument('-p','--profile',
74 dest='Global.profile',type=unicode,
73 dest='Global.profile',type=unicode,
75 help='The string name of the profile to be used. This determines '
74 help='The string name of the profile to be used. This determines '
76 'the name of the cluster dir as: cluster_<profile>. The default profile '
75 'the name of the cluster dir as: cluster_<profile>. The default profile '
77 'is named "default". The cluster directory is resolve this way '
76 'is named "default". The cluster directory is resolve this way '
78 'if the --cluster-dir option is not used.',
77 'if the --cluster-dir option is not used.',
79 default=NoConfigDefault,
80 metavar='Global.profile')
78 metavar='Global.profile')
81 parent_parser2.add_argument('--cluster-dir',
79 parent_parser2.add_argument('--cluster-dir',
82 dest='Global.cluster_dir',type=unicode,
80 dest='Global.cluster_dir',type=unicode,
83 help='Set the cluster dir. This overrides the logic used by the '
81 help='Set the cluster dir. This overrides the logic used by the '
84 '--profile option.',
82 '--profile option.',
85 default=NoConfigDefault,
86 metavar='Global.cluster_dir'),
83 metavar='Global.cluster_dir'),
87 parent_parser2.add_argument('--work-dir',
84 parent_parser2.add_argument('--work-dir',
88 dest='Global.work_dir',type=unicode,
85 dest='Global.work_dir',type=unicode,
89 help='Set the working dir for the process.',
86 help='Set the working dir for the process.',
90 default=NoConfigDefault,
91 metavar='Global.work_dir')
87 metavar='Global.work_dir')
92 parent_parser2.add_argument('--log-to-file',
88 parent_parser2.add_argument('--log-to-file',
93 action='store_true', dest='Global.log_to_file',
89 action='store_true', dest='Global.log_to_file',
94 default=NoConfigDefault,
95 help='Log to a file in the log directory (default is stdout)'
90 help='Log to a file in the log directory (default is stdout)'
96 )
91 )
97
92
98 subparsers = self.parser.add_subparsers(
93 subparsers = self.parser.add_subparsers(
99 dest='Global.subcommand',
94 dest='Global.subcommand',
100 title='ipcluster subcommands',
95 title='ipcluster subcommands',
101 description='ipcluster has a variety of subcommands. '
96 description='ipcluster has a variety of subcommands. '
102 'The general way of running ipcluster is "ipcluster <cmd> '
97 'The general way of running ipcluster is "ipcluster <cmd> '
103 ' [options]""',
98 ' [options]""',
104 help='For more help, type "ipcluster <cmd> -h"')
99 help='For more help, type "ipcluster <cmd> -h"')
105
100
106 parser_list = subparsers.add_parser(
101 parser_list = subparsers.add_parser(
107 'list',
102 'list',
108 help='List all clusters in cwd and ipython_dir.',
103 help='List all clusters in cwd and ipython_dir.',
109 parents=[parent_parser1]
104 parents=[parent_parser1]
110 )
105 )
111
106
112 parser_create = subparsers.add_parser(
107 parser_create = subparsers.add_parser(
113 'create',
108 'create',
114 help='Create a new cluster directory.',
109 help='Create a new cluster directory.',
115 parents=[parent_parser1, parent_parser2]
110 parents=[parent_parser1, parent_parser2]
116 )
111 )
117 parser_create.add_argument(
112 parser_create.add_argument(
118 '--reset-config',
113 '--reset-config',
119 dest='Global.reset_config', action='store_true',
114 dest='Global.reset_config', action='store_true',
120 default=NoConfigDefault,
115 default=NoConfigDefault,
121 help='Recopy the default config files to the cluster directory. '
116 help='Recopy the default config files to the cluster directory. '
122 'You will loose any modifications you have made to these files.'
117 'You will loose any modifications you have made to these files.'
123 )
118 )
124
119
125 parser_start = subparsers.add_parser(
120 parser_start = subparsers.add_parser(
126 'start',
121 'start',
127 help='Start a cluster.',
122 help='Start a cluster.',
128 parents=[parent_parser1, parent_parser2]
123 parents=[parent_parser1, parent_parser2]
129 )
124 )
130 parser_start.add_argument(
125 parser_start.add_argument(
131 '-n', '--number',
126 '-n', '--number',
132 type=int, dest='Global.n',
127 type=int, dest='Global.n',
133 default=NoConfigDefault,
134 help='The number of engines to start.',
128 help='The number of engines to start.',
135 metavar='Global.n'
129 metavar='Global.n'
136 )
130 )
137 parser_start.add_argument('--clean-logs',
131 parser_start.add_argument('--clean-logs',
138 dest='Global.clean_logs', action='store_true',
132 dest='Global.clean_logs', action='store_true',
139 help='Delete old log flies before starting.',
133 help='Delete old log flies before starting.',
140 default=NoConfigDefault
141 )
134 )
142 parser_start.add_argument('--no-clean-logs',
135 parser_start.add_argument('--no-clean-logs',
143 dest='Global.clean_logs', action='store_false',
136 dest='Global.clean_logs', action='store_false',
144 help="Don't delete old log flies before starting.",
137 help="Don't delete old log flies before starting.",
145 default=NoConfigDefault
146 )
138 )
147 parser_start.add_argument('--daemon',
139 parser_start.add_argument('--daemon',
148 dest='Global.daemonize', action='store_true',
140 dest='Global.daemonize', action='store_true',
149 help='Daemonize the ipcluster program. This implies --log-to-file',
141 help='Daemonize the ipcluster program. This implies --log-to-file',
150 default=NoConfigDefault
151 )
142 )
152 parser_start.add_argument('--no-daemon',
143 parser_start.add_argument('--no-daemon',
153 dest='Global.daemonize', action='store_false',
144 dest='Global.daemonize', action='store_false',
154 help="Dont't daemonize the ipcluster program.",
145 help="Dont't daemonize the ipcluster program.",
155 default=NoConfigDefault
156 )
146 )
157
147
158 parser_start = subparsers.add_parser(
148 parser_start = subparsers.add_parser(
159 'stop',
149 'stop',
160 help='Stop a cluster.',
150 help='Stop a cluster.',
161 parents=[parent_parser1, parent_parser2]
151 parents=[parent_parser1, parent_parser2]
162 )
152 )
163 parser_start.add_argument('--signal',
153 parser_start.add_argument('--signal',
164 dest='Global.signal', type=int,
154 dest='Global.signal', type=int,
165 help="The signal number to use in stopping the cluster (default=2).",
155 help="The signal number to use in stopping the cluster (default=2).",
166 metavar="Global.signal",
156 metavar="Global.signal",
167 default=NoConfigDefault
168 )
157 )
169
158
170
159
171 default_config_file_name = u'ipcluster_config.py'
160 default_config_file_name = u'ipcluster_config.py'
172
161
173
162
174 _description = """Start an IPython cluster for parallel computing.\n\n
163 _description = """Start an IPython cluster for parallel computing.\n\n
175
164
176 An IPython cluster consists of 1 controller and 1 or more engines.
165 An IPython cluster consists of 1 controller and 1 or more engines.
177 This command automates the startup of these processes using a wide
166 This command automates the startup of these processes using a wide
178 range of startup methods (SSH, local processes, PBS, mpiexec,
167 range of startup methods (SSH, local processes, PBS, mpiexec,
179 Windows HPC Server 2008). To start a cluster with 4 engines on your
168 Windows HPC Server 2008). To start a cluster with 4 engines on your
180 local host simply do "ipcluster start -n 4". For more complex usage
169 local host simply do "ipcluster start -n 4". For more complex usage
181 you will typically do "ipcluster create -p mycluster", then edit
170 you will typically do "ipcluster create -p mycluster", then edit
182 configuration files, followed by "ipcluster start -p mycluster -n 4".
171 configuration files, followed by "ipcluster start -p mycluster -n 4".
183 """
172 """
184
173
185
174
186 class IPClusterApp(ApplicationWithClusterDir):
175 class IPClusterApp(ApplicationWithClusterDir):
187
176
188 name = u'ipcluster'
177 name = u'ipcluster'
189 description = _description
178 description = _description
190 config_file_name = default_config_file_name
179 config_file_name = default_config_file_name
191 default_log_level = logging.INFO
180 default_log_level = logging.INFO
192 auto_create_cluster_dir = False
181 auto_create_cluster_dir = False
193
182
194 def create_default_config(self):
183 def create_default_config(self):
195 super(IPClusterApp, self).create_default_config()
184 super(IPClusterApp, self).create_default_config()
196 self.default_config.Global.controller_launcher = \
185 self.default_config.Global.controller_launcher = \
197 'IPython.kernel.launcher.LocalControllerLauncher'
186 'IPython.kernel.launcher.LocalControllerLauncher'
198 self.default_config.Global.engine_launcher = \
187 self.default_config.Global.engine_launcher = \
199 'IPython.kernel.launcher.LocalEngineSetLauncher'
188 'IPython.kernel.launcher.LocalEngineSetLauncher'
200 self.default_config.Global.n = 2
189 self.default_config.Global.n = 2
201 self.default_config.Global.reset_config = False
190 self.default_config.Global.reset_config = False
202 self.default_config.Global.clean_logs = True
191 self.default_config.Global.clean_logs = True
203 self.default_config.Global.signal = 2
192 self.default_config.Global.signal = 2
204 self.default_config.Global.daemonize = False
193 self.default_config.Global.daemonize = False
205
194
206 def create_command_line_config(self):
195 def create_command_line_config(self):
207 """Create and return a command line config loader."""
196 """Create and return a command line config loader."""
208 return IPClusterCLLoader(
197 return IPClusterCLLoader(
209 description=self.description,
198 description=self.description,
210 version=release.version
199 version=release.version
211 )
200 )
212
201
213 def find_resources(self):
202 def find_resources(self):
214 subcommand = self.command_line_config.Global.subcommand
203 subcommand = self.command_line_config.Global.subcommand
215 if subcommand=='list':
204 if subcommand=='list':
216 self.list_cluster_dirs()
205 self.list_cluster_dirs()
217 # Exit immediately because there is nothing left to do.
206 # Exit immediately because there is nothing left to do.
218 self.exit()
207 self.exit()
219 elif subcommand=='create':
208 elif subcommand=='create':
220 self.auto_create_cluster_dir = True
209 self.auto_create_cluster_dir = True
221 super(IPClusterApp, self).find_resources()
210 super(IPClusterApp, self).find_resources()
222 elif subcommand=='start' or subcommand=='stop':
211 elif subcommand=='start' or subcommand=='stop':
223 self.auto_create_cluster_dir = True
212 self.auto_create_cluster_dir = True
224 try:
213 try:
225 super(IPClusterApp, self).find_resources()
214 super(IPClusterApp, self).find_resources()
226 except ClusterDirError:
215 except ClusterDirError:
227 raise ClusterDirError(
216 raise ClusterDirError(
228 "Could not find a cluster directory. A cluster dir must "
217 "Could not find a cluster directory. A cluster dir must "
229 "be created before running 'ipcluster start'. Do "
218 "be created before running 'ipcluster start'. Do "
230 "'ipcluster create -h' or 'ipcluster list -h' for more "
219 "'ipcluster create -h' or 'ipcluster list -h' for more "
231 "information about creating and listing cluster dirs."
220 "information about creating and listing cluster dirs."
232 )
221 )
233
222
234 def list_cluster_dirs(self):
223 def list_cluster_dirs(self):
235 # Find the search paths
224 # Find the search paths
236 cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
225 cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','')
237 if cluster_dir_paths:
226 if cluster_dir_paths:
238 cluster_dir_paths = cluster_dir_paths.split(':')
227 cluster_dir_paths = cluster_dir_paths.split(':')
239 else:
228 else:
240 cluster_dir_paths = []
229 cluster_dir_paths = []
241 try:
230 try:
242 ipython_dir = self.command_line_config.Global.ipython_dir
231 ipython_dir = self.command_line_config.Global.ipython_dir
243 except AttributeError:
232 except AttributeError:
244 ipython_dir = self.default_config.Global.ipython_dir
233 ipython_dir = self.default_config.Global.ipython_dir
245 paths = [os.getcwd(), ipython_dir] + \
234 paths = [os.getcwd(), ipython_dir] + \
246 cluster_dir_paths
235 cluster_dir_paths
247 paths = list(set(paths))
236 paths = list(set(paths))
248
237
249 self.log.info('Searching for cluster dirs in paths: %r' % paths)
238 self.log.info('Searching for cluster dirs in paths: %r' % paths)
250 for path in paths:
239 for path in paths:
251 files = os.listdir(path)
240 files = os.listdir(path)
252 for f in files:
241 for f in files:
253 full_path = os.path.join(path, f)
242 full_path = os.path.join(path, f)
254 if os.path.isdir(full_path) and f.startswith('cluster_'):
243 if os.path.isdir(full_path) and f.startswith('cluster_'):
255 profile = full_path.split('_')[-1]
244 profile = full_path.split('_')[-1]
256 start_cmd = 'ipcluster start -p %s -n 4' % profile
245 start_cmd = 'ipcluster start -p %s -n 4' % profile
257 print start_cmd + " ==> " + full_path
246 print start_cmd + " ==> " + full_path
258
247
259 def pre_construct(self):
248 def pre_construct(self):
260 # IPClusterApp.pre_construct() is where we cd to the working directory.
249 # IPClusterApp.pre_construct() is where we cd to the working directory.
261 super(IPClusterApp, self).pre_construct()
250 super(IPClusterApp, self).pre_construct()
262 config = self.master_config
251 config = self.master_config
263 try:
252 try:
264 daemon = config.Global.daemonize
253 daemon = config.Global.daemonize
265 if daemon:
254 if daemon:
266 config.Global.log_to_file = True
255 config.Global.log_to_file = True
267 except AttributeError:
256 except AttributeError:
268 pass
257 pass
269
258
270 def construct(self):
259 def construct(self):
271 config = self.master_config
260 config = self.master_config
272 subcmd = config.Global.subcommand
261 subcmd = config.Global.subcommand
273 reset = config.Global.reset_config
262 reset = config.Global.reset_config
274 if subcmd == 'list':
263 if subcmd == 'list':
275 return
264 return
276 if subcmd == 'create':
265 if subcmd == 'create':
277 self.log.info('Copying default config files to cluster directory '
266 self.log.info('Copying default config files to cluster directory '
278 '[overwrite=%r]' % (reset,))
267 '[overwrite=%r]' % (reset,))
279 self.cluster_dir_obj.copy_all_config_files(overwrite=reset)
268 self.cluster_dir_obj.copy_all_config_files(overwrite=reset)
280 if subcmd =='start':
269 if subcmd =='start':
281 self.cluster_dir_obj.copy_all_config_files(overwrite=False)
270 self.cluster_dir_obj.copy_all_config_files(overwrite=False)
282 self.start_logging()
271 self.start_logging()
283 reactor.callWhenRunning(self.start_launchers)
272 reactor.callWhenRunning(self.start_launchers)
284
273
285 def start_launchers(self):
274 def start_launchers(self):
286 config = self.master_config
275 config = self.master_config
287
276
288 # Create the launchers. In both bases, we set the work_dir of
277 # Create the launchers. In both bases, we set the work_dir of
289 # the launcher to the cluster_dir. This is where the launcher's
278 # the launcher to the cluster_dir. This is where the launcher's
290 # subprocesses will be launched. It is not where the controller
279 # subprocesses will be launched. It is not where the controller
291 # and engine will be launched.
280 # and engine will be launched.
292 el_class = import_item(config.Global.engine_launcher)
281 el_class = import_item(config.Global.engine_launcher)
293 self.engine_launcher = el_class(
282 self.engine_launcher = el_class(
294 work_dir=self.cluster_dir, config=config
283 work_dir=self.cluster_dir, config=config
295 )
284 )
296 cl_class = import_item(config.Global.controller_launcher)
285 cl_class = import_item(config.Global.controller_launcher)
297 self.controller_launcher = cl_class(
286 self.controller_launcher = cl_class(
298 work_dir=self.cluster_dir, config=config
287 work_dir=self.cluster_dir, config=config
299 )
288 )
300
289
301 # Setup signals
290 # Setup signals
302 signal.signal(signal.SIGINT, self.sigint_handler)
291 signal.signal(signal.SIGINT, self.sigint_handler)
303
292
304 # Setup the observing of stopping. If the controller dies, shut
293 # Setup the observing of stopping. If the controller dies, shut
305 # everything down as that will be completely fatal for the engines.
294 # everything down as that will be completely fatal for the engines.
306 d1 = self.controller_launcher.observe_stop()
295 d1 = self.controller_launcher.observe_stop()
307 d1.addCallback(self.stop_launchers)
296 d1.addCallback(self.stop_launchers)
308 # But, we don't monitor the stopping of engines. An engine dying
297 # But, we don't monitor the stopping of engines. An engine dying
309 # is just fine and in principle a user could start a new engine.
298 # is just fine and in principle a user could start a new engine.
310 # Also, if we did monitor engine stopping, it is difficult to
299 # Also, if we did monitor engine stopping, it is difficult to
311 # know what to do when only some engines die. Currently, the
300 # know what to do when only some engines die. Currently, the
312 # observing of engine stopping is inconsistent. Some launchers
301 # observing of engine stopping is inconsistent. Some launchers
313 # might trigger on a single engine stopping, other wait until
302 # might trigger on a single engine stopping, other wait until
314 # all stop. TODO: think more about how to handle this.
303 # all stop. TODO: think more about how to handle this.
315
304
316 # Start the controller and engines
305 # Start the controller and engines
317 self._stopping = False # Make sure stop_launchers is not called 2x.
306 self._stopping = False # Make sure stop_launchers is not called 2x.
318 d = self.start_controller()
307 d = self.start_controller()
319 d.addCallback(self.start_engines)
308 d.addCallback(self.start_engines)
320 d.addCallback(self.startup_message)
309 d.addCallback(self.startup_message)
321 # If the controller or engines fail to start, stop everything
310 # If the controller or engines fail to start, stop everything
322 d.addErrback(self.stop_launchers)
311 d.addErrback(self.stop_launchers)
323 return d
312 return d
324
313
325 def startup_message(self, r=None):
314 def startup_message(self, r=None):
326 log.msg("IPython cluster: started")
315 log.msg("IPython cluster: started")
327 return r
316 return r
328
317
329 def start_controller(self, r=None):
318 def start_controller(self, r=None):
330 # log.msg("In start_controller")
319 # log.msg("In start_controller")
331 config = self.master_config
320 config = self.master_config
332 d = self.controller_launcher.start(
321 d = self.controller_launcher.start(
333 cluster_dir=config.Global.cluster_dir
322 cluster_dir=config.Global.cluster_dir
334 )
323 )
335 return d
324 return d
336
325
337 def start_engines(self, r=None):
326 def start_engines(self, r=None):
338 # log.msg("In start_engines")
327 # log.msg("In start_engines")
339 config = self.master_config
328 config = self.master_config
340 d = self.engine_launcher.start(
329 d = self.engine_launcher.start(
341 config.Global.n,
330 config.Global.n,
342 cluster_dir=config.Global.cluster_dir
331 cluster_dir=config.Global.cluster_dir
343 )
332 )
344 return d
333 return d
345
334
346 def stop_controller(self, r=None):
335 def stop_controller(self, r=None):
347 # log.msg("In stop_controller")
336 # log.msg("In stop_controller")
348 if self.controller_launcher.running:
337 if self.controller_launcher.running:
349 d = self.controller_launcher.stop()
338 d = self.controller_launcher.stop()
350 d.addErrback(self.log_err)
339 d.addErrback(self.log_err)
351 return d
340 return d
352 else:
341 else:
353 return defer.succeed(None)
342 return defer.succeed(None)
354
343
355 def stop_engines(self, r=None):
344 def stop_engines(self, r=None):
356 # log.msg("In stop_engines")
345 # log.msg("In stop_engines")
357 if self.engine_launcher.running:
346 if self.engine_launcher.running:
358 d = self.engine_launcher.stop()
347 d = self.engine_launcher.stop()
359 d.addErrback(self.log_err)
348 d.addErrback(self.log_err)
360 return d
349 return d
361 else:
350 else:
362 return defer.succeed(None)
351 return defer.succeed(None)
363
352
364 def log_err(self, f):
353 def log_err(self, f):
365 log.msg(f.getTraceback())
354 log.msg(f.getTraceback())
366 return None
355 return None
367
356
368 def stop_launchers(self, r=None):
357 def stop_launchers(self, r=None):
369 if not self._stopping:
358 if not self._stopping:
370 self._stopping = True
359 self._stopping = True
371 if isinstance(r, failure.Failure):
360 if isinstance(r, failure.Failure):
372 log.msg('Unexpected error in ipcluster:')
361 log.msg('Unexpected error in ipcluster:')
373 log.msg(r.getTraceback())
362 log.msg(r.getTraceback())
374 log.msg("IPython cluster: stopping")
363 log.msg("IPython cluster: stopping")
375 d= self.stop_engines()
364 self.stop_engines()
376 d2 = self.stop_controller()
365 self.stop_controller()
377 # Wait a few seconds to let things shut down.
366 # Wait a few seconds to let things shut down.
378 reactor.callLater(4.0, reactor.stop)
367 reactor.callLater(4.0, reactor.stop)
379
368
380 def sigint_handler(self, signum, frame):
369 def sigint_handler(self, signum, frame):
381 self.stop_launchers()
370 self.stop_launchers()
382
371
383 def start_logging(self):
372 def start_logging(self):
384 # Remove old log files of the controller and engine
373 # Remove old log files of the controller and engine
385 if self.master_config.Global.clean_logs:
374 if self.master_config.Global.clean_logs:
386 log_dir = self.master_config.Global.log_dir
375 log_dir = self.master_config.Global.log_dir
387 for f in os.listdir(log_dir):
376 for f in os.listdir(log_dir):
388 if f.startswith('ipengine' + '-'):
377 if f.startswith('ipengine' + '-'):
389 if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
378 if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
390 os.remove(os.path.join(log_dir, f))
379 os.remove(os.path.join(log_dir, f))
391 if f.startswith('ipcontroller' + '-'):
380 if f.startswith('ipcontroller' + '-'):
392 if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
381 if f.endswith('.log') or f.endswith('.out') or f.endswith('.err'):
393 os.remove(os.path.join(log_dir, f))
382 os.remove(os.path.join(log_dir, f))
394 # This will remote old log files for ipcluster itself
383 # This will remote old log files for ipcluster itself
395 super(IPClusterApp, self).start_logging()
384 super(IPClusterApp, self).start_logging()
396
385
397 def start_app(self):
386 def start_app(self):
398 """Start the application, depending on what subcommand is used."""
387 """Start the application, depending on what subcommand is used."""
399 subcmd = self.master_config.Global.subcommand
388 subcmd = self.master_config.Global.subcommand
400 if subcmd=='create' or subcmd=='list':
389 if subcmd=='create' or subcmd=='list':
401 return
390 return
402 elif subcmd=='start':
391 elif subcmd=='start':
403 self.start_app_start()
392 self.start_app_start()
404 elif subcmd=='stop':
393 elif subcmd=='stop':
405 self.start_app_stop()
394 self.start_app_stop()
406
395
407 def start_app_start(self):
396 def start_app_start(self):
408 """Start the app for the start subcommand."""
397 """Start the app for the start subcommand."""
409 config = self.master_config
398 config = self.master_config
410 # First see if the cluster is already running
399 # First see if the cluster is already running
411 try:
400 try:
412 pid = self.get_pid_from_file()
401 pid = self.get_pid_from_file()
413 except PIDFileError:
402 except PIDFileError:
414 pass
403 pass
415 else:
404 else:
416 self.log.critical(
405 self.log.critical(
417 'Cluster is already running with [pid=%s]. '
406 'Cluster is already running with [pid=%s]. '
418 'use "ipcluster stop" to stop the cluster.' % pid
407 'use "ipcluster stop" to stop the cluster.' % pid
419 )
408 )
420 # Here I exit with a unusual exit status that other processes
409 # Here I exit with a unusual exit status that other processes
421 # can watch for to learn how I existed.
410 # can watch for to learn how I existed.
422 self.exit(ALREADY_STARTED)
411 self.exit(ALREADY_STARTED)
423
412
424 # Now log and daemonize
413 # Now log and daemonize
425 self.log.info(
414 self.log.info(
426 'Starting ipcluster with [daemon=%r]' % config.Global.daemonize
415 'Starting ipcluster with [daemon=%r]' % config.Global.daemonize
427 )
416 )
428 # TODO: Get daemonize working on Windows or as a Windows Server.
417 # TODO: Get daemonize working on Windows or as a Windows Server.
429 if config.Global.daemonize:
418 if config.Global.daemonize:
430 if os.name=='posix':
419 if os.name=='posix':
431 daemonize()
420 daemonize()
432
421
433 # Now write the new pid file AFTER our new forked pid is active.
422 # Now write the new pid file AFTER our new forked pid is active.
434 self.write_pid_file()
423 self.write_pid_file()
435 reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
424 reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
436 reactor.run()
425 reactor.run()
437
426
438 def start_app_stop(self):
427 def start_app_stop(self):
439 """Start the app for the stop subcommand."""
428 """Start the app for the stop subcommand."""
440 config = self.master_config
429 config = self.master_config
441 try:
430 try:
442 pid = self.get_pid_from_file()
431 pid = self.get_pid_from_file()
443 except PIDFileError:
432 except PIDFileError:
444 self.log.critical(
433 self.log.critical(
445 'Problem reading pid file, cluster is probably not running.'
434 'Problem reading pid file, cluster is probably not running.'
446 )
435 )
447 # Here I exit with a unusual exit status that other processes
436 # Here I exit with a unusual exit status that other processes
448 # can watch for to learn how I existed.
437 # can watch for to learn how I existed.
449 self.exit(ALREADY_STOPPED)
438 self.exit(ALREADY_STOPPED)
450 else:
439 else:
451 if os.name=='posix':
440 if os.name=='posix':
452 sig = config.Global.signal
441 sig = config.Global.signal
453 self.log.info(
442 self.log.info(
454 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
443 "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig)
455 )
444 )
456 os.kill(pid, sig)
445 os.kill(pid, sig)
457 elif os.name=='nt':
446 elif os.name=='nt':
458 # As of right now, we don't support daemonize on Windows, so
447 # As of right now, we don't support daemonize on Windows, so
459 # stop will not do anything. Minimally, it should clean up the
448 # stop will not do anything. Minimally, it should clean up the
460 # old .pid files.
449 # old .pid files.
461 self.remove_pid_file()
450 self.remove_pid_file()
462
451
463 def launch_new_instance():
452 def launch_new_instance():
464 """Create and run the IPython cluster."""
453 """Create and run the IPython cluster."""
465 app = IPClusterApp()
454 app = IPClusterApp()
466 app.start()
455 app.start()
467
456
468
457
469 if __name__ == '__main__':
458 if __name__ == '__main__':
470 launch_new_instance()
459 launch_new_instance()
471
460
@@ -1,275 +1,255 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The IPython controller application.
4 The IPython controller application.
5 """
5 """
6
6
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Copyright (C) 2008-2009 The IPython Development Team
8 # Copyright (C) 2008-2009 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 from __future__ import with_statement
18 from __future__ import with_statement
19
19
20 import copy
20 import copy
21 import os
21 import os
22 import sys
22 import sys
23
23
24 from twisted.application import service
24 from twisted.application import service
25 from twisted.internet import reactor
25 from twisted.internet import reactor
26 from twisted.python import log
26 from twisted.python import log
27
27
28 from IPython.config.loader import Config, NoConfigDefault
28 from IPython.config.loader import Config, NoConfigDefault
29
30 from IPython.kernel.clusterdir import (
31 ApplicationWithClusterDir,
32 AppWithClusterDirArgParseConfigLoader
33 )
34
35 from IPython.core import release
29 from IPython.core import release
36
30 from IPython.core.application import Application
37 from IPython.utils.traitlets import Str, Instance, Unicode
38
39 from IPython.kernel import controllerservice
31 from IPython.kernel import controllerservice
40
32 from IPython.kernel.clusterdir import ApplicationWithClusterDir
41 from IPython.kernel.fcutil import FCServiceFactory
33 from IPython.kernel.fcutil import FCServiceFactory
34 from IPython.utils.traitlets import Str, Instance, Unicode
42
35
43 #-----------------------------------------------------------------------------
36 #-----------------------------------------------------------------------------
44 # Default interfaces
37 # Default interfaces
45 #-----------------------------------------------------------------------------
38 #-----------------------------------------------------------------------------
46
39
47
48 # The default client interfaces for FCClientServiceFactory.interfaces
40 # The default client interfaces for FCClientServiceFactory.interfaces
49 default_client_interfaces = Config()
41 default_client_interfaces = Config()
50 default_client_interfaces.Task.interface_chain = [
42 default_client_interfaces.Task.interface_chain = [
51 'IPython.kernel.task.ITaskController',
43 'IPython.kernel.task.ITaskController',
52 'IPython.kernel.taskfc.IFCTaskController'
44 'IPython.kernel.taskfc.IFCTaskController'
53 ]
45 ]
54
46
55 default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
47 default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl'
56
48
57 default_client_interfaces.MultiEngine.interface_chain = [
49 default_client_interfaces.MultiEngine.interface_chain = [
58 'IPython.kernel.multiengine.IMultiEngine',
50 'IPython.kernel.multiengine.IMultiEngine',
59 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
51 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine'
60 ]
52 ]
61
53
62 default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
54 default_client_interfaces.MultiEngine.furl_file = u'ipcontroller-mec.furl'
63
55
64 # Make this a dict we can pass to Config.__init__ for the default
56 # Make this a dict we can pass to Config.__init__ for the default
65 default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
57 default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items()))
66
58
67
59
68
60
69 # The default engine interfaces for FCEngineServiceFactory.interfaces
61 # The default engine interfaces for FCEngineServiceFactory.interfaces
70 default_engine_interfaces = Config()
62 default_engine_interfaces = Config()
71 default_engine_interfaces.Default.interface_chain = [
63 default_engine_interfaces.Default.interface_chain = [
72 'IPython.kernel.enginefc.IFCControllerBase'
64 'IPython.kernel.enginefc.IFCControllerBase'
73 ]
65 ]
74
66
75 default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
67 default_engine_interfaces.Default.furl_file = u'ipcontroller-engine.furl'
76
68
77 # Make this a dict we can pass to Config.__init__ for the default
69 # Make this a dict we can pass to Config.__init__ for the default
78 default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
70 default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items()))
79
71
80
72
81 #-----------------------------------------------------------------------------
73 #-----------------------------------------------------------------------------
82 # Service factories
74 # Service factories
83 #-----------------------------------------------------------------------------
75 #-----------------------------------------------------------------------------
84
76
85
77
86 class FCClientServiceFactory(FCServiceFactory):
78 class FCClientServiceFactory(FCServiceFactory):
87 """A Foolscap implementation of the client services."""
79 """A Foolscap implementation of the client services."""
88
80
89 cert_file = Unicode(u'ipcontroller-client.pem', config=True)
81 cert_file = Unicode(u'ipcontroller-client.pem', config=True)
90 interfaces = Instance(klass=Config, kw=default_client_interfaces,
82 interfaces = Instance(klass=Config, kw=default_client_interfaces,
91 allow_none=False, config=True)
83 allow_none=False, config=True)
92
84
93
85
94 class FCEngineServiceFactory(FCServiceFactory):
86 class FCEngineServiceFactory(FCServiceFactory):
95 """A Foolscap implementation of the engine services."""
87 """A Foolscap implementation of the engine services."""
96
88
97 cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
89 cert_file = Unicode(u'ipcontroller-engine.pem', config=True)
98 interfaces = Instance(klass=dict, kw=default_engine_interfaces,
90 interfaces = Instance(klass=dict, kw=default_engine_interfaces,
99 allow_none=False, config=True)
91 allow_none=False, config=True)
100
92
101
93
102 #-----------------------------------------------------------------------------
94 #-----------------------------------------------------------------------------
103 # The main application
95 # The main application
104 #-----------------------------------------------------------------------------
96 #-----------------------------------------------------------------------------
105
97
106
98
107 cl_args = (
99 cl_args = (
108 # Client config
100 # Client config
109 (('--client-ip',), dict(
101 (('--client-ip',), dict(
110 type=str, dest='FCClientServiceFactory.ip', default=NoConfigDefault,
102 type=str, dest='FCClientServiceFactory.ip',
111 help='The IP address or hostname the controller will listen on for '
103 help='The IP address or hostname the controller will listen on for '
112 'client connections.',
104 'client connections.',
113 metavar='FCClientServiceFactory.ip')
105 metavar='FCClientServiceFactory.ip')
114 ),
106 ),
115 (('--client-port',), dict(
107 (('--client-port',), dict(
116 type=int, dest='FCClientServiceFactory.port', default=NoConfigDefault,
108 type=int, dest='FCClientServiceFactory.port',
117 help='The port the controller will listen on for client connections. '
109 help='The port the controller will listen on for client connections. '
118 'The default is to use 0, which will autoselect an open port.',
110 'The default is to use 0, which will autoselect an open port.',
119 metavar='FCClientServiceFactory.port')
111 metavar='FCClientServiceFactory.port')
120 ),
112 ),
121 (('--client-location',), dict(
113 (('--client-location',), dict(
122 type=str, dest='FCClientServiceFactory.location', default=NoConfigDefault,
114 type=str, dest='FCClientServiceFactory.location',
123 help='The hostname or IP that clients should connect to. This does '
115 help='The hostname or IP that clients should connect to. This does '
124 'not control which interface the controller listens on. Instead, this '
116 'not control which interface the controller listens on. Instead, this '
125 'determines the hostname/IP that is listed in the FURL, which is how '
117 'determines the hostname/IP that is listed in the FURL, which is how '
126 'clients know where to connect. Useful if the controller is listening '
118 'clients know where to connect. Useful if the controller is listening '
127 'on multiple interfaces.',
119 'on multiple interfaces.',
128 metavar='FCClientServiceFactory.location')
120 metavar='FCClientServiceFactory.location')
129 ),
121 ),
130 # Engine config
122 # Engine config
131 (('--engine-ip',), dict(
123 (('--engine-ip',), dict(
132 type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault,
124 type=str, dest='FCEngineServiceFactory.ip',
133 help='The IP address or hostname the controller will listen on for '
125 help='The IP address or hostname the controller will listen on for '
134 'engine connections.',
126 'engine connections.',
135 metavar='FCEngineServiceFactory.ip')
127 metavar='FCEngineServiceFactory.ip')
136 ),
128 ),
137 (('--engine-port',), dict(
129 (('--engine-port',), dict(
138 type=int, dest='FCEngineServiceFactory.port', default=NoConfigDefault,
130 type=int, dest='FCEngineServiceFactory.port',
139 help='The port the controller will listen on for engine connections. '
131 help='The port the controller will listen on for engine connections. '
140 'The default is to use 0, which will autoselect an open port.',
132 'The default is to use 0, which will autoselect an open port.',
141 metavar='FCEngineServiceFactory.port')
133 metavar='FCEngineServiceFactory.port')
142 ),
134 ),
143 (('--engine-location',), dict(
135 (('--engine-location',), dict(
144 type=str, dest='FCEngineServiceFactory.location', default=NoConfigDefault,
136 type=str, dest='FCEngineServiceFactory.location',
145 help='The hostname or IP that engines should connect to. This does '
137 help='The hostname or IP that engines should connect to. This does '
146 'not control which interface the controller listens on. Instead, this '
138 'not control which interface the controller listens on. Instead, this '
147 'determines the hostname/IP that is listed in the FURL, which is how '
139 'determines the hostname/IP that is listed in the FURL, which is how '
148 'engines know where to connect. Useful if the controller is listening '
140 'engines know where to connect. Useful if the controller is listening '
149 'on multiple interfaces.',
141 'on multiple interfaces.',
150 metavar='FCEngineServiceFactory.location')
142 metavar='FCEngineServiceFactory.location')
151 ),
143 ),
152 # Global config
144 # Global config
153 (('--log-to-file',), dict(
145 (('--log-to-file',), dict(
154 action='store_true', dest='Global.log_to_file', default=NoConfigDefault,
146 action='store_true', dest='Global.log_to_file',
155 help='Log to a file in the log directory (default is stdout)')
147 help='Log to a file in the log directory (default is stdout)')
156 ),
148 ),
157 (('-r','--reuse-furls'), dict(
149 (('-r','--reuse-furls'), dict(
158 action='store_true', dest='Global.reuse_furls', default=NoConfigDefault,
150 action='store_true', dest='Global.reuse_furls',
159 help='Try to reuse all FURL files. If this is not set all FURL files '
151 help='Try to reuse all FURL files. If this is not set all FURL files '
160 'are deleted before the controller starts. This must be set if '
152 'are deleted before the controller starts. This must be set if '
161 'specific ports are specified by --engine-port or --client-port.')
153 'specific ports are specified by --engine-port or --client-port.')
162 ),
154 ),
163 (('--no-secure',), dict(
155 (('--no-secure',), dict(
164 action='store_false', dest='Global.secure', default=NoConfigDefault,
156 action='store_false', dest='Global.secure',
165 help='Turn off SSL encryption for all connections.')
157 help='Turn off SSL encryption for all connections.')
166 ),
158 ),
167 (('--secure',), dict(
159 (('--secure',), dict(
168 action='store_true', dest='Global.secure', default=NoConfigDefault,
160 action='store_true', dest='Global.secure',
169 help='Turn off SSL encryption for all connections.')
161 help='Turn off SSL encryption for all connections.')
170 )
162 )
171 )
163 )
172
164
173
165
174 class IPControllerAppCLConfigLoader(AppWithClusterDirArgParseConfigLoader):
175
176 arguments = cl_args
177
178
179 _description = """Start the IPython controller for parallel computing.
166 _description = """Start the IPython controller for parallel computing.
180
167
181 The IPython controller provides a gateway between the IPython engines and
168 The IPython controller provides a gateway between the IPython engines and
182 clients. The controller needs to be started before the engines and can be
169 clients. The controller needs to be started before the engines and can be
183 configured using command line options or using a cluster directory. Cluster
170 configured using command line options or using a cluster directory. Cluster
184 directories contain config, log and security files and are usually located in
171 directories contain config, log and security files and are usually located in
185 your .ipython directory and named as "cluster_<profile>". See the --profile
172 your .ipython directory and named as "cluster_<profile>". See the --profile
186 and --cluster-dir options for details.
173 and --cluster-dir options for details.
187 """
174 """
188
175
189 default_config_file_name = u'ipcontroller_config.py'
176 default_config_file_name = u'ipcontroller_config.py'
190
177
191
178
192 class IPControllerApp(ApplicationWithClusterDir):
179 class IPControllerApp(ApplicationWithClusterDir):
193
180
194 name = u'ipcontroller'
181 name = u'ipcontroller'
195 description = _description
182 description = _description
196 config_file_name = default_config_file_name
183 config_file_name = default_config_file_name
197 auto_create_cluster_dir = True
184 auto_create_cluster_dir = True
185 cl_arguments = Application.cl_arguments + cl_args
198
186
199 def create_default_config(self):
187 def create_default_config(self):
200 super(IPControllerApp, self).create_default_config()
188 super(IPControllerApp, self).create_default_config()
201 self.default_config.Global.reuse_furls = False
189 self.default_config.Global.reuse_furls = False
202 self.default_config.Global.secure = True
190 self.default_config.Global.secure = True
203 self.default_config.Global.import_statements = []
191 self.default_config.Global.import_statements = []
204 self.default_config.Global.clean_logs = True
192 self.default_config.Global.clean_logs = True
205
193
206 def create_command_line_config(self):
207 """Create and return a command line config loader."""
208 return IPControllerAppCLConfigLoader(
209 description=self.description,
210 version=release.version
211 )
212
213 def post_load_command_line_config(self):
194 def post_load_command_line_config(self):
214 # Now setup reuse_furls
195 # Now setup reuse_furls
215 c = self.command_line_config
196 c = self.command_line_config
216 if hasattr(c.Global, 'reuse_furls'):
197 if hasattr(c.Global, 'reuse_furls'):
217 c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
198 c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls
218 c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
199 c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls
219 del c.Global.reuse_furls
200 del c.Global.reuse_furls
220 if hasattr(c.Global, 'secure'):
201 if hasattr(c.Global, 'secure'):
221 c.FCClientServiceFactory.secure = c.Global.secure
202 c.FCClientServiceFactory.secure = c.Global.secure
222 c.FCEngineServiceFactory.secure = c.Global.secure
203 c.FCEngineServiceFactory.secure = c.Global.secure
223 del c.Global.secure
204 del c.Global.secure
224
205
225 def construct(self):
206 def construct(self):
226 # This is the working dir by now.
207 # This is the working dir by now.
227 sys.path.insert(0, '')
208 sys.path.insert(0, '')
228
209
229 self.start_logging()
210 self.start_logging()
230 self.import_statements()
211 self.import_statements()
231
212
232 # Create the service hierarchy
213 # Create the service hierarchy
233 self.main_service = service.MultiService()
214 self.main_service = service.MultiService()
234 # The controller service
215 # The controller service
235 controller_service = controllerservice.ControllerService()
216 controller_service = controllerservice.ControllerService()
236 controller_service.setServiceParent(self.main_service)
217 controller_service.setServiceParent(self.main_service)
237 # The client tub and all its refereceables
218 # The client tub and all its refereceables
238 csfactory = FCClientServiceFactory(self.master_config, controller_service)
219 csfactory = FCClientServiceFactory(self.master_config, controller_service)
239 client_service = csfactory.create()
220 client_service = csfactory.create()
240 client_service.setServiceParent(self.main_service)
221 client_service.setServiceParent(self.main_service)
241 # The engine tub
222 # The engine tub
242 esfactory = FCEngineServiceFactory(self.master_config, controller_service)
223 esfactory = FCEngineServiceFactory(self.master_config, controller_service)
243 engine_service = esfactory.create()
224 engine_service = esfactory.create()
244 engine_service.setServiceParent(self.main_service)
225 engine_service.setServiceParent(self.main_service)
245
226
246 def import_statements(self):
227 def import_statements(self):
247 statements = self.master_config.Global.import_statements
228 statements = self.master_config.Global.import_statements
248 for s in statements:
229 for s in statements:
249 try:
230 try:
250 log.msg("Executing statement: '%s'" % s)
231 log.msg("Executing statement: '%s'" % s)
251 exec s in globals(), locals()
232 exec s in globals(), locals()
252 except:
233 except:
253 log.msg("Error running statement: %s" % s)
234 log.msg("Error running statement: %s" % s)
254
235
255 def start_app(self):
236 def start_app(self):
256 # Start the controller service.
237 # Start the controller service.
257 self.main_service.startService()
238 self.main_service.startService()
258 # Write the .pid file overwriting old ones. This allow multiple
239 # Write the .pid file overwriting old ones. This allow multiple
259 # controllers to clober each other. But Windows is not cleaning
240 # controllers to clober each other. But Windows is not cleaning
260 # these up properly.
241 # these up properly.
261 self.write_pid_file(overwrite=True)
242 self.write_pid_file(overwrite=True)
262 # Add a trigger to delete the .pid file upon shutting down.
243 # Add a trigger to delete the .pid file upon shutting down.
263 reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
244 reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
264 reactor.run()
245 reactor.run()
265
246
266
247
267 def launch_new_instance():
248 def launch_new_instance():
268 """Create and run the IPython controller"""
249 """Create and run the IPython controller"""
269 app = IPControllerApp()
250 app = IPControllerApp()
270 app.start()
251 app.start()
271
252
272
253
273 if __name__ == '__main__':
254 if __name__ == '__main__':
274 launch_new_instance()
255 launch_new_instance()
275
@@ -1,248 +1,229 b''
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 # encoding: utf-8
2 # encoding: utf-8
3 """
3 """
4 The IPython controller application
4 The IPython controller application
5 """
5 """
6
6
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Copyright (C) 2008-2009 The IPython Development Team
8 # Copyright (C) 2008-2009 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17
17
18 import os
18 import os
19 import sys
19 import sys
20
20
21 from twisted.application import service
21 from twisted.application import service
22 from twisted.internet import reactor
22 from twisted.internet import reactor
23 from twisted.python import log
23 from twisted.python import log
24
24
25 from IPython.config.loader import NoConfigDefault
25 from IPython.core.application import Application
26
26 from IPython.kernel.clusterdir import ApplicationWithClusterDir
27 from IPython.kernel.clusterdir import (
27 from IPython.kernel.engineconnector import EngineConnector
28 ApplicationWithClusterDir,
29 AppWithClusterDirArgParseConfigLoader
30 )
31 from IPython.core import release
32
33 from IPython.utils.importstring import import_item
34
35 from IPython.kernel.engineservice import EngineService
28 from IPython.kernel.engineservice import EngineService
36 from IPython.kernel.fcutil import Tub
29 from IPython.kernel.fcutil import Tub
37 from IPython.kernel.engineconnector import EngineConnector
30 from IPython.utils.importstring import import_item
38
31
39 #-----------------------------------------------------------------------------
32 #-----------------------------------------------------------------------------
40 # The main application
33 # The main application
41 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
42
35
43
44 cl_args = (
36 cl_args = (
45 # Controller config
37 # Controller config
46 (('--furl-file',), dict(
38 (('--furl-file',), dict(
47 type=unicode, dest='Global.furl_file', default=NoConfigDefault,
39 type=unicode, dest='Global.furl_file',
48 help='The full location of the file containing the FURL of the '
40 help='The full location of the file containing the FURL of the '
49 'controller. If this is not given, the FURL file must be in the '
41 'controller. If this is not given, the FURL file must be in the '
50 'security directory of the cluster directory. This location is '
42 'security directory of the cluster directory. This location is '
51 'resolved using the --profile and --app-dir options.',
43 'resolved using the --profile and --app-dir options.',
52 metavar='Global.furl_file')
44 metavar='Global.furl_file')
53 ),
45 ),
54 # MPI
46 # MPI
55 (('--mpi',), dict(
47 (('--mpi',), dict(
56 type=str, dest='MPI.use', default=NoConfigDefault,
48 type=str, dest='MPI.use',
57 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
49 help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).',
58 metavar='MPI.use')
50 metavar='MPI.use')
59 ),
51 ),
60 # Global config
52 # Global config
61 (('--log-to-file',), dict(
53 (('--log-to-file',), dict(
62 action='store_true', dest='Global.log_to_file', default=NoConfigDefault,
54 action='store_true', dest='Global.log_to_file',
63 help='Log to a file in the log directory (default is stdout)')
55 help='Log to a file in the log directory (default is stdout)')
64 )
56 )
65 )
57 )
66
58
67
59
68 class IPEngineAppCLConfigLoader(AppWithClusterDirArgParseConfigLoader):
69
70 arguments = cl_args
71
72
73 mpi4py_init = """from mpi4py import MPI as mpi
60 mpi4py_init = """from mpi4py import MPI as mpi
74 mpi.size = mpi.COMM_WORLD.Get_size()
61 mpi.size = mpi.COMM_WORLD.Get_size()
75 mpi.rank = mpi.COMM_WORLD.Get_rank()
62 mpi.rank = mpi.COMM_WORLD.Get_rank()
76 """
63 """
77
64
78 pytrilinos_init = """from PyTrilinos import Epetra
65 pytrilinos_init = """from PyTrilinos import Epetra
79 class SimpleStruct:
66 class SimpleStruct:
80 pass
67 pass
81 mpi = SimpleStruct()
68 mpi = SimpleStruct()
82 mpi.rank = 0
69 mpi.rank = 0
83 mpi.size = 0
70 mpi.size = 0
84 """
71 """
85
72
86
73
87 default_config_file_name = u'ipengine_config.py'
74 default_config_file_name = u'ipengine_config.py'
88
75
89
76
90 _description = """Start an IPython engine for parallel computing.\n\n
77 _description = """Start an IPython engine for parallel computing.\n\n
91
78
92 IPython engines run in parallel and perform computations on behalf of a client
79 IPython engines run in parallel and perform computations on behalf of a client
93 and controller. A controller needs to be started before the engines. The
80 and controller. A controller needs to be started before the engines. The
94 engine can be configured using command line options or using a cluster
81 engine can be configured using command line options or using a cluster
95 directory. Cluster directories contain config, log and security files and are
82 directory. Cluster directories contain config, log and security files and are
96 usually located in your .ipython directory and named as "cluster_<profile>".
83 usually located in your .ipython directory and named as "cluster_<profile>".
97 See the --profile and --cluster-dir options for details.
84 See the --profile and --cluster-dir options for details.
98 """
85 """
99
86
100
87
101 class IPEngineApp(ApplicationWithClusterDir):
88 class IPEngineApp(ApplicationWithClusterDir):
102
89
103 name = u'ipengine'
90 name = u'ipengine'
104 description = _description
91 description = _description
105 config_file_name = default_config_file_name
92 config_file_name = default_config_file_name
106 auto_create_cluster_dir = True
93 auto_create_cluster_dir = True
94 cl_arguments = Application.cl_arguments + cl_args
107
95
108 def create_default_config(self):
96 def create_default_config(self):
109 super(IPEngineApp, self).create_default_config()
97 super(IPEngineApp, self).create_default_config()
110
98
111 # The engine should not clean logs as we don't want to remove the
99 # The engine should not clean logs as we don't want to remove the
112 # active log files of other running engines.
100 # active log files of other running engines.
113 self.default_config.Global.clean_logs = False
101 self.default_config.Global.clean_logs = False
114
102
115 # Global config attributes
103 # Global config attributes
116 self.default_config.Global.exec_lines = []
104 self.default_config.Global.exec_lines = []
117 self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
105 self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
118
106
119 # Configuration related to the controller
107 # Configuration related to the controller
120 # This must match the filename (path not included) that the controller
108 # This must match the filename (path not included) that the controller
121 # used for the FURL file.
109 # used for the FURL file.
122 self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl'
110 self.default_config.Global.furl_file_name = u'ipcontroller-engine.furl'
123 # If given, this is the actual location of the controller's FURL file.
111 # If given, this is the actual location of the controller's FURL file.
124 # If not, this is computed using the profile, app_dir and furl_file_name
112 # If not, this is computed using the profile, app_dir and furl_file_name
125 self.default_config.Global.furl_file = u''
113 self.default_config.Global.furl_file = u''
126
114
127 # The max number of connection attemps and the initial delay between
115 # The max number of connection attemps and the initial delay between
128 # those attemps.
116 # those attemps.
129 self.default_config.Global.connect_delay = 0.1
117 self.default_config.Global.connect_delay = 0.1
130 self.default_config.Global.connect_max_tries = 15
118 self.default_config.Global.connect_max_tries = 15
131
119
132 # MPI related config attributes
120 # MPI related config attributes
133 self.default_config.MPI.use = ''
121 self.default_config.MPI.use = ''
134 self.default_config.MPI.mpi4py = mpi4py_init
122 self.default_config.MPI.mpi4py = mpi4py_init
135 self.default_config.MPI.pytrilinos = pytrilinos_init
123 self.default_config.MPI.pytrilinos = pytrilinos_init
136
124
137 def create_command_line_config(self):
138 """Create and return a command line config loader."""
139 return IPEngineAppCLConfigLoader(
140 description=self.description,
141 version=release.version
142 )
143
144 def post_load_command_line_config(self):
125 def post_load_command_line_config(self):
145 pass
126 pass
146
127
147 def pre_construct(self):
128 def pre_construct(self):
148 super(IPEngineApp, self).pre_construct()
129 super(IPEngineApp, self).pre_construct()
149 self.find_cont_furl_file()
130 self.find_cont_furl_file()
150
131
151 def find_cont_furl_file(self):
132 def find_cont_furl_file(self):
152 """Set the furl file.
133 """Set the furl file.
153
134
154 Here we don't try to actually see if it exists for is valid as that
135 Here we don't try to actually see if it exists for is valid as that
155 is hadled by the connection logic.
136 is hadled by the connection logic.
156 """
137 """
157 config = self.master_config
138 config = self.master_config
158 # Find the actual controller FURL file
139 # Find the actual controller FURL file
159 if not config.Global.furl_file:
140 if not config.Global.furl_file:
160 try_this = os.path.join(
141 try_this = os.path.join(
161 config.Global.cluster_dir,
142 config.Global.cluster_dir,
162 config.Global.security_dir,
143 config.Global.security_dir,
163 config.Global.furl_file_name
144 config.Global.furl_file_name
164 )
145 )
165 config.Global.furl_file = try_this
146 config.Global.furl_file = try_this
166
147
167 def construct(self):
148 def construct(self):
168 # This is the working dir by now.
149 # This is the working dir by now.
169 sys.path.insert(0, '')
150 sys.path.insert(0, '')
170
151
171 self.start_mpi()
152 self.start_mpi()
172 self.start_logging()
153 self.start_logging()
173
154
174 # Create the underlying shell class and EngineService
155 # Create the underlying shell class and EngineService
175 shell_class = import_item(self.master_config.Global.shell_class)
156 shell_class = import_item(self.master_config.Global.shell_class)
176 self.engine_service = EngineService(shell_class, mpi=mpi)
157 self.engine_service = EngineService(shell_class, mpi=mpi)
177
158
178 self.exec_lines()
159 self.exec_lines()
179
160
180 # Create the service hierarchy
161 # Create the service hierarchy
181 self.main_service = service.MultiService()
162 self.main_service = service.MultiService()
182 self.engine_service.setServiceParent(self.main_service)
163 self.engine_service.setServiceParent(self.main_service)
183 self.tub_service = Tub()
164 self.tub_service = Tub()
184 self.tub_service.setServiceParent(self.main_service)
165 self.tub_service.setServiceParent(self.main_service)
185 # This needs to be called before the connection is initiated
166 # This needs to be called before the connection is initiated
186 self.main_service.startService()
167 self.main_service.startService()
187
168
188 # This initiates the connection to the controller and calls
169 # This initiates the connection to the controller and calls
189 # register_engine to tell the controller we are ready to do work
170 # register_engine to tell the controller we are ready to do work
190 self.engine_connector = EngineConnector(self.tub_service)
171 self.engine_connector = EngineConnector(self.tub_service)
191
172
192 log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
173 log.msg("Using furl file: %s" % self.master_config.Global.furl_file)
193
174
194 reactor.callWhenRunning(self.call_connect)
175 reactor.callWhenRunning(self.call_connect)
195
176
196 def call_connect(self):
177 def call_connect(self):
197 d = self.engine_connector.connect_to_controller(
178 d = self.engine_connector.connect_to_controller(
198 self.engine_service,
179 self.engine_service,
199 self.master_config.Global.furl_file,
180 self.master_config.Global.furl_file,
200 self.master_config.Global.connect_delay,
181 self.master_config.Global.connect_delay,
201 self.master_config.Global.connect_max_tries
182 self.master_config.Global.connect_max_tries
202 )
183 )
203
184
204 def handle_error(f):
185 def handle_error(f):
205 log.msg('Error connecting to controller. This usually means that '
186 log.msg('Error connecting to controller. This usually means that '
206 'i) the controller was not started, ii) a firewall was blocking '
187 'i) the controller was not started, ii) a firewall was blocking '
207 'the engine from connecting to the controller or iii) the engine '
188 'the engine from connecting to the controller or iii) the engine '
208 ' was not pointed at the right FURL file:')
189 ' was not pointed at the right FURL file:')
209 log.msg(f.getErrorMessage())
190 log.msg(f.getErrorMessage())
210 reactor.callLater(0.1, reactor.stop)
191 reactor.callLater(0.1, reactor.stop)
211
192
212 d.addErrback(handle_error)
193 d.addErrback(handle_error)
213
194
214 def start_mpi(self):
195 def start_mpi(self):
215 global mpi
196 global mpi
216 mpikey = self.master_config.MPI.use
197 mpikey = self.master_config.MPI.use
217 mpi_import_statement = self.master_config.MPI.get(mpikey, None)
198 mpi_import_statement = self.master_config.MPI.get(mpikey, None)
218 if mpi_import_statement is not None:
199 if mpi_import_statement is not None:
219 try:
200 try:
220 self.log.info("Initializing MPI:")
201 self.log.info("Initializing MPI:")
221 self.log.info(mpi_import_statement)
202 self.log.info(mpi_import_statement)
222 exec mpi_import_statement in globals()
203 exec mpi_import_statement in globals()
223 except:
204 except:
224 mpi = None
205 mpi = None
225 else:
206 else:
226 mpi = None
207 mpi = None
227
208
228 def exec_lines(self):
209 def exec_lines(self):
229 for line in self.master_config.Global.exec_lines:
210 for line in self.master_config.Global.exec_lines:
230 try:
211 try:
231 log.msg("Executing statement: '%s'" % line)
212 log.msg("Executing statement: '%s'" % line)
232 self.engine_service.execute(line)
213 self.engine_service.execute(line)
233 except:
214 except:
234 log.msg("Error executing statement: %s" % line)
215 log.msg("Error executing statement: %s" % line)
235
216
236 def start_app(self):
217 def start_app(self):
237 reactor.run()
218 reactor.run()
238
219
239
220
240 def launch_new_instance():
221 def launch_new_instance():
241 """Create and run the IPython controller"""
222 """Create and run the IPython controller"""
242 app = IPEngineApp()
223 app = IPEngineApp()
243 app.start()
224 app.start()
244
225
245
226
246 if __name__ == '__main__':
227 if __name__ == '__main__':
247 launch_new_instance()
228 launch_new_instance()
248
229
General Comments 0
You need to be logged in to leave comments. Login now