Show More
@@ -1,364 +1,364 b'' | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | An application for IPython. |
|
4 | An application for IPython. | |
5 |
|
5 | |||
6 | All top-level applications should use the classes in this module for |
|
6 | All top-level applications should use the classes in this module for | |
7 | handling configuration and creating componenets. |
|
7 | handling configuration and creating componenets. | |
8 |
|
8 | |||
9 | The job of an :class:`Application` is to create the master configuration |
|
9 | The job of an :class:`Application` is to create the master configuration | |
10 | object and then create the components, passing the config to them. |
|
10 | object and then create the components, passing the config to them. | |
11 |
|
11 | |||
12 | Authors: |
|
12 | Authors: | |
13 |
|
13 | |||
14 | * Brian Granger |
|
14 | * Brian Granger | |
15 | * Fernando Perez |
|
15 | * Fernando Perez | |
16 |
|
16 | |||
17 | Notes |
|
17 | Notes | |
18 | ----- |
|
18 | ----- | |
19 | """ |
|
19 | """ | |
20 |
|
20 | |||
21 | #----------------------------------------------------------------------------- |
|
21 | #----------------------------------------------------------------------------- | |
22 | # Copyright (C) 2008-2009 The IPython Development Team |
|
22 | # Copyright (C) 2008-2009 The IPython Development Team | |
23 | # |
|
23 | # | |
24 | # Distributed under the terms of the BSD License. The full license is in |
|
24 | # Distributed under the terms of the BSD License. The full license is in | |
25 | # the file COPYING, distributed as part of this software. |
|
25 | # the file COPYING, distributed as part of this software. | |
26 | #----------------------------------------------------------------------------- |
|
26 | #----------------------------------------------------------------------------- | |
27 |
|
27 | |||
28 | #----------------------------------------------------------------------------- |
|
28 | #----------------------------------------------------------------------------- | |
29 | # Imports |
|
29 | # Imports | |
30 | #----------------------------------------------------------------------------- |
|
30 | #----------------------------------------------------------------------------- | |
31 |
|
31 | |||
32 | import logging |
|
32 | import logging | |
33 | import os |
|
33 | import os | |
34 | import sys |
|
34 | import sys | |
35 |
|
35 | |||
36 | from IPython.core import release |
|
36 | from IPython.core import release | |
37 | from IPython.utils.genutils import get_ipython_dir |
|
37 | from IPython.utils.genutils import get_ipython_dir | |
38 | from IPython.config.loader import ( |
|
38 | from IPython.config.loader import ( | |
39 | PyFileConfigLoader, |
|
39 | PyFileConfigLoader, | |
40 | ArgParseConfigLoader, |
|
40 | ArgParseConfigLoader, | |
41 | Config, |
|
41 | Config, | |
42 | NoConfigDefault |
|
42 | NoConfigDefault | |
43 | ) |
|
43 | ) | |
44 |
|
44 | |||
45 | #----------------------------------------------------------------------------- |
|
45 | #----------------------------------------------------------------------------- | |
46 | # Classes and functions |
|
46 | # Classes and functions | |
47 | #----------------------------------------------------------------------------- |
|
47 | #----------------------------------------------------------------------------- | |
48 |
|
48 | |||
49 |
|
49 | |||
50 | class BaseAppArgParseConfigLoader(ArgParseConfigLoader): |
|
50 | class BaseAppArgParseConfigLoader(ArgParseConfigLoader): | |
51 | """Default command line options for IPython based applications.""" |
|
51 | """Default command line options for IPython based applications.""" | |
52 |
|
52 | |||
53 | def _add_other_arguments(self): |
|
53 | def _add_other_arguments(self): | |
54 | self.parser.add_argument('--ipython-dir', |
|
54 | self.parser.add_argument('--ipython-dir', | |
55 | dest='Global.ipython_dir',type=str, |
|
55 | dest='Global.ipython_dir',type=str, | |
56 | help='Set to override default location of Global.ipython_dir.', |
|
56 | help='Set to override default location of Global.ipython_dir.', | |
57 | default=NoConfigDefault, |
|
57 | default=NoConfigDefault, | |
58 | metavar='Global.ipython_dir') |
|
58 | metavar='Global.ipython_dir') | |
59 | self.parser.add_argument('-p', '--profile', |
|
59 | self.parser.add_argument('-p', '--profile', | |
60 | dest='Global.profile',type=str, |
|
60 | dest='Global.profile',type=str, | |
61 | help='The string name of the ipython profile to be used.', |
|
61 | help='The string name of the ipython profile to be used.', | |
62 | default=NoConfigDefault, |
|
62 | default=NoConfigDefault, | |
63 | metavar='Global.profile') |
|
63 | metavar='Global.profile') | |
64 | self.parser.add_argument('--log-level', |
|
64 | self.parser.add_argument('--log-level', | |
65 | dest="Global.log_level",type=int, |
|
65 | dest="Global.log_level",type=int, | |
66 | help='Set the log level (0,10,20,30,40,50). Default is 30.', |
|
66 | help='Set the log level (0,10,20,30,40,50). Default is 30.', | |
67 | default=NoConfigDefault, |
|
67 | default=NoConfigDefault, | |
68 | metavar='Global.log_level') |
|
68 | metavar='Global.log_level') | |
69 | self.parser.add_argument('--config-file', |
|
69 | self.parser.add_argument('--config-file', | |
70 | dest='Global.config_file',type=str, |
|
70 | dest='Global.config_file',type=str, | |
71 | help='Set the config file name to override default.', |
|
71 | help='Set the config file name to override default.', | |
72 | default=NoConfigDefault, |
|
72 | default=NoConfigDefault, | |
73 | metavar='Global.config_file') |
|
73 | metavar='Global.config_file') | |
74 |
|
74 | |||
75 |
|
75 | |||
76 | class ApplicationError(Exception): |
|
76 | class ApplicationError(Exception): | |
77 | pass |
|
77 | pass | |
78 |
|
78 | |||
79 |
|
79 | |||
80 | class Application(object): |
|
80 | class Application(object): | |
81 | """Load a config, construct components and set them running.""" |
|
81 | """Load a config, construct components and set them running.""" | |
82 |
|
82 | |||
83 | name = 'ipython' |
|
83 | name = 'ipython' | |
84 | description = 'IPython: an enhanced interactive Python shell.' |
|
84 | description = 'IPython: an enhanced interactive Python shell.' | |
85 | config_file_name = 'ipython_config.py' |
|
85 | config_file_name = 'ipython_config.py' | |
86 | default_log_level = logging.WARN |
|
86 | default_log_level = logging.WARN | |
87 |
|
87 | |||
88 | def __init__(self): |
|
88 | def __init__(self): | |
89 | self._exiting = False |
|
89 | self._exiting = False | |
90 | self.init_logger() |
|
90 | self.init_logger() | |
91 | # Track the default and actual separately because some messages are |
|
91 | # Track the default and actual separately because some messages are | |
92 | # only printed if we aren't using the default. |
|
92 | # only printed if we aren't using the default. | |
93 | self.default_config_file_name = self.config_file_name |
|
93 | self.default_config_file_name = self.config_file_name | |
94 |
|
94 | |||
95 | def init_logger(self): |
|
95 | def init_logger(self): | |
96 | self.log = logging.getLogger(self.__class__.__name__) |
|
96 | self.log = logging.getLogger(self.__class__.__name__) | |
97 | # This is used as the default until the command line arguments are read. |
|
97 | # This is used as the default until the command line arguments are read. | |
98 | self.log.setLevel(self.default_log_level) |
|
98 | self.log.setLevel(self.default_log_level) | |
99 | self._log_handler = logging.StreamHandler() |
|
99 | self._log_handler = logging.StreamHandler() | |
100 | self._log_formatter = logging.Formatter("[%(name)s] %(message)s") |
|
100 | self._log_formatter = logging.Formatter("[%(name)s] %(message)s") | |
101 | self._log_handler.setFormatter(self._log_formatter) |
|
101 | self._log_handler.setFormatter(self._log_formatter) | |
102 | self.log.addHandler(self._log_handler) |
|
102 | self.log.addHandler(self._log_handler) | |
103 |
|
103 | |||
104 | def _set_log_level(self, level): |
|
104 | def _set_log_level(self, level): | |
105 | self.log.setLevel(level) |
|
105 | self.log.setLevel(level) | |
106 |
|
106 | |||
107 | def _get_log_level(self): |
|
107 | def _get_log_level(self): | |
108 | return self.log.level |
|
108 | return self.log.level | |
109 |
|
109 | |||
110 | log_level = property(_get_log_level, _set_log_level) |
|
110 | log_level = property(_get_log_level, _set_log_level) | |
111 |
|
111 | |||
112 | def start(self): |
|
112 | def start(self): | |
113 | """Start the application.""" |
|
113 | """Start the application.""" | |
114 | self.attempt(self.create_default_config) |
|
114 | self.attempt(self.create_default_config) | |
115 | self.log_default_config() |
|
115 | self.log_default_config() | |
116 | self.set_default_config_log_level() |
|
116 | self.set_default_config_log_level() | |
117 | self.attempt(self.pre_load_command_line_config) |
|
117 | self.attempt(self.pre_load_command_line_config) | |
118 | self.attempt(self.load_command_line_config, action='abort') |
|
118 | self.attempt(self.load_command_line_config, action='abort') | |
119 | self.set_command_line_config_log_level() |
|
119 | self.set_command_line_config_log_level() | |
120 | self.attempt(self.post_load_command_line_config) |
|
120 | self.attempt(self.post_load_command_line_config) | |
121 | self.log_command_line_config() |
|
121 | self.log_command_line_config() | |
122 | self.attempt(self.find_ipython_dir) |
|
122 | self.attempt(self.find_ipython_dir) | |
123 | self.attempt(self.find_resources) |
|
123 | self.attempt(self.find_resources) | |
124 | self.attempt(self.find_config_file_name) |
|
124 | self.attempt(self.find_config_file_name) | |
125 | self.attempt(self.find_config_file_paths) |
|
125 | self.attempt(self.find_config_file_paths) | |
126 | self.attempt(self.pre_load_file_config) |
|
126 | self.attempt(self.pre_load_file_config) | |
127 | self.attempt(self.load_file_config) |
|
127 | self.attempt(self.load_file_config) | |
128 | self.set_file_config_log_level() |
|
128 | self.set_file_config_log_level() | |
129 | self.attempt(self.post_load_file_config) |
|
129 | self.attempt(self.post_load_file_config) | |
130 | self.log_file_config() |
|
130 | self.log_file_config() | |
131 | self.attempt(self.merge_configs) |
|
131 | self.attempt(self.merge_configs) | |
132 | self.log_master_config() |
|
132 | self.log_master_config() | |
133 | self.attempt(self.pre_construct) |
|
133 | self.attempt(self.pre_construct) | |
134 | self.attempt(self.construct) |
|
134 | self.attempt(self.construct) | |
135 | self.attempt(self.post_construct) |
|
135 | self.attempt(self.post_construct) | |
136 | self.attempt(self.start_app) |
|
136 | self.attempt(self.start_app) | |
137 |
|
137 | |||
138 | #------------------------------------------------------------------------- |
|
138 | #------------------------------------------------------------------------- | |
139 | # Various stages of Application creation |
|
139 | # Various stages of Application creation | |
140 | #------------------------------------------------------------------------- |
|
140 | #------------------------------------------------------------------------- | |
141 |
|
141 | |||
142 | def create_default_config(self): |
|
142 | def create_default_config(self): | |
143 | """Create defaults that can't be set elsewhere. |
|
143 | """Create defaults that can't be set elsewhere. | |
144 |
|
144 | |||
145 | For the most part, we try to set default in the class attributes |
|
145 | For the most part, we try to set default in the class attributes | |
146 | of Components. But, defaults the top-level Application (which is |
|
146 | of Components. But, defaults the top-level Application (which is | |
147 | not a HasTraitlets or Component) are not set in this way. Instead |
|
147 | not a HasTraitlets or Component) are not set in this way. Instead | |
148 | we set them here. The Global section is for variables like this that |
|
148 | we set them here. The Global section is for variables like this that | |
149 | don't belong to a particular component. |
|
149 | don't belong to a particular component. | |
150 | """ |
|
150 | """ | |
151 | self.default_config = Config() |
|
151 | self.default_config = Config() | |
152 | self.default_config.Global.ipython_dir = get_ipython_dir() |
|
152 | self.default_config.Global.ipython_dir = get_ipython_dir() | |
153 | self.default_config.Global.log_level = self.log_level |
|
153 | self.default_config.Global.log_level = self.log_level | |
154 |
|
154 | |||
155 | def log_default_config(self): |
|
155 | def log_default_config(self): | |
156 | self.log.debug('Default config loaded:') |
|
156 | self.log.debug('Default config loaded:') | |
157 | self.log.debug(repr(self.default_config)) |
|
157 | self.log.debug(repr(self.default_config)) | |
158 |
|
158 | |||
159 | def set_default_config_log_level(self): |
|
159 | def set_default_config_log_level(self): | |
160 | try: |
|
160 | try: | |
161 | self.log_level = self.default_config.Global.log_level |
|
161 | self.log_level = self.default_config.Global.log_level | |
162 | except AttributeError: |
|
162 | except AttributeError: | |
163 | # Fallback to the default_log_level class attribute |
|
163 | # Fallback to the default_log_level class attribute | |
164 | pass |
|
164 | pass | |
165 |
|
165 | |||
166 | def create_command_line_config(self): |
|
166 | def create_command_line_config(self): | |
167 | """Create and return a command line config loader.""" |
|
167 | """Create and return a command line config loader.""" | |
168 | return BaseAppArgParseConfigLoader( |
|
168 | return BaseAppArgParseConfigLoader( | |
169 | description=self.description, |
|
169 | description=self.description, | |
170 | version=release.version |
|
170 | version=release.version | |
171 | ) |
|
171 | ) | |
172 |
|
172 | |||
173 | def pre_load_command_line_config(self): |
|
173 | def pre_load_command_line_config(self): | |
174 | """Do actions just before loading the command line config.""" |
|
174 | """Do actions just before loading the command line config.""" | |
175 | pass |
|
175 | pass | |
176 |
|
176 | |||
177 | def load_command_line_config(self): |
|
177 | def load_command_line_config(self): | |
178 | """Load the command line config.""" |
|
178 | """Load the command line config.""" | |
179 | loader = self.create_command_line_config() |
|
179 | loader = self.create_command_line_config() | |
180 | self.command_line_config = loader.load_config() |
|
180 | self.command_line_config = loader.load_config() | |
181 | self.extra_args = loader.get_extra_args() |
|
181 | self.extra_args = loader.get_extra_args() | |
182 |
|
182 | |||
183 | def set_command_line_config_log_level(self): |
|
183 | def set_command_line_config_log_level(self): | |
184 | try: |
|
184 | try: | |
185 | self.log_level = self.command_line_config.Global.log_level |
|
185 | self.log_level = self.command_line_config.Global.log_level | |
186 | except AttributeError: |
|
186 | except AttributeError: | |
187 | pass |
|
187 | pass | |
188 |
|
188 | |||
189 | def post_load_command_line_config(self): |
|
189 | def post_load_command_line_config(self): | |
190 | """Do actions just after loading the command line config.""" |
|
190 | """Do actions just after loading the command line config.""" | |
191 | pass |
|
191 | pass | |
192 |
|
192 | |||
193 | def log_command_line_config(self): |
|
193 | def log_command_line_config(self): | |
194 | self.log.debug("Command line config loaded:") |
|
194 | self.log.debug("Command line config loaded:") | |
195 | self.log.debug(repr(self.command_line_config)) |
|
195 | self.log.debug(repr(self.command_line_config)) | |
196 |
|
196 | |||
197 | def find_ipython_dir(self): |
|
197 | def find_ipython_dir(self): | |
198 | """Set the IPython directory. |
|
198 | """Set the IPython directory. | |
199 |
|
199 | |||
200 | This sets ``self.ipython_dir``, but the actual value that is passed |
|
200 | This sets ``self.ipython_dir``, but the actual value that is passed | |
201 | to the application is kept in either ``self.default_config`` or |
|
201 | to the application is kept in either ``self.default_config`` or | |
202 | ``self.command_line_config``. This also adds ``self.ipython_dir`` to |
|
202 | ``self.command_line_config``. This also adds ``self.ipython_dir`` to | |
203 | ``sys.path`` so config files there can be references by other config |
|
203 | ``sys.path`` so config files there can be references by other config | |
204 | files. |
|
204 | files. | |
205 | """ |
|
205 | """ | |
206 |
|
206 | |||
207 | try: |
|
207 | try: | |
208 | self.ipython_dir = self.command_line_config.Global.ipython_dir |
|
208 | self.ipython_dir = self.command_line_config.Global.ipython_dir | |
209 | except AttributeError: |
|
209 | except AttributeError: | |
210 | self.ipython_dir = self.default_config.Global.ipython_dir |
|
210 | self.ipython_dir = self.default_config.Global.ipython_dir | |
211 | sys.path.append(os.path.abspath(self.ipython_dir)) |
|
211 | sys.path.append(os.path.abspath(self.ipython_dir)) | |
212 | if not os.path.isdir(self.ipython_dir): |
|
212 | if not os.path.isdir(self.ipython_dir): | |
213 | os.makedirs(self.ipython_dir, mode=0777) |
|
213 | os.makedirs(self.ipython_dir, mode=0777) | |
214 | self.log.debug("IPYTHON_DIR set to: %s" % self.ipython_dir) |
|
214 | self.log.debug("IPYTHON_DIR set to: %s" % self.ipython_dir) | |
215 |
|
215 | |||
216 | def find_resources(self): |
|
216 | def find_resources(self): | |
217 | """Find other resources that need to be in place. |
|
217 | """Find other resources that need to be in place. | |
218 |
|
218 | |||
219 | Things like cluster directories need to be in place to find the |
|
219 | Things like cluster directories need to be in place to find the | |
220 | config file. These happen right after the IPython directory has |
|
220 | config file. These happen right after the IPython directory has | |
221 | been set. |
|
221 | been set. | |
222 | """ |
|
222 | """ | |
223 | pass |
|
223 | pass | |
224 |
|
224 | |||
225 | def find_config_file_name(self): |
|
225 | def find_config_file_name(self): | |
226 | """Find the config file name for this application. |
|
226 | """Find the config file name for this application. | |
227 |
|
227 | |||
228 | This must set ``self.config_file_name`` to the filename of the |
|
228 | This must set ``self.config_file_name`` to the filename of the | |
229 | config file to use (just the filename). The search paths for the |
|
229 | config file to use (just the filename). The search paths for the | |
230 | config file are set in :meth:`find_config_file_paths` and then passed |
|
230 | config file are set in :meth:`find_config_file_paths` and then passed | |
231 | to the config file loader where they are resolved to an absolute path. |
|
231 | to the config file loader where they are resolved to an absolute path. | |
232 |
|
232 | |||
233 | If a profile has been set at the command line, this will resolve |
|
233 | If a profile has been set at the command line, this will resolve | |
234 | it. |
|
234 | it. | |
235 | """ |
|
235 | """ | |
236 |
|
236 | |||
237 | try: |
|
237 | try: | |
238 | self.config_file_name = self.command_line_config.Global.config_file |
|
238 | self.config_file_name = self.command_line_config.Global.config_file | |
239 | except AttributeError: |
|
239 | except AttributeError: | |
240 | pass |
|
240 | pass | |
241 |
|
241 | |||
242 | try: |
|
242 | try: | |
243 | self.profile_name = self.command_line_config.Global.profile |
|
243 | self.profile_name = self.command_line_config.Global.profile | |
244 | name_parts = self.config_file_name.split('.') |
|
244 | name_parts = self.config_file_name.split('.') | |
245 | name_parts.insert(1, '_' + self.profile_name + '.') |
|
245 | name_parts.insert(1, '_' + self.profile_name + '.') | |
246 | self.config_file_name = ''.join(name_parts) |
|
246 | self.config_file_name = ''.join(name_parts) | |
247 | except AttributeError: |
|
247 | except AttributeError: | |
248 | pass |
|
248 | pass | |
249 |
|
249 | |||
250 | def find_config_file_paths(self): |
|
250 | def find_config_file_paths(self): | |
251 | """Set the search paths for resolving the config file. |
|
251 | """Set the search paths for resolving the config file. | |
252 |
|
252 | |||
253 | This must set ``self.config_file_paths`` to a sequence of search |
|
253 | This must set ``self.config_file_paths`` to a sequence of search | |
254 | paths to pass to the config file loader. |
|
254 | paths to pass to the config file loader. | |
255 | """ |
|
255 | """ | |
256 | self.config_file_paths = (os.getcwd(), self.ipython_dir) |
|
256 | self.config_file_paths = (os.getcwd(), self.ipython_dir) | |
257 |
|
257 | |||
258 | def pre_load_file_config(self): |
|
258 | def pre_load_file_config(self): | |
259 | """Do actions before the config file is loaded.""" |
|
259 | """Do actions before the config file is loaded.""" | |
260 | pass |
|
260 | pass | |
261 |
|
261 | |||
262 | def load_file_config(self): |
|
262 | def load_file_config(self): | |
263 | """Load the config file. |
|
263 | """Load the config file. | |
264 |
|
264 | |||
265 | This tries to load the config file from disk. If successful, the |
|
265 | This tries to load the config file from disk. If successful, the | |
266 | ``CONFIG_FILE`` config variable is set to the resolved config file |
|
266 | ``CONFIG_FILE`` config variable is set to the resolved config file | |
267 | location. If not successful, an empty config is used. |
|
267 | location. If not successful, an empty config is used. | |
268 | """ |
|
268 | """ | |
269 | self.log.debug("Attempting to load config file: %s" % self.config_file_name) |
|
269 | self.log.debug("Attempting to load config file: %s" % self.config_file_name) | |
270 | loader = PyFileConfigLoader(self.config_file_name, |
|
270 | loader = PyFileConfigLoader(self.config_file_name, | |
271 | path=self.config_file_paths) |
|
271 | path=self.config_file_paths) | |
272 | try: |
|
272 | try: | |
273 | self.file_config = loader.load_config() |
|
273 | self.file_config = loader.load_config() | |
274 | self.file_config.Global.config_file = loader.full_filename |
|
274 | self.file_config.Global.config_file = loader.full_filename | |
275 | except IOError: |
|
275 | except IOError: | |
276 | # Only warn if the default config file was NOT being used. |
|
276 | # Only warn if the default config file was NOT being used. | |
277 | if not self.config_file_name==self.default_config_file_name: |
|
277 | if not self.config_file_name==self.default_config_file_name: | |
278 | self.log.warn("Config file not found, skipping: %s" % \ |
|
278 | self.log.warn("Config file not found, skipping: %s" % \ | |
279 | self.config_file_name, exc_info=True) |
|
279 | self.config_file_name, exc_info=True) | |
280 | self.file_config = Config() |
|
280 | self.file_config = Config() | |
281 | except: |
|
281 | except: | |
282 | self.log.warn("Error loading config file: %s" % \ |
|
282 | self.log.warn("Error loading config file: %s" % \ | |
283 | self.config_file_name, exc_info=True) |
|
283 | self.config_file_name, exc_info=True) | |
284 | self.file_config = Config() |
|
284 | self.file_config = Config() | |
285 |
|
285 | |||
286 | def set_file_config_log_level(self): |
|
286 | def set_file_config_log_level(self): | |
287 | # We need to keeep self.log_level updated. But we only use the value |
|
287 | # We need to keeep self.log_level updated. But we only use the value | |
288 | # of the file_config if a value was not specified at the command |
|
288 | # of the file_config if a value was not specified at the command | |
289 | # line, because the command line overrides everything. |
|
289 | # line, because the command line overrides everything. | |
290 | if not hasattr(self.command_line_config.Global, 'log_level'): |
|
290 | if not hasattr(self.command_line_config.Global, 'log_level'): | |
291 | try: |
|
291 | try: | |
292 | self.log_level = self.file_config.Global.log_level |
|
292 | self.log_level = self.file_config.Global.log_level | |
293 | except AttributeError: |
|
293 | except AttributeError: | |
294 | pass # Use existing value |
|
294 | pass # Use existing value | |
295 |
|
295 | |||
296 | def post_load_file_config(self): |
|
296 | def post_load_file_config(self): | |
297 | """Do actions after the config file is loaded.""" |
|
297 | """Do actions after the config file is loaded.""" | |
298 | pass |
|
298 | pass | |
299 |
|
299 | |||
300 | def log_file_config(self): |
|
300 | def log_file_config(self): | |
301 | if hasattr(self.file_config.Global, 'config_file'): |
|
301 | if hasattr(self.file_config.Global, 'config_file'): | |
302 | self.log.debug("Config file loaded: %s" % self.file_config.Global.config_file) |
|
302 | self.log.debug("Config file loaded: %s" % self.file_config.Global.config_file) | |
303 | self.log.debug(repr(self.file_config)) |
|
303 | self.log.debug(repr(self.file_config)) | |
304 |
|
304 | |||
305 | def merge_configs(self): |
|
305 | def merge_configs(self): | |
306 | """Merge the default, command line and file config objects.""" |
|
306 | """Merge the default, command line and file config objects.""" | |
307 | config = Config() |
|
307 | config = Config() | |
308 | config._merge(self.default_config) |
|
308 | config._merge(self.default_config) | |
309 | config._merge(self.file_config) |
|
309 | config._merge(self.file_config) | |
310 | config._merge(self.command_line_config) |
|
310 | config._merge(self.command_line_config) | |
311 | self.master_config = config |
|
311 | self.master_config = config | |
312 |
|
312 | |||
313 | def log_master_config(self): |
|
313 | def log_master_config(self): | |
314 | self.log.debug("Master config created:") |
|
314 | self.log.debug("Master config created:") | |
315 | self.log.debug(repr(self.master_config)) |
|
315 | self.log.debug(repr(self.master_config)) | |
316 |
|
316 | |||
317 | def pre_construct(self): |
|
317 | def pre_construct(self): | |
318 | """Do actions after the config has been built, but before construct.""" |
|
318 | """Do actions after the config has been built, but before construct.""" | |
319 | pass |
|
319 | pass | |
320 |
|
320 | |||
321 | def construct(self): |
|
321 | def construct(self): | |
322 | """Construct the main components that make up this app.""" |
|
322 | """Construct the main components that make up this app.""" | |
323 | self.log.debug("Constructing components for application") |
|
323 | self.log.debug("Constructing components for application") | |
324 |
|
324 | |||
325 | def post_construct(self): |
|
325 | def post_construct(self): | |
326 | """Do actions after construct, but before starting the app.""" |
|
326 | """Do actions after construct, but before starting the app.""" | |
327 | pass |
|
327 | pass | |
328 |
|
328 | |||
329 | def start_app(self): |
|
329 | def start_app(self): | |
330 | """Actually start the app.""" |
|
330 | """Actually start the app.""" | |
331 | self.log.debug("Starting application") |
|
331 | self.log.debug("Starting application") | |
332 |
|
332 | |||
333 | #------------------------------------------------------------------------- |
|
333 | #------------------------------------------------------------------------- | |
334 | # Utility methods |
|
334 | # Utility methods | |
335 | #------------------------------------------------------------------------- |
|
335 | #------------------------------------------------------------------------- | |
336 |
|
336 | |||
337 | def abort(self): |
|
337 | def abort(self): | |
338 | """Abort the starting of the application.""" |
|
338 | """Abort the starting of the application.""" | |
339 | if self._exiting: |
|
339 | if self._exiting: | |
340 | pass |
|
340 | pass | |
341 | else: |
|
341 | else: | |
342 | self.log.critical("Aborting application: %s" % self.name, exc_info=True) |
|
342 | self.log.critical("Aborting application: %s" % self.name, exc_info=True) | |
343 | self._exiting = True |
|
343 | self._exiting = True | |
344 | sys.exit(1) |
|
344 | sys.exit(1) | |
345 |
|
345 | |||
346 | def exit(self): |
|
346 | def exit(self, exit_status=0): | |
347 | if self._exiting: |
|
347 | if self._exiting: | |
348 | pass |
|
348 | pass | |
349 | else: |
|
349 | else: | |
350 | self.log.debug("Exiting application: %s" % self.name) |
|
350 | self.log.debug("Exiting application: %s" % self.name) | |
351 | self._exiting = True |
|
351 | self._exiting = True | |
352 |
sys.exit( |
|
352 | sys.exit(exit_status) | |
353 |
|
353 | |||
354 | def attempt(self, func, action='abort'): |
|
354 | def attempt(self, func, action='abort'): | |
355 | try: |
|
355 | try: | |
356 | func() |
|
356 | func() | |
357 | except SystemExit: |
|
357 | except SystemExit: | |
358 | raise |
|
358 | raise | |
359 | except: |
|
359 | except: | |
360 | if action == 'abort': |
|
360 | if action == 'abort': | |
361 | self.abort() |
|
361 | self.abort() | |
362 | elif action == 'exit': |
|
362 | elif action == 'exit': | |
363 | self.exit() |
|
363 | self.exit(0) | |
364 |
|
364 |
@@ -1,461 +1,463 b'' | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | The IPython cluster directory |
|
4 | The IPython cluster directory | |
5 | """ |
|
5 | """ | |
6 |
|
6 | |||
7 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
8 | # Copyright (C) 2008-2009 The IPython Development Team |
|
8 | # Copyright (C) 2008-2009 The IPython Development Team | |
9 | # |
|
9 | # | |
10 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | # Distributed under the terms of the BSD License. The full license is in | |
11 | # the file COPYING, distributed as part of this software. |
|
11 | # the file COPYING, distributed as part of this software. | |
12 | #----------------------------------------------------------------------------- |
|
12 | #----------------------------------------------------------------------------- | |
13 |
|
13 | |||
14 | #----------------------------------------------------------------------------- |
|
14 | #----------------------------------------------------------------------------- | |
15 | # Imports |
|
15 | # Imports | |
16 | #----------------------------------------------------------------------------- |
|
16 | #----------------------------------------------------------------------------- | |
17 |
|
17 | |||
18 | from __future__ import with_statement |
|
18 | from __future__ import with_statement | |
19 |
|
19 | |||
20 | import os |
|
20 | import os | |
21 | import shutil |
|
21 | import shutil | |
22 | import sys |
|
22 | import sys | |
23 |
|
23 | |||
24 | from twisted.python import log |
|
24 | from twisted.python import log | |
25 |
|
25 | |||
26 | from IPython.core import release |
|
26 | from IPython.core import release | |
27 | from IPython.config.loader import PyFileConfigLoader |
|
27 | from IPython.config.loader import PyFileConfigLoader | |
28 | from IPython.core.application import Application |
|
28 | from IPython.core.application import Application | |
29 | from IPython.core.component import Component |
|
29 | from IPython.core.component import Component | |
30 | from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault |
|
30 | from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault | |
31 | from IPython.utils.traitlets import Unicode, Bool |
|
31 | from IPython.utils.traitlets import Unicode, Bool | |
32 |
|
32 | |||
33 | #----------------------------------------------------------------------------- |
|
33 | #----------------------------------------------------------------------------- | |
34 | # Imports |
|
34 | # Imports | |
35 | #----------------------------------------------------------------------------- |
|
35 | #----------------------------------------------------------------------------- | |
36 |
|
36 | |||
37 |
|
37 | |||
38 | class ClusterDirError(Exception): |
|
38 | class ClusterDirError(Exception): | |
39 | pass |
|
39 | pass | |
40 |
|
40 | |||
41 |
|
41 | |||
42 | class PIDFileError(Exception): |
|
42 | class PIDFileError(Exception): | |
43 | pass |
|
43 | pass | |
44 |
|
44 | |||
45 |
|
45 | |||
46 | class ClusterDir(Component): |
|
46 | class ClusterDir(Component): | |
47 | """An object to manage the cluster directory and its resources. |
|
47 | """An object to manage the cluster directory and its resources. | |
48 |
|
48 | |||
49 | The cluster directory is used by :command:`ipcontroller`, |
|
49 | The cluster directory is used by :command:`ipcontroller`, | |
50 | :command:`ipcontroller` and :command:`ipcontroller` to manage the |
|
50 | :command:`ipcontroller` and :command:`ipcontroller` to manage the | |
51 | configuration, logging and security of these applications. |
|
51 | configuration, logging and security of these applications. | |
52 |
|
52 | |||
53 | This object knows how to find, create and manage these directories. This |
|
53 | This object knows how to find, create and manage these directories. This | |
54 | should be used by any code that want's to handle cluster directories. |
|
54 | should be used by any code that want's to handle cluster directories. | |
55 | """ |
|
55 | """ | |
56 |
|
56 | |||
57 | security_dir_name = Unicode('security') |
|
57 | security_dir_name = Unicode('security') | |
58 | log_dir_name = Unicode('log') |
|
58 | log_dir_name = Unicode('log') | |
59 | pid_dir_name = Unicode('pid') |
|
59 | pid_dir_name = Unicode('pid') | |
60 | security_dir = Unicode(u'') |
|
60 | security_dir = Unicode(u'') | |
61 | log_dir = Unicode(u'') |
|
61 | log_dir = Unicode(u'') | |
62 | pid_dir = Unicode(u'') |
|
62 | pid_dir = Unicode(u'') | |
63 | location = Unicode(u'') |
|
63 | location = Unicode(u'') | |
64 |
|
64 | |||
65 | def __init__(self, location): |
|
65 | def __init__(self, location): | |
66 | super(ClusterDir, self).__init__(None) |
|
66 | super(ClusterDir, self).__init__(None) | |
67 | self.location = location |
|
67 | self.location = location | |
68 |
|
68 | |||
69 | def _location_changed(self, name, old, new): |
|
69 | def _location_changed(self, name, old, new): | |
70 | if not os.path.isdir(new): |
|
70 | if not os.path.isdir(new): | |
71 | os.makedirs(new, mode=0777) |
|
71 | os.makedirs(new, mode=0777) | |
72 | else: |
|
72 | else: | |
73 | os.chmod(new, 0777) |
|
73 | os.chmod(new, 0777) | |
74 | self.security_dir = os.path.join(new, self.security_dir_name) |
|
74 | self.security_dir = os.path.join(new, self.security_dir_name) | |
75 | self.log_dir = os.path.join(new, self.log_dir_name) |
|
75 | self.log_dir = os.path.join(new, self.log_dir_name) | |
76 | self.pid_dir = os.path.join(new, self.pid_dir_name) |
|
76 | self.pid_dir = os.path.join(new, self.pid_dir_name) | |
77 | self.check_dirs() |
|
77 | self.check_dirs() | |
78 |
|
78 | |||
79 | def _log_dir_changed(self, name, old, new): |
|
79 | def _log_dir_changed(self, name, old, new): | |
80 | self.check_log_dir() |
|
80 | self.check_log_dir() | |
81 |
|
81 | |||
82 | def check_log_dir(self): |
|
82 | def check_log_dir(self): | |
83 | if not os.path.isdir(self.log_dir): |
|
83 | if not os.path.isdir(self.log_dir): | |
84 | os.mkdir(self.log_dir, 0777) |
|
84 | os.mkdir(self.log_dir, 0777) | |
85 | else: |
|
85 | else: | |
86 | os.chmod(self.log_dir, 0777) |
|
86 | os.chmod(self.log_dir, 0777) | |
87 |
|
87 | |||
88 | def _security_dir_changed(self, name, old, new): |
|
88 | def _security_dir_changed(self, name, old, new): | |
89 | self.check_security_dir() |
|
89 | self.check_security_dir() | |
90 |
|
90 | |||
91 | def check_security_dir(self): |
|
91 | def check_security_dir(self): | |
92 | if not os.path.isdir(self.security_dir): |
|
92 | if not os.path.isdir(self.security_dir): | |
93 | os.mkdir(self.security_dir, 0700) |
|
93 | os.mkdir(self.security_dir, 0700) | |
94 | else: |
|
94 | else: | |
95 | os.chmod(self.security_dir, 0700) |
|
95 | os.chmod(self.security_dir, 0700) | |
96 |
|
96 | |||
97 | def _pid_dir_changed(self, name, old, new): |
|
97 | def _pid_dir_changed(self, name, old, new): | |
98 | self.check_pid_dir() |
|
98 | self.check_pid_dir() | |
99 |
|
99 | |||
100 | def check_pid_dir(self): |
|
100 | def check_pid_dir(self): | |
101 | if not os.path.isdir(self.pid_dir): |
|
101 | if not os.path.isdir(self.pid_dir): | |
102 | os.mkdir(self.pid_dir, 0700) |
|
102 | os.mkdir(self.pid_dir, 0700) | |
103 | else: |
|
103 | else: | |
104 | os.chmod(self.pid_dir, 0700) |
|
104 | os.chmod(self.pid_dir, 0700) | |
105 |
|
105 | |||
106 | def check_dirs(self): |
|
106 | def check_dirs(self): | |
107 | self.check_security_dir() |
|
107 | self.check_security_dir() | |
108 | self.check_log_dir() |
|
108 | self.check_log_dir() | |
109 | self.check_pid_dir() |
|
109 | self.check_pid_dir() | |
110 |
|
110 | |||
111 | def load_config_file(self, filename): |
|
111 | def load_config_file(self, filename): | |
112 | """Load a config file from the top level of the cluster dir. |
|
112 | """Load a config file from the top level of the cluster dir. | |
113 |
|
113 | |||
114 | Parameters |
|
114 | Parameters | |
115 | ---------- |
|
115 | ---------- | |
116 | filename : unicode or str |
|
116 | filename : unicode or str | |
117 | The filename only of the config file that must be located in |
|
117 | The filename only of the config file that must be located in | |
118 | the top-level of the cluster directory. |
|
118 | the top-level of the cluster directory. | |
119 | """ |
|
119 | """ | |
120 | loader = PyFileConfigLoader(filename, self.location) |
|
120 | loader = PyFileConfigLoader(filename, self.location) | |
121 | return loader.load_config() |
|
121 | return loader.load_config() | |
122 |
|
122 | |||
123 | def copy_config_file(self, config_file, path=None, overwrite=False): |
|
123 | def copy_config_file(self, config_file, path=None, overwrite=False): | |
124 | """Copy a default config file into the active cluster directory. |
|
124 | """Copy a default config file into the active cluster directory. | |
125 |
|
125 | |||
126 | Default configuration files are kept in :mod:`IPython.config.default`. |
|
126 | Default configuration files are kept in :mod:`IPython.config.default`. | |
127 | This function moves these from that location to the working cluster |
|
127 | This function moves these from that location to the working cluster | |
128 | directory. |
|
128 | directory. | |
129 | """ |
|
129 | """ | |
130 | if path is None: |
|
130 | if path is None: | |
131 | import IPython.config.default |
|
131 | import IPython.config.default | |
132 | path = IPython.config.default.__file__.split(os.path.sep)[:-1] |
|
132 | path = IPython.config.default.__file__.split(os.path.sep)[:-1] | |
133 | path = os.path.sep.join(path) |
|
133 | path = os.path.sep.join(path) | |
134 | src = os.path.join(path, config_file) |
|
134 | src = os.path.join(path, config_file) | |
135 | dst = os.path.join(self.location, config_file) |
|
135 | dst = os.path.join(self.location, config_file) | |
136 | if not os.path.isfile(dst) or overwrite: |
|
136 | if not os.path.isfile(dst) or overwrite: | |
137 | shutil.copy(src, dst) |
|
137 | shutil.copy(src, dst) | |
138 |
|
138 | |||
139 | def copy_all_config_files(self, path=None, overwrite=False): |
|
139 | def copy_all_config_files(self, path=None, overwrite=False): | |
140 | """Copy all config files into the active cluster directory.""" |
|
140 | """Copy all config files into the active cluster directory.""" | |
141 | for f in ['ipcontroller_config.py', 'ipengine_config.py', |
|
141 | for f in ['ipcontroller_config.py', 'ipengine_config.py', | |
142 | 'ipcluster_config.py']: |
|
142 | 'ipcluster_config.py']: | |
143 | self.copy_config_file(f, path=path, overwrite=overwrite) |
|
143 | self.copy_config_file(f, path=path, overwrite=overwrite) | |
144 |
|
144 | |||
145 | @classmethod |
|
145 | @classmethod | |
146 | def create_cluster_dir(csl, cluster_dir): |
|
146 | def create_cluster_dir(csl, cluster_dir): | |
147 | """Create a new cluster directory given a full path. |
|
147 | """Create a new cluster directory given a full path. | |
148 |
|
148 | |||
149 | Parameters |
|
149 | Parameters | |
150 | ---------- |
|
150 | ---------- | |
151 | cluster_dir : str |
|
151 | cluster_dir : str | |
152 | The full path to the cluster directory. If it does exist, it will |
|
152 | The full path to the cluster directory. If it does exist, it will | |
153 | be used. If not, it will be created. |
|
153 | be used. If not, it will be created. | |
154 | """ |
|
154 | """ | |
155 | return ClusterDir(cluster_dir) |
|
155 | return ClusterDir(cluster_dir) | |
156 |
|
156 | |||
157 | @classmethod |
|
157 | @classmethod | |
158 | def create_cluster_dir_by_profile(cls, path, profile='default'): |
|
158 | def create_cluster_dir_by_profile(cls, path, profile='default'): | |
159 | """Create a cluster dir by profile name and path. |
|
159 | """Create a cluster dir by profile name and path. | |
160 |
|
160 | |||
161 | Parameters |
|
161 | Parameters | |
162 | ---------- |
|
162 | ---------- | |
163 | path : str |
|
163 | path : str | |
164 | The path (directory) to put the cluster directory in. |
|
164 | The path (directory) to put the cluster directory in. | |
165 | profile : str |
|
165 | profile : str | |
166 | The name of the profile. The name of the cluster directory will |
|
166 | The name of the profile. The name of the cluster directory will | |
167 | be "cluster_<profile>". |
|
167 | be "cluster_<profile>". | |
168 | """ |
|
168 | """ | |
169 | if not os.path.isdir(path): |
|
169 | if not os.path.isdir(path): | |
170 | raise ClusterDirError('Directory not found: %s' % path) |
|
170 | raise ClusterDirError('Directory not found: %s' % path) | |
171 | cluster_dir = os.path.join(path, 'cluster_' + profile) |
|
171 | cluster_dir = os.path.join(path, 'cluster_' + profile) | |
172 | return ClusterDir(cluster_dir) |
|
172 | return ClusterDir(cluster_dir) | |
173 |
|
173 | |||
174 | @classmethod |
|
174 | @classmethod | |
175 | def find_cluster_dir_by_profile(cls, ipython_dir, profile='default'): |
|
175 | def find_cluster_dir_by_profile(cls, ipython_dir, profile='default'): | |
176 | """Find an existing cluster dir by profile name, return its ClusterDir. |
|
176 | """Find an existing cluster dir by profile name, return its ClusterDir. | |
177 |
|
177 | |||
178 | This searches through a sequence of paths for a cluster dir. If it |
|
178 | This searches through a sequence of paths for a cluster dir. If it | |
179 | is not found, a :class:`ClusterDirError` exception will be raised. |
|
179 | is not found, a :class:`ClusterDirError` exception will be raised. | |
180 |
|
180 | |||
181 | The search path algorithm is: |
|
181 | The search path algorithm is: | |
182 | 1. ``os.getcwd()`` |
|
182 | 1. ``os.getcwd()`` | |
183 | 2. ``ipython_dir`` |
|
183 | 2. ``ipython_dir`` | |
184 | 3. The directories found in the ":" separated |
|
184 | 3. The directories found in the ":" separated | |
185 | :env:`IPCLUSTER_DIR_PATH` environment variable. |
|
185 | :env:`IPCLUSTER_DIR_PATH` environment variable. | |
186 |
|
186 | |||
187 | Parameters |
|
187 | Parameters | |
188 | ---------- |
|
188 | ---------- | |
189 | ipython_dir : unicode or str |
|
189 | ipython_dir : unicode or str | |
190 | The IPython directory to use. |
|
190 | The IPython directory to use. | |
191 | profile : unicode or str |
|
191 | profile : unicode or str | |
192 | The name of the profile. The name of the cluster directory |
|
192 | The name of the profile. The name of the cluster directory | |
193 | will be "cluster_<profile>". |
|
193 | will be "cluster_<profile>". | |
194 | """ |
|
194 | """ | |
195 | dirname = 'cluster_' + profile |
|
195 | dirname = 'cluster_' + profile | |
196 | cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','') |
|
196 | cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','') | |
197 | if cluster_dir_paths: |
|
197 | if cluster_dir_paths: | |
198 | cluster_dir_paths = cluster_dir_paths.split(':') |
|
198 | cluster_dir_paths = cluster_dir_paths.split(':') | |
199 | else: |
|
199 | else: | |
200 | cluster_dir_paths = [] |
|
200 | cluster_dir_paths = [] | |
201 | paths = [os.getcwd(), ipython_dir] + cluster_dir_paths |
|
201 | paths = [os.getcwd(), ipython_dir] + cluster_dir_paths | |
202 | for p in paths: |
|
202 | for p in paths: | |
203 | cluster_dir = os.path.join(p, dirname) |
|
203 | cluster_dir = os.path.join(p, dirname) | |
204 | if os.path.isdir(cluster_dir): |
|
204 | if os.path.isdir(cluster_dir): | |
205 | return ClusterDir(cluster_dir) |
|
205 | return ClusterDir(cluster_dir) | |
206 | else: |
|
206 | else: | |
207 | raise ClusterDirError('Cluster directory not found in paths: %s' % dirname) |
|
207 | raise ClusterDirError('Cluster directory not found in paths: %s' % dirname) | |
208 |
|
208 | |||
209 | @classmethod |
|
209 | @classmethod | |
210 | def find_cluster_dir(cls, cluster_dir): |
|
210 | def find_cluster_dir(cls, cluster_dir): | |
211 | """Find/create a cluster dir and return its ClusterDir. |
|
211 | """Find/create a cluster dir and return its ClusterDir. | |
212 |
|
212 | |||
213 | This will create the cluster directory if it doesn't exist. |
|
213 | This will create the cluster directory if it doesn't exist. | |
214 |
|
214 | |||
215 | Parameters |
|
215 | Parameters | |
216 | ---------- |
|
216 | ---------- | |
217 | cluster_dir : unicode or str |
|
217 | cluster_dir : unicode or str | |
218 | The path of the cluster directory. This is expanded using |
|
218 | The path of the cluster directory. This is expanded using | |
219 | :func:`os.path.expandvars` and :func:`os.path.expanduser`. |
|
219 | :func:`os.path.expandvars` and :func:`os.path.expanduser`. | |
220 | """ |
|
220 | """ | |
221 | cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir)) |
|
221 | cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir)) | |
222 | if not os.path.isdir(cluster_dir): |
|
222 | if not os.path.isdir(cluster_dir): | |
223 | raise ClusterDirError('Cluster directory not found: %s' % cluster_dir) |
|
223 | raise ClusterDirError('Cluster directory not found: %s' % cluster_dir) | |
224 | return ClusterDir(cluster_dir) |
|
224 | return ClusterDir(cluster_dir) | |
225 |
|
225 | |||
226 |
|
226 | |||
227 | class AppWithClusterDirArgParseConfigLoader(ArgParseConfigLoader): |
|
227 | class AppWithClusterDirArgParseConfigLoader(ArgParseConfigLoader): | |
228 | """Default command line options for IPython cluster applications.""" |
|
228 | """Default command line options for IPython cluster applications.""" | |
229 |
|
229 | |||
230 | def _add_other_arguments(self): |
|
230 | def _add_other_arguments(self): | |
231 | self.parser.add_argument('--ipython-dir', |
|
231 | self.parser.add_argument('--ipython-dir', | |
232 | dest='Global.ipython_dir',type=str, |
|
232 | dest='Global.ipython_dir',type=str, | |
233 | help='Set to override default location of Global.ipython_dir.', |
|
233 | help='Set to override default location of Global.ipython_dir.', | |
234 | default=NoConfigDefault, |
|
234 | default=NoConfigDefault, | |
235 | metavar='Global.ipython_dir' |
|
235 | metavar='Global.ipython_dir' | |
236 | ) |
|
236 | ) | |
237 | self.parser.add_argument('-p', '--profile', |
|
237 | self.parser.add_argument('-p', '--profile', | |
238 | dest='Global.profile',type=str, |
|
238 | dest='Global.profile',type=str, | |
239 | help='The string name of the profile to be used. This determines ' |
|
239 | help='The string name of the profile to be used. This determines ' | |
240 | 'the name of the cluster dir as: cluster_<profile>. The default profile ' |
|
240 | 'the name of the cluster dir as: cluster_<profile>. The default profile ' | |
241 | 'is named "default". The cluster directory is resolve this way ' |
|
241 | 'is named "default". The cluster directory is resolve this way ' | |
242 | 'if the --cluster-dir option is not used.', |
|
242 | 'if the --cluster-dir option is not used.', | |
243 | default=NoConfigDefault, |
|
243 | default=NoConfigDefault, | |
244 | metavar='Global.profile' |
|
244 | metavar='Global.profile' | |
245 | ) |
|
245 | ) | |
246 | self.parser.add_argument('--log-level', |
|
246 | self.parser.add_argument('--log-level', | |
247 | dest="Global.log_level",type=int, |
|
247 | dest="Global.log_level",type=int, | |
248 | help='Set the log level (0,10,20,30,40,50). Default is 30.', |
|
248 | help='Set the log level (0,10,20,30,40,50). Default is 30.', | |
249 | default=NoConfigDefault, |
|
249 | default=NoConfigDefault, | |
250 | metavar="Global.log_level" |
|
250 | metavar="Global.log_level" | |
251 | ) |
|
251 | ) | |
252 | self.parser.add_argument('--cluster-dir', |
|
252 | self.parser.add_argument('--cluster-dir', | |
253 | dest='Global.cluster_dir',type=str, |
|
253 | dest='Global.cluster_dir',type=str, | |
254 | help='Set the cluster dir. This overrides the logic used by the ' |
|
254 | help='Set the cluster dir. This overrides the logic used by the ' | |
255 | '--profile option.', |
|
255 | '--profile option.', | |
256 | default=NoConfigDefault, |
|
256 | default=NoConfigDefault, | |
257 | metavar='Global.cluster_dir' |
|
257 | metavar='Global.cluster_dir' | |
258 | ) |
|
258 | ) | |
259 | self.parser.add_argument('--clean-logs', |
|
259 | self.parser.add_argument('--clean-logs', | |
260 | dest='Global.clean_logs', action='store_true', |
|
260 | dest='Global.clean_logs', action='store_true', | |
261 | help='Delete old log flies before starting.', |
|
261 | help='Delete old log flies before starting.', | |
262 | default=NoConfigDefault |
|
262 | default=NoConfigDefault | |
263 | ) |
|
263 | ) | |
264 | self.parser.add_argument('--no-clean-logs', |
|
264 | self.parser.add_argument('--no-clean-logs', | |
265 | dest='Global.clean_logs', action='store_false', |
|
265 | dest='Global.clean_logs', action='store_false', | |
266 | help="Don't Delete old log flies before starting.", |
|
266 | help="Don't Delete old log flies before starting.", | |
267 | default=NoConfigDefault |
|
267 | default=NoConfigDefault | |
268 | ) |
|
268 | ) | |
269 |
|
269 | |||
270 | class ApplicationWithClusterDir(Application): |
|
270 | class ApplicationWithClusterDir(Application): | |
271 | """An application that puts everything into a cluster directory. |
|
271 | """An application that puts everything into a cluster directory. | |
272 |
|
272 | |||
273 | Instead of looking for things in the ipython_dir, this type of application |
|
273 | Instead of looking for things in the ipython_dir, this type of application | |
274 | will use its own private directory called the "cluster directory" |
|
274 | will use its own private directory called the "cluster directory" | |
275 | for things like config files, log files, etc. |
|
275 | for things like config files, log files, etc. | |
276 |
|
276 | |||
277 | The cluster directory is resolved as follows: |
|
277 | The cluster directory is resolved as follows: | |
278 |
|
278 | |||
279 | * If the ``--cluster-dir`` option is given, it is used. |
|
279 | * If the ``--cluster-dir`` option is given, it is used. | |
280 | * If ``--cluster-dir`` is not given, the application directory is |
|
280 | * If ``--cluster-dir`` is not given, the application directory is | |
281 | resolve using the profile name as ``cluster_<profile>``. The search |
|
281 | resolve using the profile name as ``cluster_<profile>``. The search | |
282 | path for this directory is then i) cwd if it is found there |
|
282 | path for this directory is then i) cwd if it is found there | |
283 | and ii) in ipython_dir otherwise. |
|
283 | and ii) in ipython_dir otherwise. | |
284 |
|
284 | |||
285 | The config file for the application is to be put in the cluster |
|
285 | The config file for the application is to be put in the cluster | |
286 | dir and named the value of the ``config_file_name`` class attribute. |
|
286 | dir and named the value of the ``config_file_name`` class attribute. | |
287 | """ |
|
287 | """ | |
288 |
|
288 | |||
289 | auto_create_cluster_dir = True |
|
289 | auto_create_cluster_dir = True | |
290 |
|
290 | |||
291 | def create_default_config(self): |
|
291 | def create_default_config(self): | |
292 | super(ApplicationWithClusterDir, self).create_default_config() |
|
292 | super(ApplicationWithClusterDir, self).create_default_config() | |
293 | self.default_config.Global.profile = 'default' |
|
293 | self.default_config.Global.profile = 'default' | |
294 | self.default_config.Global.cluster_dir = '' |
|
294 | self.default_config.Global.cluster_dir = '' | |
295 | self.default_config.Global.log_to_file = False |
|
295 | self.default_config.Global.log_to_file = False | |
296 | self.default_config.Global.clean_logs = False |
|
296 | self.default_config.Global.clean_logs = False | |
297 |
|
297 | |||
298 | def create_command_line_config(self): |
|
298 | def create_command_line_config(self): | |
299 | """Create and return a command line config loader.""" |
|
299 | """Create and return a command line config loader.""" | |
300 | return AppWithClusterDirArgParseConfigLoader( |
|
300 | return AppWithClusterDirArgParseConfigLoader( | |
301 | description=self.description, |
|
301 | description=self.description, | |
302 | version=release.version |
|
302 | version=release.version | |
303 | ) |
|
303 | ) | |
304 |
|
304 | |||
305 | def find_resources(self): |
|
305 | def find_resources(self): | |
306 | """This resolves the cluster directory. |
|
306 | """This resolves the cluster directory. | |
307 |
|
307 | |||
308 | This tries to find the cluster directory and if successful, it will |
|
308 | This tries to find the cluster directory and if successful, it will | |
309 | have done: |
|
309 | have done: | |
310 | * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for |
|
310 | * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for | |
311 | the application. |
|
311 | the application. | |
312 | * Sets ``self.cluster_dir`` attribute of the application and config |
|
312 | * Sets ``self.cluster_dir`` attribute of the application and config | |
313 | objects. |
|
313 | objects. | |
314 |
|
314 | |||
315 | The algorithm used for this is as follows: |
|
315 | The algorithm used for this is as follows: | |
316 | 1. Try ``Global.cluster_dir``. |
|
316 | 1. Try ``Global.cluster_dir``. | |
317 | 2. Try using ``Global.profile``. |
|
317 | 2. Try using ``Global.profile``. | |
318 | 3. If both of these fail and ``self.auto_create_cluster_dir`` is |
|
318 | 3. If both of these fail and ``self.auto_create_cluster_dir`` is | |
319 | ``True``, then create the new cluster dir in the IPython directory. |
|
319 | ``True``, then create the new cluster dir in the IPython directory. | |
320 | 4. If all fails, then raise :class:`ClusterDirError`. |
|
320 | 4. If all fails, then raise :class:`ClusterDirError`. | |
321 | """ |
|
321 | """ | |
322 |
|
322 | |||
323 | try: |
|
323 | try: | |
324 | cluster_dir = self.command_line_config.Global.cluster_dir |
|
324 | cluster_dir = self.command_line_config.Global.cluster_dir | |
325 | except AttributeError: |
|
325 | except AttributeError: | |
326 | cluster_dir = self.default_config.Global.cluster_dir |
|
326 | cluster_dir = self.default_config.Global.cluster_dir | |
327 | cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir)) |
|
327 | cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir)) | |
328 | try: |
|
328 | try: | |
329 | self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir) |
|
329 | self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir) | |
330 | except ClusterDirError: |
|
330 | except ClusterDirError: | |
331 | pass |
|
331 | pass | |
332 | else: |
|
332 | else: | |
333 | self.log.info('Using existing cluster dir: %s' % \ |
|
333 | self.log.info('Using existing cluster dir: %s' % \ | |
334 | self.cluster_dir_obj.location |
|
334 | self.cluster_dir_obj.location | |
335 | ) |
|
335 | ) | |
336 | self.finish_cluster_dir() |
|
336 | self.finish_cluster_dir() | |
337 | return |
|
337 | return | |
338 |
|
338 | |||
339 | try: |
|
339 | try: | |
340 | self.profile = self.command_line_config.Global.profile |
|
340 | self.profile = self.command_line_config.Global.profile | |
341 | except AttributeError: |
|
341 | except AttributeError: | |
342 | self.profile = self.default_config.Global.profile |
|
342 | self.profile = self.default_config.Global.profile | |
343 | try: |
|
343 | try: | |
344 | self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile( |
|
344 | self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile( | |
345 | self.ipython_dir, self.profile) |
|
345 | self.ipython_dir, self.profile) | |
346 | except ClusterDirError: |
|
346 | except ClusterDirError: | |
347 | pass |
|
347 | pass | |
348 | else: |
|
348 | else: | |
349 | self.log.info('Using existing cluster dir: %s' % \ |
|
349 | self.log.info('Using existing cluster dir: %s' % \ | |
350 | self.cluster_dir_obj.location |
|
350 | self.cluster_dir_obj.location | |
351 | ) |
|
351 | ) | |
352 | self.finish_cluster_dir() |
|
352 | self.finish_cluster_dir() | |
353 | return |
|
353 | return | |
354 |
|
354 | |||
355 | if self.auto_create_cluster_dir: |
|
355 | if self.auto_create_cluster_dir: | |
356 | self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile( |
|
356 | self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile( | |
357 | self.ipython_dir, self.profile |
|
357 | self.ipython_dir, self.profile | |
358 | ) |
|
358 | ) | |
359 | self.log.info('Creating new cluster dir: %s' % \ |
|
359 | self.log.info('Creating new cluster dir: %s' % \ | |
360 | self.cluster_dir_obj.location |
|
360 | self.cluster_dir_obj.location | |
361 | ) |
|
361 | ) | |
362 | self.finish_cluster_dir() |
|
362 | self.finish_cluster_dir() | |
363 | else: |
|
363 | else: | |
364 | raise ClusterDirError('Could not find a valid cluster directory.') |
|
364 | raise ClusterDirError('Could not find a valid cluster directory.') | |
365 |
|
365 | |||
366 | def finish_cluster_dir(self): |
|
366 | def finish_cluster_dir(self): | |
367 | # Set the cluster directory |
|
367 | # Set the cluster directory | |
368 | self.cluster_dir = self.cluster_dir_obj.location |
|
368 | self.cluster_dir = self.cluster_dir_obj.location | |
369 |
|
369 | |||
370 | # These have to be set because they could be different from the one |
|
370 | # These have to be set because they could be different from the one | |
371 | # that we just computed. Because command line has the highest |
|
371 | # that we just computed. Because command line has the highest | |
372 | # priority, this will always end up in the master_config. |
|
372 | # priority, this will always end up in the master_config. | |
373 | self.default_config.Global.cluster_dir = self.cluster_dir |
|
373 | self.default_config.Global.cluster_dir = self.cluster_dir | |
374 | self.command_line_config.Global.cluster_dir = self.cluster_dir |
|
374 | self.command_line_config.Global.cluster_dir = self.cluster_dir | |
375 |
|
375 | |||
376 | # Set the search path to the cluster directory |
|
376 | # Set the search path to the cluster directory | |
377 | self.config_file_paths = (self.cluster_dir,) |
|
377 | self.config_file_paths = (self.cluster_dir,) | |
378 |
|
378 | |||
379 | def find_config_file_name(self): |
|
379 | def find_config_file_name(self): | |
380 | """Find the config file name for this application.""" |
|
380 | """Find the config file name for this application.""" | |
381 | # For this type of Application it should be set as a class attribute. |
|
381 | # For this type of Application it should be set as a class attribute. | |
382 | if not hasattr(self, 'config_file_name'): |
|
382 | if not hasattr(self, 'config_file_name'): | |
383 | self.log.critical("No config filename found") |
|
383 | self.log.critical("No config filename found") | |
384 |
|
384 | |||
385 | def find_config_file_paths(self): |
|
385 | def find_config_file_paths(self): | |
386 | # Set the search path to the cluster directory |
|
386 | # Set the search path to the cluster directory | |
387 | self.config_file_paths = (self.cluster_dir,) |
|
387 | self.config_file_paths = (self.cluster_dir,) | |
388 |
|
388 | |||
389 | def pre_construct(self): |
|
389 | def pre_construct(self): | |
390 | # The log and security dirs were set earlier, but here we put them |
|
390 | # The log and security dirs were set earlier, but here we put them | |
391 | # into the config and log them. |
|
391 | # into the config and log them. | |
392 | config = self.master_config |
|
392 | config = self.master_config | |
393 | sdir = self.cluster_dir_obj.security_dir |
|
393 | sdir = self.cluster_dir_obj.security_dir | |
394 | self.security_dir = config.Global.security_dir = sdir |
|
394 | self.security_dir = config.Global.security_dir = sdir | |
395 | ldir = self.cluster_dir_obj.log_dir |
|
395 | ldir = self.cluster_dir_obj.log_dir | |
396 | self.log_dir = config.Global.log_dir = ldir |
|
396 | self.log_dir = config.Global.log_dir = ldir | |
397 | pdir = self.cluster_dir_obj.pid_dir |
|
397 | pdir = self.cluster_dir_obj.pid_dir | |
398 | self.pid_dir = config.Global.pid_dir = pdir |
|
398 | self.pid_dir = config.Global.pid_dir = pdir | |
399 | self.log.info("Cluster directory set to: %s" % self.cluster_dir) |
|
399 | self.log.info("Cluster directory set to: %s" % self.cluster_dir) | |
400 |
|
400 | |||
401 | def start_logging(self): |
|
401 | def start_logging(self): | |
402 | # Remove old log files |
|
402 | # Remove old log files | |
403 | if self.master_config.Global.clean_logs: |
|
403 | if self.master_config.Global.clean_logs: | |
404 | log_dir = self.master_config.Global.log_dir |
|
404 | log_dir = self.master_config.Global.log_dir | |
405 | for f in os.listdir(log_dir): |
|
405 | for f in os.listdir(log_dir): | |
406 | if f.startswith(self.name + '-') and f.endswith('.log'): |
|
406 | if f.startswith(self.name + '-') and f.endswith('.log'): | |
407 | os.remove(os.path.join(log_dir, f)) |
|
407 | os.remove(os.path.join(log_dir, f)) | |
408 | # Start logging to the new log file |
|
408 | # Start logging to the new log file | |
409 | if self.master_config.Global.log_to_file: |
|
409 | if self.master_config.Global.log_to_file: | |
410 | log_filename = self.name + '-' + str(os.getpid()) + '.log' |
|
410 | log_filename = self.name + '-' + str(os.getpid()) + '.log' | |
411 | logfile = os.path.join(self.log_dir, log_filename) |
|
411 | logfile = os.path.join(self.log_dir, log_filename) | |
412 | open_log_file = open(logfile, 'w') |
|
412 | open_log_file = open(logfile, 'w') | |
413 | else: |
|
413 | else: | |
414 | open_log_file = sys.stdout |
|
414 | open_log_file = sys.stdout | |
415 | log.startLogging(open_log_file) |
|
415 | log.startLogging(open_log_file) | |
416 |
|
416 | |||
417 | def write_pid_file(self, overwrite=False): |
|
417 | def write_pid_file(self, overwrite=False): | |
418 | """Create a .pid file in the pid_dir with my pid. |
|
418 | """Create a .pid file in the pid_dir with my pid. | |
419 |
|
419 | |||
420 | This must be called after pre_construct, which sets `self.pid_dir`. |
|
420 | This must be called after pre_construct, which sets `self.pid_dir`. | |
421 | This raises :exc:`PIDFileError` if the pid file exists already. |
|
421 | This raises :exc:`PIDFileError` if the pid file exists already. | |
422 | """ |
|
422 | """ | |
423 | pid_file = os.path.join(self.pid_dir, self.name + '.pid') |
|
423 | pid_file = os.path.join(self.pid_dir, self.name + '.pid') | |
424 | if os.path.isfile(pid_file): |
|
424 | if os.path.isfile(pid_file): | |
425 | pid = self.get_pid_from_file() |
|
425 | pid = self.get_pid_from_file() | |
426 | if not overwrite: |
|
426 | if not overwrite: | |
427 | raise PIDFileError( |
|
427 | raise PIDFileError( | |
428 | 'The pid file [%s] already exists. \nThis could mean that this ' |
|
428 | 'The pid file [%s] already exists. \nThis could mean that this ' | |
429 | 'server is already running with [pid=%s].' % (pid_file, pid) |
|
429 | 'server is already running with [pid=%s].' % (pid_file, pid) | |
430 | ) |
|
430 | ) | |
431 | with open(pid_file, 'w') as f: |
|
431 | with open(pid_file, 'w') as f: | |
432 | self.log.info("Creating pid file: %s" % pid_file) |
|
432 | self.log.info("Creating pid file: %s" % pid_file) | |
433 | f.write(repr(os.getpid())+'\n') |
|
433 | f.write(repr(os.getpid())+'\n') | |
434 |
|
434 | |||
435 | def remove_pid_file(self): |
|
435 | def remove_pid_file(self): | |
436 | """Remove the pid file. |
|
436 | """Remove the pid file. | |
437 |
|
437 | |||
438 | This should be called at shutdown by registering a callback with |
|
438 | This should be called at shutdown by registering a callback with | |
439 | :func:`reactor.addSystemEventTrigger`. |
|
439 | :func:`reactor.addSystemEventTrigger`. | |
440 | """ |
|
440 | """ | |
441 | pid_file = os.path.join(self.pid_dir, self.name + '.pid') |
|
441 | pid_file = os.path.join(self.pid_dir, self.name + '.pid') | |
442 | if os.path.isfile(pid_file): |
|
442 | if os.path.isfile(pid_file): | |
443 | try: |
|
443 | try: | |
444 | self.log.info("Removing pid file: %s" % pid_file) |
|
444 | self.log.info("Removing pid file: %s" % pid_file) | |
445 | os.remove(pid_file) |
|
445 | os.remove(pid_file) | |
446 | except: |
|
446 | except: | |
447 | self.log.warn("Error removing the pid file: %s" % pid_file) |
|
447 | self.log.warn("Error removing the pid file: %s" % pid_file) | |
448 | raise |
|
448 | raise | |
449 |
|
449 | |||
450 | def get_pid_from_file(self): |
|
450 | def get_pid_from_file(self): | |
451 | """Get the pid from the pid file. |
|
451 | """Get the pid from the pid file. | |
452 |
|
452 | |||
453 | If the pid file doesn't exist a :exc:`PIDFileError` is raised. |
|
453 | If the pid file doesn't exist a :exc:`PIDFileError` is raised. | |
454 | """ |
|
454 | """ | |
455 | pid_file = os.path.join(self.pid_dir, self.name + '.pid') |
|
455 | pid_file = os.path.join(self.pid_dir, self.name + '.pid') | |
456 | if os.path.isfile(pid_file): |
|
456 | if os.path.isfile(pid_file): | |
457 | with open(pid_file, 'r') as f: |
|
457 | with open(pid_file, 'r') as f: | |
458 | pid = int(f.read().strip()) |
|
458 | pid = int(f.read().strip()) | |
459 | return pid |
|
459 | return pid | |
460 | else: |
|
460 | else: | |
461 | raise PIDFileError('pid file not found: %s' % pid_file) No newline at end of file |
|
461 | raise PIDFileError('pid file not found: %s' % pid_file) | |
|
462 | ||||
|
463 |
@@ -1,266 +1,273 b'' | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | Foolscap related utilities. |
|
4 | Foolscap related utilities. | |
5 | """ |
|
5 | """ | |
6 |
|
6 | |||
7 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
8 | # Copyright (C) 2008-2009 The IPython Development Team |
|
8 | # Copyright (C) 2008-2009 The IPython Development Team | |
9 | # |
|
9 | # | |
10 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | # Distributed under the terms of the BSD License. The full license is in | |
11 | # the file COPYING, distributed as part of this software. |
|
11 | # the file COPYING, distributed as part of this software. | |
12 | #----------------------------------------------------------------------------- |
|
12 | #----------------------------------------------------------------------------- | |
13 |
|
13 | |||
14 | #----------------------------------------------------------------------------- |
|
14 | #----------------------------------------------------------------------------- | |
15 | # Imports |
|
15 | # Imports | |
16 | #----------------------------------------------------------------------------- |
|
16 | #----------------------------------------------------------------------------- | |
17 |
|
17 | |||
18 | from __future__ import with_statement |
|
18 | from __future__ import with_statement | |
19 |
|
19 | |||
20 | import os |
|
20 | import os | |
21 | import tempfile |
|
21 | import tempfile | |
22 |
|
22 | |||
23 | from twisted.internet import reactor, defer |
|
23 | from twisted.internet import reactor, defer | |
24 | from twisted.python import log |
|
24 | from twisted.python import log | |
25 |
|
25 | |||
26 | from foolscap import Tub, UnauthenticatedTub |
|
26 | from foolscap import Tub, UnauthenticatedTub | |
27 |
|
27 | |||
28 | from IPython.config.loader import Config |
|
28 | from IPython.config.loader import Config | |
29 |
|
29 | |||
30 | from IPython.kernel.configobjfactory import AdaptedConfiguredObjectFactory |
|
30 | from IPython.kernel.configobjfactory import AdaptedConfiguredObjectFactory | |
31 |
|
31 | |||
32 | from IPython.kernel.error import SecurityError |
|
32 | from IPython.kernel.error import SecurityError | |
33 |
|
33 | |||
34 | from IPython.utils.traitlets import Int, Str, Bool, Instance |
|
34 | from IPython.utils.traitlets import Int, Str, Bool, Instance | |
35 | from IPython.utils.importstring import import_item |
|
35 | from IPython.utils.importstring import import_item | |
36 |
|
36 | |||
37 | #----------------------------------------------------------------------------- |
|
37 | #----------------------------------------------------------------------------- | |
38 | # Code |
|
38 | # Code | |
39 | #----------------------------------------------------------------------------- |
|
39 | #----------------------------------------------------------------------------- | |
40 |
|
40 | |||
41 |
|
41 | |||
42 | # We do this so if a user doesn't have OpenSSL installed, it will try to use |
|
42 | # We do this so if a user doesn't have OpenSSL installed, it will try to use | |
43 | # an UnauthenticatedTub. But, they will still run into problems if they |
|
43 | # an UnauthenticatedTub. But, they will still run into problems if they | |
44 | # try to use encrypted furls. |
|
44 | # try to use encrypted furls. | |
45 | try: |
|
45 | try: | |
46 | import OpenSSL |
|
46 | import OpenSSL | |
47 | except: |
|
47 | except: | |
48 | Tub = UnauthenticatedTub |
|
48 | Tub = UnauthenticatedTub | |
49 | have_crypto = False |
|
49 | have_crypto = False | |
50 | else: |
|
50 | else: | |
51 | have_crypto = True |
|
51 | have_crypto = True | |
52 |
|
52 | |||
53 |
|
53 | |||
54 | class FURLError(Exception): |
|
54 | class FURLError(Exception): | |
55 | pass |
|
55 | pass | |
56 |
|
56 | |||
57 |
|
57 | |||
58 | def check_furl_file_security(furl_file, secure): |
|
58 | def check_furl_file_security(furl_file, secure): | |
59 | """Remove the old furl_file if changing security modes.""" |
|
59 | """Remove the old furl_file if changing security modes.""" | |
60 | if os.path.isfile(furl_file): |
|
60 | if os.path.isfile(furl_file): | |
61 | f = open(furl_file, 'r') |
|
61 | f = open(furl_file, 'r') | |
62 | oldfurl = f.read().strip() |
|
62 | oldfurl = f.read().strip() | |
63 | f.close() |
|
63 | f.close() | |
64 | if (oldfurl.startswith('pb://') and not secure) or (oldfurl.startswith('pbu://') and secure): |
|
64 | if (oldfurl.startswith('pb://') and not secure) or (oldfurl.startswith('pbu://') and secure): | |
65 | os.remove(furl_file) |
|
65 | os.remove(furl_file) | |
66 |
|
66 | |||
67 |
|
67 | |||
68 | def is_secure(furl): |
|
68 | def is_secure(furl): | |
69 | """Is the given FURL secure or not.""" |
|
69 | """Is the given FURL secure or not.""" | |
70 | if is_valid(furl): |
|
70 | if is_valid(furl): | |
71 | if furl.startswith("pb://"): |
|
71 | if furl.startswith("pb://"): | |
72 | return True |
|
72 | return True | |
73 | elif furl.startswith("pbu://"): |
|
73 | elif furl.startswith("pbu://"): | |
74 | return False |
|
74 | return False | |
75 | else: |
|
75 | else: | |
76 | raise FURLError("invalid FURL: %s" % furl) |
|
76 | raise FURLError("invalid FURL: %s" % furl) | |
77 |
|
77 | |||
78 |
|
78 | |||
79 | def is_valid(furl): |
|
79 | def is_valid(furl): | |
80 | """Is the str a valid FURL or not.""" |
|
80 | """Is the str a valid FURL or not.""" | |
81 | if isinstance(furl, str): |
|
81 | if isinstance(furl, str): | |
82 | if furl.startswith("pb://") or furl.startswith("pbu://"): |
|
82 | if furl.startswith("pb://") or furl.startswith("pbu://"): | |
83 | return True |
|
83 | return True | |
84 | else: |
|
84 | else: | |
85 | return False |
|
85 | return False | |
86 |
|
86 | |||
87 |
|
87 | |||
88 | def find_furl(furl_or_file): |
|
88 | def find_furl(furl_or_file): | |
89 | """Find, validate and return a FURL in a string or file.""" |
|
89 | """Find, validate and return a FURL in a string or file.""" | |
90 | if isinstance(furl_or_file, str): |
|
90 | if isinstance(furl_or_file, str): | |
91 | if is_valid(furl_or_file): |
|
91 | if is_valid(furl_or_file): | |
92 | return furl_or_file |
|
92 | return furl_or_file | |
93 | if os.path.isfile(furl_or_file): |
|
93 | if os.path.isfile(furl_or_file): | |
94 | with open(furl_or_file, 'r') as f: |
|
94 | with open(furl_or_file, 'r') as f: | |
95 | furl = f.read().strip() |
|
95 | furl = f.read().strip() | |
96 | if is_valid(furl): |
|
96 | if is_valid(furl): | |
97 | return furl |
|
97 | return furl | |
98 | raise FURLError("Not a valid FURL or FURL file: %s" % furl_or_file) |
|
98 | raise FURLError("Not a valid FURL or FURL file: %s" % furl_or_file) | |
99 |
|
99 | |||
100 |
|
100 | |||
101 | def is_valid_furl_or_file(furl_or_file): |
|
101 | def is_valid_furl_or_file(furl_or_file): | |
102 | """Validate a FURL or a FURL file. |
|
102 | """Validate a FURL or a FURL file. | |
103 |
|
103 | |||
104 | If ``furl_or_file`` looks like a file, we simply make sure its directory |
|
104 | If ``furl_or_file`` looks like a file, we simply make sure its directory | |
105 | exists and that it has a ``.furl`` file extension. We don't try to see |
|
105 | exists and that it has a ``.furl`` file extension. We don't try to see | |
106 | if the FURL file exists or to read its contents. This is useful for |
|
106 | if the FURL file exists or to read its contents. This is useful for | |
107 | cases where auto re-connection is being used. |
|
107 | cases where auto re-connection is being used. | |
108 | """ |
|
108 | """ | |
109 | if isinstance(furl_or_file, str): |
|
109 | if isinstance(furl_or_file, str): | |
110 | if is_valid(furl_or_file): |
|
110 | if is_valid(furl_or_file): | |
111 | return True |
|
111 | return True | |
112 | if isinstance(furl_or_file, (str, unicode)): |
|
112 | if isinstance(furl_or_file, (str, unicode)): | |
113 | path, furl_filename = os.path.split(furl_or_file) |
|
113 | path, furl_filename = os.path.split(furl_or_file) | |
114 | if os.path.isdir(path) and furl_filename.endswith('.furl'): |
|
114 | if os.path.isdir(path) and furl_filename.endswith('.furl'): | |
115 | return True |
|
115 | return True | |
116 | return False |
|
116 | return False | |
117 |
|
117 | |||
118 |
|
118 | |||
119 | def validate_furl_or_file(furl_or_file): |
|
119 | def validate_furl_or_file(furl_or_file): | |
120 | if not is_valid_furl_or_file(furl_or_file): |
|
120 | if not is_valid_furl_or_file(furl_or_file): | |
121 | raise FURLError('Not a valid FURL or FURL file: %r' % furl_or_file) |
|
121 | raise FURLError('Not a valid FURL or FURL file: %r' % furl_or_file) | |
122 |
|
122 | |||
123 |
|
123 | |||
124 | def get_temp_furlfile(filename): |
|
124 | def get_temp_furlfile(filename): | |
125 | """Return a temporary FURL file.""" |
|
125 | """Return a temporary FURL file.""" | |
126 | return tempfile.mktemp(dir=os.path.dirname(filename), |
|
126 | return tempfile.mktemp(dir=os.path.dirname(filename), | |
127 | prefix=os.path.basename(filename)) |
|
127 | prefix=os.path.basename(filename)) | |
128 |
|
128 | |||
129 |
|
129 | |||
130 | def make_tub(ip, port, secure, cert_file): |
|
130 | def make_tub(ip, port, secure, cert_file): | |
131 | """Create a listening tub given an ip, port, and cert_file location. |
|
131 | """Create a listening tub given an ip, port, and cert_file location. | |
132 |
|
132 | |||
133 | Parameters |
|
133 | Parameters | |
134 | ---------- |
|
134 | ---------- | |
135 | ip : str |
|
135 | ip : str | |
136 | The ip address or hostname that the tub should listen on. |
|
136 | The ip address or hostname that the tub should listen on. | |
137 | Empty means all interfaces. |
|
137 | Empty means all interfaces. | |
138 | port : int |
|
138 | port : int | |
139 | The port that the tub should listen on. A value of 0 means |
|
139 | The port that the tub should listen on. A value of 0 means | |
140 | pick a random port |
|
140 | pick a random port | |
141 | secure: bool |
|
141 | secure: bool | |
142 | Will the connection be secure (in the Foolscap sense). |
|
142 | Will the connection be secure (in the Foolscap sense). | |
143 | cert_file: str |
|
143 | cert_file: str | |
144 | A filename of a file to be used for theSSL certificate. |
|
144 | A filename of a file to be used for theSSL certificate. | |
145 |
|
145 | |||
146 | Returns |
|
146 | Returns | |
147 | ------- |
|
147 | ------- | |
148 | A tub, listener tuple. |
|
148 | A tub, listener tuple. | |
149 | """ |
|
149 | """ | |
150 | if secure: |
|
150 | if secure: | |
151 | if have_crypto: |
|
151 | if have_crypto: | |
152 | tub = Tub(certFile=cert_file) |
|
152 | tub = Tub(certFile=cert_file) | |
153 | else: |
|
153 | else: | |
154 | raise SecurityError("OpenSSL/pyOpenSSL is not available, so we " |
|
154 | raise SecurityError("OpenSSL/pyOpenSSL is not available, so we " | |
155 | "can't run in secure mode. Try running without " |
|
155 | "can't run in secure mode. Try running without " | |
156 | "security using 'ipcontroller -xy'.") |
|
156 | "security using 'ipcontroller -xy'.") | |
157 | else: |
|
157 | else: | |
158 | tub = UnauthenticatedTub() |
|
158 | tub = UnauthenticatedTub() | |
159 |
|
159 | |||
160 | # Set the strport based on the ip and port and start listening |
|
160 | # Set the strport based on the ip and port and start listening | |
161 | if ip == '': |
|
161 | if ip == '': | |
162 | strport = "tcp:%i" % port |
|
162 | strport = "tcp:%i" % port | |
163 | else: |
|
163 | else: | |
164 | strport = "tcp:%i:interface=%s" % (port, ip) |
|
164 | strport = "tcp:%i:interface=%s" % (port, ip) | |
165 | log.msg("Starting listener with [secure=%r] on: %s" % (secure, strport)) |
|
165 | log.msg("Starting listener with [secure=%r] on: %s" % (secure, strport)) | |
166 | listener = tub.listenOn(strport) |
|
166 | listener = tub.listenOn(strport) | |
167 |
|
167 | |||
168 | return tub, listener |
|
168 | return tub, listener | |
169 |
|
169 | |||
170 |
|
170 | |||
171 | class FCServiceFactory(AdaptedConfiguredObjectFactory): |
|
171 | class FCServiceFactory(AdaptedConfiguredObjectFactory): | |
172 | """This class creates a tub with various services running in it. |
|
172 | """This class creates a tub with various services running in it. | |
173 |
|
173 | |||
174 | The basic idea is that :meth:`create` returns a running :class:`Tub` |
|
174 | The basic idea is that :meth:`create` returns a running :class:`Tub` | |
175 | instance that has a number of Foolscap references registered in it. |
|
175 | instance that has a number of Foolscap references registered in it. | |
176 | This class is a subclass of :class:`IPython.core.component.Component` |
|
176 | This class is a subclass of :class:`IPython.core.component.Component` | |
177 | so the IPython configuration and component system are used. |
|
177 | so the IPython configuration and component system are used. | |
178 |
|
178 | |||
179 | Attributes |
|
179 | Attributes | |
180 | ---------- |
|
180 | ---------- | |
181 | interfaces : Config |
|
181 | interfaces : Config | |
182 | A Config instance whose values are sub-Config objects having two |
|
182 | A Config instance whose values are sub-Config objects having two | |
183 | keys: furl_file and interface_chain. |
|
183 | keys: furl_file and interface_chain. | |
184 |
|
184 | |||
185 | The other attributes are the standard ones for Foolscap. |
|
185 | The other attributes are the standard ones for Foolscap. | |
186 | """ |
|
186 | """ | |
187 |
|
187 | |||
188 | ip = Str('', config=True) |
|
188 | ip = Str('', config=True) | |
189 | port = Int(0, config=True) |
|
189 | port = Int(0, config=True) | |
190 | secure = Bool(True, config=True) |
|
190 | secure = Bool(True, config=True) | |
191 | cert_file = Str('', config=True) |
|
191 | cert_file = Str('', config=True) | |
192 | location = Str('', config=True) |
|
192 | location = Str('', config=True) | |
193 | reuse_furls = Bool(False, config=True) |
|
193 | reuse_furls = Bool(False, config=True) | |
194 | interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True) |
|
194 | interfaces = Instance(klass=Config, kw={}, allow_none=False, config=True) | |
195 |
|
195 | |||
196 | def __init__(self, config, adaptee): |
|
196 | def __init__(self, config, adaptee): | |
197 | super(FCServiceFactory, self).__init__(config, adaptee) |
|
197 | super(FCServiceFactory, self).__init__(config, adaptee) | |
198 | self._check_reuse_furls() |
|
198 | self._check_reuse_furls() | |
199 |
|
199 | |||
200 | def _ip_changed(self, name, old, new): |
|
200 | def _ip_changed(self, name, old, new): | |
201 | if new == 'localhost' or new == '127.0.0.1': |
|
201 | if new == 'localhost' or new == '127.0.0.1': | |
202 | self.location = '127.0.0.1' |
|
202 | self.location = '127.0.0.1' | |
203 |
|
203 | |||
204 | def _check_reuse_furls(self): |
|
204 | def _check_reuse_furls(self): | |
205 | furl_files = [i.furl_file for i in self.interfaces.values()] |
|
205 | furl_files = [i.furl_file for i in self.interfaces.values()] | |
206 | for ff in furl_files: |
|
206 | for ff in furl_files: | |
207 | fullfile = self._get_security_file(ff) |
|
207 | fullfile = self._get_security_file(ff) | |
208 | if self.reuse_furls: |
|
208 | if self.reuse_furls: | |
209 | log.msg("Reusing FURL file: %s" % fullfile) |
|
209 | if self.port==0: | |
|
210 | raise FURLError("You are trying to reuse the FURL file " | |||
|
211 | "for this connection, but the port for this connection " | |||
|
212 | "is set to 0 (autoselect). To reuse the FURL file " | |||
|
213 | "you need to specify specific port to listen on." | |||
|
214 | ) | |||
|
215 | else: | |||
|
216 | log.msg("Reusing FURL file: %s" % fullfile) | |||
210 | else: |
|
217 | else: | |
211 | if os.path.isfile(fullfile): |
|
218 | if os.path.isfile(fullfile): | |
212 | log.msg("Removing old FURL file: %s" % fullfile) |
|
219 | log.msg("Removing old FURL file: %s" % fullfile) | |
213 | os.remove(fullfile) |
|
220 | os.remove(fullfile) | |
214 |
|
221 | |||
215 | def _get_security_file(self, filename): |
|
222 | def _get_security_file(self, filename): | |
216 | return os.path.join(self.config.Global.security_dir, filename) |
|
223 | return os.path.join(self.config.Global.security_dir, filename) | |
217 |
|
224 | |||
218 | def create(self): |
|
225 | def create(self): | |
219 | """Create and return the Foolscap tub with everything running.""" |
|
226 | """Create and return the Foolscap tub with everything running.""" | |
220 |
|
227 | |||
221 | self.tub, self.listener = make_tub( |
|
228 | self.tub, self.listener = make_tub( | |
222 | self.ip, self.port, self.secure, |
|
229 | self.ip, self.port, self.secure, | |
223 | self._get_security_file(self.cert_file) |
|
230 | self._get_security_file(self.cert_file) | |
224 | ) |
|
231 | ) | |
225 | # log.msg("Interfaces to register [%r]: %r" % \ |
|
232 | # log.msg("Interfaces to register [%r]: %r" % \ | |
226 | # (self.__class__, self.interfaces)) |
|
233 | # (self.__class__, self.interfaces)) | |
227 | if not self.secure: |
|
234 | if not self.secure: | |
228 | log.msg("WARNING: running with no security: %s" % \ |
|
235 | log.msg("WARNING: running with no security: %s" % \ | |
229 | self.__class__.__name__) |
|
236 | self.__class__.__name__) | |
230 | reactor.callWhenRunning(self.set_location_and_register) |
|
237 | reactor.callWhenRunning(self.set_location_and_register) | |
231 | return self.tub |
|
238 | return self.tub | |
232 |
|
239 | |||
233 | def set_location_and_register(self): |
|
240 | def set_location_and_register(self): | |
234 | """Set the location for the tub and return a deferred.""" |
|
241 | """Set the location for the tub and return a deferred.""" | |
235 |
|
242 | |||
236 | if self.location == '': |
|
243 | if self.location == '': | |
237 | d = self.tub.setLocationAutomatically() |
|
244 | d = self.tub.setLocationAutomatically() | |
238 | else: |
|
245 | else: | |
239 | d = defer.maybeDeferred(self.tub.setLocation, |
|
246 | d = defer.maybeDeferred(self.tub.setLocation, | |
240 | "%s:%i" % (self.location, self.listener.getPortnum())) |
|
247 | "%s:%i" % (self.location, self.listener.getPortnum())) | |
241 | self.adapt_to_interfaces(d) |
|
248 | self.adapt_to_interfaces(d) | |
242 |
|
249 | |||
243 | def adapt_to_interfaces(self, d): |
|
250 | def adapt_to_interfaces(self, d): | |
244 | """Run through the interfaces, adapt and register.""" |
|
251 | """Run through the interfaces, adapt and register.""" | |
245 |
|
252 | |||
246 | for ifname, ifconfig in self.interfaces.iteritems(): |
|
253 | for ifname, ifconfig in self.interfaces.iteritems(): | |
247 | ff = self._get_security_file(ifconfig.furl_file) |
|
254 | ff = self._get_security_file(ifconfig.furl_file) | |
248 | log.msg("Adapting [%s] to interface: %s" % \ |
|
255 | log.msg("Adapting [%s] to interface: %s" % \ | |
249 | (self.adaptee.__class__.__name__, ifname)) |
|
256 | (self.adaptee.__class__.__name__, ifname)) | |
250 | log.msg("Saving FURL for interface [%s] to file: %s" % (ifname, ff)) |
|
257 | log.msg("Saving FURL for interface [%s] to file: %s" % (ifname, ff)) | |
251 | check_furl_file_security(ff, self.secure) |
|
258 | check_furl_file_security(ff, self.secure) | |
252 | adaptee = self.adaptee |
|
259 | adaptee = self.adaptee | |
253 | for i in ifconfig.interface_chain: |
|
260 | for i in ifconfig.interface_chain: | |
254 | adaptee = import_item(i)(adaptee) |
|
261 | adaptee = import_item(i)(adaptee) | |
255 | d.addCallback(self.register, adaptee, furl_file=ff) |
|
262 | d.addCallback(self.register, adaptee, furl_file=ff) | |
256 |
|
263 | |||
257 | def register(self, empty, ref, furl_file): |
|
264 | def register(self, empty, ref, furl_file): | |
258 | """Register the reference with the FURL file. |
|
265 | """Register the reference with the FURL file. | |
259 |
|
266 | |||
260 | The FURL file is created and then moved to make sure that when the |
|
267 | The FURL file is created and then moved to make sure that when the | |
261 | file appears, the buffer has been flushed and the file closed. |
|
268 | file appears, the buffer has been flushed and the file closed. | |
262 | """ |
|
269 | """ | |
263 | temp_furl_file = get_temp_furlfile(furl_file) |
|
270 | temp_furl_file = get_temp_furlfile(furl_file) | |
264 | self.tub.registerReference(ref, furlFile=temp_furl_file) |
|
271 | self.tub.registerReference(ref, furlFile=temp_furl_file) | |
265 | os.rename(temp_furl_file, furl_file) |
|
272 | os.rename(temp_furl_file, furl_file) | |
266 |
|
273 |
@@ -1,384 +1,403 b'' | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | The ipcluster application. |
|
4 | The ipcluster application. | |
5 | """ |
|
5 | """ | |
6 |
|
6 | |||
7 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
8 | # Copyright (C) 2008-2009 The IPython Development Team |
|
8 | # Copyright (C) 2008-2009 The IPython Development Team | |
9 | # |
|
9 | # | |
10 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | # Distributed under the terms of the BSD License. The full license is in | |
11 | # the file COPYING, distributed as part of this software. |
|
11 | # the file COPYING, distributed as part of this software. | |
12 | #----------------------------------------------------------------------------- |
|
12 | #----------------------------------------------------------------------------- | |
13 |
|
13 | |||
14 | #----------------------------------------------------------------------------- |
|
14 | #----------------------------------------------------------------------------- | |
15 | # Imports |
|
15 | # Imports | |
16 | #----------------------------------------------------------------------------- |
|
16 | #----------------------------------------------------------------------------- | |
17 |
|
17 | |||
18 | import logging |
|
18 | import logging | |
19 | import os |
|
19 | import os | |
20 | import signal |
|
20 | import signal | |
21 | import sys |
|
21 | import sys | |
22 |
|
22 | |||
23 | if os.name=='posix': |
|
23 | if os.name=='posix': | |
24 | from twisted.scripts._twistd_unix import daemonize |
|
24 | from twisted.scripts._twistd_unix import daemonize | |
25 |
|
25 | |||
26 | from IPython.core import release |
|
26 | from IPython.core import release | |
27 | from IPython.external import argparse |
|
27 | from IPython.external import argparse | |
28 | from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault |
|
28 | from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault | |
29 | from IPython.utils.importstring import import_item |
|
29 | from IPython.utils.importstring import import_item | |
30 |
|
30 | |||
31 | from IPython.kernel.clusterdir import ( |
|
31 | from IPython.kernel.clusterdir import ( | |
32 | ApplicationWithClusterDir, ClusterDirError, PIDFileError |
|
32 | ApplicationWithClusterDir, ClusterDirError, PIDFileError | |
33 | ) |
|
33 | ) | |
34 |
|
34 | |||
35 |
from twisted.internet import reactor |
|
35 | from twisted.internet import reactor | |
36 | from twisted.python import log |
|
36 | from twisted.python import log | |
37 |
|
37 | |||
|
38 | ||||
38 | #----------------------------------------------------------------------------- |
|
39 | #----------------------------------------------------------------------------- | |
39 | # Code for launchers |
|
40 | # The ipcluster application | |
40 | #----------------------------------------------------------------------------- |
|
41 | #----------------------------------------------------------------------------- | |
41 |
|
42 | |||
42 |
|
43 | |||
|
44 | # Exit codes for ipcluster | |||
43 |
|
45 | |||
44 | #----------------------------------------------------------------------------- |
|
46 | # This will be the exit code if the ipcluster appears to be running because | |
45 | # The ipcluster application |
|
47 | # a .pid file exists | |
46 | #----------------------------------------------------------------------------- |
|
48 | ALREADY_STARTED = 10 | |
|
49 | ||||
|
50 | # This will be the exit code if ipcluster stop is run, but there is not .pid | |||
|
51 | # file to be found. | |||
|
52 | ALREADY_STOPPED = 11 | |||
47 |
|
53 | |||
48 |
|
54 | |||
49 | class IPClusterCLLoader(ArgParseConfigLoader): |
|
55 | class IPClusterCLLoader(ArgParseConfigLoader): | |
50 |
|
56 | |||
51 | def _add_arguments(self): |
|
57 | def _add_arguments(self): | |
52 | # This has all the common options that all subcommands use |
|
58 | # This has all the common options that all subcommands use | |
53 | parent_parser1 = argparse.ArgumentParser(add_help=False) |
|
59 | parent_parser1 = argparse.ArgumentParser(add_help=False) | |
54 | parent_parser1.add_argument('--ipython-dir', |
|
60 | parent_parser1.add_argument('--ipython-dir', | |
55 | dest='Global.ipython_dir',type=str, |
|
61 | dest='Global.ipython_dir',type=str, | |
56 | help='Set to override default location of Global.ipython_dir.', |
|
62 | help='Set to override default location of Global.ipython_dir.', | |
57 | default=NoConfigDefault, |
|
63 | default=NoConfigDefault, | |
58 | metavar='Global.ipython_dir') |
|
64 | metavar='Global.ipython_dir') | |
59 | parent_parser1.add_argument('--log-level', |
|
65 | parent_parser1.add_argument('--log-level', | |
60 | dest="Global.log_level",type=int, |
|
66 | dest="Global.log_level",type=int, | |
61 | help='Set the log level (0,10,20,30,40,50). Default is 30.', |
|
67 | help='Set the log level (0,10,20,30,40,50). Default is 30.', | |
62 | default=NoConfigDefault, |
|
68 | default=NoConfigDefault, | |
63 | metavar='Global.log_level') |
|
69 | metavar='Global.log_level') | |
64 |
|
70 | |||
65 | # This has all the common options that other subcommands use |
|
71 | # This has all the common options that other subcommands use | |
66 | parent_parser2 = argparse.ArgumentParser(add_help=False) |
|
72 | parent_parser2 = argparse.ArgumentParser(add_help=False) | |
67 | parent_parser2.add_argument('-p','--profile', |
|
73 | parent_parser2.add_argument('-p','--profile', | |
68 | dest='Global.profile',type=str, |
|
74 | dest='Global.profile',type=str, | |
69 | default=NoConfigDefault, |
|
75 | default=NoConfigDefault, | |
70 | help='The string name of the profile to be used. This determines ' |
|
76 | help='The string name of the profile to be used. This determines ' | |
71 | 'the name of the cluster dir as: cluster_<profile>. The default profile ' |
|
77 | 'the name of the cluster dir as: cluster_<profile>. The default profile ' | |
72 | 'is named "default". The cluster directory is resolve this way ' |
|
78 | 'is named "default". The cluster directory is resolve this way ' | |
73 | 'if the --cluster-dir option is not used.', |
|
79 | 'if the --cluster-dir option is not used.', | |
74 | default=NoConfigDefault, |
|
80 | default=NoConfigDefault, | |
75 | metavar='Global.profile') |
|
81 | metavar='Global.profile') | |
76 | parent_parser2.add_argument('--cluster-dir', |
|
82 | parent_parser2.add_argument('--cluster-dir', | |
77 | dest='Global.cluster_dir',type=str, |
|
83 | dest='Global.cluster_dir',type=str, | |
78 | default=NoConfigDefault, |
|
84 | default=NoConfigDefault, | |
79 | help='Set the cluster dir. This overrides the logic used by the ' |
|
85 | help='Set the cluster dir. This overrides the logic used by the ' | |
80 | '--profile option.', |
|
86 | '--profile option.', | |
81 | default=NoConfigDefault, |
|
87 | default=NoConfigDefault, | |
82 | metavar='Global.cluster_dir') |
|
88 | metavar='Global.cluster_dir') | |
83 | parent_parser2.add_argument('--log-to-file', |
|
89 | parent_parser2.add_argument('--log-to-file', | |
84 | action='store_true', dest='Global.log_to_file', |
|
90 | action='store_true', dest='Global.log_to_file', | |
85 | default=NoConfigDefault, |
|
91 | default=NoConfigDefault, | |
86 | help='Log to a file in the log directory (default is stdout)' |
|
92 | help='Log to a file in the log directory (default is stdout)' | |
87 | ) |
|
93 | ) | |
88 |
|
94 | |||
89 | subparsers = self.parser.add_subparsers( |
|
95 | subparsers = self.parser.add_subparsers( | |
90 | dest='Global.subcommand', |
|
96 | dest='Global.subcommand', | |
91 | title='ipcluster subcommands', |
|
97 | title='ipcluster subcommands', | |
92 | description='ipcluster has a variety of subcommands. ' |
|
98 | description='ipcluster has a variety of subcommands. ' | |
93 | 'The general way of running ipcluster is "ipcluster <cmd> ' |
|
99 | 'The general way of running ipcluster is "ipcluster <cmd> ' | |
94 | ' [options]""', |
|
100 | ' [options]""', | |
95 | help='For more help, type "ipcluster <cmd> -h"') |
|
101 | help='For more help, type "ipcluster <cmd> -h"') | |
96 |
|
102 | |||
97 | parser_list = subparsers.add_parser( |
|
103 | parser_list = subparsers.add_parser( | |
98 | 'list', |
|
104 | 'list', | |
99 | help='List all clusters in cwd and ipython_dir.', |
|
105 | help='List all clusters in cwd and ipython_dir.', | |
100 | parents=[parent_parser1] |
|
106 | parents=[parent_parser1] | |
101 | ) |
|
107 | ) | |
102 |
|
108 | |||
103 | parser_create = subparsers.add_parser( |
|
109 | parser_create = subparsers.add_parser( | |
104 | 'create', |
|
110 | 'create', | |
105 | help='Create a new cluster directory.', |
|
111 | help='Create a new cluster directory.', | |
106 | parents=[parent_parser1, parent_parser2] |
|
112 | parents=[parent_parser1, parent_parser2] | |
107 | ) |
|
113 | ) | |
108 | parser_create.add_argument( |
|
114 | parser_create.add_argument( | |
109 | '--reset-config', |
|
115 | '--reset-config', | |
110 | dest='Global.reset_config', action='store_true', |
|
116 | dest='Global.reset_config', action='store_true', | |
111 | default=NoConfigDefault, |
|
117 | default=NoConfigDefault, | |
112 | help='Recopy the default config files to the cluster directory. ' |
|
118 | help='Recopy the default config files to the cluster directory. ' | |
113 | 'You will loose any modifications you have made to these files.' |
|
119 | 'You will loose any modifications you have made to these files.' | |
114 | ) |
|
120 | ) | |
115 |
|
121 | |||
116 | parser_start = subparsers.add_parser( |
|
122 | parser_start = subparsers.add_parser( | |
117 | 'start', |
|
123 | 'start', | |
118 | help='Start a cluster.', |
|
124 | help='Start a cluster.', | |
119 | parents=[parent_parser1, parent_parser2] |
|
125 | parents=[parent_parser1, parent_parser2] | |
120 | ) |
|
126 | ) | |
121 | parser_start.add_argument( |
|
127 | parser_start.add_argument( | |
122 | '-n', '--number', |
|
128 | '-n', '--number', | |
123 | type=int, dest='Global.n', |
|
129 | type=int, dest='Global.n', | |
124 | default=NoConfigDefault, |
|
130 | default=NoConfigDefault, | |
125 | help='The number of engines to start.', |
|
131 | help='The number of engines to start.', | |
126 | metavar='Global.n' |
|
132 | metavar='Global.n' | |
127 | ) |
|
133 | ) | |
128 | parser_start.add_argument('--clean-logs', |
|
134 | parser_start.add_argument('--clean-logs', | |
129 | dest='Global.clean_logs', action='store_true', |
|
135 | dest='Global.clean_logs', action='store_true', | |
130 | help='Delete old log flies before starting.', |
|
136 | help='Delete old log flies before starting.', | |
131 | default=NoConfigDefault |
|
137 | default=NoConfigDefault | |
132 | ) |
|
138 | ) | |
133 | parser_start.add_argument('--no-clean-logs', |
|
139 | parser_start.add_argument('--no-clean-logs', | |
134 | dest='Global.clean_logs', action='store_false', |
|
140 | dest='Global.clean_logs', action='store_false', | |
135 | help="Don't delete old log flies before starting.", |
|
141 | help="Don't delete old log flies before starting.", | |
136 | default=NoConfigDefault |
|
142 | default=NoConfigDefault | |
137 | ) |
|
143 | ) | |
138 | parser_start.add_argument('--daemon', |
|
144 | parser_start.add_argument('--daemon', | |
139 | dest='Global.daemonize', action='store_true', |
|
145 | dest='Global.daemonize', action='store_true', | |
140 | help='Daemonize the ipcluster program. This implies --log-to-file', |
|
146 | help='Daemonize the ipcluster program. This implies --log-to-file', | |
141 | default=NoConfigDefault |
|
147 | default=NoConfigDefault | |
142 | ) |
|
148 | ) | |
143 | parser_start.add_argument('--no-daemon', |
|
149 | parser_start.add_argument('--no-daemon', | |
144 | dest='Global.daemonize', action='store_false', |
|
150 | dest='Global.daemonize', action='store_false', | |
145 | help="Dont't daemonize the ipcluster program.", |
|
151 | help="Dont't daemonize the ipcluster program.", | |
146 | default=NoConfigDefault |
|
152 | default=NoConfigDefault | |
147 | ) |
|
153 | ) | |
148 |
|
154 | |||
149 | parser_start = subparsers.add_parser( |
|
155 | parser_start = subparsers.add_parser( | |
150 | 'stop', |
|
156 | 'stop', | |
151 | help='Stop a cluster.', |
|
157 | help='Stop a cluster.', | |
152 | parents=[parent_parser1, parent_parser2] |
|
158 | parents=[parent_parser1, parent_parser2] | |
153 | ) |
|
159 | ) | |
154 |
parser_start.add_argument('--signal |
|
160 | parser_start.add_argument('--signal', | |
155 |
dest='Global.s |
|
161 | dest='Global.signal', type=int, | |
156 | help="The signal number to use in stopping the cluster (default=2).", |
|
162 | help="The signal number to use in stopping the cluster (default=2).", | |
157 |
metavar="Global.s |
|
163 | metavar="Global.signal", | |
158 | default=NoConfigDefault |
|
164 | default=NoConfigDefault | |
159 | ) |
|
165 | ) | |
160 |
|
166 | |||
|
167 | ||||
161 | default_config_file_name = 'ipcluster_config.py' |
|
168 | default_config_file_name = 'ipcluster_config.py' | |
162 |
|
169 | |||
163 |
|
170 | |||
164 | class IPClusterApp(ApplicationWithClusterDir): |
|
171 | class IPClusterApp(ApplicationWithClusterDir): | |
165 |
|
172 | |||
166 | name = 'ipcluster' |
|
173 | name = 'ipcluster' | |
167 | description = 'Start an IPython cluster (controller and engines).' |
|
174 | description = 'Start an IPython cluster (controller and engines).' | |
168 | config_file_name = default_config_file_name |
|
175 | config_file_name = default_config_file_name | |
169 | default_log_level = logging.INFO |
|
176 | default_log_level = logging.INFO | |
170 | auto_create_cluster_dir = False |
|
177 | auto_create_cluster_dir = False | |
171 |
|
178 | |||
172 | def create_default_config(self): |
|
179 | def create_default_config(self): | |
173 | super(IPClusterApp, self).create_default_config() |
|
180 | super(IPClusterApp, self).create_default_config() | |
174 | self.default_config.Global.controller_launcher = \ |
|
181 | self.default_config.Global.controller_launcher = \ | |
175 | 'IPython.kernel.launcher.LocalControllerLauncher' |
|
182 | 'IPython.kernel.launcher.LocalControllerLauncher' | |
176 | self.default_config.Global.engine_launcher = \ |
|
183 | self.default_config.Global.engine_launcher = \ | |
177 | 'IPython.kernel.launcher.LocalEngineSetLauncher' |
|
184 | 'IPython.kernel.launcher.LocalEngineSetLauncher' | |
178 | self.default_config.Global.n = 2 |
|
185 | self.default_config.Global.n = 2 | |
179 | self.default_config.Global.reset_config = False |
|
186 | self.default_config.Global.reset_config = False | |
180 | self.default_config.Global.clean_logs = True |
|
187 | self.default_config.Global.clean_logs = True | |
181 |
self.default_config.Global. |
|
188 | self.default_config.Global.signal = 2 | |
182 | self.default_config.Global.daemonize = False |
|
189 | self.default_config.Global.daemonize = False | |
183 |
|
190 | |||
184 | def create_command_line_config(self): |
|
191 | def create_command_line_config(self): | |
185 | """Create and return a command line config loader.""" |
|
192 | """Create and return a command line config loader.""" | |
186 | return IPClusterCLLoader( |
|
193 | return IPClusterCLLoader( | |
187 | description=self.description, |
|
194 | description=self.description, | |
188 | version=release.version |
|
195 | version=release.version | |
189 | ) |
|
196 | ) | |
190 |
|
197 | |||
191 | def find_resources(self): |
|
198 | def find_resources(self): | |
192 | subcommand = self.command_line_config.Global.subcommand |
|
199 | subcommand = self.command_line_config.Global.subcommand | |
193 | if subcommand=='list': |
|
200 | if subcommand=='list': | |
194 | self.list_cluster_dirs() |
|
201 | self.list_cluster_dirs() | |
195 | # Exit immediately because there is nothing left to do. |
|
202 | # Exit immediately because there is nothing left to do. | |
196 | self.exit() |
|
203 | self.exit() | |
197 | elif subcommand=='create': |
|
204 | elif subcommand=='create': | |
198 | self.auto_create_cluster_dir = True |
|
205 | self.auto_create_cluster_dir = True | |
199 | super(IPClusterApp, self).find_resources() |
|
206 | super(IPClusterApp, self).find_resources() | |
200 | elif subcommand=='start' or subcommand=='stop': |
|
207 | elif subcommand=='start' or subcommand=='stop': | |
201 | self.auto_create_cluster_dir = False |
|
208 | self.auto_create_cluster_dir = False | |
202 | try: |
|
209 | try: | |
203 | super(IPClusterApp, self).find_resources() |
|
210 | super(IPClusterApp, self).find_resources() | |
204 | except ClusterDirError: |
|
211 | except ClusterDirError: | |
205 | raise ClusterDirError( |
|
212 | raise ClusterDirError( | |
206 | "Could not find a cluster directory. A cluster dir must " |
|
213 | "Could not find a cluster directory. A cluster dir must " | |
207 | "be created before running 'ipcluster start'. Do " |
|
214 | "be created before running 'ipcluster start'. Do " | |
208 | "'ipcluster create -h' or 'ipcluster list -h' for more " |
|
215 | "'ipcluster create -h' or 'ipcluster list -h' for more " | |
209 | "information about creating and listing cluster dirs." |
|
216 | "information about creating and listing cluster dirs." | |
210 | ) |
|
217 | ) | |
211 |
|
218 | |||
212 | def pre_construct(self): |
|
|||
213 | super(IPClusterApp, self).pre_construct() |
|
|||
214 | config = self.master_config |
|
|||
215 | try: |
|
|||
216 | daemon = config.Global.daemonize |
|
|||
217 | if daemon: |
|
|||
218 | config.Global.log_to_file = True |
|
|||
219 | except AttributeError: |
|
|||
220 | pass |
|
|||
221 |
|
||||
222 | def construct(self): |
|
|||
223 | config = self.master_config |
|
|||
224 | if config.Global.subcommand=='list': |
|
|||
225 | pass |
|
|||
226 | elif config.Global.subcommand=='create': |
|
|||
227 | self.log.info('Copying default config files to cluster directory ' |
|
|||
228 | '[overwrite=%r]' % (config.Global.reset_config,)) |
|
|||
229 | self.cluster_dir_obj.copy_all_config_files(overwrite=config.Global.reset_config) |
|
|||
230 | elif config.Global.subcommand=='start': |
|
|||
231 | self.start_logging() |
|
|||
232 | reactor.callWhenRunning(self.start_launchers) |
|
|||
233 |
|
||||
234 | def list_cluster_dirs(self): |
|
219 | def list_cluster_dirs(self): | |
235 | # Find the search paths |
|
220 | # Find the search paths | |
236 | cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','') |
|
221 | cluster_dir_paths = os.environ.get('IPCLUSTER_DIR_PATH','') | |
237 | if cluster_dir_paths: |
|
222 | if cluster_dir_paths: | |
238 | cluster_dir_paths = cluster_dir_paths.split(':') |
|
223 | cluster_dir_paths = cluster_dir_paths.split(':') | |
239 | else: |
|
224 | else: | |
240 | cluster_dir_paths = [] |
|
225 | cluster_dir_paths = [] | |
241 | try: |
|
226 | try: | |
242 | ipython_dir = self.command_line_config.Global.ipython_dir |
|
227 | ipython_dir = self.command_line_config.Global.ipython_dir | |
243 | except AttributeError: |
|
228 | except AttributeError: | |
244 | ipython_dir = self.default_config.Global.ipython_dir |
|
229 | ipython_dir = self.default_config.Global.ipython_dir | |
245 | paths = [os.getcwd(), ipython_dir] + \ |
|
230 | paths = [os.getcwd(), ipython_dir] + \ | |
246 | cluster_dir_paths |
|
231 | cluster_dir_paths | |
247 | paths = list(set(paths)) |
|
232 | paths = list(set(paths)) | |
248 |
|
233 | |||
249 | self.log.info('Searching for cluster dirs in paths: %r' % paths) |
|
234 | self.log.info('Searching for cluster dirs in paths: %r' % paths) | |
250 | for path in paths: |
|
235 | for path in paths: | |
251 | files = os.listdir(path) |
|
236 | files = os.listdir(path) | |
252 | for f in files: |
|
237 | for f in files: | |
253 | full_path = os.path.join(path, f) |
|
238 | full_path = os.path.join(path, f) | |
254 | if os.path.isdir(full_path) and f.startswith('cluster_'): |
|
239 | if os.path.isdir(full_path) and f.startswith('cluster_'): | |
255 | profile = full_path.split('_')[-1] |
|
240 | profile = full_path.split('_')[-1] | |
256 | start_cmd = '"ipcluster start -n 4 -p %s"' % profile |
|
241 | start_cmd = '"ipcluster start -n 4 -p %s"' % profile | |
257 | print start_cmd + " ==> " + full_path |
|
242 | print start_cmd + " ==> " + full_path | |
258 |
|
243 | |||
|
244 | def pre_construct(self): | |||
|
245 | super(IPClusterApp, self).pre_construct() | |||
|
246 | config = self.master_config | |||
|
247 | try: | |||
|
248 | daemon = config.Global.daemonize | |||
|
249 | if daemon: | |||
|
250 | config.Global.log_to_file = True | |||
|
251 | except AttributeError: | |||
|
252 | pass | |||
|
253 | ||||
|
254 | def construct(self): | |||
|
255 | config = self.master_config | |||
|
256 | if config.Global.subcommand=='list': | |||
|
257 | pass | |||
|
258 | elif config.Global.subcommand=='create': | |||
|
259 | self.log.info('Copying default config files to cluster directory ' | |||
|
260 | '[overwrite=%r]' % (config.Global.reset_config,)) | |||
|
261 | self.cluster_dir_obj.copy_all_config_files(overwrite=config.Global.reset_config) | |||
|
262 | elif config.Global.subcommand=='start': | |||
|
263 | self.start_logging() | |||
|
264 | reactor.callWhenRunning(self.start_launchers) | |||
|
265 | ||||
259 | def start_launchers(self): |
|
266 | def start_launchers(self): | |
260 | config = self.master_config |
|
267 | config = self.master_config | |
261 |
|
268 | |||
262 | # Create the launchers |
|
269 | # Create the launchers | |
263 | el_class = import_item(config.Global.engine_launcher) |
|
270 | el_class = import_item(config.Global.engine_launcher) | |
264 | self.engine_launcher = el_class( |
|
271 | self.engine_launcher = el_class( | |
265 | self.cluster_dir, config=config |
|
272 | self.cluster_dir, config=config | |
266 | ) |
|
273 | ) | |
267 | cl_class = import_item(config.Global.controller_launcher) |
|
274 | cl_class = import_item(config.Global.controller_launcher) | |
268 | self.controller_launcher = cl_class( |
|
275 | self.controller_launcher = cl_class( | |
269 | self.cluster_dir, config=config |
|
276 | self.cluster_dir, config=config | |
270 | ) |
|
277 | ) | |
271 |
|
278 | |||
272 | # Setup signals |
|
279 | # Setup signals | |
273 | signal.signal(signal.SIGINT, self.stop_launchers) |
|
280 | signal.signal(signal.SIGINT, self.stop_launchers) | |
274 | # signal.signal(signal.SIGKILL, self.stop_launchers) |
|
281 | # signal.signal(signal.SIGKILL, self.stop_launchers) | |
275 |
|
282 | |||
276 | # Setup the observing of stopping |
|
283 | # Setup the observing of stopping | |
277 | d1 = self.controller_launcher.observe_stop() |
|
284 | d1 = self.controller_launcher.observe_stop() | |
278 | d1.addCallback(self.stop_engines) |
|
285 | d1.addCallback(self.stop_engines) | |
279 | d1.addErrback(self.err_and_stop) |
|
286 | d1.addErrback(self.err_and_stop) | |
280 | # If this triggers, just let them die |
|
287 | # If this triggers, just let them die | |
281 | # d2 = self.engine_launcher.observe_stop() |
|
288 | # d2 = self.engine_launcher.observe_stop() | |
282 |
|
289 | |||
283 | # Start the controller and engines |
|
290 | # Start the controller and engines | |
284 | d = self.controller_launcher.start( |
|
291 | d = self.controller_launcher.start( | |
285 | profile=None, cluster_dir=config.Global.cluster_dir |
|
292 | profile=None, cluster_dir=config.Global.cluster_dir | |
286 | ) |
|
293 | ) | |
287 | d.addCallback(lambda _: self.start_engines()) |
|
294 | d.addCallback(lambda _: self.start_engines()) | |
288 | d.addErrback(self.err_and_stop) |
|
295 | d.addErrback(self.err_and_stop) | |
289 |
|
296 | |||
290 | def err_and_stop(self, f): |
|
297 | def err_and_stop(self, f): | |
291 | log.msg('Unexpected error in ipcluster:') |
|
298 | log.msg('Unexpected error in ipcluster:') | |
292 | log.err(f) |
|
299 | log.err(f) | |
293 | reactor.stop() |
|
300 | reactor.stop() | |
294 |
|
301 | |||
295 | def stop_engines(self, r): |
|
302 | def stop_engines(self, r): | |
296 | return self.engine_launcher.stop() |
|
303 | return self.engine_launcher.stop() | |
297 |
|
304 | |||
298 | def start_engines(self): |
|
305 | def start_engines(self): | |
299 | config = self.master_config |
|
306 | config = self.master_config | |
300 | d = self.engine_launcher.start( |
|
307 | d = self.engine_launcher.start( | |
301 | config.Global.n, |
|
308 | config.Global.n, | |
302 | profile=None, cluster_dir=config.Global.cluster_dir |
|
309 | profile=None, cluster_dir=config.Global.cluster_dir | |
303 | ) |
|
310 | ) | |
304 | return d |
|
311 | return d | |
305 |
|
312 | |||
306 | def stop_launchers(self, signum, frame): |
|
313 | def stop_launchers(self, signum, frame): | |
307 | log.msg("Stopping cluster") |
|
314 | log.msg("Stopping cluster") | |
308 | d1 = self.engine_launcher.stop() |
|
315 | d1 = self.engine_launcher.stop() | |
309 | d2 = self.controller_launcher.stop() |
|
316 | d2 = self.controller_launcher.stop() | |
310 | # d1.addCallback(lambda _: self.controller_launcher.stop) |
|
317 | # d1.addCallback(lambda _: self.controller_launcher.stop) | |
311 | d1.addErrback(self.err_and_stop) |
|
318 | d1.addErrback(self.err_and_stop) | |
312 | d2.addErrback(self.err_and_stop) |
|
319 | d2.addErrback(self.err_and_stop) | |
313 | reactor.callLater(2.0, reactor.stop) |
|
320 | reactor.callLater(2.0, reactor.stop) | |
314 |
|
321 | |||
315 | def start_logging(self): |
|
322 | def start_logging(self): | |
316 | # Remove old log files |
|
323 | # Remove old log files | |
317 | if self.master_config.Global.clean_logs: |
|
324 | if self.master_config.Global.clean_logs: | |
318 | log_dir = self.master_config.Global.log_dir |
|
325 | log_dir = self.master_config.Global.log_dir | |
319 | for f in os.listdir(log_dir): |
|
326 | for f in os.listdir(log_dir): | |
320 | if f.startswith('ipengine' + '-') and f.endswith('.log'): |
|
327 | if f.startswith('ipengine' + '-') and f.endswith('.log'): | |
321 | os.remove(os.path.join(log_dir, f)) |
|
328 | os.remove(os.path.join(log_dir, f)) | |
322 | for f in os.listdir(log_dir): |
|
329 | for f in os.listdir(log_dir): | |
323 | if f.startswith('ipcontroller' + '-') and f.endswith('.log'): |
|
330 | if f.startswith('ipcontroller' + '-') and f.endswith('.log'): | |
324 | os.remove(os.path.join(log_dir, f)) |
|
331 | os.remove(os.path.join(log_dir, f)) | |
325 | super(IPClusterApp, self).start_logging() |
|
332 | super(IPClusterApp, self).start_logging() | |
326 |
|
333 | |||
327 | def start_app(self): |
|
334 | def start_app(self): | |
328 | """Start the application, depending on what subcommand is used.""" |
|
335 | """Start the application, depending on what subcommand is used.""" | |
329 |
|
|
336 | subcmd = self.master_config.Global.subcommand | |
330 | subcmd = config.Global.subcommand |
|
|||
331 | if subcmd=='create' or subcmd=='list': |
|
337 | if subcmd=='create' or subcmd=='list': | |
332 | return |
|
338 | return | |
333 | elif subcmd=='start': |
|
339 | elif subcmd=='start': | |
334 | # First see if the cluster is already running |
|
340 | self.start_app_start() | |
335 | try: |
|
|||
336 | pid = self.get_pid_from_file() |
|
|||
337 | except: |
|
|||
338 | pass |
|
|||
339 | else: |
|
|||
340 | self.log.critical( |
|
|||
341 | 'Cluster is already running with [pid=%s]. ' |
|
|||
342 | 'use "ipcluster stop" to stop the cluster.' % pid |
|
|||
343 | ) |
|
|||
344 | # Here I exit with a unusual exit status that other processes |
|
|||
345 | # can watch for to learn how I existed. |
|
|||
346 | sys.exit(10) |
|
|||
347 | # Now log and daemonize |
|
|||
348 | self.log.info('Starting ipcluster with [daemon=%r]' % config.Global.daemonize) |
|
|||
349 | if config.Global.daemonize: |
|
|||
350 | if os.name=='posix': |
|
|||
351 | os.chdir(config.Global.cluster_dir) |
|
|||
352 | self.log_level = 40 |
|
|||
353 | daemonize() |
|
|||
354 |
|
||||
355 | # Now write the new pid file after our new forked pid is active. |
|
|||
356 | self.write_pid_file() |
|
|||
357 | reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file) |
|
|||
358 | reactor.run() |
|
|||
359 | elif subcmd=='stop': |
|
341 | elif subcmd=='stop': | |
360 | try: |
|
342 | self.start_app_stop() | |
361 | pid = self.get_pid_from_file() |
|
343 | ||
362 | except PIDFileError: |
|
344 | def start_app_start(self): | |
363 | self.log.critical( |
|
345 | """Start the app for the start subcommand.""" | |
364 | 'Problem reading pid file, cluster is probably not running.' |
|
346 | config = self.master_config | |
365 | ) |
|
347 | # First see if the cluster is already running | |
366 | # Here I exit with a unusual exit status that other processes |
|
348 | try: | |
367 | # can watch for to learn how I existed. |
|
349 | pid = self.get_pid_from_file() | |
368 | sys.exit(11) |
|
350 | except PIDFileError: | |
369 | sig = config.Global.stop_signal |
|
351 | pass | |
370 | self.log.info( |
|
352 | else: | |
371 | "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig) |
|
353 | self.log.critical( | |
|
354 | 'Cluster is already running with [pid=%s]. ' | |||
|
355 | 'use "ipcluster stop" to stop the cluster.' % pid | |||
|
356 | ) | |||
|
357 | # Here I exit with a unusual exit status that other processes | |||
|
358 | # can watch for to learn how I existed. | |||
|
359 | self.exit(ALREADY_STARTED) | |||
|
360 | ||||
|
361 | # Now log and daemonize | |||
|
362 | self.log.info( | |||
|
363 | 'Starting ipcluster with [daemon=%r]' % config.Global.daemonize | |||
|
364 | ) | |||
|
365 | if config.Global.daemonize: | |||
|
366 | if os.name=='posix': | |||
|
367 | daemonize() | |||
|
368 | ||||
|
369 | # Now write the new pid file AFTER our new forked pid is active. | |||
|
370 | self.write_pid_file() | |||
|
371 | # cd to the cluster_dir as our working directory. | |||
|
372 | os.chdir(config.Global.cluster_dir) | |||
|
373 | reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file) | |||
|
374 | reactor.run() | |||
|
375 | ||||
|
376 | def start_app_stop(self): | |||
|
377 | """Start the app for the stop subcommand.""" | |||
|
378 | config = self.master_config | |||
|
379 | try: | |||
|
380 | pid = self.get_pid_from_file() | |||
|
381 | except PIDFileError: | |||
|
382 | self.log.critical( | |||
|
383 | 'Problem reading pid file, cluster is probably not running.' | |||
372 | ) |
|
384 | ) | |
373 | os.kill(pid, sig) |
|
385 | # Here I exit with a unusual exit status that other processes | |
|
386 | # can watch for to learn how I existed. | |||
|
387 | self.exit(ALREADY_STOPPED) | |||
|
388 | sig = config.Global.signal | |||
|
389 | self.log.info( | |||
|
390 | "Stopping cluster [pid=%r] with [signal=%r]" % (pid, sig) | |||
|
391 | ) | |||
|
392 | os.kill(pid, sig) | |||
374 |
|
393 | |||
375 |
|
394 | |||
376 | def launch_new_instance(): |
|
395 | def launch_new_instance(): | |
377 | """Create and run the IPython cluster.""" |
|
396 | """Create and run the IPython cluster.""" | |
378 | app = IPClusterApp() |
|
397 | app = IPClusterApp() | |
379 | app.start() |
|
398 | app.start() | |
380 |
|
399 | |||
381 |
|
400 | |||
382 | if __name__ == '__main__': |
|
401 | if __name__ == '__main__': | |
383 | launch_new_instance() |
|
402 | launch_new_instance() | |
384 |
|
403 |
@@ -1,262 +1,268 b'' | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | The IPython controller application. |
|
4 | The IPython controller application. | |
5 | """ |
|
5 | """ | |
6 |
|
6 | |||
7 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
8 | # Copyright (C) 2008-2009 The IPython Development Team |
|
8 | # Copyright (C) 2008-2009 The IPython Development Team | |
9 | # |
|
9 | # | |
10 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | # Distributed under the terms of the BSD License. The full license is in | |
11 | # the file COPYING, distributed as part of this software. |
|
11 | # the file COPYING, distributed as part of this software. | |
12 | #----------------------------------------------------------------------------- |
|
12 | #----------------------------------------------------------------------------- | |
13 |
|
13 | |||
14 | #----------------------------------------------------------------------------- |
|
14 | #----------------------------------------------------------------------------- | |
15 | # Imports |
|
15 | # Imports | |
16 | #----------------------------------------------------------------------------- |
|
16 | #----------------------------------------------------------------------------- | |
17 |
|
17 | |||
18 | from __future__ import with_statement |
|
18 | from __future__ import with_statement | |
19 |
|
19 | |||
20 | import copy |
|
20 | import copy | |
21 | import os |
|
21 | import os | |
22 | import sys |
|
22 | import sys | |
23 |
|
23 | |||
24 | from twisted.application import service |
|
24 | from twisted.application import service | |
25 | from twisted.internet import reactor |
|
25 | from twisted.internet import reactor | |
26 | from twisted.python import log |
|
26 | from twisted.python import log | |
27 |
|
27 | |||
28 | from IPython.config.loader import Config, NoConfigDefault |
|
28 | from IPython.config.loader import Config, NoConfigDefault | |
29 |
|
29 | |||
30 | from IPython.kernel.clusterdir import ( |
|
30 | from IPython.kernel.clusterdir import ( | |
31 | ApplicationWithClusterDir, |
|
31 | ApplicationWithClusterDir, | |
32 | AppWithClusterDirArgParseConfigLoader |
|
32 | AppWithClusterDirArgParseConfigLoader | |
33 | ) |
|
33 | ) | |
34 |
|
34 | |||
35 | from IPython.core import release |
|
35 | from IPython.core import release | |
36 |
|
36 | |||
37 | from IPython.utils.traitlets import Str, Instance |
|
37 | from IPython.utils.traitlets import Str, Instance | |
38 |
|
38 | |||
39 | from IPython.kernel import controllerservice |
|
39 | from IPython.kernel import controllerservice | |
40 |
|
40 | |||
41 | from IPython.kernel.fcutil import FCServiceFactory |
|
41 | from IPython.kernel.fcutil import FCServiceFactory | |
42 |
|
42 | |||
43 | #----------------------------------------------------------------------------- |
|
43 | #----------------------------------------------------------------------------- | |
44 | # Default interfaces |
|
44 | # Default interfaces | |
45 | #----------------------------------------------------------------------------- |
|
45 | #----------------------------------------------------------------------------- | |
46 |
|
46 | |||
47 |
|
47 | |||
48 | # The default client interfaces for FCClientServiceFactory.interfaces |
|
48 | # The default client interfaces for FCClientServiceFactory.interfaces | |
49 | default_client_interfaces = Config() |
|
49 | default_client_interfaces = Config() | |
50 | default_client_interfaces.Task.interface_chain = [ |
|
50 | default_client_interfaces.Task.interface_chain = [ | |
51 | 'IPython.kernel.task.ITaskController', |
|
51 | 'IPython.kernel.task.ITaskController', | |
52 | 'IPython.kernel.taskfc.IFCTaskController' |
|
52 | 'IPython.kernel.taskfc.IFCTaskController' | |
53 | ] |
|
53 | ] | |
54 |
|
54 | |||
55 | default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl' |
|
55 | default_client_interfaces.Task.furl_file = 'ipcontroller-tc.furl' | |
56 |
|
56 | |||
57 | default_client_interfaces.MultiEngine.interface_chain = [ |
|
57 | default_client_interfaces.MultiEngine.interface_chain = [ | |
58 | 'IPython.kernel.multiengine.IMultiEngine', |
|
58 | 'IPython.kernel.multiengine.IMultiEngine', | |
59 | 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine' |
|
59 | 'IPython.kernel.multienginefc.IFCSynchronousMultiEngine' | |
60 | ] |
|
60 | ] | |
61 |
|
61 | |||
62 | default_client_interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl' |
|
62 | default_client_interfaces.MultiEngine.furl_file = 'ipcontroller-mec.furl' | |
63 |
|
63 | |||
64 | # Make this a dict we can pass to Config.__init__ for the default |
|
64 | # Make this a dict we can pass to Config.__init__ for the default | |
65 | default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items())) |
|
65 | default_client_interfaces = dict(copy.deepcopy(default_client_interfaces.items())) | |
66 |
|
66 | |||
67 |
|
67 | |||
68 |
|
68 | |||
69 | # The default engine interfaces for FCEngineServiceFactory.interfaces |
|
69 | # The default engine interfaces for FCEngineServiceFactory.interfaces | |
70 | default_engine_interfaces = Config() |
|
70 | default_engine_interfaces = Config() | |
71 | default_engine_interfaces.Default.interface_chain = [ |
|
71 | default_engine_interfaces.Default.interface_chain = [ | |
72 | 'IPython.kernel.enginefc.IFCControllerBase' |
|
72 | 'IPython.kernel.enginefc.IFCControllerBase' | |
73 | ] |
|
73 | ] | |
74 |
|
74 | |||
75 | default_engine_interfaces.Default.furl_file = 'ipcontroller-engine.furl' |
|
75 | default_engine_interfaces.Default.furl_file = 'ipcontroller-engine.furl' | |
76 |
|
76 | |||
77 | # Make this a dict we can pass to Config.__init__ for the default |
|
77 | # Make this a dict we can pass to Config.__init__ for the default | |
78 | default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items())) |
|
78 | default_engine_interfaces = dict(copy.deepcopy(default_engine_interfaces.items())) | |
79 |
|
79 | |||
80 |
|
80 | |||
81 | #----------------------------------------------------------------------------- |
|
81 | #----------------------------------------------------------------------------- | |
82 | # Service factories |
|
82 | # Service factories | |
83 | #----------------------------------------------------------------------------- |
|
83 | #----------------------------------------------------------------------------- | |
84 |
|
84 | |||
85 |
|
85 | |||
86 | class FCClientServiceFactory(FCServiceFactory): |
|
86 | class FCClientServiceFactory(FCServiceFactory): | |
87 | """A Foolscap implementation of the client services.""" |
|
87 | """A Foolscap implementation of the client services.""" | |
88 |
|
88 | |||
89 | cert_file = Str('ipcontroller-client.pem', config=True) |
|
89 | cert_file = Str('ipcontroller-client.pem', config=True) | |
90 | interfaces = Instance(klass=Config, kw=default_client_interfaces, |
|
90 | interfaces = Instance(klass=Config, kw=default_client_interfaces, | |
91 | allow_none=False, config=True) |
|
91 | allow_none=False, config=True) | |
92 |
|
92 | |||
93 |
|
93 | |||
94 | class FCEngineServiceFactory(FCServiceFactory): |
|
94 | class FCEngineServiceFactory(FCServiceFactory): | |
95 | """A Foolscap implementation of the engine services.""" |
|
95 | """A Foolscap implementation of the engine services.""" | |
96 |
|
96 | |||
97 | cert_file = Str('ipcontroller-engine.pem', config=True) |
|
97 | cert_file = Str('ipcontroller-engine.pem', config=True) | |
98 | interfaces = Instance(klass=dict, kw=default_engine_interfaces, |
|
98 | interfaces = Instance(klass=dict, kw=default_engine_interfaces, | |
99 | allow_none=False, config=True) |
|
99 | allow_none=False, config=True) | |
100 |
|
100 | |||
101 |
|
101 | |||
102 | #----------------------------------------------------------------------------- |
|
102 | #----------------------------------------------------------------------------- | |
103 | # The main application |
|
103 | # The main application | |
104 | #----------------------------------------------------------------------------- |
|
104 | #----------------------------------------------------------------------------- | |
105 |
|
105 | |||
106 |
|
106 | |||
107 | cl_args = ( |
|
107 | cl_args = ( | |
108 | # Client config |
|
108 | # Client config | |
109 | (('--client-ip',), dict( |
|
109 | (('--client-ip',), dict( | |
110 | type=str, dest='FCClientServiceFactory.ip', default=NoConfigDefault, |
|
110 | type=str, dest='FCClientServiceFactory.ip', default=NoConfigDefault, | |
111 | help='The IP address or hostname the controller will listen on for ' |
|
111 | help='The IP address or hostname the controller will listen on for ' | |
112 | 'client connections.', |
|
112 | 'client connections.', | |
113 | metavar='FCClientServiceFactory.ip') |
|
113 | metavar='FCClientServiceFactory.ip') | |
114 | ), |
|
114 | ), | |
115 | (('--client-port',), dict( |
|
115 | (('--client-port',), dict( | |
116 | type=int, dest='FCClientServiceFactory.port', default=NoConfigDefault, |
|
116 | type=int, dest='FCClientServiceFactory.port', default=NoConfigDefault, | |
117 | help='The port the controller will listen on for client connections. ' |
|
117 | help='The port the controller will listen on for client connections. ' | |
118 | 'The default is to use 0, which will autoselect an open port.', |
|
118 | 'The default is to use 0, which will autoselect an open port.', | |
119 | metavar='FCClientServiceFactory.port') |
|
119 | metavar='FCClientServiceFactory.port') | |
120 | ), |
|
120 | ), | |
121 | (('--client-location',), dict( |
|
121 | (('--client-location',), dict( | |
122 | type=str, dest='FCClientServiceFactory.location', default=NoConfigDefault, |
|
122 | type=str, dest='FCClientServiceFactory.location', default=NoConfigDefault, | |
123 | help='The hostname or IP that clients should connect to. This does ' |
|
123 | help='The hostname or IP that clients should connect to. This does ' | |
124 | 'not control which interface the controller listens on. Instead, this ' |
|
124 | 'not control which interface the controller listens on. Instead, this ' | |
125 | 'determines the hostname/IP that is listed in the FURL, which is how ' |
|
125 | 'determines the hostname/IP that is listed in the FURL, which is how ' | |
126 | 'clients know where to connect. Useful if the controller is listening ' |
|
126 | 'clients know where to connect. Useful if the controller is listening ' | |
127 | 'on multiple interfaces.', |
|
127 | 'on multiple interfaces.', | |
128 | metavar='FCClientServiceFactory.location') |
|
128 | metavar='FCClientServiceFactory.location') | |
129 | ), |
|
129 | ), | |
130 | # Engine config |
|
130 | # Engine config | |
131 | (('--engine-ip',), dict( |
|
131 | (('--engine-ip',), dict( | |
132 | type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault, |
|
132 | type=str, dest='FCEngineServiceFactory.ip', default=NoConfigDefault, | |
133 | help='The IP address or hostname the controller will listen on for ' |
|
133 | help='The IP address or hostname the controller will listen on for ' | |
134 | 'engine connections.', |
|
134 | 'engine connections.', | |
135 | metavar='FCEngineServiceFactory.ip') |
|
135 | metavar='FCEngineServiceFactory.ip') | |
136 | ), |
|
136 | ), | |
137 | (('--engine-port',), dict( |
|
137 | (('--engine-port',), dict( | |
138 | type=int, dest='FCEngineServiceFactory.port', default=NoConfigDefault, |
|
138 | type=int, dest='FCEngineServiceFactory.port', default=NoConfigDefault, | |
139 | help='The port the controller will listen on for engine connections. ' |
|
139 | help='The port the controller will listen on for engine connections. ' | |
140 | 'The default is to use 0, which will autoselect an open port.', |
|
140 | 'The default is to use 0, which will autoselect an open port.', | |
141 | metavar='FCEngineServiceFactory.port') |
|
141 | metavar='FCEngineServiceFactory.port') | |
142 | ), |
|
142 | ), | |
143 | (('--engine-location',), dict( |
|
143 | (('--engine-location',), dict( | |
144 | type=str, dest='FCEngineServiceFactory.location', default=NoConfigDefault, |
|
144 | type=str, dest='FCEngineServiceFactory.location', default=NoConfigDefault, | |
145 | help='The hostname or IP that engines should connect to. This does ' |
|
145 | help='The hostname or IP that engines should connect to. This does ' | |
146 | 'not control which interface the controller listens on. Instead, this ' |
|
146 | 'not control which interface the controller listens on. Instead, this ' | |
147 | 'determines the hostname/IP that is listed in the FURL, which is how ' |
|
147 | 'determines the hostname/IP that is listed in the FURL, which is how ' | |
148 | 'engines know where to connect. Useful if the controller is listening ' |
|
148 | 'engines know where to connect. Useful if the controller is listening ' | |
149 | 'on multiple interfaces.', |
|
149 | 'on multiple interfaces.', | |
150 | metavar='FCEngineServiceFactory.location') |
|
150 | metavar='FCEngineServiceFactory.location') | |
151 | ), |
|
151 | ), | |
152 | # Global config |
|
152 | # Global config | |
153 | (('--log-to-file',), dict( |
|
153 | (('--log-to-file',), dict( | |
154 | action='store_true', dest='Global.log_to_file', default=NoConfigDefault, |
|
154 | action='store_true', dest='Global.log_to_file', default=NoConfigDefault, | |
155 | help='Log to a file in the log directory (default is stdout)') |
|
155 | help='Log to a file in the log directory (default is stdout)') | |
156 | ), |
|
156 | ), | |
157 | (('-r','--reuse-furls'), dict( |
|
157 | (('-r','--reuse-furls'), dict( | |
158 | action='store_true', dest='Global.reuse_furls', default=NoConfigDefault, |
|
158 | action='store_true', dest='Global.reuse_furls', default=NoConfigDefault, | |
159 | help='Try to reuse all FURL files. If this is not set all FURL files ' |
|
159 | help='Try to reuse all FURL files. If this is not set all FURL files ' | |
160 | 'are deleted before the controller starts. This must be set if ' |
|
160 | 'are deleted before the controller starts. This must be set if ' | |
161 | 'specific ports are specified by --engine-port or --client-port.') |
|
161 | 'specific ports are specified by --engine-port or --client-port.') | |
162 | ), |
|
162 | ), | |
163 | (('--no-secure',), dict( |
|
163 | (('--no-secure',), dict( | |
164 | action='store_false', dest='Global.secure', default=NoConfigDefault, |
|
164 | action='store_false', dest='Global.secure', default=NoConfigDefault, | |
165 | help='Turn off SSL encryption for all connections.') |
|
165 | help='Turn off SSL encryption for all connections.') | |
166 | ), |
|
166 | ), | |
167 | (('--secure',), dict( |
|
167 | (('--secure',), dict( | |
168 | action='store_true', dest='Global.secure', default=NoConfigDefault, |
|
168 | action='store_true', dest='Global.secure', default=NoConfigDefault, | |
169 | help='Turn off SSL encryption for all connections.') |
|
169 | help='Turn off SSL encryption for all connections.') | |
170 | ) |
|
170 | ) | |
171 | ) |
|
171 | ) | |
172 |
|
172 | |||
173 |
|
173 | |||
174 | class IPControllerAppCLConfigLoader(AppWithClusterDirArgParseConfigLoader): |
|
174 | class IPControllerAppCLConfigLoader(AppWithClusterDirArgParseConfigLoader): | |
175 |
|
175 | |||
176 | arguments = cl_args |
|
176 | arguments = cl_args | |
177 |
|
177 | |||
178 |
|
178 | |||
179 | default_config_file_name = 'ipcontroller_config.py' |
|
179 | default_config_file_name = 'ipcontroller_config.py' | |
180 |
|
180 | |||
181 |
|
181 | |||
182 | class IPControllerApp(ApplicationWithClusterDir): |
|
182 | class IPControllerApp(ApplicationWithClusterDir): | |
183 |
|
183 | |||
184 | name = 'ipcontroller' |
|
184 | name = 'ipcontroller' | |
185 | description = 'Start the IPython controller for parallel computing.' |
|
185 | description = 'Start the IPython controller for parallel computing.' | |
186 | config_file_name = default_config_file_name |
|
186 | config_file_name = default_config_file_name | |
187 | auto_create_cluster_dir = True |
|
187 | auto_create_cluster_dir = True | |
188 |
|
188 | |||
189 | def create_default_config(self): |
|
189 | def create_default_config(self): | |
190 | super(IPControllerApp, self).create_default_config() |
|
190 | super(IPControllerApp, self).create_default_config() | |
191 | self.default_config.Global.reuse_furls = False |
|
191 | self.default_config.Global.reuse_furls = False | |
192 | self.default_config.Global.secure = True |
|
192 | self.default_config.Global.secure = True | |
193 | self.default_config.Global.import_statements = [] |
|
193 | self.default_config.Global.import_statements = [] | |
194 | self.default_config.Global.clean_logs = True |
|
194 | self.default_config.Global.clean_logs = True | |
195 |
|
195 | |||
196 | def create_command_line_config(self): |
|
196 | def create_command_line_config(self): | |
197 | """Create and return a command line config loader.""" |
|
197 | """Create and return a command line config loader.""" | |
198 | return IPControllerAppCLConfigLoader( |
|
198 | return IPControllerAppCLConfigLoader( | |
199 | description=self.description, |
|
199 | description=self.description, | |
200 | version=release.version |
|
200 | version=release.version | |
201 | ) |
|
201 | ) | |
202 |
|
202 | |||
203 | def post_load_command_line_config(self): |
|
203 | def post_load_command_line_config(self): | |
204 | # Now setup reuse_furls |
|
204 | # Now setup reuse_furls | |
205 | c = self.command_line_config |
|
205 | c = self.command_line_config | |
206 | if hasattr(c.Global, 'reuse_furls'): |
|
206 | if hasattr(c.Global, 'reuse_furls'): | |
207 | c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls |
|
207 | c.FCClientServiceFactory.reuse_furls = c.Global.reuse_furls | |
208 | c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls |
|
208 | c.FCEngineServiceFactory.reuse_furls = c.Global.reuse_furls | |
209 | del c.Global.reuse_furls |
|
209 | del c.Global.reuse_furls | |
210 | if hasattr(c.Global, 'secure'): |
|
210 | if hasattr(c.Global, 'secure'): | |
211 | c.FCClientServiceFactory.secure = c.Global.secure |
|
211 | c.FCClientServiceFactory.secure = c.Global.secure | |
212 | c.FCEngineServiceFactory.secure = c.Global.secure |
|
212 | c.FCEngineServiceFactory.secure = c.Global.secure | |
213 | del c.Global.secure |
|
213 | del c.Global.secure | |
214 |
|
214 | |||
215 | def construct(self): |
|
215 | def construct(self): | |
216 | # I am a little hesitant to put these into InteractiveShell itself. |
|
216 | # I am a little hesitant to put these into InteractiveShell itself. | |
217 | # But that might be the place for them |
|
217 | # But that might be the place for them | |
218 | sys.path.insert(0, '') |
|
218 | sys.path.insert(0, '') | |
219 |
|
219 | |||
220 | self.start_logging() |
|
220 | self.start_logging() | |
221 | self.import_statements() |
|
221 | self.import_statements() | |
222 |
|
222 | |||
223 | # Create the service hierarchy |
|
223 | # Create the service hierarchy | |
224 | self.main_service = service.MultiService() |
|
224 | self.main_service = service.MultiService() | |
225 | # The controller service |
|
225 | # The controller service | |
226 | controller_service = controllerservice.ControllerService() |
|
226 | controller_service = controllerservice.ControllerService() | |
227 | controller_service.setServiceParent(self.main_service) |
|
227 | controller_service.setServiceParent(self.main_service) | |
228 | # The client tub and all its refereceables |
|
228 | # The client tub and all its refereceables | |
229 | csfactory = FCClientServiceFactory(self.master_config, controller_service) |
|
229 | csfactory = FCClientServiceFactory(self.master_config, controller_service) | |
230 | client_service = csfactory.create() |
|
230 | client_service = csfactory.create() | |
231 | client_service.setServiceParent(self.main_service) |
|
231 | client_service.setServiceParent(self.main_service) | |
232 | # The engine tub |
|
232 | # The engine tub | |
233 | esfactory = FCEngineServiceFactory(self.master_config, controller_service) |
|
233 | esfactory = FCEngineServiceFactory(self.master_config, controller_service) | |
234 | engine_service = esfactory.create() |
|
234 | engine_service = esfactory.create() | |
235 | engine_service.setServiceParent(self.main_service) |
|
235 | engine_service.setServiceParent(self.main_service) | |
236 |
|
236 | |||
237 | def import_statements(self): |
|
237 | def import_statements(self): | |
238 | statements = self.master_config.Global.import_statements |
|
238 | statements = self.master_config.Global.import_statements | |
239 | for s in statements: |
|
239 | for s in statements: | |
240 | try: |
|
240 | try: | |
241 | log.msg("Executing statement: '%s'" % s) |
|
241 | log.msg("Executing statement: '%s'" % s) | |
242 | exec s in globals(), locals() |
|
242 | exec s in globals(), locals() | |
243 | except: |
|
243 | except: | |
244 | log.msg("Error running statement: %s" % s) |
|
244 | log.msg("Error running statement: %s" % s) | |
245 |
|
245 | |||
246 | def start_app(self): |
|
246 | def start_app(self): | |
247 |
# Start the controller service |
|
247 | # Start the controller service. | |
248 | self.main_service.startService() |
|
248 | self.main_service.startService() | |
|
249 | # Write the .pid file overwriting old ones. This allow multiple | |||
|
250 | # controllers to clober each other. But Windows is not cleaning | |||
|
251 | # these up properly. | |||
249 | self.write_pid_file(overwrite=True) |
|
252 | self.write_pid_file(overwrite=True) | |
|
253 | # cd to the cluster_dir as our working directory. | |||
|
254 | os.chdir(self.master_config.Global.cluster_dir) | |||
|
255 | # Add a trigger to delete the .pid file upon shutting down. | |||
250 | reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file) |
|
256 | reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file) | |
251 | reactor.run() |
|
257 | reactor.run() | |
252 |
|
258 | |||
253 |
|
259 | |||
254 | def launch_new_instance(): |
|
260 | def launch_new_instance(): | |
255 | """Create and run the IPython controller""" |
|
261 | """Create and run the IPython controller""" | |
256 | app = IPControllerApp() |
|
262 | app = IPControllerApp() | |
257 | app.start() |
|
263 | app.start() | |
258 |
|
264 | |||
259 |
|
265 | |||
260 | if __name__ == '__main__': |
|
266 | if __name__ == '__main__': | |
261 | launch_new_instance() |
|
267 | launch_new_instance() | |
262 |
|
268 |
@@ -1,239 +1,240 b'' | |||||
1 | #!/usr/bin/env python |
|
1 | #!/usr/bin/env python | |
2 | # encoding: utf-8 |
|
2 | # encoding: utf-8 | |
3 | """ |
|
3 | """ | |
4 | The IPython controller application |
|
4 | The IPython controller application | |
5 | """ |
|
5 | """ | |
6 |
|
6 | |||
7 | #----------------------------------------------------------------------------- |
|
7 | #----------------------------------------------------------------------------- | |
8 | # Copyright (C) 2008-2009 The IPython Development Team |
|
8 | # Copyright (C) 2008-2009 The IPython Development Team | |
9 | # |
|
9 | # | |
10 | # Distributed under the terms of the BSD License. The full license is in |
|
10 | # Distributed under the terms of the BSD License. The full license is in | |
11 | # the file COPYING, distributed as part of this software. |
|
11 | # the file COPYING, distributed as part of this software. | |
12 | #----------------------------------------------------------------------------- |
|
12 | #----------------------------------------------------------------------------- | |
13 |
|
13 | |||
14 | #----------------------------------------------------------------------------- |
|
14 | #----------------------------------------------------------------------------- | |
15 | # Imports |
|
15 | # Imports | |
16 | #----------------------------------------------------------------------------- |
|
16 | #----------------------------------------------------------------------------- | |
17 |
|
17 | |||
18 | import os |
|
18 | import os | |
19 | import sys |
|
19 | import sys | |
20 |
|
20 | |||
21 | from twisted.application import service |
|
21 | from twisted.application import service | |
22 | from twisted.internet import reactor |
|
22 | from twisted.internet import reactor | |
23 | from twisted.python import log |
|
23 | from twisted.python import log | |
24 |
|
24 | |||
25 | from IPython.config.loader import NoConfigDefault |
|
25 | from IPython.config.loader import NoConfigDefault | |
26 |
|
26 | |||
27 | from IPython.kernel.clusterdir import ( |
|
27 | from IPython.kernel.clusterdir import ( | |
28 | ApplicationWithClusterDir, |
|
28 | ApplicationWithClusterDir, | |
29 | AppWithClusterDirArgParseConfigLoader |
|
29 | AppWithClusterDirArgParseConfigLoader | |
30 | ) |
|
30 | ) | |
31 | from IPython.core import release |
|
31 | from IPython.core import release | |
32 |
|
32 | |||
33 | from IPython.utils.importstring import import_item |
|
33 | from IPython.utils.importstring import import_item | |
34 |
|
34 | |||
35 | from IPython.kernel.engineservice import EngineService |
|
35 | from IPython.kernel.engineservice import EngineService | |
36 | from IPython.kernel.fcutil import Tub |
|
36 | from IPython.kernel.fcutil import Tub | |
37 | from IPython.kernel.engineconnector import EngineConnector |
|
37 | from IPython.kernel.engineconnector import EngineConnector | |
38 |
|
38 | |||
39 | #----------------------------------------------------------------------------- |
|
39 | #----------------------------------------------------------------------------- | |
40 | # The main application |
|
40 | # The main application | |
41 | #----------------------------------------------------------------------------- |
|
41 | #----------------------------------------------------------------------------- | |
42 |
|
42 | |||
43 |
|
43 | |||
44 | cl_args = ( |
|
44 | cl_args = ( | |
45 | # Controller config |
|
45 | # Controller config | |
46 | (('--furl-file',), dict( |
|
46 | (('--furl-file',), dict( | |
47 | type=str, dest='Global.furl_file', default=NoConfigDefault, |
|
47 | type=str, dest='Global.furl_file', default=NoConfigDefault, | |
48 | help='The full location of the file containing the FURL of the ' |
|
48 | help='The full location of the file containing the FURL of the ' | |
49 | 'controller. If this is not given, the FURL file must be in the ' |
|
49 | 'controller. If this is not given, the FURL file must be in the ' | |
50 | 'security directory of the cluster directory. This location is ' |
|
50 | 'security directory of the cluster directory. This location is ' | |
51 | 'resolved using the --profile and --app-dir options.', |
|
51 | 'resolved using the --profile and --app-dir options.', | |
52 | metavar='Global.furl_file') |
|
52 | metavar='Global.furl_file') | |
53 | ), |
|
53 | ), | |
54 | # MPI |
|
54 | # MPI | |
55 | (('--mpi',), dict( |
|
55 | (('--mpi',), dict( | |
56 | type=str, dest='MPI.use', default=NoConfigDefault, |
|
56 | type=str, dest='MPI.use', default=NoConfigDefault, | |
57 | help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).', |
|
57 | help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).', | |
58 | metavar='MPI.use') |
|
58 | metavar='MPI.use') | |
59 | ), |
|
59 | ), | |
60 | # Global config |
|
60 | # Global config | |
61 | (('--log-to-file',), dict( |
|
61 | (('--log-to-file',), dict( | |
62 | action='store_true', dest='Global.log_to_file', default=NoConfigDefault, |
|
62 | action='store_true', dest='Global.log_to_file', default=NoConfigDefault, | |
63 | help='Log to a file in the log directory (default is stdout)') |
|
63 | help='Log to a file in the log directory (default is stdout)') | |
64 | ) |
|
64 | ) | |
65 | ) |
|
65 | ) | |
66 |
|
66 | |||
67 |
|
67 | |||
68 | class IPEngineAppCLConfigLoader(AppWithClusterDirArgParseConfigLoader): |
|
68 | class IPEngineAppCLConfigLoader(AppWithClusterDirArgParseConfigLoader): | |
69 |
|
69 | |||
70 | arguments = cl_args |
|
70 | arguments = cl_args | |
71 |
|
71 | |||
72 |
|
72 | |||
73 | mpi4py_init = """from mpi4py import MPI as mpi |
|
73 | mpi4py_init = """from mpi4py import MPI as mpi | |
74 | mpi.size = mpi.COMM_WORLD.Get_size() |
|
74 | mpi.size = mpi.COMM_WORLD.Get_size() | |
75 | mpi.rank = mpi.COMM_WORLD.Get_rank() |
|
75 | mpi.rank = mpi.COMM_WORLD.Get_rank() | |
76 | """ |
|
76 | """ | |
77 |
|
77 | |||
78 | pytrilinos_init = """from PyTrilinos import Epetra |
|
78 | pytrilinos_init = """from PyTrilinos import Epetra | |
79 | class SimpleStruct: |
|
79 | class SimpleStruct: | |
80 | pass |
|
80 | pass | |
81 | mpi = SimpleStruct() |
|
81 | mpi = SimpleStruct() | |
82 | mpi.rank = 0 |
|
82 | mpi.rank = 0 | |
83 | mpi.size = 0 |
|
83 | mpi.size = 0 | |
84 | """ |
|
84 | """ | |
85 |
|
85 | |||
86 |
|
86 | |||
87 | default_config_file_name = 'ipengine_config.py' |
|
87 | default_config_file_name = 'ipengine_config.py' | |
88 |
|
88 | |||
89 |
|
89 | |||
90 | class IPEngineApp(ApplicationWithClusterDir): |
|
90 | class IPEngineApp(ApplicationWithClusterDir): | |
91 |
|
91 | |||
92 | name = 'ipengine' |
|
92 | name = 'ipengine' | |
93 | description = 'Start the IPython engine for parallel computing.' |
|
93 | description = 'Start the IPython engine for parallel computing.' | |
94 | config_file_name = default_config_file_name |
|
94 | config_file_name = default_config_file_name | |
95 | auto_create_cluster_dir = True |
|
95 | auto_create_cluster_dir = True | |
96 |
|
96 | |||
97 | def create_default_config(self): |
|
97 | def create_default_config(self): | |
98 | super(IPEngineApp, self).create_default_config() |
|
98 | super(IPEngineApp, self).create_default_config() | |
99 |
|
99 | |||
100 | # The engine should not clean logs as we don't want to remove the |
|
100 | # The engine should not clean logs as we don't want to remove the | |
101 | # active log files of other running engines. |
|
101 | # active log files of other running engines. | |
102 | self.default_config.Global.clean_logs = False |
|
102 | self.default_config.Global.clean_logs = False | |
103 |
|
103 | |||
104 | # Global config attributes |
|
104 | # Global config attributes | |
105 | self.default_config.Global.exec_lines = [] |
|
105 | self.default_config.Global.exec_lines = [] | |
106 | self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter' |
|
106 | self.default_config.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter' | |
107 |
|
107 | |||
108 | # Configuration related to the controller |
|
108 | # Configuration related to the controller | |
109 | # This must match the filename (path not included) that the controller |
|
109 | # This must match the filename (path not included) that the controller | |
110 | # used for the FURL file. |
|
110 | # used for the FURL file. | |
111 | self.default_config.Global.furl_file_name = 'ipcontroller-engine.furl' |
|
111 | self.default_config.Global.furl_file_name = 'ipcontroller-engine.furl' | |
112 | # If given, this is the actual location of the controller's FURL file. |
|
112 | # If given, this is the actual location of the controller's FURL file. | |
113 | # If not, this is computed using the profile, app_dir and furl_file_name |
|
113 | # If not, this is computed using the profile, app_dir and furl_file_name | |
114 | self.default_config.Global.furl_file = '' |
|
114 | self.default_config.Global.furl_file = '' | |
115 |
|
115 | |||
116 | # The max number of connection attemps and the initial delay between |
|
116 | # The max number of connection attemps and the initial delay between | |
117 | # those attemps. |
|
117 | # those attemps. | |
118 | self.default_config.Global.connect_delay = 0.1 |
|
118 | self.default_config.Global.connect_delay = 0.1 | |
119 | self.default_config.Global.connect_max_tries = 15 |
|
119 | self.default_config.Global.connect_max_tries = 15 | |
120 |
|
120 | |||
121 | # MPI related config attributes |
|
121 | # MPI related config attributes | |
122 | self.default_config.MPI.use = '' |
|
122 | self.default_config.MPI.use = '' | |
123 | self.default_config.MPI.mpi4py = mpi4py_init |
|
123 | self.default_config.MPI.mpi4py = mpi4py_init | |
124 | self.default_config.MPI.pytrilinos = pytrilinos_init |
|
124 | self.default_config.MPI.pytrilinos = pytrilinos_init | |
125 |
|
125 | |||
126 | def create_command_line_config(self): |
|
126 | def create_command_line_config(self): | |
127 | """Create and return a command line config loader.""" |
|
127 | """Create and return a command line config loader.""" | |
128 | return IPEngineAppCLConfigLoader( |
|
128 | return IPEngineAppCLConfigLoader( | |
129 | description=self.description, |
|
129 | description=self.description, | |
130 | version=release.version |
|
130 | version=release.version | |
131 | ) |
|
131 | ) | |
132 |
|
132 | |||
133 | def post_load_command_line_config(self): |
|
133 | def post_load_command_line_config(self): | |
134 | pass |
|
134 | pass | |
135 |
|
135 | |||
136 | def pre_construct(self): |
|
136 | def pre_construct(self): | |
137 | super(IPEngineApp, self).pre_construct() |
|
137 | super(IPEngineApp, self).pre_construct() | |
138 | self.find_cont_furl_file() |
|
138 | self.find_cont_furl_file() | |
139 |
|
139 | |||
140 | def find_cont_furl_file(self): |
|
140 | def find_cont_furl_file(self): | |
141 | """Set the furl file. |
|
141 | """Set the furl file. | |
142 |
|
142 | |||
143 | Here we don't try to actually see if it exists for is valid as that |
|
143 | Here we don't try to actually see if it exists for is valid as that | |
144 | is hadled by the connection logic. |
|
144 | is hadled by the connection logic. | |
145 | """ |
|
145 | """ | |
146 | config = self.master_config |
|
146 | config = self.master_config | |
147 | # Find the actual controller FURL file |
|
147 | # Find the actual controller FURL file | |
148 | if not config.Global.furl_file: |
|
148 | if not config.Global.furl_file: | |
149 | try_this = os.path.join( |
|
149 | try_this = os.path.join( | |
150 | config.Global.cluster_dir, |
|
150 | config.Global.cluster_dir, | |
151 | config.Global.security_dir, |
|
151 | config.Global.security_dir, | |
152 | config.Global.furl_file_name |
|
152 | config.Global.furl_file_name | |
153 | ) |
|
153 | ) | |
154 | config.Global.furl_file = try_this |
|
154 | config.Global.furl_file = try_this | |
155 |
|
155 | |||
156 | def construct(self): |
|
156 | def construct(self): | |
157 | # I am a little hesitant to put these into InteractiveShell itself. |
|
157 | # I am a little hesitant to put these into InteractiveShell itself. | |
158 | # But that might be the place for them |
|
158 | # But that might be the place for them | |
159 | sys.path.insert(0, '') |
|
159 | sys.path.insert(0, '') | |
160 |
|
160 | |||
161 | self.start_mpi() |
|
161 | self.start_mpi() | |
162 | self.start_logging() |
|
162 | self.start_logging() | |
163 |
|
163 | |||
164 | # Create the underlying shell class and EngineService |
|
164 | # Create the underlying shell class and EngineService | |
165 | shell_class = import_item(self.master_config.Global.shell_class) |
|
165 | shell_class = import_item(self.master_config.Global.shell_class) | |
166 | self.engine_service = EngineService(shell_class, mpi=mpi) |
|
166 | self.engine_service = EngineService(shell_class, mpi=mpi) | |
167 |
|
167 | |||
168 | self.exec_lines() |
|
168 | self.exec_lines() | |
169 |
|
169 | |||
170 | # Create the service hierarchy |
|
170 | # Create the service hierarchy | |
171 | self.main_service = service.MultiService() |
|
171 | self.main_service = service.MultiService() | |
172 | self.engine_service.setServiceParent(self.main_service) |
|
172 | self.engine_service.setServiceParent(self.main_service) | |
173 | self.tub_service = Tub() |
|
173 | self.tub_service = Tub() | |
174 | self.tub_service.setServiceParent(self.main_service) |
|
174 | self.tub_service.setServiceParent(self.main_service) | |
175 | # This needs to be called before the connection is initiated |
|
175 | # This needs to be called before the connection is initiated | |
176 | self.main_service.startService() |
|
176 | self.main_service.startService() | |
177 |
|
177 | |||
178 | # This initiates the connection to the controller and calls |
|
178 | # This initiates the connection to the controller and calls | |
179 | # register_engine to tell the controller we are ready to do work |
|
179 | # register_engine to tell the controller we are ready to do work | |
180 | self.engine_connector = EngineConnector(self.tub_service) |
|
180 | self.engine_connector = EngineConnector(self.tub_service) | |
181 |
|
181 | |||
182 | log.msg("Using furl file: %s" % self.master_config.Global.furl_file) |
|
182 | log.msg("Using furl file: %s" % self.master_config.Global.furl_file) | |
183 |
|
183 | |||
184 | reactor.callWhenRunning(self.call_connect) |
|
184 | reactor.callWhenRunning(self.call_connect) | |
185 |
|
185 | |||
186 | def call_connect(self): |
|
186 | def call_connect(self): | |
187 | d = self.engine_connector.connect_to_controller( |
|
187 | d = self.engine_connector.connect_to_controller( | |
188 | self.engine_service, |
|
188 | self.engine_service, | |
189 | self.master_config.Global.furl_file, |
|
189 | self.master_config.Global.furl_file, | |
190 | self.master_config.Global.connect_delay, |
|
190 | self.master_config.Global.connect_delay, | |
191 | self.master_config.Global.connect_max_tries |
|
191 | self.master_config.Global.connect_max_tries | |
192 | ) |
|
192 | ) | |
193 |
|
193 | |||
194 | def handle_error(f): |
|
194 | def handle_error(f): | |
195 | log.msg('Error connecting to controller. This usually means that ' |
|
195 | log.msg('Error connecting to controller. This usually means that ' | |
196 | 'i) the controller was not started, ii) a firewall was blocking ' |
|
196 | 'i) the controller was not started, ii) a firewall was blocking ' | |
197 | 'the engine from connecting to the controller or iii) the engine ' |
|
197 | 'the engine from connecting to the controller or iii) the engine ' | |
198 | ' was not pointed at the right FURL file:') |
|
198 | ' was not pointed at the right FURL file:') | |
199 | log.msg(f.getErrorMessage()) |
|
199 | log.msg(f.getErrorMessage()) | |
200 | reactor.callLater(0.1, reactor.stop) |
|
200 | reactor.callLater(0.1, reactor.stop) | |
201 |
|
201 | |||
202 | d.addErrback(handle_error) |
|
202 | d.addErrback(handle_error) | |
203 |
|
203 | |||
204 | def start_mpi(self): |
|
204 | def start_mpi(self): | |
205 | global mpi |
|
205 | global mpi | |
206 | mpikey = self.master_config.MPI.use |
|
206 | mpikey = self.master_config.MPI.use | |
207 | mpi_import_statement = self.master_config.MPI.get(mpikey, None) |
|
207 | mpi_import_statement = self.master_config.MPI.get(mpikey, None) | |
208 | if mpi_import_statement is not None: |
|
208 | if mpi_import_statement is not None: | |
209 | try: |
|
209 | try: | |
210 | self.log.info("Initializing MPI:") |
|
210 | self.log.info("Initializing MPI:") | |
211 | self.log.info(mpi_import_statement) |
|
211 | self.log.info(mpi_import_statement) | |
212 | exec mpi_import_statement in globals() |
|
212 | exec mpi_import_statement in globals() | |
213 | except: |
|
213 | except: | |
214 | mpi = None |
|
214 | mpi = None | |
215 | else: |
|
215 | else: | |
216 | mpi = None |
|
216 | mpi = None | |
217 |
|
217 | |||
218 | def exec_lines(self): |
|
218 | def exec_lines(self): | |
219 | for line in self.master_config.Global.exec_lines: |
|
219 | for line in self.master_config.Global.exec_lines: | |
220 | try: |
|
220 | try: | |
221 | log.msg("Executing statement: '%s'" % line) |
|
221 | log.msg("Executing statement: '%s'" % line) | |
222 | self.engine_service.execute(line) |
|
222 | self.engine_service.execute(line) | |
223 | except: |
|
223 | except: | |
224 | log.msg("Error executing statement: %s" % line) |
|
224 | log.msg("Error executing statement: %s" % line) | |
225 |
|
225 | |||
226 | def start_app(self): |
|
226 | def start_app(self): | |
227 | # Start the controller service and set things running |
|
227 | # cd to the cluster_dir as our working directory. | |
|
228 | os.chdir(self.master_config.Global.cluster_dir) | |||
228 | reactor.run() |
|
229 | reactor.run() | |
229 |
|
230 | |||
230 |
|
231 | |||
231 | def launch_new_instance(): |
|
232 | def launch_new_instance(): | |
232 | """Create and run the IPython controller""" |
|
233 | """Create and run the IPython controller""" | |
233 | app = IPEngineApp() |
|
234 | app = IPEngineApp() | |
234 | app.start() |
|
235 | app.start() | |
235 |
|
236 | |||
236 |
|
237 | |||
237 | if __name__ == '__main__': |
|
238 | if __name__ == '__main__': | |
238 | launch_new_instance() |
|
239 | launch_new_instance() | |
239 |
|
240 |
General Comments 0
You need to be logged in to leave comments.
Login now