Show More
@@ -1,32 +1,24 b'' | |||
|
1 | 1 | c = get_config() |
|
2 | 2 | |
|
3 | c.MPI.use = '' | |
|
3 | # c.Global.log_to_file = False | |
|
4 | # c.Global.exec_lines = ['import numpy'] | |
|
5 | # c.Global.log_dir_name = 'log' | |
|
6 | # c.Global.security_dir_name = 'security' | |
|
7 | # c.Global.log_level = 10 | |
|
8 | # c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter' | |
|
9 | # c.Global.furl_file_name = 'ipcontroller-engine.furl' | |
|
10 | # c.Global.furl_file = '' | |
|
11 | ||
|
12 | # c.MPI.use = '' | |
|
13 | # c.MPI.mpi4py = """from mpi4py import MPI as mpi | |
|
14 | # mpi.size = mpi.COMM_WORLD.Get_size() | |
|
15 | # mpi.rank = mpi.COMM_WORLD.Get_rank() | |
|
16 | # """ | |
|
17 | # c.MPI.pytrilinos = """from PyTrilinos import Epetra | |
|
18 | # class SimpleStruct: | |
|
19 | # pass | |
|
20 | # mpi = SimpleStruct() | |
|
21 | # mpi.rank = 0 | |
|
22 | # mpi.size = 0 | |
|
23 | # """ | |
|
4 | 24 | |
|
5 | c.MPI.mpi4py = """from mpi4py import MPI as mpi | |
|
6 | mpi.size = mpi.COMM_WORLD.Get_size() | |
|
7 | mpi.rank = mpi.COMM_WORLD.Get_rank() | |
|
8 | """ | |
|
9 | ||
|
10 | c.MPI.pytrilinos = """from PyTrilinos import Epetra | |
|
11 | class SimpleStruct: | |
|
12 | pass | |
|
13 | mpi = SimpleStruct() | |
|
14 | mpi.rank = 0 | |
|
15 | mpi.size = 0 | |
|
16 | """ | |
|
17 | ||
|
18 | c.Global.log_to_file = False | |
|
19 | ||
|
20 | c.Global.exec_lines = ['import numpy'] | |
|
21 | ||
|
22 | c.Global.log_dir_name = 'log' | |
|
23 | ||
|
24 | c.Global.security_dir_name = 'security' | |
|
25 | ||
|
26 | c.Global.log_level = 10 | |
|
27 | ||
|
28 | c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter' | |
|
29 | ||
|
30 | c.Global.furl_file_name = 'ipcontroller-engine.furl' | |
|
31 | ||
|
32 | c.Global.furl_file = '' |
@@ -86,6 +86,7 b' class Application(object):' | |||
|
86 | 86 | default_log_level = logging.WARN |
|
87 | 87 | |
|
88 | 88 | def __init__(self): |
|
89 | self._exiting = False | |
|
89 | 90 | self.init_logger() |
|
90 | 91 | # Track the default and actual separately because some messages are |
|
91 | 92 | # only printed if we aren't using the default. |
@@ -119,6 +120,7 b' class Application(object):' | |||
|
119 | 120 | self.attempt(self.post_load_command_line_config) |
|
120 | 121 | self.log_command_line_config() |
|
121 | 122 | self.attempt(self.find_ipythondir) |
|
123 | self.attempt(self.find_resources) | |
|
122 | 124 | self.attempt(self.find_config_file_name) |
|
123 | 125 | self.attempt(self.find_config_file_paths) |
|
124 | 126 | self.attempt(self.pre_load_file_config) |
@@ -211,6 +213,15 b' class Application(object):' | |||
|
211 | 213 | os.makedirs(self.ipythondir, mode=0777) |
|
212 | 214 | self.log.debug("IPYTHONDIR set to: %s" % self.ipythondir) |
|
213 | 215 | |
|
216 | def find_resources(self): | |
|
217 | """Find other resources that need to be in place. | |
|
218 | ||
|
219 | Things like cluster directories need to be in place to find the | |
|
220 | config file. These happen right after the IPython directory has | |
|
221 | been set. | |
|
222 | """ | |
|
223 | pass | |
|
224 | ||
|
214 | 225 | def find_config_file_name(self): |
|
215 | 226 | """Find the config file name for this application. |
|
216 | 227 | |
@@ -325,18 +336,26 b' class Application(object):' | |||
|
325 | 336 | |
|
326 | 337 | def abort(self): |
|
327 | 338 | """Abort the starting of the application.""" |
|
328 | self.log.critical("Aborting application: %s" % self.name, exc_info=True) | |
|
329 | sys.exit(1) | |
|
339 | if self._exiting: | |
|
340 | pass | |
|
341 | else: | |
|
342 | self.log.critical("Aborting application: %s" % self.name, exc_info=True) | |
|
343 | self._exiting = True | |
|
344 | sys.exit(1) | |
|
330 | 345 | |
|
331 | 346 | def exit(self): |
|
332 | self.log.critical("Aborting application: %s" % self.name) | |
|
333 | sys.exit(1) | |
|
347 | if self._exiting: | |
|
348 | pass | |
|
349 | else: | |
|
350 | self.log.debug("Exiting application: %s" % self.name) | |
|
351 | self._exiting = True | |
|
352 | sys.exit(1) | |
|
334 | 353 | |
|
335 | 354 | def attempt(self, func, action='abort'): |
|
336 | 355 | try: |
|
337 | 356 | func() |
|
338 | 357 | except SystemExit: |
|
339 |
se |
|
|
358 | raise | |
|
340 | 359 | except: |
|
341 | 360 | if action == 'abort': |
|
342 | 361 | self.abort() |
@@ -23,13 +23,17 b' from IPython.config.loader import PyFileConfigLoader' | |||
|
23 | 23 | from IPython.core.application import Application |
|
24 | 24 | from IPython.core.component import Component |
|
25 | 25 | from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault |
|
26 | from IPython.utils.traitlets import Unicode | |
|
26 | from IPython.utils.traitlets import Unicode, Bool | |
|
27 | 27 | |
|
28 | 28 | #----------------------------------------------------------------------------- |
|
29 | 29 | # Imports |
|
30 | 30 | #----------------------------------------------------------------------------- |
|
31 | 31 | |
|
32 | 32 | |
|
33 | class ClusterDirError(Exception): | |
|
34 | pass | |
|
35 | ||
|
36 | ||
|
33 | 37 | class ClusterDir(Component): |
|
34 | 38 | """An object to manage the cluster directory and its resources. |
|
35 | 39 | |
@@ -117,28 +121,68 b' class ClusterDir(Component):' | |||
|
117 | 121 | self.copy_config_file(f, path=path, overwrite=overwrite) |
|
118 | 122 | |
|
119 | 123 | @classmethod |
|
120 | def find_cluster_dir_by_profile(cls, path, profile='default'): | |
|
121 | """Find/create a cluster dir by profile name and return its ClusterDir. | |
|
124 | def create_cluster_dir(csl, cluster_dir): | |
|
125 | """Create a new cluster directory given a full path. | |
|
122 | 126 | |
|
123 | This will create the cluster directory if it doesn't exist. | |
|
127 | Parameters | |
|
128 | ---------- | |
|
129 | cluster_dir : str | |
|
130 | The full path to the cluster directory. If it does exist, it will | |
|
131 | be used. If not, it will be created. | |
|
132 | """ | |
|
133 | return ClusterDir(cluster_dir) | |
|
134 | ||
|
135 | @classmethod | |
|
136 | def create_cluster_dir_by_profile(cls, path, profile='default'): | |
|
137 | """Create a cluster dir by profile name and path. | |
|
138 | ||
|
139 | Parameters | |
|
140 | ---------- | |
|
141 | path : str | |
|
142 | The path (directory) to put the cluster directory in. | |
|
143 | profile : str | |
|
144 | The name of the profile. The name of the cluster directory will | |
|
145 | be "cluster_<profile>". | |
|
146 | """ | |
|
147 | if not os.path.isdir(path): | |
|
148 | raise ClusterDirError('Directory not found: %s' % path) | |
|
149 | cluster_dir = os.path.join(path, 'cluster_' + profile) | |
|
150 | return ClusterDir(cluster_dir) | |
|
151 | ||
|
152 | @classmethod | |
|
153 | def find_cluster_dir_by_profile(cls, ipythondir, profile='default'): | |
|
154 | """Find an existing cluster dir by profile name, return its ClusterDir. | |
|
155 | ||
|
156 | This searches through a sequence of paths for a cluster dir. If it | |
|
157 | is not found, a :class:`ClusterDirError` exception will be raised. | |
|
158 | ||
|
159 | The search path algorithm is: | |
|
160 | 1. ``os.getcwd()`` | |
|
161 | 2. ``ipythondir`` | |
|
162 | 3. The directories found in the ":" separated | |
|
163 | :env:`IPCLUSTERDIR_PATH` environment variable. | |
|
124 | 164 | |
|
125 | 165 | Parameters |
|
126 | 166 | ---------- |
|
127 |
|
|
|
128 | The directory path to look for the cluster directory in. | |
|
167 | ipythondir : unicode or str | |
|
168 | The IPython directory to use. | |
|
129 | 169 | profile : unicode or str |
|
130 | 170 | The name of the profile. The name of the cluster directory |
|
131 | 171 | will be "cluster_<profile>". |
|
132 | 172 | """ |
|
133 | 173 | dirname = 'cluster_' + profile |
|
134 | cluster_dir = os.path.join(os.getcwd(), dirname) | |
|
135 |
if |
|
|
136 | return ClusterDir(cluster_dir) | |
|
174 | cluster_dir_paths = os.environ.get('IPCLUSTERDIR_PATH','') | |
|
175 | if cluster_dir_paths: | |
|
176 | cluster_dir_paths = cluster_dir_paths.split(':') | |
|
137 | 177 | else: |
|
138 | if not os.path.isdir(path): | |
|
139 | raise IOError("Directory doesn't exist: %s" % path) | |
|
140 | cluster_dir = os.path.join(path, dirname) | |
|
141 | return ClusterDir(cluster_dir) | |
|
178 | cluster_dir_paths = [] | |
|
179 | paths = [os.getcwd(), ipythondir] + cluster_dir_paths | |
|
180 | for p in paths: | |
|
181 | cluster_dir = os.path.join(p, dirname) | |
|
182 | if os.path.isdir(cluster_dir): | |
|
183 | return ClusterDir(cluster_dir) | |
|
184 | else: | |
|
185 | raise ClusterDirError('Cluster directory not found in paths: %s' % dirname) | |
|
142 | 186 | |
|
143 | 187 | @classmethod |
|
144 | 188 | def find_cluster_dir(cls, cluster_dir): |
@@ -153,6 +197,8 b' class ClusterDir(Component):' | |||
|
153 | 197 | :func:`os.path.expandvars` and :func:`os.path.expanduser`. |
|
154 | 198 | """ |
|
155 | 199 | cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir)) |
|
200 | if not os.path.isdir(cluster_dir): | |
|
201 | raise ClusterDirError('Cluster directory not found: %s' % cluster_dir) | |
|
156 | 202 | return ClusterDir(cluster_dir) |
|
157 | 203 | |
|
158 | 204 | |
@@ -176,7 +222,8 b' class AppWithClusterDirArgParseConfigLoader(ArgParseConfigLoader):' | |||
|
176 | 222 | self.parser.add_argument('-log_level', '--log-level', |
|
177 | 223 | dest="Global.log_level",type=int, |
|
178 | 224 | help='Set the log level (0,10,20,30,40,50). Default is 30.', |
|
179 |
default=NoConfigDefault |
|
|
225 | default=NoConfigDefault, | |
|
226 | metavar="Global.log_level") | |
|
180 | 227 | self.parser.add_argument('-cluster_dir', '--cluster-dir', |
|
181 | 228 | dest='Global.cluster_dir',type=str, |
|
182 | 229 | help='Set the cluster dir. This overrides the logic used by the ' |
@@ -204,6 +251,8 b' class ApplicationWithClusterDir(Application):' | |||
|
204 | 251 | dir and named the value of the ``config_file_name`` class attribute. |
|
205 | 252 | """ |
|
206 | 253 | |
|
254 | auto_create_cluster_dir = True | |
|
255 | ||
|
207 | 256 | def create_default_config(self): |
|
208 | 257 | super(ApplicationWithClusterDir, self).create_default_config() |
|
209 | 258 | self.default_config.Global.profile = 'default' |
@@ -216,40 +265,68 b' class ApplicationWithClusterDir(Application):' | |||
|
216 | 265 | version=release.version |
|
217 | 266 | ) |
|
218 | 267 | |
|
219 |
def find_ |
|
|
220 | """Find the config file name for this application.""" | |
|
221 | # For this type of Application it should be set as a class attribute. | |
|
222 | if not hasattr(self, 'config_file_name'): | |
|
223 | self.log.critical("No config filename found") | |
|
224 | ||
|
225 | def find_config_file_paths(self): | |
|
226 | """This resolves the cluster directory and sets ``config_file_paths``. | |
|
268 | def find_resources(self): | |
|
269 | """This resolves the cluster directory. | |
|
227 | 270 | |
|
228 | This does the following: | |
|
229 | * Create the :class:`ClusterDir` object for the application. | |
|
230 | * Set the ``cluster_dir`` attribute of the application and config | |
|
271 | This tries to find the cluster directory and if successful, it will | |
|
272 | have done: | |
|
273 | * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for | |
|
274 | the application. | |
|
275 | * Sets ``self.cluster_dir`` attribute of the application and config | |
|
231 | 276 | objects. |
|
232 | * Set ``config_file_paths`` to point to the cluster directory. | |
|
277 | ||
|
278 | The algorithm used for this is as follows: | |
|
279 | 1. Try ``Global.cluster_dir``. | |
|
280 | 2. Try using ``Global.profile``. | |
|
281 | 3. If both of these fail and ``self.auto_create_cluster_dir`` is | |
|
282 | ``True``, then create the new cluster dir in the IPython directory. | |
|
283 | 4. If all fails, then raise :class:`ClusterDirError`. | |
|
233 | 284 | """ |
|
234 | 285 | |
|
235 | # Create the ClusterDir object for managing everything | |
|
236 | 286 | try: |
|
237 | 287 | cluster_dir = self.command_line_config.Global.cluster_dir |
|
238 | 288 | except AttributeError: |
|
239 | 289 | cluster_dir = self.default_config.Global.cluster_dir |
|
240 | 290 | cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir)) |
|
241 |
|
|
|
242 | # Just use cluster_dir | |
|
291 | try: | |
|
243 | 292 | self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir) |
|
293 | except ClusterDirError: | |
|
294 | pass | |
|
244 | 295 | else: |
|
245 | # Then look for a profile | |
|
246 | try: | |
|
247 | self.profile = self.command_line_config.Global.profile | |
|
248 | except AttributeError: | |
|
249 | self.profile = self.default_config.Global.profile | |
|
296 | self.log.info('Using existing cluster dir: %s' % \ | |
|
297 | self.cluster_dir_obj.location | |
|
298 | ) | |
|
299 | self.finish_cluster_dir() | |
|
300 | return | |
|
301 | ||
|
302 | try: | |
|
303 | self.profile = self.command_line_config.Global.profile | |
|
304 | except AttributeError: | |
|
305 | self.profile = self.default_config.Global.profile | |
|
306 | try: | |
|
250 | 307 | self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile( |
|
251 | 308 | self.ipythondir, self.profile) |
|
309 | except ClusterDirError: | |
|
310 | pass | |
|
311 | else: | |
|
312 | self.log.info('Using existing cluster dir: %s' % \ | |
|
313 | self.cluster_dir_obj.location | |
|
314 | ) | |
|
315 | self.finish_cluster_dir() | |
|
316 | return | |
|
317 | ||
|
318 | if self.auto_create_cluster_dir: | |
|
319 | self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile( | |
|
320 | self.ipythondir, self.profile | |
|
321 | ) | |
|
322 | self.log.info('Creating new cluster dir: %s' % \ | |
|
323 | self.cluster_dir_obj.location | |
|
324 | ) | |
|
325 | self.finish_cluster_dir() | |
|
326 | else: | |
|
327 | raise ClusterDirError('Could not find a valid cluster directory.') | |
|
252 | 328 | |
|
329 | def finish_cluster_dir(self): | |
|
253 | 330 | # Set the cluster directory |
|
254 | 331 | self.cluster_dir = self.cluster_dir_obj.location |
|
255 | 332 | |
@@ -261,3 +338,15 b' class ApplicationWithClusterDir(Application):' | |||
|
261 | 338 | |
|
262 | 339 | # Set the search path to the cluster directory |
|
263 | 340 | self.config_file_paths = (self.cluster_dir,) |
|
341 | ||
|
342 | def find_config_file_name(self): | |
|
343 | """Find the config file name for this application.""" | |
|
344 | # For this type of Application it should be set as a class attribute. | |
|
345 | if not hasattr(self, 'config_file_name'): | |
|
346 | self.log.critical("No config filename found") | |
|
347 | ||
|
348 | def find_config_file_paths(self): | |
|
349 | # Set the search path to the cluster directory | |
|
350 | self.config_file_paths = (self.cluster_dir,) | |
|
351 | ||
|
352 |
@@ -178,6 +178,7 b' class IPControllerApp(ApplicationWithClusterDir):' | |||
|
178 | 178 | name = 'ipcontroller' |
|
179 | 179 | description = 'Start the IPython controller for parallel computing.' |
|
180 | 180 | config_file_name = default_config_file_name |
|
181 | auto_create_cluster_dir = True | |
|
181 | 182 | |
|
182 | 183 | def create_default_config(self): |
|
183 | 184 | super(IPControllerApp, self).create_default_config() |
@@ -92,6 +92,7 b' class IPEngineApp(ApplicationWithClusterDir):' | |||
|
92 | 92 | name = 'ipengine' |
|
93 | 93 | description = 'Start the IPython engine for parallel computing.' |
|
94 | 94 | config_file_name = default_config_file_name |
|
95 | auto_create_cluster_dir = True | |
|
95 | 96 | |
|
96 | 97 | def create_default_config(self): |
|
97 | 98 | super(IPEngineApp, self).create_default_config() |
@@ -254,3 +254,12 b' def sleep_deferred(seconds):' | |||
|
254 | 254 | d = defer.Deferred() |
|
255 | 255 | reactor.callLater(seconds, d.callback, seconds) |
|
256 | 256 | return d |
|
257 | ||
|
258 | ||
|
259 | def make_deferred(func): | |
|
260 | """A decorator that calls a function with :func`maybeDeferred`.""" | |
|
261 | ||
|
262 | def _wrapper(*args, **kwargs): | |
|
263 | return defer.maybeDeferred(func, *args, **kwargs) | |
|
264 | ||
|
265 | return _wrapper No newline at end of file |
General Comments 0
You need to be logged in to leave comments.
Login now