##// END OF EJS Templates
Lots more work on the kernel scripts.
Brian Granger -
Show More
@@ -1,32 +1,24 b''
1 c = get_config()
1 c = get_config()
2
2
3 c.MPI.use = ''
3 # c.Global.log_to_file = False
4 # c.Global.exec_lines = ['import numpy']
5 # c.Global.log_dir_name = 'log'
6 # c.Global.security_dir_name = 'security'
7 # c.Global.log_level = 10
8 # c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
9 # c.Global.furl_file_name = 'ipcontroller-engine.furl'
10 # c.Global.furl_file = ''
11
12 # c.MPI.use = ''
13 # c.MPI.mpi4py = """from mpi4py import MPI as mpi
14 # mpi.size = mpi.COMM_WORLD.Get_size()
15 # mpi.rank = mpi.COMM_WORLD.Get_rank()
16 # """
17 # c.MPI.pytrilinos = """from PyTrilinos import Epetra
18 # class SimpleStruct:
19 # pass
20 # mpi = SimpleStruct()
21 # mpi.rank = 0
22 # mpi.size = 0
23 # """
4
24
5 c.MPI.mpi4py = """from mpi4py import MPI as mpi
6 mpi.size = mpi.COMM_WORLD.Get_size()
7 mpi.rank = mpi.COMM_WORLD.Get_rank()
8 """
9
10 c.MPI.pytrilinos = """from PyTrilinos import Epetra
11 class SimpleStruct:
12 pass
13 mpi = SimpleStruct()
14 mpi.rank = 0
15 mpi.size = 0
16 """
17
18 c.Global.log_to_file = False
19
20 c.Global.exec_lines = ['import numpy']
21
22 c.Global.log_dir_name = 'log'
23
24 c.Global.security_dir_name = 'security'
25
26 c.Global.log_level = 10
27
28 c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
29
30 c.Global.furl_file_name = 'ipcontroller-engine.furl'
31
32 c.Global.furl_file = ''
@@ -86,6 +86,7 b' class Application(object):'
86 default_log_level = logging.WARN
86 default_log_level = logging.WARN
87
87
88 def __init__(self):
88 def __init__(self):
89 self._exiting = False
89 self.init_logger()
90 self.init_logger()
90 # Track the default and actual separately because some messages are
91 # Track the default and actual separately because some messages are
91 # only printed if we aren't using the default.
92 # only printed if we aren't using the default.
@@ -119,6 +120,7 b' class Application(object):'
119 self.attempt(self.post_load_command_line_config)
120 self.attempt(self.post_load_command_line_config)
120 self.log_command_line_config()
121 self.log_command_line_config()
121 self.attempt(self.find_ipythondir)
122 self.attempt(self.find_ipythondir)
123 self.attempt(self.find_resources)
122 self.attempt(self.find_config_file_name)
124 self.attempt(self.find_config_file_name)
123 self.attempt(self.find_config_file_paths)
125 self.attempt(self.find_config_file_paths)
124 self.attempt(self.pre_load_file_config)
126 self.attempt(self.pre_load_file_config)
@@ -211,6 +213,15 b' class Application(object):'
211 os.makedirs(self.ipythondir, mode=0777)
213 os.makedirs(self.ipythondir, mode=0777)
212 self.log.debug("IPYTHONDIR set to: %s" % self.ipythondir)
214 self.log.debug("IPYTHONDIR set to: %s" % self.ipythondir)
213
215
216 def find_resources(self):
217 """Find other resources that need to be in place.
218
219 Things like cluster directories need to be in place to find the
220 config file. These happen right after the IPython directory has
221 been set.
222 """
223 pass
224
214 def find_config_file_name(self):
225 def find_config_file_name(self):
215 """Find the config file name for this application.
226 """Find the config file name for this application.
216
227
@@ -325,18 +336,26 b' class Application(object):'
325
336
326 def abort(self):
337 def abort(self):
327 """Abort the starting of the application."""
338 """Abort the starting of the application."""
339 if self._exiting:
340 pass
341 else:
328 self.log.critical("Aborting application: %s" % self.name, exc_info=True)
342 self.log.critical("Aborting application: %s" % self.name, exc_info=True)
343 self._exiting = True
329 sys.exit(1)
344 sys.exit(1)
330
345
331 def exit(self):
346 def exit(self):
332 self.log.critical("Aborting application: %s" % self.name)
347 if self._exiting:
348 pass
349 else:
350 self.log.debug("Exiting application: %s" % self.name)
351 self._exiting = True
333 sys.exit(1)
352 sys.exit(1)
334
353
335 def attempt(self, func, action='abort'):
354 def attempt(self, func, action='abort'):
336 try:
355 try:
337 func()
356 func()
338 except SystemExit:
357 except SystemExit:
339 self.exit()
358 raise
340 except:
359 except:
341 if action == 'abort':
360 if action == 'abort':
342 self.abort()
361 self.abort()
@@ -23,13 +23,17 b' from IPython.config.loader import PyFileConfigLoader'
23 from IPython.core.application import Application
23 from IPython.core.application import Application
24 from IPython.core.component import Component
24 from IPython.core.component import Component
25 from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault
25 from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault
26 from IPython.utils.traitlets import Unicode
26 from IPython.utils.traitlets import Unicode, Bool
27
27
28 #-----------------------------------------------------------------------------
28 #-----------------------------------------------------------------------------
29 # Imports
29 # Imports
30 #-----------------------------------------------------------------------------
30 #-----------------------------------------------------------------------------
31
31
32
32
33 class ClusterDirError(Exception):
34 pass
35
36
33 class ClusterDir(Component):
37 class ClusterDir(Component):
34 """An object to manage the cluster directory and its resources.
38 """An object to manage the cluster directory and its resources.
35
39
@@ -117,28 +121,68 b' class ClusterDir(Component):'
117 self.copy_config_file(f, path=path, overwrite=overwrite)
121 self.copy_config_file(f, path=path, overwrite=overwrite)
118
122
119 @classmethod
123 @classmethod
120 def find_cluster_dir_by_profile(cls, path, profile='default'):
124 def create_cluster_dir(csl, cluster_dir):
121 """Find/create a cluster dir by profile name and return its ClusterDir.
125 """Create a new cluster directory given a full path.
122
126
123 This will create the cluster directory if it doesn't exist.
127 Parameters
128 ----------
129 cluster_dir : str
130 The full path to the cluster directory. If it does exist, it will
131 be used. If not, it will be created.
132 """
133 return ClusterDir(cluster_dir)
134
135 @classmethod
136 def create_cluster_dir_by_profile(cls, path, profile='default'):
137 """Create a cluster dir by profile name and path.
124
138
125 Parameters
139 Parameters
126 ----------
140 ----------
127 path : unicode or str
141 path : str
128 The directory path to look for the cluster directory in.
142 The path (directory) to put the cluster directory in.
143 profile : str
144 The name of the profile. The name of the cluster directory will
145 be "cluster_<profile>".
146 """
147 if not os.path.isdir(path):
148 raise ClusterDirError('Directory not found: %s' % path)
149 cluster_dir = os.path.join(path, 'cluster_' + profile)
150 return ClusterDir(cluster_dir)
151
152 @classmethod
153 def find_cluster_dir_by_profile(cls, ipythondir, profile='default'):
154 """Find an existing cluster dir by profile name, return its ClusterDir.
155
156 This searches through a sequence of paths for a cluster dir. If it
157 is not found, a :class:`ClusterDirError` exception will be raised.
158
159 The search path algorithm is:
160 1. ``os.getcwd()``
161 2. ``ipythondir``
162 3. The directories found in the ":" separated
163 :env:`IPCLUSTERDIR_PATH` environment variable.
164
165 Parameters
166 ----------
167 ipythondir : unicode or str
168 The IPython directory to use.
129 profile : unicode or str
169 profile : unicode or str
130 The name of the profile. The name of the cluster directory
170 The name of the profile. The name of the cluster directory
131 will be "cluster_<profile>".
171 will be "cluster_<profile>".
132 """
172 """
133 dirname = 'cluster_' + profile
173 dirname = 'cluster_' + profile
134 cluster_dir = os.path.join(os.getcwd(), dirname)
174 cluster_dir_paths = os.environ.get('IPCLUSTERDIR_PATH','')
175 if cluster_dir_paths:
176 cluster_dir_paths = cluster_dir_paths.split(':')
177 else:
178 cluster_dir_paths = []
179 paths = [os.getcwd(), ipythondir] + cluster_dir_paths
180 for p in paths:
181 cluster_dir = os.path.join(p, dirname)
135 if os.path.isdir(cluster_dir):
182 if os.path.isdir(cluster_dir):
136 return ClusterDir(cluster_dir)
183 return ClusterDir(cluster_dir)
137 else:
184 else:
138 if not os.path.isdir(path):
185 raise ClusterDirError('Cluster directory not found in paths: %s' % dirname)
139 raise IOError("Directory doesn't exist: %s" % path)
140 cluster_dir = os.path.join(path, dirname)
141 return ClusterDir(cluster_dir)
142
186
143 @classmethod
187 @classmethod
144 def find_cluster_dir(cls, cluster_dir):
188 def find_cluster_dir(cls, cluster_dir):
@@ -153,6 +197,8 b' class ClusterDir(Component):'
153 :func:`os.path.expandvars` and :func:`os.path.expanduser`.
197 :func:`os.path.expandvars` and :func:`os.path.expanduser`.
154 """
198 """
155 cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir))
199 cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir))
200 if not os.path.isdir(cluster_dir):
201 raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
156 return ClusterDir(cluster_dir)
202 return ClusterDir(cluster_dir)
157
203
158
204
@@ -176,7 +222,8 b' class AppWithClusterDirArgParseConfigLoader(ArgParseConfigLoader):'
176 self.parser.add_argument('-log_level', '--log-level',
222 self.parser.add_argument('-log_level', '--log-level',
177 dest="Global.log_level",type=int,
223 dest="Global.log_level",type=int,
178 help='Set the log level (0,10,20,30,40,50). Default is 30.',
224 help='Set the log level (0,10,20,30,40,50). Default is 30.',
179 default=NoConfigDefault)
225 default=NoConfigDefault,
226 metavar="Global.log_level")
180 self.parser.add_argument('-cluster_dir', '--cluster-dir',
227 self.parser.add_argument('-cluster_dir', '--cluster-dir',
181 dest='Global.cluster_dir',type=str,
228 dest='Global.cluster_dir',type=str,
182 help='Set the cluster dir. This overrides the logic used by the '
229 help='Set the cluster dir. This overrides the logic used by the '
@@ -204,6 +251,8 b' class ApplicationWithClusterDir(Application):'
204 dir and named the value of the ``config_file_name`` class attribute.
251 dir and named the value of the ``config_file_name`` class attribute.
205 """
252 """
206
253
254 auto_create_cluster_dir = True
255
207 def create_default_config(self):
256 def create_default_config(self):
208 super(ApplicationWithClusterDir, self).create_default_config()
257 super(ApplicationWithClusterDir, self).create_default_config()
209 self.default_config.Global.profile = 'default'
258 self.default_config.Global.profile = 'default'
@@ -216,40 +265,68 b' class ApplicationWithClusterDir(Application):'
216 version=release.version
265 version=release.version
217 )
266 )
218
267
219 def find_config_file_name(self):
268 def find_resources(self):
220 """Find the config file name for this application."""
269 """This resolves the cluster directory.
221 # For this type of Application it should be set as a class attribute.
222 if not hasattr(self, 'config_file_name'):
223 self.log.critical("No config filename found")
224
225 def find_config_file_paths(self):
226 """This resolves the cluster directory and sets ``config_file_paths``.
227
270
228 This does the following:
271 This tries to find the cluster directory and if successful, it will
229 * Create the :class:`ClusterDir` object for the application.
272 have done:
230 * Set the ``cluster_dir`` attribute of the application and config
273 * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for
274 the application.
275 * Sets ``self.cluster_dir`` attribute of the application and config
231 objects.
276 objects.
232 * Set ``config_file_paths`` to point to the cluster directory.
277
278 The algorithm used for this is as follows:
279 1. Try ``Global.cluster_dir``.
280 2. Try using ``Global.profile``.
281 3. If both of these fail and ``self.auto_create_cluster_dir`` is
282 ``True``, then create the new cluster dir in the IPython directory.
283 4. If all fails, then raise :class:`ClusterDirError`.
233 """
284 """
234
285
235 # Create the ClusterDir object for managing everything
236 try:
286 try:
237 cluster_dir = self.command_line_config.Global.cluster_dir
287 cluster_dir = self.command_line_config.Global.cluster_dir
238 except AttributeError:
288 except AttributeError:
239 cluster_dir = self.default_config.Global.cluster_dir
289 cluster_dir = self.default_config.Global.cluster_dir
240 cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir))
290 cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir))
241 if cluster_dir:
291 try:
242 # Just use cluster_dir
243 self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
292 self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
293 except ClusterDirError:
294 pass
244 else:
295 else:
245 # Then look for a profile
296 self.log.info('Using existing cluster dir: %s' % \
297 self.cluster_dir_obj.location
298 )
299 self.finish_cluster_dir()
300 return
301
246 try:
302 try:
247 self.profile = self.command_line_config.Global.profile
303 self.profile = self.command_line_config.Global.profile
248 except AttributeError:
304 except AttributeError:
249 self.profile = self.default_config.Global.profile
305 self.profile = self.default_config.Global.profile
306 try:
250 self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
307 self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
251 self.ipythondir, self.profile)
308 self.ipythondir, self.profile)
309 except ClusterDirError:
310 pass
311 else:
312 self.log.info('Using existing cluster dir: %s' % \
313 self.cluster_dir_obj.location
314 )
315 self.finish_cluster_dir()
316 return
252
317
318 if self.auto_create_cluster_dir:
319 self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
320 self.ipythondir, self.profile
321 )
322 self.log.info('Creating new cluster dir: %s' % \
323 self.cluster_dir_obj.location
324 )
325 self.finish_cluster_dir()
326 else:
327 raise ClusterDirError('Could not find a valid cluster directory.')
328
329 def finish_cluster_dir(self):
253 # Set the cluster directory
330 # Set the cluster directory
254 self.cluster_dir = self.cluster_dir_obj.location
331 self.cluster_dir = self.cluster_dir_obj.location
255
332
@@ -261,3 +338,15 b' class ApplicationWithClusterDir(Application):'
261
338
262 # Set the search path to the cluster directory
339 # Set the search path to the cluster directory
263 self.config_file_paths = (self.cluster_dir,)
340 self.config_file_paths = (self.cluster_dir,)
341
342 def find_config_file_name(self):
343 """Find the config file name for this application."""
344 # For this type of Application it should be set as a class attribute.
345 if not hasattr(self, 'config_file_name'):
346 self.log.critical("No config filename found")
347
348 def find_config_file_paths(self):
349 # Set the search path to the cluster directory
350 self.config_file_paths = (self.cluster_dir,)
351
352
@@ -178,6 +178,7 b' class IPControllerApp(ApplicationWithClusterDir):'
178 name = 'ipcontroller'
178 name = 'ipcontroller'
179 description = 'Start the IPython controller for parallel computing.'
179 description = 'Start the IPython controller for parallel computing.'
180 config_file_name = default_config_file_name
180 config_file_name = default_config_file_name
181 auto_create_cluster_dir = True
181
182
182 def create_default_config(self):
183 def create_default_config(self):
183 super(IPControllerApp, self).create_default_config()
184 super(IPControllerApp, self).create_default_config()
@@ -92,6 +92,7 b' class IPEngineApp(ApplicationWithClusterDir):'
92 name = 'ipengine'
92 name = 'ipengine'
93 description = 'Start the IPython engine for parallel computing.'
93 description = 'Start the IPython engine for parallel computing.'
94 config_file_name = default_config_file_name
94 config_file_name = default_config_file_name
95 auto_create_cluster_dir = True
95
96
96 def create_default_config(self):
97 def create_default_config(self):
97 super(IPEngineApp, self).create_default_config()
98 super(IPEngineApp, self).create_default_config()
@@ -254,3 +254,12 b' def sleep_deferred(seconds):'
254 d = defer.Deferred()
254 d = defer.Deferred()
255 reactor.callLater(seconds, d.callback, seconds)
255 reactor.callLater(seconds, d.callback, seconds)
256 return d
256 return d
257
258
259 def make_deferred(func):
260 """A decorator that calls a function with :func`maybeDeferred`."""
261
262 def _wrapper(*args, **kwargs):
263 return defer.maybeDeferred(func, *args, **kwargs)
264
265 return _wrapper No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now