##// END OF EJS Templates
Lots more work on the kernel scripts.
Brian Granger -
Show More
@@ -1,32 +1,24 b''
1 1 c = get_config()
2 2
3 c.MPI.use = ''
3 # c.Global.log_to_file = False
4 # c.Global.exec_lines = ['import numpy']
5 # c.Global.log_dir_name = 'log'
6 # c.Global.security_dir_name = 'security'
7 # c.Global.log_level = 10
8 # c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
9 # c.Global.furl_file_name = 'ipcontroller-engine.furl'
10 # c.Global.furl_file = ''
11
12 # c.MPI.use = ''
13 # c.MPI.mpi4py = """from mpi4py import MPI as mpi
14 # mpi.size = mpi.COMM_WORLD.Get_size()
15 # mpi.rank = mpi.COMM_WORLD.Get_rank()
16 # """
17 # c.MPI.pytrilinos = """from PyTrilinos import Epetra
18 # class SimpleStruct:
19 # pass
20 # mpi = SimpleStruct()
21 # mpi.rank = 0
22 # mpi.size = 0
23 # """
4 24
5 c.MPI.mpi4py = """from mpi4py import MPI as mpi
6 mpi.size = mpi.COMM_WORLD.Get_size()
7 mpi.rank = mpi.COMM_WORLD.Get_rank()
8 """
9
10 c.MPI.pytrilinos = """from PyTrilinos import Epetra
11 class SimpleStruct:
12 pass
13 mpi = SimpleStruct()
14 mpi.rank = 0
15 mpi.size = 0
16 """
17
18 c.Global.log_to_file = False
19
20 c.Global.exec_lines = ['import numpy']
21
22 c.Global.log_dir_name = 'log'
23
24 c.Global.security_dir_name = 'security'
25
26 c.Global.log_level = 10
27
28 c.Global.shell_class = 'IPython.kernel.core.interpreter.Interpreter'
29
30 c.Global.furl_file_name = 'ipcontroller-engine.furl'
31
32 c.Global.furl_file = ''
@@ -86,6 +86,7 b' class Application(object):'
86 86 default_log_level = logging.WARN
87 87
88 88 def __init__(self):
89 self._exiting = False
89 90 self.init_logger()
90 91 # Track the default and actual separately because some messages are
91 92 # only printed if we aren't using the default.
@@ -119,6 +120,7 b' class Application(object):'
119 120 self.attempt(self.post_load_command_line_config)
120 121 self.log_command_line_config()
121 122 self.attempt(self.find_ipythondir)
123 self.attempt(self.find_resources)
122 124 self.attempt(self.find_config_file_name)
123 125 self.attempt(self.find_config_file_paths)
124 126 self.attempt(self.pre_load_file_config)
@@ -211,6 +213,15 b' class Application(object):'
211 213 os.makedirs(self.ipythondir, mode=0777)
212 214 self.log.debug("IPYTHONDIR set to: %s" % self.ipythondir)
213 215
216 def find_resources(self):
217 """Find other resources that need to be in place.
218
219 Things like cluster directories need to be in place to find the
220 config file. These happen right after the IPython directory has
221 been set.
222 """
223 pass
224
214 225 def find_config_file_name(self):
215 226 """Find the config file name for this application.
216 227
@@ -325,18 +336,26 b' class Application(object):'
325 336
326 337 def abort(self):
327 338 """Abort the starting of the application."""
328 self.log.critical("Aborting application: %s" % self.name, exc_info=True)
329 sys.exit(1)
339 if self._exiting:
340 pass
341 else:
342 self.log.critical("Aborting application: %s" % self.name, exc_info=True)
343 self._exiting = True
344 sys.exit(1)
330 345
331 346 def exit(self):
332 self.log.critical("Aborting application: %s" % self.name)
333 sys.exit(1)
347 if self._exiting:
348 pass
349 else:
350 self.log.debug("Exiting application: %s" % self.name)
351 self._exiting = True
352 sys.exit(1)
334 353
335 354 def attempt(self, func, action='abort'):
336 355 try:
337 356 func()
338 357 except SystemExit:
339 self.exit()
358 raise
340 359 except:
341 360 if action == 'abort':
342 361 self.abort()
@@ -23,13 +23,17 b' from IPython.config.loader import PyFileConfigLoader'
23 23 from IPython.core.application import Application
24 24 from IPython.core.component import Component
25 25 from IPython.config.loader import ArgParseConfigLoader, NoConfigDefault
26 from IPython.utils.traitlets import Unicode
26 from IPython.utils.traitlets import Unicode, Bool
27 27
28 28 #-----------------------------------------------------------------------------
29 29 # Imports
30 30 #-----------------------------------------------------------------------------
31 31
32 32
33 class ClusterDirError(Exception):
34 pass
35
36
33 37 class ClusterDir(Component):
34 38 """An object to manage the cluster directory and its resources.
35 39
@@ -117,28 +121,68 b' class ClusterDir(Component):'
117 121 self.copy_config_file(f, path=path, overwrite=overwrite)
118 122
119 123 @classmethod
120 def find_cluster_dir_by_profile(cls, path, profile='default'):
121 """Find/create a cluster dir by profile name and return its ClusterDir.
124 def create_cluster_dir(csl, cluster_dir):
125 """Create a new cluster directory given a full path.
122 126
123 This will create the cluster directory if it doesn't exist.
127 Parameters
128 ----------
129 cluster_dir : str
130 The full path to the cluster directory. If it does exist, it will
131 be used. If not, it will be created.
132 """
133 return ClusterDir(cluster_dir)
134
135 @classmethod
136 def create_cluster_dir_by_profile(cls, path, profile='default'):
137 """Create a cluster dir by profile name and path.
138
139 Parameters
140 ----------
141 path : str
142 The path (directory) to put the cluster directory in.
143 profile : str
144 The name of the profile. The name of the cluster directory will
145 be "cluster_<profile>".
146 """
147 if not os.path.isdir(path):
148 raise ClusterDirError('Directory not found: %s' % path)
149 cluster_dir = os.path.join(path, 'cluster_' + profile)
150 return ClusterDir(cluster_dir)
151
152 @classmethod
153 def find_cluster_dir_by_profile(cls, ipythondir, profile='default'):
154 """Find an existing cluster dir by profile name, return its ClusterDir.
155
156 This searches through a sequence of paths for a cluster dir. If it
157 is not found, a :class:`ClusterDirError` exception will be raised.
158
159 The search path algorithm is:
160 1. ``os.getcwd()``
161 2. ``ipythondir``
162 3. The directories found in the ":" separated
163 :env:`IPCLUSTERDIR_PATH` environment variable.
124 164
125 165 Parameters
126 166 ----------
127 path : unicode or str
128 The directory path to look for the cluster directory in.
167 ipythondir : unicode or str
168 The IPython directory to use.
129 169 profile : unicode or str
130 170 The name of the profile. The name of the cluster directory
131 171 will be "cluster_<profile>".
132 172 """
133 173 dirname = 'cluster_' + profile
134 cluster_dir = os.path.join(os.getcwd(), dirname)
135 if os.path.isdir(cluster_dir):
136 return ClusterDir(cluster_dir)
174 cluster_dir_paths = os.environ.get('IPCLUSTERDIR_PATH','')
175 if cluster_dir_paths:
176 cluster_dir_paths = cluster_dir_paths.split(':')
137 177 else:
138 if not os.path.isdir(path):
139 raise IOError("Directory doesn't exist: %s" % path)
140 cluster_dir = os.path.join(path, dirname)
141 return ClusterDir(cluster_dir)
178 cluster_dir_paths = []
179 paths = [os.getcwd(), ipythondir] + cluster_dir_paths
180 for p in paths:
181 cluster_dir = os.path.join(p, dirname)
182 if os.path.isdir(cluster_dir):
183 return ClusterDir(cluster_dir)
184 else:
185 raise ClusterDirError('Cluster directory not found in paths: %s' % dirname)
142 186
143 187 @classmethod
144 188 def find_cluster_dir(cls, cluster_dir):
@@ -153,6 +197,8 b' class ClusterDir(Component):'
153 197 :func:`os.path.expandvars` and :func:`os.path.expanduser`.
154 198 """
155 199 cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir))
200 if not os.path.isdir(cluster_dir):
201 raise ClusterDirError('Cluster directory not found: %s' % cluster_dir)
156 202 return ClusterDir(cluster_dir)
157 203
158 204
@@ -176,7 +222,8 b' class AppWithClusterDirArgParseConfigLoader(ArgParseConfigLoader):'
176 222 self.parser.add_argument('-log_level', '--log-level',
177 223 dest="Global.log_level",type=int,
178 224 help='Set the log level (0,10,20,30,40,50). Default is 30.',
179 default=NoConfigDefault)
225 default=NoConfigDefault,
226 metavar="Global.log_level")
180 227 self.parser.add_argument('-cluster_dir', '--cluster-dir',
181 228 dest='Global.cluster_dir',type=str,
182 229 help='Set the cluster dir. This overrides the logic used by the '
@@ -204,6 +251,8 b' class ApplicationWithClusterDir(Application):'
204 251 dir and named the value of the ``config_file_name`` class attribute.
205 252 """
206 253
254 auto_create_cluster_dir = True
255
207 256 def create_default_config(self):
208 257 super(ApplicationWithClusterDir, self).create_default_config()
209 258 self.default_config.Global.profile = 'default'
@@ -216,40 +265,68 b' class ApplicationWithClusterDir(Application):'
216 265 version=release.version
217 266 )
218 267
219 def find_config_file_name(self):
220 """Find the config file name for this application."""
221 # For this type of Application it should be set as a class attribute.
222 if not hasattr(self, 'config_file_name'):
223 self.log.critical("No config filename found")
224
225 def find_config_file_paths(self):
226 """This resolves the cluster directory and sets ``config_file_paths``.
268 def find_resources(self):
269 """This resolves the cluster directory.
227 270
228 This does the following:
229 * Create the :class:`ClusterDir` object for the application.
230 * Set the ``cluster_dir`` attribute of the application and config
271 This tries to find the cluster directory and if successful, it will
272 have done:
273 * Sets ``self.cluster_dir_obj`` to the :class:`ClusterDir` object for
274 the application.
275 * Sets ``self.cluster_dir`` attribute of the application and config
231 276 objects.
232 * Set ``config_file_paths`` to point to the cluster directory.
277
278 The algorithm used for this is as follows:
279 1. Try ``Global.cluster_dir``.
280 2. Try using ``Global.profile``.
281 3. If both of these fail and ``self.auto_create_cluster_dir`` is
282 ``True``, then create the new cluster dir in the IPython directory.
283 4. If all fails, then raise :class:`ClusterDirError`.
233 284 """
234 285
235 # Create the ClusterDir object for managing everything
236 286 try:
237 287 cluster_dir = self.command_line_config.Global.cluster_dir
238 288 except AttributeError:
239 289 cluster_dir = self.default_config.Global.cluster_dir
240 290 cluster_dir = os.path.expandvars(os.path.expanduser(cluster_dir))
241 if cluster_dir:
242 # Just use cluster_dir
291 try:
243 292 self.cluster_dir_obj = ClusterDir.find_cluster_dir(cluster_dir)
293 except ClusterDirError:
294 pass
244 295 else:
245 # Then look for a profile
246 try:
247 self.profile = self.command_line_config.Global.profile
248 except AttributeError:
249 self.profile = self.default_config.Global.profile
296 self.log.info('Using existing cluster dir: %s' % \
297 self.cluster_dir_obj.location
298 )
299 self.finish_cluster_dir()
300 return
301
302 try:
303 self.profile = self.command_line_config.Global.profile
304 except AttributeError:
305 self.profile = self.default_config.Global.profile
306 try:
250 307 self.cluster_dir_obj = ClusterDir.find_cluster_dir_by_profile(
251 308 self.ipythondir, self.profile)
309 except ClusterDirError:
310 pass
311 else:
312 self.log.info('Using existing cluster dir: %s' % \
313 self.cluster_dir_obj.location
314 )
315 self.finish_cluster_dir()
316 return
317
318 if self.auto_create_cluster_dir:
319 self.cluster_dir_obj = ClusterDir.create_cluster_dir_by_profile(
320 self.ipythondir, self.profile
321 )
322 self.log.info('Creating new cluster dir: %s' % \
323 self.cluster_dir_obj.location
324 )
325 self.finish_cluster_dir()
326 else:
327 raise ClusterDirError('Could not find a valid cluster directory.')
252 328
329 def finish_cluster_dir(self):
253 330 # Set the cluster directory
254 331 self.cluster_dir = self.cluster_dir_obj.location
255 332
@@ -261,3 +338,15 b' class ApplicationWithClusterDir(Application):'
261 338
262 339 # Set the search path to the cluster directory
263 340 self.config_file_paths = (self.cluster_dir,)
341
342 def find_config_file_name(self):
343 """Find the config file name for this application."""
344 # For this type of Application it should be set as a class attribute.
345 if not hasattr(self, 'config_file_name'):
346 self.log.critical("No config filename found")
347
348 def find_config_file_paths(self):
349 # Set the search path to the cluster directory
350 self.config_file_paths = (self.cluster_dir,)
351
352
@@ -178,6 +178,7 b' class IPControllerApp(ApplicationWithClusterDir):'
178 178 name = 'ipcontroller'
179 179 description = 'Start the IPython controller for parallel computing.'
180 180 config_file_name = default_config_file_name
181 auto_create_cluster_dir = True
181 182
182 183 def create_default_config(self):
183 184 super(IPControllerApp, self).create_default_config()
@@ -92,6 +92,7 b' class IPEngineApp(ApplicationWithClusterDir):'
92 92 name = 'ipengine'
93 93 description = 'Start the IPython engine for parallel computing.'
94 94 config_file_name = default_config_file_name
95 auto_create_cluster_dir = True
95 96
96 97 def create_default_config(self):
97 98 super(IPEngineApp, self).create_default_config()
@@ -254,3 +254,12 b' def sleep_deferred(seconds):'
254 254 d = defer.Deferred()
255 255 reactor.callLater(seconds, d.callback, seconds)
256 256 return d
257
258
259 def make_deferred(func):
260 """A decorator that calls a function with :func`maybeDeferred`."""
261
262 def _wrapper(*args, **kwargs):
263 return defer.maybeDeferred(func, *args, **kwargs)
264
265 return _wrapper No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now