##// END OF EJS Templates
Merge pull request #11752 from gooverdian/fix_background_job_indexing...
Matthias Bussonnier -
r25048:78886a92 merge
parent child Browse files
Show More
@@ -1,489 +1,491 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """Manage background (threaded) jobs conveniently from an interactive shell.
2 """Manage background (threaded) jobs conveniently from an interactive shell.
3
3
4 This module provides a BackgroundJobManager class. This is the main class
4 This module provides a BackgroundJobManager class. This is the main class
5 meant for public usage, it implements an object which can create and manage
5 meant for public usage, it implements an object which can create and manage
6 new background jobs.
6 new background jobs.
7
7
8 It also provides the actual job classes managed by these BackgroundJobManager
8 It also provides the actual job classes managed by these BackgroundJobManager
9 objects, see their docstrings below.
9 objects, see their docstrings below.
10
10
11
11
12 This system was inspired by discussions with B. Granger and the
12 This system was inspired by discussions with B. Granger and the
13 BackgroundCommand class described in the book Python Scripting for
13 BackgroundCommand class described in the book Python Scripting for
14 Computational Science, by H. P. Langtangen:
14 Computational Science, by H. P. Langtangen:
15
15
16 http://folk.uio.no/hpl/scripting
16 http://folk.uio.no/hpl/scripting
17
17
18 (although ultimately no code from this text was used, as IPython's system is a
18 (although ultimately no code from this text was used, as IPython's system is a
19 separate implementation).
19 separate implementation).
20
20
21 An example notebook is provided in our documentation illustrating interactive
21 An example notebook is provided in our documentation illustrating interactive
22 use of the system.
22 use of the system.
23 """
23 """
24
24
25 #*****************************************************************************
25 #*****************************************************************************
26 # Copyright (C) 2005-2006 Fernando Perez <fperez@colorado.edu>
26 # Copyright (C) 2005-2006 Fernando Perez <fperez@colorado.edu>
27 #
27 #
28 # Distributed under the terms of the BSD License. The full license is in
28 # Distributed under the terms of the BSD License. The full license is in
29 # the file COPYING, distributed as part of this software.
29 # the file COPYING, distributed as part of this software.
30 #*****************************************************************************
30 #*****************************************************************************
31
31
32 # Code begins
32 # Code begins
33 import sys
33 import sys
34 import threading
34 import threading
35
35
36 from IPython import get_ipython
36 from IPython import get_ipython
37 from IPython.core.ultratb import AutoFormattedTB
37 from IPython.core.ultratb import AutoFormattedTB
38 from logging import error, debug
38 from logging import error, debug
39
39
40
40
41 class BackgroundJobManager(object):
41 class BackgroundJobManager(object):
42 """Class to manage a pool of backgrounded threaded jobs.
42 """Class to manage a pool of backgrounded threaded jobs.
43
43
44 Below, we assume that 'jobs' is a BackgroundJobManager instance.
44 Below, we assume that 'jobs' is a BackgroundJobManager instance.
45
45
46 Usage summary (see the method docstrings for details):
46 Usage summary (see the method docstrings for details):
47
47
48 jobs.new(...) -> start a new job
48 jobs.new(...) -> start a new job
49
49
50 jobs() or jobs.status() -> print status summary of all jobs
50 jobs() or jobs.status() -> print status summary of all jobs
51
51
52 jobs[N] -> returns job number N.
52 jobs[N] -> returns job number N.
53
53
54 foo = jobs[N].result -> assign to variable foo the result of job N
54 foo = jobs[N].result -> assign to variable foo the result of job N
55
55
56 jobs[N].traceback() -> print the traceback of dead job N
56 jobs[N].traceback() -> print the traceback of dead job N
57
57
58 jobs.remove(N) -> remove (finished) job N
58 jobs.remove(N) -> remove (finished) job N
59
59
60 jobs.flush() -> remove all finished jobs
60 jobs.flush() -> remove all finished jobs
61
61
62 As a convenience feature, BackgroundJobManager instances provide the
62 As a convenience feature, BackgroundJobManager instances provide the
63 utility result and traceback methods which retrieve the corresponding
63 utility result and traceback methods which retrieve the corresponding
64 information from the jobs list:
64 information from the jobs list:
65
65
66 jobs.result(N) <--> jobs[N].result
66 jobs.result(N) <--> jobs[N].result
67 jobs.traceback(N) <--> jobs[N].traceback()
67 jobs.traceback(N) <--> jobs[N].traceback()
68
68
69 While this appears minor, it allows you to use tab completion
69 While this appears minor, it allows you to use tab completion
70 interactively on the job manager instance.
70 interactively on the job manager instance.
71 """
71 """
72
72
73 def __init__(self):
73 def __init__(self):
74 # Lists for job management, accessed via a property to ensure they're
74 # Lists for job management, accessed via a property to ensure they're
75 # up to date.x
75 # up to date.x
76 self._running = []
76 self._running = []
77 self._completed = []
77 self._completed = []
78 self._dead = []
78 self._dead = []
79 # A dict of all jobs, so users can easily access any of them
79 # A dict of all jobs, so users can easily access any of them
80 self.all = {}
80 self.all = {}
81 # For reporting
81 # For reporting
82 self._comp_report = []
82 self._comp_report = []
83 self._dead_report = []
83 self._dead_report = []
84 # Store status codes locally for fast lookups
84 # Store status codes locally for fast lookups
85 self._s_created = BackgroundJobBase.stat_created_c
85 self._s_created = BackgroundJobBase.stat_created_c
86 self._s_running = BackgroundJobBase.stat_running_c
86 self._s_running = BackgroundJobBase.stat_running_c
87 self._s_completed = BackgroundJobBase.stat_completed_c
87 self._s_completed = BackgroundJobBase.stat_completed_c
88 self._s_dead = BackgroundJobBase.stat_dead_c
88 self._s_dead = BackgroundJobBase.stat_dead_c
89 self._current_job_id = 0
89
90
90 @property
91 @property
91 def running(self):
92 def running(self):
92 self._update_status()
93 self._update_status()
93 return self._running
94 return self._running
94
95
95 @property
96 @property
96 def dead(self):
97 def dead(self):
97 self._update_status()
98 self._update_status()
98 return self._dead
99 return self._dead
99
100
100 @property
101 @property
101 def completed(self):
102 def completed(self):
102 self._update_status()
103 self._update_status()
103 return self._completed
104 return self._completed
104
105
105 def new(self, func_or_exp, *args, **kwargs):
106 def new(self, func_or_exp, *args, **kwargs):
106 """Add a new background job and start it in a separate thread.
107 """Add a new background job and start it in a separate thread.
107
108
108 There are two types of jobs which can be created:
109 There are two types of jobs which can be created:
109
110
110 1. Jobs based on expressions which can be passed to an eval() call.
111 1. Jobs based on expressions which can be passed to an eval() call.
111 The expression must be given as a string. For example:
112 The expression must be given as a string. For example:
112
113
113 job_manager.new('myfunc(x,y,z=1)'[,glob[,loc]])
114 job_manager.new('myfunc(x,y,z=1)'[,glob[,loc]])
114
115
115 The given expression is passed to eval(), along with the optional
116 The given expression is passed to eval(), along with the optional
116 global/local dicts provided. If no dicts are given, they are
117 global/local dicts provided. If no dicts are given, they are
117 extracted automatically from the caller's frame.
118 extracted automatically from the caller's frame.
118
119
119 A Python statement is NOT a valid eval() expression. Basically, you
120 A Python statement is NOT a valid eval() expression. Basically, you
120 can only use as an eval() argument something which can go on the right
121 can only use as an eval() argument something which can go on the right
121 of an '=' sign and be assigned to a variable.
122 of an '=' sign and be assigned to a variable.
122
123
123 For example,"print 'hello'" is not valid, but '2+3' is.
124 For example,"print 'hello'" is not valid, but '2+3' is.
124
125
125 2. Jobs given a function object, optionally passing additional
126 2. Jobs given a function object, optionally passing additional
126 positional arguments:
127 positional arguments:
127
128
128 job_manager.new(myfunc, x, y)
129 job_manager.new(myfunc, x, y)
129
130
130 The function is called with the given arguments.
131 The function is called with the given arguments.
131
132
132 If you need to pass keyword arguments to your function, you must
133 If you need to pass keyword arguments to your function, you must
133 supply them as a dict named kw:
134 supply them as a dict named kw:
134
135
135 job_manager.new(myfunc, x, y, kw=dict(z=1))
136 job_manager.new(myfunc, x, y, kw=dict(z=1))
136
137
137 The reason for this assymmetry is that the new() method needs to
138 The reason for this assymmetry is that the new() method needs to
138 maintain access to its own keywords, and this prevents name collisions
139 maintain access to its own keywords, and this prevents name collisions
139 between arguments to new() and arguments to your own functions.
140 between arguments to new() and arguments to your own functions.
140
141
141 In both cases, the result is stored in the job.result field of the
142 In both cases, the result is stored in the job.result field of the
142 background job object.
143 background job object.
143
144
144 You can set `daemon` attribute of the thread by giving the keyword
145 You can set `daemon` attribute of the thread by giving the keyword
145 argument `daemon`.
146 argument `daemon`.
146
147
147 Notes and caveats:
148 Notes and caveats:
148
149
149 1. All threads running share the same standard output. Thus, if your
150 1. All threads running share the same standard output. Thus, if your
150 background jobs generate output, it will come out on top of whatever
151 background jobs generate output, it will come out on top of whatever
151 you are currently writing. For this reason, background jobs are best
152 you are currently writing. For this reason, background jobs are best
152 used with silent functions which simply return their output.
153 used with silent functions which simply return their output.
153
154
154 2. Threads also all work within the same global namespace, and this
155 2. Threads also all work within the same global namespace, and this
155 system does not lock interactive variables. So if you send job to the
156 system does not lock interactive variables. So if you send job to the
156 background which operates on a mutable object for a long time, and
157 background which operates on a mutable object for a long time, and
157 start modifying that same mutable object interactively (or in another
158 start modifying that same mutable object interactively (or in another
158 backgrounded job), all sorts of bizarre behaviour will occur.
159 backgrounded job), all sorts of bizarre behaviour will occur.
159
160
160 3. If a background job is spending a lot of time inside a C extension
161 3. If a background job is spending a lot of time inside a C extension
161 module which does not release the Python Global Interpreter Lock
162 module which does not release the Python Global Interpreter Lock
162 (GIL), this will block the IPython prompt. This is simply because the
163 (GIL), this will block the IPython prompt. This is simply because the
163 Python interpreter can only switch between threads at Python
164 Python interpreter can only switch between threads at Python
164 bytecodes. While the execution is inside C code, the interpreter must
165 bytecodes. While the execution is inside C code, the interpreter must
165 simply wait unless the extension module releases the GIL.
166 simply wait unless the extension module releases the GIL.
166
167
167 4. There is no way, due to limitations in the Python threads library,
168 4. There is no way, due to limitations in the Python threads library,
168 to kill a thread once it has started."""
169 to kill a thread once it has started."""
169
170
170 if callable(func_or_exp):
171 if callable(func_or_exp):
171 kw = kwargs.get('kw',{})
172 kw = kwargs.get('kw',{})
172 job = BackgroundJobFunc(func_or_exp,*args,**kw)
173 job = BackgroundJobFunc(func_or_exp,*args,**kw)
173 elif isinstance(func_or_exp, str):
174 elif isinstance(func_or_exp, str):
174 if not args:
175 if not args:
175 frame = sys._getframe(1)
176 frame = sys._getframe(1)
176 glob, loc = frame.f_globals, frame.f_locals
177 glob, loc = frame.f_globals, frame.f_locals
177 elif len(args)==1:
178 elif len(args)==1:
178 glob = loc = args[0]
179 glob = loc = args[0]
179 elif len(args)==2:
180 elif len(args)==2:
180 glob,loc = args
181 glob,loc = args
181 else:
182 else:
182 raise ValueError(
183 raise ValueError(
183 'Expression jobs take at most 2 args (globals,locals)')
184 'Expression jobs take at most 2 args (globals,locals)')
184 job = BackgroundJobExpr(func_or_exp, glob, loc)
185 job = BackgroundJobExpr(func_or_exp, glob, loc)
185 else:
186 else:
186 raise TypeError('invalid args for new job')
187 raise TypeError('invalid args for new job')
187
188
188 if kwargs.get('daemon', False):
189 if kwargs.get('daemon', False):
189 job.daemon = True
190 job.daemon = True
190 job.num = len(self.all)+1 if self.all else 0
191 job.num = self._current_job_id
192 self._current_job_id += 1
191 self.running.append(job)
193 self.running.append(job)
192 self.all[job.num] = job
194 self.all[job.num] = job
193 debug('Starting job # %s in a separate thread.' % job.num)
195 debug('Starting job # %s in a separate thread.' % job.num)
194 job.start()
196 job.start()
195 return job
197 return job
196
198
197 def __getitem__(self, job_key):
199 def __getitem__(self, job_key):
198 num = job_key if isinstance(job_key, int) else job_key.num
200 num = job_key if isinstance(job_key, int) else job_key.num
199 return self.all[num]
201 return self.all[num]
200
202
201 def __call__(self):
203 def __call__(self):
202 """An alias to self.status(),
204 """An alias to self.status(),
203
205
204 This allows you to simply call a job manager instance much like the
206 This allows you to simply call a job manager instance much like the
205 Unix `jobs` shell command."""
207 Unix `jobs` shell command."""
206
208
207 return self.status()
209 return self.status()
208
210
209 def _update_status(self):
211 def _update_status(self):
210 """Update the status of the job lists.
212 """Update the status of the job lists.
211
213
212 This method moves finished jobs to one of two lists:
214 This method moves finished jobs to one of two lists:
213 - self.completed: jobs which completed successfully
215 - self.completed: jobs which completed successfully
214 - self.dead: jobs which finished but died.
216 - self.dead: jobs which finished but died.
215
217
216 It also copies those jobs to corresponding _report lists. These lists
218 It also copies those jobs to corresponding _report lists. These lists
217 are used to report jobs completed/dead since the last update, and are
219 are used to report jobs completed/dead since the last update, and are
218 then cleared by the reporting function after each call."""
220 then cleared by the reporting function after each call."""
219
221
220 # Status codes
222 # Status codes
221 srun, scomp, sdead = self._s_running, self._s_completed, self._s_dead
223 srun, scomp, sdead = self._s_running, self._s_completed, self._s_dead
222 # State lists, use the actual lists b/c the public names are properties
224 # State lists, use the actual lists b/c the public names are properties
223 # that call this very function on access
225 # that call this very function on access
224 running, completed, dead = self._running, self._completed, self._dead
226 running, completed, dead = self._running, self._completed, self._dead
225
227
226 # Now, update all state lists
228 # Now, update all state lists
227 for num, job in enumerate(running):
229 for num, job in enumerate(running):
228 stat = job.stat_code
230 stat = job.stat_code
229 if stat == srun:
231 if stat == srun:
230 continue
232 continue
231 elif stat == scomp:
233 elif stat == scomp:
232 completed.append(job)
234 completed.append(job)
233 self._comp_report.append(job)
235 self._comp_report.append(job)
234 running[num] = False
236 running[num] = False
235 elif stat == sdead:
237 elif stat == sdead:
236 dead.append(job)
238 dead.append(job)
237 self._dead_report.append(job)
239 self._dead_report.append(job)
238 running[num] = False
240 running[num] = False
239 # Remove dead/completed jobs from running list
241 # Remove dead/completed jobs from running list
240 running[:] = filter(None, running)
242 running[:] = filter(None, running)
241
243
242 def _group_report(self,group,name):
244 def _group_report(self,group,name):
243 """Report summary for a given job group.
245 """Report summary for a given job group.
244
246
245 Return True if the group had any elements."""
247 Return True if the group had any elements."""
246
248
247 if group:
249 if group:
248 print('%s jobs:' % name)
250 print('%s jobs:' % name)
249 for job in group:
251 for job in group:
250 print('%s : %s' % (job.num,job))
252 print('%s : %s' % (job.num,job))
251 print()
253 print()
252 return True
254 return True
253
255
254 def _group_flush(self,group,name):
256 def _group_flush(self,group,name):
255 """Flush a given job group
257 """Flush a given job group
256
258
257 Return True if the group had any elements."""
259 Return True if the group had any elements."""
258
260
259 njobs = len(group)
261 njobs = len(group)
260 if njobs:
262 if njobs:
261 plural = {1:''}.setdefault(njobs,'s')
263 plural = {1:''}.setdefault(njobs,'s')
262 print('Flushing %s %s job%s.' % (njobs,name,plural))
264 print('Flushing %s %s job%s.' % (njobs,name,plural))
263 group[:] = []
265 group[:] = []
264 return True
266 return True
265
267
266 def _status_new(self):
268 def _status_new(self):
267 """Print the status of newly finished jobs.
269 """Print the status of newly finished jobs.
268
270
269 Return True if any new jobs are reported.
271 Return True if any new jobs are reported.
270
272
271 This call resets its own state every time, so it only reports jobs
273 This call resets its own state every time, so it only reports jobs
272 which have finished since the last time it was called."""
274 which have finished since the last time it was called."""
273
275
274 self._update_status()
276 self._update_status()
275 new_comp = self._group_report(self._comp_report, 'Completed')
277 new_comp = self._group_report(self._comp_report, 'Completed')
276 new_dead = self._group_report(self._dead_report,
278 new_dead = self._group_report(self._dead_report,
277 'Dead, call jobs.traceback() for details')
279 'Dead, call jobs.traceback() for details')
278 self._comp_report[:] = []
280 self._comp_report[:] = []
279 self._dead_report[:] = []
281 self._dead_report[:] = []
280 return new_comp or new_dead
282 return new_comp or new_dead
281
283
282 def status(self,verbose=0):
284 def status(self,verbose=0):
283 """Print a status of all jobs currently being managed."""
285 """Print a status of all jobs currently being managed."""
284
286
285 self._update_status()
287 self._update_status()
286 self._group_report(self.running,'Running')
288 self._group_report(self.running,'Running')
287 self._group_report(self.completed,'Completed')
289 self._group_report(self.completed,'Completed')
288 self._group_report(self.dead,'Dead')
290 self._group_report(self.dead,'Dead')
289 # Also flush the report queues
291 # Also flush the report queues
290 self._comp_report[:] = []
292 self._comp_report[:] = []
291 self._dead_report[:] = []
293 self._dead_report[:] = []
292
294
293 def remove(self,num):
295 def remove(self,num):
294 """Remove a finished (completed or dead) job."""
296 """Remove a finished (completed or dead) job."""
295
297
296 try:
298 try:
297 job = self.all[num]
299 job = self.all[num]
298 except KeyError:
300 except KeyError:
299 error('Job #%s not found' % num)
301 error('Job #%s not found' % num)
300 else:
302 else:
301 stat_code = job.stat_code
303 stat_code = job.stat_code
302 if stat_code == self._s_running:
304 if stat_code == self._s_running:
303 error('Job #%s is still running, it can not be removed.' % num)
305 error('Job #%s is still running, it can not be removed.' % num)
304 return
306 return
305 elif stat_code == self._s_completed:
307 elif stat_code == self._s_completed:
306 self.completed.remove(job)
308 self.completed.remove(job)
307 elif stat_code == self._s_dead:
309 elif stat_code == self._s_dead:
308 self.dead.remove(job)
310 self.dead.remove(job)
309
311
310 def flush(self):
312 def flush(self):
311 """Flush all finished jobs (completed and dead) from lists.
313 """Flush all finished jobs (completed and dead) from lists.
312
314
313 Running jobs are never flushed.
315 Running jobs are never flushed.
314
316
315 It first calls _status_new(), to update info. If any jobs have
317 It first calls _status_new(), to update info. If any jobs have
316 completed since the last _status_new() call, the flush operation
318 completed since the last _status_new() call, the flush operation
317 aborts."""
319 aborts."""
318
320
319 # Remove the finished jobs from the master dict
321 # Remove the finished jobs from the master dict
320 alljobs = self.all
322 alljobs = self.all
321 for job in self.completed+self.dead:
323 for job in self.completed+self.dead:
322 del(alljobs[job.num])
324 del(alljobs[job.num])
323
325
324 # Now flush these lists completely
326 # Now flush these lists completely
325 fl_comp = self._group_flush(self.completed, 'Completed')
327 fl_comp = self._group_flush(self.completed, 'Completed')
326 fl_dead = self._group_flush(self.dead, 'Dead')
328 fl_dead = self._group_flush(self.dead, 'Dead')
327 if not (fl_comp or fl_dead):
329 if not (fl_comp or fl_dead):
328 print('No jobs to flush.')
330 print('No jobs to flush.')
329
331
330 def result(self,num):
332 def result(self,num):
331 """result(N) -> return the result of job N."""
333 """result(N) -> return the result of job N."""
332 try:
334 try:
333 return self.all[num].result
335 return self.all[num].result
334 except KeyError:
336 except KeyError:
335 error('Job #%s not found' % num)
337 error('Job #%s not found' % num)
336
338
337 def _traceback(self, job):
339 def _traceback(self, job):
338 num = job if isinstance(job, int) else job.num
340 num = job if isinstance(job, int) else job.num
339 try:
341 try:
340 self.all[num].traceback()
342 self.all[num].traceback()
341 except KeyError:
343 except KeyError:
342 error('Job #%s not found' % num)
344 error('Job #%s not found' % num)
343
345
344 def traceback(self, job=None):
346 def traceback(self, job=None):
345 if job is None:
347 if job is None:
346 self._update_status()
348 self._update_status()
347 for deadjob in self.dead:
349 for deadjob in self.dead:
348 print("Traceback for: %r" % deadjob)
350 print("Traceback for: %r" % deadjob)
349 self._traceback(deadjob)
351 self._traceback(deadjob)
350 print()
352 print()
351 else:
353 else:
352 self._traceback(job)
354 self._traceback(job)
353
355
354
356
355 class BackgroundJobBase(threading.Thread):
357 class BackgroundJobBase(threading.Thread):
356 """Base class to build BackgroundJob classes.
358 """Base class to build BackgroundJob classes.
357
359
358 The derived classes must implement:
360 The derived classes must implement:
359
361
360 - Their own __init__, since the one here raises NotImplementedError. The
362 - Their own __init__, since the one here raises NotImplementedError. The
361 derived constructor must call self._init() at the end, to provide common
363 derived constructor must call self._init() at the end, to provide common
362 initialization.
364 initialization.
363
365
364 - A strform attribute used in calls to __str__.
366 - A strform attribute used in calls to __str__.
365
367
366 - A call() method, which will make the actual execution call and must
368 - A call() method, which will make the actual execution call and must
367 return a value to be held in the 'result' field of the job object.
369 return a value to be held in the 'result' field of the job object.
368 """
370 """
369
371
370 # Class constants for status, in string and as numerical codes (when
372 # Class constants for status, in string and as numerical codes (when
371 # updating jobs lists, we don't want to do string comparisons). This will
373 # updating jobs lists, we don't want to do string comparisons). This will
372 # be done at every user prompt, so it has to be as fast as possible
374 # be done at every user prompt, so it has to be as fast as possible
373 stat_created = 'Created'; stat_created_c = 0
375 stat_created = 'Created'; stat_created_c = 0
374 stat_running = 'Running'; stat_running_c = 1
376 stat_running = 'Running'; stat_running_c = 1
375 stat_completed = 'Completed'; stat_completed_c = 2
377 stat_completed = 'Completed'; stat_completed_c = 2
376 stat_dead = 'Dead (Exception), call jobs.traceback() for details'
378 stat_dead = 'Dead (Exception), call jobs.traceback() for details'
377 stat_dead_c = -1
379 stat_dead_c = -1
378
380
379 def __init__(self):
381 def __init__(self):
380 """Must be implemented in subclasses.
382 """Must be implemented in subclasses.
381
383
382 Subclasses must call :meth:`_init` for standard initialisation.
384 Subclasses must call :meth:`_init` for standard initialisation.
383 """
385 """
384 raise NotImplementedError("This class can not be instantiated directly.")
386 raise NotImplementedError("This class can not be instantiated directly.")
385
387
386 def _init(self):
388 def _init(self):
387 """Common initialization for all BackgroundJob objects"""
389 """Common initialization for all BackgroundJob objects"""
388
390
389 for attr in ['call','strform']:
391 for attr in ['call','strform']:
390 assert hasattr(self,attr), "Missing attribute <%s>" % attr
392 assert hasattr(self,attr), "Missing attribute <%s>" % attr
391
393
392 # The num tag can be set by an external job manager
394 # The num tag can be set by an external job manager
393 self.num = None
395 self.num = None
394
396
395 self.status = BackgroundJobBase.stat_created
397 self.status = BackgroundJobBase.stat_created
396 self.stat_code = BackgroundJobBase.stat_created_c
398 self.stat_code = BackgroundJobBase.stat_created_c
397 self.finished = False
399 self.finished = False
398 self.result = '<BackgroundJob has not completed>'
400 self.result = '<BackgroundJob has not completed>'
399
401
400 # reuse the ipython traceback handler if we can get to it, otherwise
402 # reuse the ipython traceback handler if we can get to it, otherwise
401 # make a new one
403 # make a new one
402 try:
404 try:
403 make_tb = get_ipython().InteractiveTB.text
405 make_tb = get_ipython().InteractiveTB.text
404 except:
406 except:
405 make_tb = AutoFormattedTB(mode = 'Context',
407 make_tb = AutoFormattedTB(mode = 'Context',
406 color_scheme='NoColor',
408 color_scheme='NoColor',
407 tb_offset = 1).text
409 tb_offset = 1).text
408 # Note that the actual API for text() requires the three args to be
410 # Note that the actual API for text() requires the three args to be
409 # passed in, so we wrap it in a simple lambda.
411 # passed in, so we wrap it in a simple lambda.
410 self._make_tb = lambda : make_tb(None, None, None)
412 self._make_tb = lambda : make_tb(None, None, None)
411
413
412 # Hold a formatted traceback if one is generated.
414 # Hold a formatted traceback if one is generated.
413 self._tb = None
415 self._tb = None
414
416
415 threading.Thread.__init__(self)
417 threading.Thread.__init__(self)
416
418
417 def __str__(self):
419 def __str__(self):
418 return self.strform
420 return self.strform
419
421
420 def __repr__(self):
422 def __repr__(self):
421 return '<BackgroundJob #%d: %s>' % (self.num, self.strform)
423 return '<BackgroundJob #%d: %s>' % (self.num, self.strform)
422
424
423 def traceback(self):
425 def traceback(self):
424 print(self._tb)
426 print(self._tb)
425
427
426 def run(self):
428 def run(self):
427 try:
429 try:
428 self.status = BackgroundJobBase.stat_running
430 self.status = BackgroundJobBase.stat_running
429 self.stat_code = BackgroundJobBase.stat_running_c
431 self.stat_code = BackgroundJobBase.stat_running_c
430 self.result = self.call()
432 self.result = self.call()
431 except:
433 except:
432 self.status = BackgroundJobBase.stat_dead
434 self.status = BackgroundJobBase.stat_dead
433 self.stat_code = BackgroundJobBase.stat_dead_c
435 self.stat_code = BackgroundJobBase.stat_dead_c
434 self.finished = None
436 self.finished = None
435 self.result = ('<BackgroundJob died, call jobs.traceback() for details>')
437 self.result = ('<BackgroundJob died, call jobs.traceback() for details>')
436 self._tb = self._make_tb()
438 self._tb = self._make_tb()
437 else:
439 else:
438 self.status = BackgroundJobBase.stat_completed
440 self.status = BackgroundJobBase.stat_completed
439 self.stat_code = BackgroundJobBase.stat_completed_c
441 self.stat_code = BackgroundJobBase.stat_completed_c
440 self.finished = True
442 self.finished = True
441
443
442
444
443 class BackgroundJobExpr(BackgroundJobBase):
445 class BackgroundJobExpr(BackgroundJobBase):
444 """Evaluate an expression as a background job (uses a separate thread)."""
446 """Evaluate an expression as a background job (uses a separate thread)."""
445
447
446 def __init__(self, expression, glob=None, loc=None):
448 def __init__(self, expression, glob=None, loc=None):
447 """Create a new job from a string which can be fed to eval().
449 """Create a new job from a string which can be fed to eval().
448
450
449 global/locals dicts can be provided, which will be passed to the eval
451 global/locals dicts can be provided, which will be passed to the eval
450 call."""
452 call."""
451
453
452 # fail immediately if the given expression can't be compiled
454 # fail immediately if the given expression can't be compiled
453 self.code = compile(expression,'<BackgroundJob compilation>','eval')
455 self.code = compile(expression,'<BackgroundJob compilation>','eval')
454
456
455 glob = {} if glob is None else glob
457 glob = {} if glob is None else glob
456 loc = {} if loc is None else loc
458 loc = {} if loc is None else loc
457 self.expression = self.strform = expression
459 self.expression = self.strform = expression
458 self.glob = glob
460 self.glob = glob
459 self.loc = loc
461 self.loc = loc
460 self._init()
462 self._init()
461
463
462 def call(self):
464 def call(self):
463 return eval(self.code,self.glob,self.loc)
465 return eval(self.code,self.glob,self.loc)
464
466
465
467
466 class BackgroundJobFunc(BackgroundJobBase):
468 class BackgroundJobFunc(BackgroundJobBase):
467 """Run a function call as a background job (uses a separate thread)."""
469 """Run a function call as a background job (uses a separate thread)."""
468
470
469 def __init__(self, func, *args, **kwargs):
471 def __init__(self, func, *args, **kwargs):
470 """Create a new job from a callable object.
472 """Create a new job from a callable object.
471
473
472 Any positional arguments and keyword args given to this constructor
474 Any positional arguments and keyword args given to this constructor
473 after the initial callable are passed directly to it."""
475 after the initial callable are passed directly to it."""
474
476
475 if not callable(func):
477 if not callable(func):
476 raise TypeError(
478 raise TypeError(
477 'first argument to BackgroundJobFunc must be callable')
479 'first argument to BackgroundJobFunc must be callable')
478
480
479 self.func = func
481 self.func = func
480 self.args = args
482 self.args = args
481 self.kwargs = kwargs
483 self.kwargs = kwargs
482 # The string form will only include the function passed, because
484 # The string form will only include the function passed, because
483 # generating string representations of the arguments is a potentially
485 # generating string representations of the arguments is a potentially
484 # _very_ expensive operation (e.g. with large arrays).
486 # _very_ expensive operation (e.g. with large arrays).
485 self.strform = str(func)
487 self.strform = str(func)
486 self._init()
488 self._init()
487
489
488 def call(self):
490 def call(self):
489 return self.func(*self.args, **self.kwargs)
491 return self.func(*self.args, **self.kwargs)
General Comments 0
You need to be logged in to leave comments. Login now