##// END OF EJS Templates
Add basic test suite to background jobs library.
Fernando Perez -
Show More
@@ -0,0 +1,91 b''
1 """Tests for pylab tools module.
2 """
3 #-----------------------------------------------------------------------------
4 # Copyright (c) 2011, the IPython Development Team.
5 #
6 # Distributed under the terms of the Modified BSD License.
7 #
8 # The full license is in the file COPYING.txt, distributed with this software.
9 #-----------------------------------------------------------------------------
10
11 #-----------------------------------------------------------------------------
12 # Imports
13 #-----------------------------------------------------------------------------
14 from __future__ import print_function
15
16 # Stdlib imports
17 import sys
18 import time
19
20 # Third-party imports
21 import nose.tools as nt
22
23 # Our own imports
24 from IPython.lib import backgroundjobs as bg
25 from IPython.testing import decorators as dec
26
27 #-----------------------------------------------------------------------------
28 # Globals and constants
29 #-----------------------------------------------------------------------------
30 t_short = 0.0001 # very short interval to wait on jobs
31
32 #-----------------------------------------------------------------------------
33 # Local utilities
34 #-----------------------------------------------------------------------------
35 def sleeper(interval=t_short, *a, **kw):
36 args = dict(interval=interval,
37 other_args=a,
38 kw_args=kw)
39 time.sleep(interval)
40 return args
41
42 def crasher(interval=t_short, *a, **kw):
43 time.sleep(interval)
44 raise Exception("Dead job with interval %s" % interval)
45
46 #-----------------------------------------------------------------------------
47 # Classes and functions
48 #-----------------------------------------------------------------------------
49
50 def test_result():
51 """Test job submission and result retrieval"""
52 jobs = bg.BackgroundJobManager()
53 j = jobs.new(sleeper)
54 j.join()
55 nt.assert_equals(j.result['interval'], t_short)
56
57
58 def test_flush():
59 """Test job control"""
60 jobs = bg.BackgroundJobManager()
61 j = jobs.new(sleeper)
62 j.join()
63 nt.assert_equals(len(jobs.completed), 1)
64 nt.assert_equals(len(jobs.dead), 0)
65 jobs.flush()
66 nt.assert_equals(len(jobs.completed), 0)
67
68
69 def test_dead():
70 """Test control of dead jobs"""
71 jobs = bg.BackgroundJobManager()
72 j = jobs.new(crasher)
73 j.join()
74 nt.assert_equals(len(jobs.completed), 0)
75 nt.assert_equals(len(jobs.dead), 1)
76 jobs.flush()
77 nt.assert_equals(len(jobs.dead), 0)
78
79
80 def test_longer():
81 """Test control of longer-running jobs"""
82 jobs = bg.BackgroundJobManager()
83 # Sleep for long enough for the following two checks to still report the
84 # job as running, but not so long that it makes the test suite noticeably
85 # slower.
86 j = jobs.new(sleeper, 0.1)
87 nt.assert_equals(len(jobs.running), 1)
88 nt.assert_equals(len(jobs.completed), 0)
89 j.join()
90 nt.assert_equals(len(jobs.running), 0)
91 nt.assert_equals(len(jobs.completed), 1)
@@ -1,464 +1,480 b''
1 1 # -*- coding: utf-8 -*-
2 2 """Manage background (threaded) jobs conveniently from an interactive shell.
3 3
4 4 This module provides a BackgroundJobManager class. This is the main class
5 5 meant for public usage, it implements an object which can create and manage
6 6 new background jobs.
7 7
8 8 It also provides the actual job classes managed by these BackgroundJobManager
9 9 objects, see their docstrings below.
10 10
11 11
12 12 This system was inspired by discussions with B. Granger and the
13 13 BackgroundCommand class described in the book Python Scripting for
14 14 Computational Science, by H. P. Langtangen:
15 15
16 16 http://folk.uio.no/hpl/scripting
17 17
18 18 (although ultimately no code from this text was used, as IPython's system is a
19 19 separate implementation).
20 20
21 21 An example notebook is provided in our documentation illustrating interactive
22 22 use of the system.
23 23 """
24 24
25 25 #*****************************************************************************
26 26 # Copyright (C) 2005-2006 Fernando Perez <fperez@colorado.edu>
27 27 #
28 28 # Distributed under the terms of the BSD License. The full license is in
29 29 # the file COPYING, distributed as part of this software.
30 30 #*****************************************************************************
31 31
32 32 # Code begins
33 33 import sys
34 34 import threading
35 35
36 36 from IPython.core.ultratb import AutoFormattedTB
37 37 from IPython.utils.warn import warn, error
38 38
39 39
40 40 class BackgroundJobManager(object):
41 41 """Class to manage a pool of backgrounded threaded jobs.
42 42
43 43 Below, we assume that 'jobs' is a BackgroundJobManager instance.
44 44
45 45 Usage summary (see the method docstrings for details):
46 46
47 47 jobs.new(...) -> start a new job
48 48
49 49 jobs() or jobs.status() -> print status summary of all jobs
50 50
51 51 jobs[N] -> returns job number N.
52 52
53 53 foo = jobs[N].result -> assign to variable foo the result of job N
54 54
55 55 jobs[N].traceback() -> print the traceback of dead job N
56 56
57 57 jobs.remove(N) -> remove (finished) job N
58 58
59 59 jobs.flush() -> remove all finished jobs
60 60
61 61 As a convenience feature, BackgroundJobManager instances provide the
62 62 utility result and traceback methods which retrieve the corresponding
63 63 information from the jobs list:
64 64
65 65 jobs.result(N) <--> jobs[N].result
66 66 jobs.traceback(N) <--> jobs[N].traceback()
67 67
68 68 While this appears minor, it allows you to use tab completion
69 69 interactively on the job manager instance.
70 70 """
71 71
72 72 def __init__(self):
73 # Lists for job management
74 self.running = []
75 self.completed = []
76 self.dead = []
73 # Lists for job management, accessed via a property to ensure they're
74 # up to date.x
75 self._running = []
76 self._completed = []
77 self._dead = []
77 78 # A dict of all jobs, so users can easily access any of them
78 79 self.all = {}
79 80 # For reporting
80 81 self._comp_report = []
81 82 self._dead_report = []
82 83 # Store status codes locally for fast lookups
83 84 self._s_created = BackgroundJobBase.stat_created_c
84 85 self._s_running = BackgroundJobBase.stat_running_c
85 86 self._s_completed = BackgroundJobBase.stat_completed_c
86 87 self._s_dead = BackgroundJobBase.stat_dead_c
87 88
88 def new(self,func_or_exp,*args,**kwargs):
89 @property
90 def running(self):
91 self._update_status()
92 return self._running
93
94 @property
95 def dead(self):
96 self._update_status()
97 return self._dead
98
99 @property
100 def completed(self):
101 self._update_status()
102 return self._completed
103
104 def new(self, func_or_exp, *args, **kwargs):
89 105 """Add a new background job and start it in a separate thread.
90 106
91 107 There are two types of jobs which can be created:
92 108
93 109 1. Jobs based on expressions which can be passed to an eval() call.
94 110 The expression must be given as a string. For example:
95 111
96 112 job_manager.new('myfunc(x,y,z=1)'[,glob[,loc]])
97 113
98 114 The given expression is passed to eval(), along with the optional
99 115 global/local dicts provided. If no dicts are given, they are
100 116 extracted automatically from the caller's frame.
101 117
102 118 A Python statement is NOT a valid eval() expression. Basically, you
103 119 can only use as an eval() argument something which can go on the right
104 120 of an '=' sign and be assigned to a variable.
105 121
106 122 For example,"print 'hello'" is not valid, but '2+3' is.
107 123
108 124 2. Jobs given a function object, optionally passing additional
109 125 positional arguments:
110 126
111 job_manager.new(myfunc,x,y)
127 job_manager.new(myfunc, x, y)
112 128
113 129 The function is called with the given arguments.
114 130
115 131 If you need to pass keyword arguments to your function, you must
116 132 supply them as a dict named kw:
117 133
118 job_manager.new(myfunc,x,y,kw=dict(z=1))
134 job_manager.new(myfunc, x, y, kw=dict(z=1))
119 135
120 136 The reason for this assymmetry is that the new() method needs to
121 137 maintain access to its own keywords, and this prevents name collisions
122 138 between arguments to new() and arguments to your own functions.
123 139
124 140 In both cases, the result is stored in the job.result field of the
125 141 background job object.
126 142
127 143
128 144 Notes and caveats:
129 145
130 146 1. All threads running share the same standard output. Thus, if your
131 147 background jobs generate output, it will come out on top of whatever
132 148 you are currently writing. For this reason, background jobs are best
133 149 used with silent functions which simply return their output.
134 150
135 151 2. Threads also all work within the same global namespace, and this
136 152 system does not lock interactive variables. So if you send job to the
137 153 background which operates on a mutable object for a long time, and
138 154 start modifying that same mutable object interactively (or in another
139 155 backgrounded job), all sorts of bizarre behaviour will occur.
140 156
141 157 3. If a background job is spending a lot of time inside a C extension
142 158 module which does not release the Python Global Interpreter Lock
143 159 (GIL), this will block the IPython prompt. This is simply because the
144 160 Python interpreter can only switch between threads at Python
145 161 bytecodes. While the execution is inside C code, the interpreter must
146 162 simply wait unless the extension module releases the GIL.
147 163
148 164 4. There is no way, due to limitations in the Python threads library,
149 165 to kill a thread once it has started."""
150 166
151 167 if callable(func_or_exp):
152 168 kw = kwargs.get('kw',{})
153 169 job = BackgroundJobFunc(func_or_exp,*args,**kw)
154 170 elif isinstance(func_or_exp, basestring):
155 171 if not args:
156 172 frame = sys._getframe(1)
157 173 glob, loc = frame.f_globals, frame.f_locals
158 174 elif len(args)==1:
159 175 glob = loc = args[0]
160 176 elif len(args)==2:
161 177 glob,loc = args
162 178 else:
163 179 raise ValueError(
164 180 'Expression jobs take at most 2 args (globals,locals)')
165 181 job = BackgroundJobExpr(func_or_exp, glob, loc)
166 182 else:
167 183 raise TypeError('invalid args for new job')
168 184
169 185 job.num = len(self.all)+1 if self.all else 0
170 186 self.running.append(job)
171 187 self.all[job.num] = job
172 188 print 'Starting job # %s in a separate thread.' % job.num
173 189 job.start()
174 190 return job
175 191
176 192 def __getitem__(self, job_key):
177 193 num = job_key if isinstance(job_key, int) else job_key.num
178 194 return self.all[num]
179 195
180 196 def __call__(self):
181 197 """An alias to self.status(),
182 198
183 199 This allows you to simply call a job manager instance much like the
184 200 Unix `jobs` shell command."""
185 201
186 202 return self.status()
187 203
188 204 def _update_status(self):
189 205 """Update the status of the job lists.
190 206
191 207 This method moves finished jobs to one of two lists:
192 208 - self.completed: jobs which completed successfully
193 209 - self.dead: jobs which finished but died.
194 210
195 211 It also copies those jobs to corresponding _report lists. These lists
196 212 are used to report jobs completed/dead since the last update, and are
197 213 then cleared by the reporting function after each call."""
198
199 run,comp,dead = self._s_running,self._s_completed,self._s_dead
200 running = self.running
201 for num in range(len(running)):
202 job = running[num]
214
215 # Status codes
216 srun, scomp, sdead = self._s_running, self._s_completed, self._s_dead
217 # State lists, use the actual lists b/c the public names are properties
218 # that call this very function on access
219 running, completed, dead = self._running, self._completed, self._dead
220
221 # Now, update all state lists
222 for num, job in enumerate(running):
203 223 stat = job.stat_code
204 if stat == run:
224 if stat == srun:
205 225 continue
206 elif stat == comp:
207 self.completed.append(job)
226 elif stat == scomp:
227 completed.append(job)
208 228 self._comp_report.append(job)
209 229 running[num] = False
210 elif stat == dead:
211 self.dead.append(job)
230 elif stat == sdead:
231 dead.append(job)
212 232 self._dead_report.append(job)
213 233 running[num] = False
214 self.running = filter(None,self.running)
234 # Remove dead/completed jobs from running list
235 running[:] = filter(None, running)
215 236
216 237 def _group_report(self,group,name):
217 238 """Report summary for a given job group.
218 239
219 240 Return True if the group had any elements."""
220 241
221 242 if group:
222 243 print '%s jobs:' % name
223 244 for job in group:
224 245 print '%s : %s' % (job.num,job)
225 246 print
226 247 return True
227 248
228 249 def _group_flush(self,group,name):
229 250 """Flush a given job group
230 251
231 252 Return True if the group had any elements."""
232 253
233 254 njobs = len(group)
234 255 if njobs:
235 256 plural = {1:''}.setdefault(njobs,'s')
236 257 print 'Flushing %s %s job%s.' % (njobs,name,plural)
237 258 group[:] = []
238 259 return True
239 260
240 261 def _status_new(self):
241 262 """Print the status of newly finished jobs.
242 263
243 264 Return True if any new jobs are reported.
244 265
245 266 This call resets its own state every time, so it only reports jobs
246 267 which have finished since the last time it was called."""
247 268
248 269 self._update_status()
249 270 new_comp = self._group_report(self._comp_report, 'Completed')
250 271 new_dead = self._group_report(self._dead_report,
251 272 'Dead, call jobs.traceback() for details')
252 273 self._comp_report[:] = []
253 274 self._dead_report[:] = []
254 275 return new_comp or new_dead
255 276
256 277 def status(self,verbose=0):
257 278 """Print a status of all jobs currently being managed."""
258 279
259 280 self._update_status()
260 281 self._group_report(self.running,'Running')
261 282 self._group_report(self.completed,'Completed')
262 283 self._group_report(self.dead,'Dead')
263 284 # Also flush the report queues
264 285 self._comp_report[:] = []
265 286 self._dead_report[:] = []
266 287
267 288 def remove(self,num):
268 289 """Remove a finished (completed or dead) job."""
269 290
270 291 try:
271 292 job = self.all[num]
272 293 except KeyError:
273 294 error('Job #%s not found' % num)
274 295 else:
275 296 stat_code = job.stat_code
276 297 if stat_code == self._s_running:
277 298 error('Job #%s is still running, it can not be removed.' % num)
278 299 return
279 300 elif stat_code == self._s_completed:
280 301 self.completed.remove(job)
281 302 elif stat_code == self._s_dead:
282 303 self.dead.remove(job)
283 304
284 305 def flush(self):
285 306 """Flush all finished jobs (completed and dead) from lists.
286 307
287 308 Running jobs are never flushed.
288 309
289 310 It first calls _status_new(), to update info. If any jobs have
290 311 completed since the last _status_new() call, the flush operation
291 312 aborts."""
292 313
293 if self._status_new():
294 error('New jobs completed since last '\
295 '_status_new(), aborting flush.')
296 return
297
298 314 # Remove the finished jobs from the master dict
299 all = self.all
315 alljobs = self.all
300 316 for job in self.completed+self.dead:
301 del(all[job.num])
317 del(alljobs[job.num])
302 318
303 319 # Now flush these lists completely
304 320 fl_comp = self._group_flush(self.completed, 'Completed')
305 321 fl_dead = self._group_flush(self.dead, 'Dead')
306 322 if not (fl_comp or fl_dead):
307 323 print 'No jobs to flush.'
308 324
309 325 def result(self,num):
310 326 """result(N) -> return the result of job N."""
311 327 try:
312 328 return self.all[num].result
313 329 except KeyError:
314 330 error('Job #%s not found' % num)
315 331
316 332 def _traceback(self, job):
317 333 num = job if isinstance(job, int) else job.num
318 334 try:
319 335 self.all[num].traceback()
320 336 except KeyError:
321 337 error('Job #%s not found' % num)
322 338
323 339 def traceback(self, job=None):
324 340 if job is None:
325 341 self._update_status()
326 342 for deadjob in self.dead:
327 343 print "Traceback for: %r" % deadjob
328 344 self._traceback(deadjob)
329 345 print
330 346 else:
331 347 self._traceback(job)
332 348
333 349
334 350 class BackgroundJobBase(threading.Thread):
335 351 """Base class to build BackgroundJob classes.
336 352
337 353 The derived classes must implement:
338 354
339 355 - Their own __init__, since the one here raises NotImplementedError. The
340 356 derived constructor must call self._init() at the end, to provide common
341 357 initialization.
342 358
343 359 - A strform attribute used in calls to __str__.
344 360
345 361 - A call() method, which will make the actual execution call and must
346 362 return a value to be held in the 'result' field of the job object."""
347 363
348 364 # Class constants for status, in string and as numerical codes (when
349 365 # updating jobs lists, we don't want to do string comparisons). This will
350 366 # be done at every user prompt, so it has to be as fast as possible
351 367 stat_created = 'Created'; stat_created_c = 0
352 368 stat_running = 'Running'; stat_running_c = 1
353 369 stat_completed = 'Completed'; stat_completed_c = 2
354 370 stat_dead = 'Dead (Exception), call jobs.traceback() for details'
355 371 stat_dead_c = -1
356 372
357 373 def __init__(self):
358 374 raise NotImplementedError, \
359 375 "This class can not be instantiated directly."
360 376
361 377 def _init(self):
362 378 """Common initialization for all BackgroundJob objects"""
363 379
364 380 for attr in ['call','strform']:
365 381 assert hasattr(self,attr), "Missing attribute <%s>" % attr
366 382
367 383 # The num tag can be set by an external job manager
368 384 self.num = None
369 385
370 386 self.status = BackgroundJobBase.stat_created
371 387 self.stat_code = BackgroundJobBase.stat_created_c
372 388 self.finished = False
373 389 self.result = '<BackgroundJob has not completed>'
374 390
375 391 # reuse the ipython traceback handler if we can get to it, otherwise
376 392 # make a new one
377 393 try:
378 394 make_tb = get_ipython().InteractiveTB.text
379 395 except:
380 396 make_tb = AutoFormattedTB(mode = 'Context',
381 397 color_scheme='NoColor',
382 398 tb_offset = 1).text
383 399 # Note that the actual API for text() requires the three args to be
384 400 # passed in, so we wrap it in a simple lambda.
385 401 self._make_tb = lambda : make_tb(None, None, None)
386 402
387 403 # Hold a formatted traceback if one is generated.
388 404 self._tb = None
389 405
390 406 threading.Thread.__init__(self)
391 407
392 408 def __str__(self):
393 409 return self.strform
394 410
395 411 def __repr__(self):
396 412 return '<BackgroundJob #%d: %s>' % (self.num, self.strform)
397 413
398 414 def traceback(self):
399 415 print self._tb
400 416
401 417 def run(self):
402 418 try:
403 419 self.status = BackgroundJobBase.stat_running
404 420 self.stat_code = BackgroundJobBase.stat_running_c
405 421 self.result = self.call()
406 422 except:
407 423 self.status = BackgroundJobBase.stat_dead
408 424 self.stat_code = BackgroundJobBase.stat_dead_c
409 425 self.finished = None
410 426 self.result = ('<BackgroundJob died, call jobs.traceback() for details>')
411 427 self._tb = self._make_tb()
412 428 else:
413 429 self.status = BackgroundJobBase.stat_completed
414 430 self.stat_code = BackgroundJobBase.stat_completed_c
415 431 self.finished = True
416 432
417 433
418 434 class BackgroundJobExpr(BackgroundJobBase):
419 435 """Evaluate an expression as a background job (uses a separate thread)."""
420 436
421 437 def __init__(self, expression, glob=None, loc=None):
422 438 """Create a new job from a string which can be fed to eval().
423 439
424 440 global/locals dicts can be provided, which will be passed to the eval
425 441 call."""
426 442
427 443 # fail immediately if the given expression can't be compiled
428 444 self.code = compile(expression,'<BackgroundJob compilation>','eval')
429 445
430 446 glob = {} if glob is None else glob
431 447 loc = {} if loc is None else loc
432 448 self.expression = self.strform = expression
433 449 self.glob = glob
434 450 self.loc = loc
435 451 self._init()
436 452
437 453 def call(self):
438 454 return eval(self.code,self.glob,self.loc)
439 455
440 456
441 457 class BackgroundJobFunc(BackgroundJobBase):
442 458 """Run a function call as a background job (uses a separate thread)."""
443 459
444 460 def __init__(self, func, *args, **kwargs):
445 461 """Create a new job from a callable object.
446 462
447 463 Any positional arguments and keyword args given to this constructor
448 464 after the initial callable are passed directly to it."""
449 465
450 466 if not callable(func):
451 467 raise TypeError(
452 468 'first argument to BackgroundJobFunc must be callable')
453 469
454 470 self.func = func
455 471 self.args = args
456 472 self.kwargs = kwargs
457 473 # The string form will only include the function passed, because
458 474 # generating string representations of the arguments is a potentially
459 475 # _very_ expensive operation (e.g. with large arrays).
460 476 self.strform = str(func)
461 477 self._init()
462 478
463 479 def call(self):
464 480 return self.func(*self.args, **self.kwargs)
@@ -1,171 +1,193 b''
1 1 {
2 2 "worksheets": [
3 3 {
4 4 "cells": [
5 5 {
6 6 "source": "# Simple interactive bacgkround jobs with IPython\n\nWe start by loading the `backgroundjobs` library and defining a few trivial functions to illustrate things with.",
7 7 "cell_type": "markdown"
8 8 },
9 9 {
10 10 "cell_type": "code",
11 11 "language": "python",
12 12 "outputs": [],
13 13 "collapsed": false,
14 "prompt_number": 15,
14 "prompt_number": 35,
15 15 "input": "from IPython.lib import backgroundjobs as bg\n\nimport sys\nimport time\n\ndef sleepfunc(interval=2, *a, **kw):\n args = dict(interval=interval,\n args=a,\n kwargs=kw)\n time.sleep(interval)\n return args\n\ndef diefunc(interval=2, *a, **kw):\n time.sleep(interval)\n raise Exception(\"Dead job with interval %s\" % interval)\n\ndef printfunc(interval=1, reps=5):\n for n in range(reps):\n time.sleep(interval)\n print 'In the background...', n\n sys.stdout.flush()\n print 'All done!'\n sys.stdout.flush()"
16 16 },
17 17 {
18 18 "source": "Now, we can create a job manager (called simply `jobs`) and use it to submit new jobs.\n<br>\nRun the cell below and wait a few seconds for the whole thing to finish, until you see the \"All done!\" printout.",
19 19 "cell_type": "markdown"
20 20 },
21 21 {
22 22 "cell_type": "code",
23 23 "language": "python",
24 24 "outputs": [
25 25 {
26 26 "output_type": "stream",
27 27 "stream": "stdout",
28 28 "text": "Starting job # 0 in a separate thread.\nStarting job # 2 in a separate thread.\nStarting job # 3 in a separate thread.\nStarting job # 4 in a separate thread.\nStarting job # 5 in a separate thread.\n"
29 29 },
30 30 {
31 31 "output_type": "stream",
32 32 "stream": "stdout",
33 33 "text": "In the background... 0\n"
34 34 },
35 35 {
36 36 "output_type": "stream",
37 37 "stream": "stdout",
38 38 "text": "In the background... 1\n"
39 39 },
40 40 {
41 41 "output_type": "stream",
42 42 "stream": "stdout",
43 43 "text": "In the background... 2\n"
44 44 },
45 45 {
46 46 "output_type": "stream",
47 47 "stream": "stdout",
48 48 "text": "All done!\n"
49 49 }
50 50 ],
51 51 "collapsed": false,
52 "prompt_number": 28,
52 "prompt_number": 36,
53 53 "input": "jobs = bg.BackgroundJobManager()\n\n# Start a few jobs, the first one will have ID # 0\njobs.new(sleepfunc, 4)\njobs.new(sleepfunc, kw={'reps':2})\njobs.new('printfunc(1,3)')\n\n# This makes a couple of jobs which will die. Let's keep a reference to\n# them for easier traceback reporting later\ndiejob1 = jobs.new(diefunc, 1)\ndiejob2 = jobs.new(diefunc, 2)"
54 54 },
55 55 {
56 56 "source": "You can check the status of your jobs at any time:",
57 57 "cell_type": "markdown"
58 58 },
59 59 {
60 60 "cell_type": "code",
61 61 "language": "python",
62 62 "outputs": [
63 63 {
64 64 "output_type": "stream",
65 65 "stream": "stdout",
66 "text": "Completed jobs:\n0 : &lt;function sleepfunc at 0x30c8500&gt;\n2 : &lt;function sleepfunc at 0x30c8500&gt;\n3 : printfunc(1,3)\n\nDead jobs:\n4 : &lt;function diefunc at 0x30df758&gt;\n5 : &lt;function diefunc at 0x30df758&gt;\n\n"
66 "text": "Completed jobs:\n0 : &lt;function sleepfunc at 0x30e1578&gt;\n2 : &lt;function sleepfunc at 0x30e1578&gt;\n3 : printfunc(1,3)\n\nDead jobs:\n4 : &lt;function diefunc at 0x304d488&gt;\n5 : &lt;function diefunc at 0x304d488&gt;\n\n"
67 67 }
68 68 ],
69 69 "collapsed": false,
70 "prompt_number": 29,
70 "prompt_number": 37,
71 71 "input": "jobs.status()"
72 72 },
73 73 {
74 74 "source": "For any completed job, you can get its result easily:",
75 75 "cell_type": "markdown"
76 76 },
77 77 {
78 78 "cell_type": "code",
79 79 "language": "python",
80 "outputs": [
81 {
82 "output_type": "pyout",
83 "prompt_number": 31,
84 "text": "{&apos;args&apos;: (), &apos;interval&apos;: 4, &apos;kwargs&apos;: {}}"
85 }
86 ],
80 "outputs": [],
87 81 "collapsed": false,
88 "prompt_number": 31,
89 "input": "jobs[0].result"
82 "prompt_number": 43,
83 "input": "jobs[0].result\nj0 = jobs[0]\nj0.join?"
90 84 },
91 85 {
92 86 "source": "You can get the traceback of any dead job. Run the line\nbelow again interactively until it prints a traceback (check the status\nof the job):\n",
93 87 "cell_type": "markdown"
94 88 },
95 89 {
96 90 "cell_type": "code",
97 91 "language": "python",
98 92 "outputs": [
99 93 {
100 94 "output_type": "stream",
101 95 "stream": "stdout",
102 96 "text": "Status of diejob1: Dead (Exception), call jobs.traceback() for details\n<span class=\"ansired\">---------------------------------------------------------------------------</span>\n<span class=\"ansired\">Exception</span> Traceback (most recent call last)\n<span class=\"ansigreen\">/home/fperez/usr/lib/python2.6/site-packages/IPython/lib/backgroundjobs.py</span> in <span class=\"ansicyan\">call</span><span class=\"ansiblue\">(self)</span>\n<span class=\"ansigreen\"> 462</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 463</span> <span class=\"ansigreen\">def</span> call<span class=\"ansiyellow\">(</span>self<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">--&gt; 464</span><span class=\"ansiyellow\"> </span><span class=\"ansigreen\">return</span> self<span class=\"ansiyellow\">.</span>func<span class=\"ansiyellow\">(</span><span class=\"ansiyellow\">*</span>self<span class=\"ansiyellow\">.</span>args<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>self<span class=\"ansiyellow\">.</span>kwargs<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/home/fperez/ipython/ipython/docs/examples/lib/&lt;ipython-input-15-54795a097787&gt;</span> in <span class=\"ansicyan\">diefunc</span><span class=\"ansiblue\">(interval, *a, **kw)</span>\n<span class=\"ansigreen\"> 14</span> <span class=\"ansigreen\">def</span> diefunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">2</span><span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">*</span>a<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>kw<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 15</span> time<span class=\"ansiyellow\">.</span>sleep<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">---&gt; 16</span><span class=\"ansiyellow\"> </span><span class=\"ansigreen\">raise</span> Exception<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&quot;Dead job with interval %s&quot;</span> <span class=\"ansiyellow\">%</span> interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 17</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 18</span> <span class=\"ansigreen\">def</span> printfunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">,</span> reps<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">5</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansired\">Exception</span>: Dead job with interval 1\n"
103 97 }
104 98 ],
105 99 "collapsed": false,
106 100 "prompt_number": 34,
107 101 "input": "print \"Status of diejob1:\", diejob1.status\ndiejob1.traceback() # jobs.traceback(4) would also work here, with the job number"
108 102 },
109 103 {
110 104 "source": "This will print all tracebacks for all dead jobs:",
111 105 "cell_type": "markdown"
112 106 },
113 107 {
114 108 "cell_type": "code",
115 109 "language": "python",
116 110 "outputs": [
117 111 {
118 112 "output_type": "stream",
119 113 "stream": "stdout",
120 114 "text": "Traceback for: &lt;BackgroundJob #4: &lt;function diefunc at 0x30df758&gt;&gt;\n<span class=\"ansired\">---------------------------------------------------------------------------</span>\n<span class=\"ansired\">Exception</span> Traceback (most recent call last)\n<span class=\"ansigreen\">/home/fperez/usr/lib/python2.6/site-packages/IPython/lib/backgroundjobs.py</span> in <span class=\"ansicyan\">call</span><span class=\"ansiblue\">(self)</span>\n<span class=\"ansigreen\"> 462</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 463</span> <span class=\"ansigreen\">def</span> call<span class=\"ansiyellow\">(</span>self<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">--&gt; 464</span><span class=\"ansiyellow\"> </span><span class=\"ansigreen\">return</span> self<span class=\"ansiyellow\">.</span>func<span class=\"ansiyellow\">(</span><span class=\"ansiyellow\">*</span>self<span class=\"ansiyellow\">.</span>args<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>self<span class=\"ansiyellow\">.</span>kwargs<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/home/fperez/ipython/ipython/docs/examples/lib/&lt;ipython-input-15-54795a097787&gt;</span> in <span class=\"ansicyan\">diefunc</span><span class=\"ansiblue\">(interval, *a, **kw)</span>\n<span class=\"ansigreen\"> 14</span> <span class=\"ansigreen\">def</span> diefunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">2</span><span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">*</span>a<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>kw<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 15</span> time<span class=\"ansiyellow\">.</span>sleep<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">---&gt; 16</span><span class=\"ansiyellow\"> </span><span class=\"ansigreen\">raise</span> Exception<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&quot;Dead job with interval %s&quot;</span> <span class=\"ansiyellow\">%</span> interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 17</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 18</span> <span class=\"ansigreen\">def</span> printfunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">,</span> reps<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">5</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansired\">Exception</span>: Dead job with interval 1\n\nTraceback for: &lt;BackgroundJob #5: &lt;function diefunc at 0x30df758&gt;&gt;\n<span class=\"ansired\">---------------------------------------------------------------------------</span>\n<span class=\"ansired\">Exception</span> Traceback (most recent call last)\n<span class=\"ansigreen\">/home/fperez/usr/lib/python2.6/site-packages/IPython/lib/backgroundjobs.py</span> in <span class=\"ansicyan\">call</span><span class=\"ansiblue\">(self)</span>\n<span class=\"ansigreen\"> 462</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 463</span> <span class=\"ansigreen\">def</span> call<span class=\"ansiyellow\">(</span>self<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">--&gt; 464</span><span class=\"ansiyellow\"> </span><span class=\"ansigreen\">return</span> self<span class=\"ansiyellow\">.</span>func<span class=\"ansiyellow\">(</span><span class=\"ansiyellow\">*</span>self<span class=\"ansiyellow\">.</span>args<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>self<span class=\"ansiyellow\">.</span>kwargs<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/home/fperez/ipython/ipython/docs/examples/lib/&lt;ipython-input-15-54795a097787&gt;</span> in <span class=\"ansicyan\">diefunc</span><span class=\"ansiblue\">(interval, *a, **kw)</span>\n<span class=\"ansigreen\"> 14</span> <span class=\"ansigreen\">def</span> diefunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">2</span><span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">*</span>a<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>kw<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 15</span> time<span class=\"ansiyellow\">.</span>sleep<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">---&gt; 16</span><span class=\"ansiyellow\"> </span><span class=\"ansigreen\">raise</span> Exception<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&quot;Dead job with interval %s&quot;</span> <span class=\"ansiyellow\">%</span> interval<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 17</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\"> 18</span> <span class=\"ansigreen\">def</span> printfunc<span class=\"ansiyellow\">(</span>interval<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">,</span> reps<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">5</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansired\">Exception</span>: Dead job with interval 2\n\n"
121 115 }
122 116 ],
123 117 "collapsed": false,
124 118 "prompt_number": 33,
125 119 "input": "jobs.traceback()"
126 120 },
127 121 {
128 122 "source": "The job manager can be flushed of all completed jobs at any time:",
129 123 "cell_type": "markdown"
130 124 },
131 125 {
132 126 "cell_type": "code",
133 127 "language": "python",
134 128 "outputs": [
135 129 {
136 130 "output_type": "stream",
137 131 "stream": "stdout",
138 132 "text": "No jobs to flush.\n"
139 133 }
140 134 ],
141 135 "collapsed": false,
142 136 "prompt_number": 25,
143 137 "input": "jobs.flush()"
144 138 },
145 139 {
146 140 "source": "After that, the status is simply empty:",
147 141 "cell_type": "markdown"
148 142 },
149 143 {
150 144 "cell_type": "code",
151 145 "language": "python",
152 146 "outputs": [],
153 147 "collapsed": true,
154 148 "prompt_number": 27,
155 149 "input": "jobs.status()"
156 150 },
157 151 {
152 "source": "It's easy to wait on a job:",
153 "cell_type": "markdown"
154 },
155 {
156 "cell_type": "code",
157 "language": "python",
158 "outputs": [
159 {
160 "output_type": "stream",
161 "stream": "stdout",
162 "text": "Starting job # 7 in a separate thread.\nWill wait for j now...\n"
163 },
164 {
165 "output_type": "stream",
166 "stream": "stdout",
167 "text": "Result from j:\n"
168 },
169 {
170 "output_type": "pyout",
171 "prompt_number": 46,
172 "text": "{&apos;args&apos;: (), &apos;interval&apos;: 2, &apos;kwargs&apos;: {}}"
173 }
174 ],
175 "collapsed": false,
176 "prompt_number": 46,
177 "input": "j = jobs.new(sleepfunc, 2)\nprint \"Will wait for j now...\"\nsys.stdout.flush()\nj.join()\nprint \"Result from j:\"\nj.result"
178 },
179 {
158 180 "input": "",
159 181 "cell_type": "code",
160 182 "collapsed": true,
161 183 "language": "python",
162 184 "outputs": []
163 185 }
164 186 ]
165 187 }
166 188 ],
167 189 "metadata": {
168 190 "name": "BackgroundJobs"
169 191 },
170 192 "nbformat": 2
171 193 } No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now