##// END OF EJS Templates
Add a failing test if we %run something with multiprocessing
Matthias Bussonnier -
Show More
@@ -1,559 +1,588 b''
1 1 # encoding: utf-8
2 2 """Tests for code execution (%run and related), which is particularly tricky.
3 3
4 4 Because of how %run manages namespaces, and the fact that we are trying here to
5 5 verify subtle object deletion and reference counting issues, the %run tests
6 6 will be kept in this separate file. This makes it easier to aggregate in one
7 7 place the tricks needed to handle it; most other magics are much easier to test
8 8 and we do so in a common test_magic file.
9 9
10 10 Note that any test using `run -i` should make sure to do a `reset` afterwards,
11 11 as otherwise it may influence later tests.
12 12 """
13 13
14 14 # Copyright (c) IPython Development Team.
15 15 # Distributed under the terms of the Modified BSD License.
16 16
17 17
18 18
19 19 import functools
20 20 import os
21 21 from os.path import join as pjoin
22 22 import random
23 23 import string
24 24 import sys
25 25 import textwrap
26 26 import unittest
27 27 from unittest.mock import patch
28 28
29 29 import nose.tools as nt
30 30 from nose import SkipTest
31 31
32 32 from IPython.testing import decorators as dec
33 33 from IPython.testing import tools as tt
34 34 from IPython.utils.io import capture_output
35 35 from IPython.utils.tempdir import TemporaryDirectory
36 36 from IPython.core import debugger
37 37
38 38 def doctest_refbug():
39 39 """Very nasty problem with references held by multiple runs of a script.
40 40 See: https://github.com/ipython/ipython/issues/141
41 41
42 42 In [1]: _ip.clear_main_mod_cache()
43 43 # random
44 44
45 45 In [2]: %run refbug
46 46
47 47 In [3]: call_f()
48 48 lowercased: hello
49 49
50 50 In [4]: %run refbug
51 51
52 52 In [5]: call_f()
53 53 lowercased: hello
54 54 lowercased: hello
55 55 """
56 56
57 57
58 58 def doctest_run_builtins():
59 59 r"""Check that %run doesn't damage __builtins__.
60 60
61 61 In [1]: import tempfile
62 62
63 63 In [2]: bid1 = id(__builtins__)
64 64
65 65 In [3]: fname = tempfile.mkstemp('.py')[1]
66 66
67 67 In [3]: f = open(fname,'w')
68 68
69 69 In [4]: dummy= f.write('pass\n')
70 70
71 71 In [5]: f.flush()
72 72
73 73 In [6]: t1 = type(__builtins__)
74 74
75 75 In [7]: %run $fname
76 76
77 77 In [7]: f.close()
78 78
79 79 In [8]: bid2 = id(__builtins__)
80 80
81 81 In [9]: t2 = type(__builtins__)
82 82
83 83 In [10]: t1 == t2
84 84 Out[10]: True
85 85
86 86 In [10]: bid1 == bid2
87 87 Out[10]: True
88 88
89 89 In [12]: try:
90 90 ....: os.unlink(fname)
91 91 ....: except:
92 92 ....: pass
93 93 ....:
94 94 """
95 95
96 96
97 97 def doctest_run_option_parser():
98 98 r"""Test option parser in %run.
99 99
100 100 In [1]: %run print_argv.py
101 101 []
102 102
103 103 In [2]: %run print_argv.py print*.py
104 104 ['print_argv.py']
105 105
106 106 In [3]: %run -G print_argv.py print*.py
107 107 ['print*.py']
108 108
109 109 """
110 110
111 111
112 112 @dec.skip_win32
113 113 def doctest_run_option_parser_for_posix():
114 114 r"""Test option parser in %run (Linux/OSX specific).
115 115
116 116 You need double quote to escape glob in POSIX systems:
117 117
118 118 In [1]: %run print_argv.py print\\*.py
119 119 ['print*.py']
120 120
121 121 You can't use quote to escape glob in POSIX systems:
122 122
123 123 In [2]: %run print_argv.py 'print*.py'
124 124 ['print_argv.py']
125 125
126 126 """
127 127
128 128
129 129 @dec.skip_if_not_win32
130 130 def doctest_run_option_parser_for_windows():
131 131 r"""Test option parser in %run (Windows specific).
132 132
133 133 In Windows, you can't escape ``*` `by backslash:
134 134
135 135 In [1]: %run print_argv.py print\\*.py
136 136 ['print\\*.py']
137 137
138 138 You can use quote to escape glob:
139 139
140 140 In [2]: %run print_argv.py 'print*.py'
141 141 ['print*.py']
142 142
143 143 """
144 144
145 145
146 146 def doctest_reset_del():
147 147 """Test that resetting doesn't cause errors in __del__ methods.
148 148
149 149 In [2]: class A(object):
150 150 ...: def __del__(self):
151 151 ...: print(str("Hi"))
152 152 ...:
153 153
154 154 In [3]: a = A()
155 155
156 156 In [4]: get_ipython().reset()
157 157 Hi
158 158
159 159 In [5]: 1+1
160 160 Out[5]: 2
161 161 """
162 162
163 163 # For some tests, it will be handy to organize them in a class with a common
164 164 # setup that makes a temp file
165 165
166 166 class TestMagicRunPass(tt.TempFileMixin):
167 167
168 168 def setUp(self):
169 169 content = "a = [1,2,3]\nb = 1"
170 170 self.mktmp(content)
171 171
172 172 def run_tmpfile(self):
173 173 _ip = get_ipython()
174 174 # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
175 175 # See below and ticket https://bugs.launchpad.net/bugs/366353
176 176 _ip.magic('run %s' % self.fname)
177 177
178 178 def run_tmpfile_p(self):
179 179 _ip = get_ipython()
180 180 # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
181 181 # See below and ticket https://bugs.launchpad.net/bugs/366353
182 182 _ip.magic('run -p %s' % self.fname)
183 183
184 184 def test_builtins_id(self):
185 185 """Check that %run doesn't damage __builtins__ """
186 186 _ip = get_ipython()
187 187 # Test that the id of __builtins__ is not modified by %run
188 188 bid1 = id(_ip.user_ns['__builtins__'])
189 189 self.run_tmpfile()
190 190 bid2 = id(_ip.user_ns['__builtins__'])
191 191 nt.assert_equal(bid1, bid2)
192 192
193 193 def test_builtins_type(self):
194 194 """Check that the type of __builtins__ doesn't change with %run.
195 195
196 196 However, the above could pass if __builtins__ was already modified to
197 197 be a dict (it should be a module) by a previous use of %run. So we
198 198 also check explicitly that it really is a module:
199 199 """
200 200 _ip = get_ipython()
201 201 self.run_tmpfile()
202 202 nt.assert_equal(type(_ip.user_ns['__builtins__']),type(sys))
203 203
204 204 def test_run_profile( self ):
205 205 """Test that the option -p, which invokes the profiler, do not
206 206 crash by invoking execfile"""
207 207 self.run_tmpfile_p()
208 208
209 209 def test_run_debug_twice(self):
210 210 # https://github.com/ipython/ipython/issues/10028
211 211 _ip = get_ipython()
212 212 with tt.fake_input(['c']):
213 213 _ip.magic('run -d %s' % self.fname)
214 214 with tt.fake_input(['c']):
215 215 _ip.magic('run -d %s' % self.fname)
216 216
217 217 def test_run_debug_twice_with_breakpoint(self):
218 218 """Make a valid python temp file."""
219 219 _ip = get_ipython()
220 220 with tt.fake_input(['b 2', 'c', 'c']):
221 221 _ip.magic('run -d %s' % self.fname)
222 222
223 223 with tt.fake_input(['c']):
224 224 with tt.AssertNotPrints('KeyError'):
225 225 _ip.magic('run -d %s' % self.fname)
226 226
227 227
228 228 class TestMagicRunSimple(tt.TempFileMixin):
229 229
230 230 def test_simpledef(self):
231 231 """Test that simple class definitions work."""
232 232 src = ("class foo: pass\n"
233 233 "def f(): return foo()")
234 234 self.mktmp(src)
235 235 _ip.magic('run %s' % self.fname)
236 236 _ip.run_cell('t = isinstance(f(), foo)')
237 237 nt.assert_true(_ip.user_ns['t'])
238 238
239 239 def test_obj_del(self):
240 240 """Test that object's __del__ methods are called on exit."""
241 241 if sys.platform == 'win32':
242 242 try:
243 243 import win32api
244 244 except ImportError:
245 245 raise SkipTest("Test requires pywin32")
246 246 src = ("class A(object):\n"
247 247 " def __del__(self):\n"
248 248 " print('object A deleted')\n"
249 249 "a = A()\n")
250 250 self.mktmp(src)
251 251 if dec.module_not_available('sqlite3'):
252 252 err = 'WARNING: IPython History requires SQLite, your history will not be saved\n'
253 253 else:
254 254 err = None
255 255 tt.ipexec_validate(self.fname, 'object A deleted', err)
256 256
257 257 def test_aggressive_namespace_cleanup(self):
258 258 """Test that namespace cleanup is not too aggressive GH-238
259 259
260 260 Returning from another run magic deletes the namespace"""
261 261 # see ticket https://github.com/ipython/ipython/issues/238
262 262
263 263 with tt.TempFileMixin() as empty:
264 264 empty.mktmp('')
265 265 # On Windows, the filename will have \users in it, so we need to use the
266 266 # repr so that the \u becomes \\u.
267 267 src = ("ip = get_ipython()\n"
268 268 "for i in range(5):\n"
269 269 " try:\n"
270 270 " ip.magic(%r)\n"
271 271 " except NameError as e:\n"
272 272 " print(i)\n"
273 273 " break\n" % ('run ' + empty.fname))
274 274 self.mktmp(src)
275 275 _ip.magic('run %s' % self.fname)
276 276 _ip.run_cell('ip == get_ipython()')
277 277 nt.assert_equal(_ip.user_ns['i'], 4)
278 278
279 279 def test_run_second(self):
280 280 """Test that running a second file doesn't clobber the first, gh-3547
281 281 """
282 282 self.mktmp("avar = 1\n"
283 283 "def afunc():\n"
284 284 " return avar\n")
285 285
286 286 with tt.TempFileMixin() as empty:
287 287 empty.mktmp("")
288 288
289 289 _ip.magic('run %s' % self.fname)
290 290 _ip.magic('run %s' % empty.fname)
291 291 nt.assert_equal(_ip.user_ns['afunc'](), 1)
292 292
293 293 @dec.skip_win32
294 294 def test_tclass(self):
295 295 mydir = os.path.dirname(__file__)
296 296 tc = os.path.join(mydir, 'tclass')
297 297 src = ("%%run '%s' C-first\n"
298 298 "%%run '%s' C-second\n"
299 299 "%%run '%s' C-third\n") % (tc, tc, tc)
300 300 self.mktmp(src, '.ipy')
301 301 out = """\
302 302 ARGV 1-: ['C-first']
303 303 ARGV 1-: ['C-second']
304 304 tclass.py: deleting object: C-first
305 305 ARGV 1-: ['C-third']
306 306 tclass.py: deleting object: C-second
307 307 tclass.py: deleting object: C-third
308 308 """
309 309 if dec.module_not_available('sqlite3'):
310 310 err = 'WARNING: IPython History requires SQLite, your history will not be saved\n'
311 311 else:
312 312 err = None
313 313 tt.ipexec_validate(self.fname, out, err)
314 314
315 315 def test_run_i_after_reset(self):
316 316 """Check that %run -i still works after %reset (gh-693)"""
317 317 src = "yy = zz\n"
318 318 self.mktmp(src)
319 319 _ip.run_cell("zz = 23")
320 320 try:
321 321 _ip.magic('run -i %s' % self.fname)
322 322 nt.assert_equal(_ip.user_ns['yy'], 23)
323 323 finally:
324 324 _ip.magic('reset -f')
325 325
326 326 _ip.run_cell("zz = 23")
327 327 try:
328 328 _ip.magic('run -i %s' % self.fname)
329 329 nt.assert_equal(_ip.user_ns['yy'], 23)
330 330 finally:
331 331 _ip.magic('reset -f')
332 332
333 333 def test_unicode(self):
334 334 """Check that files in odd encodings are accepted."""
335 335 mydir = os.path.dirname(__file__)
336 336 na = os.path.join(mydir, 'nonascii.py')
337 337 _ip.magic('run "%s"' % na)
338 338 nt.assert_equal(_ip.user_ns['u'], u'ΠŽΡ‚β„–Π€')
339 339
340 340 def test_run_py_file_attribute(self):
341 341 """Test handling of `__file__` attribute in `%run <file>.py`."""
342 342 src = "t = __file__\n"
343 343 self.mktmp(src)
344 344 _missing = object()
345 345 file1 = _ip.user_ns.get('__file__', _missing)
346 346 _ip.magic('run %s' % self.fname)
347 347 file2 = _ip.user_ns.get('__file__', _missing)
348 348
349 349 # Check that __file__ was equal to the filename in the script's
350 350 # namespace.
351 351 nt.assert_equal(_ip.user_ns['t'], self.fname)
352 352
353 353 # Check that __file__ was not leaked back into user_ns.
354 354 nt.assert_equal(file1, file2)
355 355
356 356 def test_run_ipy_file_attribute(self):
357 357 """Test handling of `__file__` attribute in `%run <file.ipy>`."""
358 358 src = "t = __file__\n"
359 359 self.mktmp(src, ext='.ipy')
360 360 _missing = object()
361 361 file1 = _ip.user_ns.get('__file__', _missing)
362 362 _ip.magic('run %s' % self.fname)
363 363 file2 = _ip.user_ns.get('__file__', _missing)
364 364
365 365 # Check that __file__ was equal to the filename in the script's
366 366 # namespace.
367 367 nt.assert_equal(_ip.user_ns['t'], self.fname)
368 368
369 369 # Check that __file__ was not leaked back into user_ns.
370 370 nt.assert_equal(file1, file2)
371 371
372 372 def test_run_formatting(self):
373 373 """ Test that %run -t -N<N> does not raise a TypeError for N > 1."""
374 374 src = "pass"
375 375 self.mktmp(src)
376 376 _ip.magic('run -t -N 1 %s' % self.fname)
377 377 _ip.magic('run -t -N 10 %s' % self.fname)
378 378
379 379 def test_ignore_sys_exit(self):
380 380 """Test the -e option to ignore sys.exit()"""
381 381 src = "import sys; sys.exit(1)"
382 382 self.mktmp(src)
383 383 with tt.AssertPrints('SystemExit'):
384 384 _ip.magic('run %s' % self.fname)
385 385
386 386 with tt.AssertNotPrints('SystemExit'):
387 387 _ip.magic('run -e %s' % self.fname)
388 388
389 389 def test_run_nb(self):
390 390 """Test %run notebook.ipynb"""
391 391 from nbformat import v4, writes
392 392 nb = v4.new_notebook(
393 393 cells=[
394 394 v4.new_markdown_cell("The Ultimate Question of Everything"),
395 395 v4.new_code_cell("answer=42")
396 396 ]
397 397 )
398 398 src = writes(nb, version=4)
399 399 self.mktmp(src, ext='.ipynb')
400 400
401 401 _ip.magic("run %s" % self.fname)
402 402
403 403 nt.assert_equal(_ip.user_ns['answer'], 42)
404 404
405 405 def test_file_options(self):
406 406 src = ('import sys\n'
407 407 'a = " ".join(sys.argv[1:])\n')
408 408 self.mktmp(src)
409 409 test_opts = '-x 3 --verbose'
410 410 _ip.run_line_magic("run", '{0} {1}'.format(self.fname, test_opts))
411 411 nt.assert_equal(_ip.user_ns['a'], test_opts)
412 412
413 413
414 414 class TestMagicRunWithPackage(unittest.TestCase):
415 415
416 416 def writefile(self, name, content):
417 417 path = os.path.join(self.tempdir.name, name)
418 418 d = os.path.dirname(path)
419 419 if not os.path.isdir(d):
420 420 os.makedirs(d)
421 421 with open(path, 'w') as f:
422 422 f.write(textwrap.dedent(content))
423 423
424 424 def setUp(self):
425 425 self.package = package = 'tmp{0}'.format(''.join([random.choice(string.ascii_letters) for i in range(10)]))
426 426 """Temporary (probably) valid python package name."""
427 427
428 428 self.value = int(random.random() * 10000)
429 429
430 430 self.tempdir = TemporaryDirectory()
431 431 self.__orig_cwd = os.getcwd()
432 432 sys.path.insert(0, self.tempdir.name)
433 433
434 434 self.writefile(os.path.join(package, '__init__.py'), '')
435 435 self.writefile(os.path.join(package, 'sub.py'), """
436 436 x = {0!r}
437 437 """.format(self.value))
438 438 self.writefile(os.path.join(package, 'relative.py'), """
439 439 from .sub import x
440 440 """)
441 441 self.writefile(os.path.join(package, 'absolute.py'), """
442 442 from {0}.sub import x
443 443 """.format(package))
444 444 self.writefile(os.path.join(package, 'args.py'), """
445 445 import sys
446 446 a = " ".join(sys.argv[1:])
447 447 """.format(package))
448 448
449 449 def tearDown(self):
450 450 os.chdir(self.__orig_cwd)
451 451 sys.path[:] = [p for p in sys.path if p != self.tempdir.name]
452 452 self.tempdir.cleanup()
453 453
454 454 def check_run_submodule(self, submodule, opts=''):
455 455 _ip.user_ns.pop('x', None)
456 456 _ip.magic('run {2} -m {0}.{1}'.format(self.package, submodule, opts))
457 457 self.assertEqual(_ip.user_ns['x'], self.value,
458 458 'Variable `x` is not loaded from module `{0}`.'
459 459 .format(submodule))
460 460
461 461 def test_run_submodule_with_absolute_import(self):
462 462 self.check_run_submodule('absolute')
463 463
464 464 def test_run_submodule_with_relative_import(self):
465 465 """Run submodule that has a relative import statement (#2727)."""
466 466 self.check_run_submodule('relative')
467 467
468 468 def test_prun_submodule_with_absolute_import(self):
469 469 self.check_run_submodule('absolute', '-p')
470 470
471 471 def test_prun_submodule_with_relative_import(self):
472 472 self.check_run_submodule('relative', '-p')
473 473
474 474 def with_fake_debugger(func):
475 475 @functools.wraps(func)
476 476 def wrapper(*args, **kwds):
477 477 with patch.object(debugger.Pdb, 'run', staticmethod(eval)):
478 478 return func(*args, **kwds)
479 479 return wrapper
480 480
481 481 @with_fake_debugger
482 482 def test_debug_run_submodule_with_absolute_import(self):
483 483 self.check_run_submodule('absolute', '-d')
484 484
485 485 @with_fake_debugger
486 486 def test_debug_run_submodule_with_relative_import(self):
487 487 self.check_run_submodule('relative', '-d')
488 488
489 489 def test_module_options(self):
490 490 _ip.user_ns.pop('a', None)
491 491 test_opts = '-x abc -m test'
492 492 _ip.run_line_magic('run', '-m {0}.args {1}'.format(self.package, test_opts))
493 493 nt.assert_equal(_ip.user_ns['a'], test_opts)
494 494
495 495 def test_module_options_with_separator(self):
496 496 _ip.user_ns.pop('a', None)
497 497 test_opts = '-x abc -m test'
498 498 _ip.run_line_magic('run', '-m {0}.args -- {1}'.format(self.package, test_opts))
499 499 nt.assert_equal(_ip.user_ns['a'], test_opts)
500 500
501 501 def test_run__name__():
502 502 with TemporaryDirectory() as td:
503 503 path = pjoin(td, 'foo.py')
504 504 with open(path, 'w') as f:
505 505 f.write("q = __name__")
506 506
507 507 _ip.user_ns.pop('q', None)
508 508 _ip.magic('run {}'.format(path))
509 509 nt.assert_equal(_ip.user_ns.pop('q'), '__main__')
510 510
511 511 _ip.magic('run -n {}'.format(path))
512 512 nt.assert_equal(_ip.user_ns.pop('q'), 'foo')
513 513
514 514 try:
515 515 _ip.magic('run -i -n {}'.format(path))
516 516 nt.assert_equal(_ip.user_ns.pop('q'), 'foo')
517 517 finally:
518 518 _ip.magic('reset -f')
519 519
520 520
521 521 def test_run_tb():
522 522 """Test traceback offset in %run"""
523 523 with TemporaryDirectory() as td:
524 524 path = pjoin(td, 'foo.py')
525 525 with open(path, 'w') as f:
526 526 f.write('\n'.join([
527 527 "def foo():",
528 528 " return bar()",
529 529 "def bar():",
530 530 " raise RuntimeError('hello!')",
531 531 "foo()",
532 532 ]))
533 533 with capture_output() as io:
534 534 _ip.magic('run {}'.format(path))
535 535 out = io.stdout
536 536 nt.assert_not_in("execfile", out)
537 537 nt.assert_in("RuntimeError", out)
538 538 nt.assert_equal(out.count("---->"), 3)
539 539 del ip.user_ns['bar']
540 540 del ip.user_ns['foo']
541 541
542
543 def test_multiprocessing_run():
544 """Set we can run mutiprocesgin without messing up up main namespace
545
546 Note that import `nose.tools as nt` mdify the value s
547 sys.module['__mp_main__'] so wee need to temporarily set it to None to test
548 the issue.
549 """
550 with TemporaryDirectory() as td:
551 mpm = sys.modules.get('__mp_main__')
552 assert mpm is not None
553 sys.modules['__mp_main__'] = None
554 try:
555 path = pjoin(td, 'test.py')
556 with open(path, 'w') as f:
557 f.write("import multiprocessing\nprint('hoy')")
558 with capture_output() as io:
559 _ip.run_line_magic('run', path)
560 _ip.run_cell("i_m_undefined")
561 out = io.stdout
562 nt.assert_in("hoy", out)
563 nt.assert_not_in("AttributeError", out)
564 nt.assert_in("NameError", out)
565 nt.assert_equal(out.count("---->"), 1)
566 except:
567 raise
568 finally:
569 sys.modules['__mp_main__'] = mpm
570
542 571 @dec.knownfailureif(sys.platform == 'win32', "writes to io.stdout aren't captured on Windows")
543 572 def test_script_tb():
544 573 """Test traceback offset in `ipython script.py`"""
545 574 with TemporaryDirectory() as td:
546 575 path = pjoin(td, 'foo.py')
547 576 with open(path, 'w') as f:
548 577 f.write('\n'.join([
549 578 "def foo():",
550 579 " return bar()",
551 580 "def bar():",
552 581 " raise RuntimeError('hello!')",
553 582 "foo()",
554 583 ]))
555 584 out, err = tt.ipexec(path)
556 585 nt.assert_not_in("execfile", out)
557 586 nt.assert_in("RuntimeError", out)
558 587 nt.assert_equal(out.count("---->"), 3)
559 588
General Comments 0
You need to be logged in to leave comments. Login now