Show More
@@ -0,0 +1,38 b'' | |||||
|
1 | """Minimal script to reproduce our nasty reference counting bug. | |||
|
2 | ||||
|
3 | The problem is related to https://bugs.launchpad.net/ipython/+bug/269966 | |||
|
4 | ||||
|
5 | The original fix for that appeared to work, but JD Hunter found a matplotlib | |||
|
6 | example which, when run twice in a row, would break. The problem were | |||
|
7 | references held by open figures to internals of Tkinter. | |||
|
8 | ||||
|
9 | This code reproduces the problem that John saw, without matplotlib. We can | |||
|
10 | thus use it for our test suite. | |||
|
11 | """ | |||
|
12 | ||||
|
13 | #----------------------------------------------------------------------------- | |||
|
14 | # Module imports | |||
|
15 | #----------------------------------------------------------------------------- | |||
|
16 | import sys | |||
|
17 | ||||
|
18 | from IPython import ipapi | |||
|
19 | ||||
|
20 | #----------------------------------------------------------------------------- | |||
|
21 | # Globals | |||
|
22 | #----------------------------------------------------------------------------- | |||
|
23 | ip = ipapi.get() | |||
|
24 | ||||
|
25 | if not '_refbug_cache' in ip.user_ns: | |||
|
26 | ip.user_ns['_refbug_cache'] = [] | |||
|
27 | ||||
|
28 | ||||
|
29 | aglobal = 'Hello' | |||
|
30 | def f(): | |||
|
31 | return aglobal | |||
|
32 | ||||
|
33 | cache = ip.user_ns['_refbug_cache'] | |||
|
34 | cache.append(f) | |||
|
35 | ||||
|
36 | def call_f(): | |||
|
37 | for func in cache: | |||
|
38 | print 'lowercased:',func().lower() |
@@ -0,0 +1,17 b'' | |||||
|
1 | """Tests for the FakeModule objects. | |||
|
2 | """ | |||
|
3 | ||||
|
4 | import nose.tools as nt | |||
|
5 | ||||
|
6 | from IPython.FakeModule import FakeModule, init_fakemod_dict | |||
|
7 | ||||
|
8 | # Make a fakemod and check a few properties | |||
|
9 | def test_mk_fakemod(): | |||
|
10 | fm = FakeModule() | |||
|
11 | yield nt.assert_true,fm | |||
|
12 | yield nt.assert_true,lambda : hasattr(fm,'__file__') | |||
|
13 | ||||
|
14 | def test_mk_fakemod_fromdict(): | |||
|
15 | """Test making a FakeModule object with initial data""" | |||
|
16 | fm = FakeModule(dict(hello=True)) | |||
|
17 | nt.assert_true(fm.hello) |
@@ -15,6 +15,37 b' sessions.' | |||||
15 |
|
15 | |||
16 | import types |
|
16 | import types | |
17 |
|
17 | |||
|
18 | def init_fakemod_dict(fm,adict=None): | |||
|
19 | """Initialize a FakeModule instance __dict__. | |||
|
20 | ||||
|
21 | Kept as a standalone function and not a method so the FakeModule API can | |||
|
22 | remain basically empty. | |||
|
23 | ||||
|
24 | This should be considered for private IPython use, used in managing | |||
|
25 | namespaces for %run. | |||
|
26 | ||||
|
27 | Parameters | |||
|
28 | ---------- | |||
|
29 | ||||
|
30 | fm : FakeModule instance | |||
|
31 | ||||
|
32 | adict : dict, optional | |||
|
33 | """ | |||
|
34 | ||||
|
35 | dct = {} | |||
|
36 | # It seems pydoc (and perhaps others) needs any module instance to | |||
|
37 | # implement a __nonzero__ method, so we add it if missing: | |||
|
38 | dct.setdefault('__nonzero__',lambda : True) | |||
|
39 | dct.setdefault('__file__',__file__) | |||
|
40 | ||||
|
41 | if adict is not None: | |||
|
42 | dct.update(adict) | |||
|
43 | ||||
|
44 | # Hard assignment of the object's __dict__. This is nasty but deliberate. | |||
|
45 | fm.__dict__.clear() | |||
|
46 | fm.__dict__.update(dct) | |||
|
47 | ||||
|
48 | ||||
18 | class FakeModule(types.ModuleType): |
|
49 | class FakeModule(types.ModuleType): | |
19 | """Simple class with attribute access to fake a module. |
|
50 | """Simple class with attribute access to fake a module. | |
20 |
|
51 | |||
@@ -29,14 +60,7 b' class FakeModule(types.ModuleType):' | |||||
29 |
|
60 | |||
30 | # tmp to force __dict__ instance creation, else self.__dict__ fails |
|
61 | # tmp to force __dict__ instance creation, else self.__dict__ fails | |
31 | self.__iptmp = None |
|
62 | self.__iptmp = None | |
32 |
|
||||
33 | # It seems pydoc (and perhaps others) needs any module instance to |
|
|||
34 | # implement a __nonzero__ method, so we add it if missing: |
|
|||
35 | self.__dict__.setdefault('__nonzero__',lambda : True) |
|
|||
36 | self.__dict__.setdefault('__file__',__file__) |
|
|||
37 |
|
||||
38 | # cleanup our temp trick |
|
63 | # cleanup our temp trick | |
39 | del self.__iptmp |
|
64 | del self.__iptmp | |
40 |
|
65 | # Now, initialize the actual data in the instance dict. | ||
41 | if adict is not None: |
|
66 | init_fakemod_dict(self,adict) | |
42 | self.__dict__.update(adict) |
|
@@ -1584,23 +1584,22 b' Currently the magic system has the following functions:\\n"""' | |||||
1584 | prog_ns = self.shell.user_ns |
|
1584 | prog_ns = self.shell.user_ns | |
1585 | __name__save = self.shell.user_ns['__name__'] |
|
1585 | __name__save = self.shell.user_ns['__name__'] | |
1586 | prog_ns['__name__'] = '__main__' |
|
1586 | prog_ns['__name__'] = '__main__' | |
1587 | main_mod = FakeModule(prog_ns) |
|
1587 | ||
|
1588 | ##main_mod = FakeModule(prog_ns) | |||
|
1589 | main_mod = self.shell.new_main_mod(prog_ns) | |||
|
1590 | ||||
1588 | else: |
|
1591 | else: | |
1589 | # Run in a fresh, empty namespace |
|
1592 | # Run in a fresh, empty namespace | |
1590 | if opts.has_key('n'): |
|
1593 | if opts.has_key('n'): | |
1591 | name = os.path.splitext(os.path.basename(filename))[0] |
|
1594 | name = os.path.splitext(os.path.basename(filename))[0] | |
1592 | else: |
|
1595 | else: | |
1593 | name = '__main__' |
|
1596 | name = '__main__' | |
1594 | main_mod = FakeModule() |
|
1597 | ||
|
1598 | main_mod = self.shell.new_main_mod() | |||
|
1599 | ||||
1595 | prog_ns = main_mod.__dict__ |
|
1600 | prog_ns = main_mod.__dict__ | |
1596 | prog_ns['__name__'] = name |
|
1601 | prog_ns['__name__'] = name | |
1597 |
|
1602 | |||
1598 | # The shell MUST hold a reference to main_mod so after %run exits, |
|
|||
1599 | # the python deletion mechanism doesn't zero it out (leaving |
|
|||
1600 | # dangling references). However, we should drop old versions of |
|
|||
1601 | # main_mod. There is now a proper API to manage this caching in |
|
|||
1602 | # the main shell object, we use that. |
|
|||
1603 | self.shell.cache_main_mod(main_mod) |
|
|||
1604 |
|
1603 | |||
1605 | # Since '%run foo' emulates 'python foo.py' at the cmd line, we must |
|
1604 | # Since '%run foo' emulates 'python foo.py' at the cmd line, we must | |
1606 | # set the __file__ global in the script's namespace |
|
1605 | # set the __file__ global in the script's namespace | |
@@ -1703,9 +1702,14 b' Currently the magic system has the following functions:\\n"""' | |||||
1703 | else: |
|
1702 | else: | |
1704 | # regular execution |
|
1703 | # regular execution | |
1705 | runner(filename,prog_ns,prog_ns,exit_ignore=exit_ignore) |
|
1704 | runner(filename,prog_ns,prog_ns,exit_ignore=exit_ignore) | |
|
1705 | ||||
1706 | if opts.has_key('i'): |
|
1706 | if opts.has_key('i'): | |
1707 | self.shell.user_ns['__name__'] = __name__save |
|
1707 | self.shell.user_ns['__name__'] = __name__save | |
1708 | else: |
|
1708 | else: | |
|
1709 | # The shell MUST hold a reference to prog_ns so after %run | |||
|
1710 | # exits, the python deletion mechanism doesn't zero it out | |||
|
1711 | # (leaving dangling references). | |||
|
1712 | self.shell.cache_main_mod(prog_ns,filename) | |||
1709 | # update IPython interactive namespace |
|
1713 | # update IPython interactive namespace | |
1710 | del prog_ns['__name__'] |
|
1714 | del prog_ns['__name__'] | |
1711 | self.shell.user_ns.update(prog_ns) |
|
1715 | self.shell.user_ns.update(prog_ns) | |
@@ -1719,6 +1723,7 b' Currently the magic system has the following functions:\\n"""' | |||||
1719 | # added. Otherwise it will trap references to objects |
|
1723 | # added. Otherwise it will trap references to objects | |
1720 | # contained therein. |
|
1724 | # contained therein. | |
1721 | del sys.modules[main_mod_name] |
|
1725 | del sys.modules[main_mod_name] | |
|
1726 | ||||
1722 | self.shell.reloadhist() |
|
1727 | self.shell.reloadhist() | |
1723 |
|
1728 | |||
1724 | return stats |
|
1729 | return stats |
@@ -54,7 +54,7 b' from pprint import pprint, pformat' | |||||
54 | from IPython import Debugger,OInspect,PyColorize,ultraTB |
|
54 | from IPython import Debugger,OInspect,PyColorize,ultraTB | |
55 | from IPython.ColorANSI import ColorScheme,ColorSchemeTable # too long names |
|
55 | from IPython.ColorANSI import ColorScheme,ColorSchemeTable # too long names | |
56 | from IPython.Extensions import pickleshare |
|
56 | from IPython.Extensions import pickleshare | |
57 | from IPython.FakeModule import FakeModule |
|
57 | from IPython.FakeModule import FakeModule, init_fakemod_dict | |
58 | from IPython.Itpl import Itpl,itpl,printpl,ItplNS,itplns |
|
58 | from IPython.Itpl import Itpl,itpl,printpl,ItplNS,itplns | |
59 | from IPython.Logger import Logger |
|
59 | from IPython.Logger import Logger | |
60 | from IPython.Magic import Magic |
|
60 | from IPython.Magic import Magic | |
@@ -499,13 +499,24 b' class InteractiveShell(object,Magic):' | |||||
499 | # calling functions defined in the script that use other things from |
|
499 | # calling functions defined in the script that use other things from | |
500 | # the script will fail, because the function's closure had references |
|
500 | # the script will fail, because the function's closure had references | |
501 | # to the original objects, which are now all None. So we must protect |
|
501 | # to the original objects, which are now all None. So we must protect | |
502 |
# these modules from deletion by keeping a cache. |
|
502 | # these modules from deletion by keeping a cache. | |
503 | # stale modules around (we only need the one from the last run), we use |
|
503 | # | |
504 | # a dict keyed with the full path to the script, so only the last |
|
504 | # To avoid keeping stale modules around (we only need the one from the | |
505 | # version of the module is held in the cache. The %reset command will |
|
505 | # last run), we use a dict keyed with the full path to the script, so | |
506 | # flush this cache. See the cache_main_mod() and clear_main_mod_cache() |
|
506 | # only the last version of the module is held in the cache. Note, | |
507 | # methods for details on use. |
|
507 | # however, that we must cache the module *namespace contents* (their | |
508 | self._user_main_modules = {} |
|
508 | # __dict__). Because if we try to cache the actual modules, old ones | |
|
509 | # (uncached) could be destroyed while still holding references (such as | |||
|
510 | # those held by GUI objects that tend to be long-lived)> | |||
|
511 | # | |||
|
512 | # The %reset command will flush this cache. See the cache_main_mod() | |||
|
513 | # and clear_main_mod_cache() methods for details on use. | |||
|
514 | ||||
|
515 | # This is the cache used for 'main' namespaces | |||
|
516 | self._main_ns_cache = {} | |||
|
517 | # And this is the single instance of FakeModule whose __dict__ we keep | |||
|
518 | # copying and clearing for reuse on each %run | |||
|
519 | self._user_main_module = FakeModule() | |||
509 |
|
520 | |||
510 | # A table holding all the namespaces IPython deals with, so that |
|
521 | # A table holding all the namespaces IPython deals with, so that | |
511 | # introspection facilities can search easily. |
|
522 | # introspection facilities can search easily. | |
@@ -521,7 +532,7 b' class InteractiveShell(object,Magic):' | |||||
521 | # a simple list. |
|
532 | # a simple list. | |
522 | self.ns_refs_table = [ user_ns, user_global_ns, self.user_config_ns, |
|
533 | self.ns_refs_table = [ user_ns, user_global_ns, self.user_config_ns, | |
523 | self.alias_table, self.internal_ns, |
|
534 | self.alias_table, self.internal_ns, | |
524 |
self. |
|
535 | self._main_ns_cache ] | |
525 |
|
536 | |||
526 | # We need to insert into sys.modules something that looks like a |
|
537 | # We need to insert into sys.modules something that looks like a | |
527 | # module but which accesses the IPython namespace, for shelve and |
|
538 | # module but which accesses the IPython namespace, for shelve and | |
@@ -1487,38 +1498,53 b' class InteractiveShell(object,Magic):' | |||||
1487 | return True |
|
1498 | return True | |
1488 | return ask_yes_no(prompt,default) |
|
1499 | return ask_yes_no(prompt,default) | |
1489 |
|
1500 | |||
1490 |
def |
|
1501 | def new_main_mod(self,ns=None): | |
1491 | """Cache a main module. |
|
1502 | """Return a new 'main' module object for user code execution. | |
|
1503 | """ | |||
|
1504 | main_mod = self._user_main_module | |||
|
1505 | init_fakemod_dict(main_mod,ns) | |||
|
1506 | return main_mod | |||
|
1507 | ||||
|
1508 | def cache_main_mod(self,ns,fname): | |||
|
1509 | """Cache a main module's namespace. | |||
1492 |
|
1510 | |||
1493 |
When scripts are executed via %run, we must keep a reference to the |
|
1511 | When scripts are executed via %run, we must keep a reference to the | |
1494 |
__main__ module (a FakeModule instance) around so |
|
1512 | namespace of their __main__ module (a FakeModule instance) around so | |
1495 |
clear it, rendering objects defined therein |
|
1513 | that Python doesn't clear it, rendering objects defined therein | |
|
1514 | useless. | |||
1496 |
|
1515 | |||
1497 | This method keeps said reference in a private dict, keyed by the |
|
1516 | This method keeps said reference in a private dict, keyed by the | |
1498 | absolute path of the module object (which corresponds to the script |
|
1517 | absolute path of the module object (which corresponds to the script | |
1499 | path). This way, for multiple executions of the same script we only |
|
1518 | path). This way, for multiple executions of the same script we only | |
1500 |
keep one copy of |
|
1519 | keep one copy of the namespace (the last one), thus preventing memory | |
1501 |
from old references while allowing the objects from the last |
|
1520 | leaks from old references while allowing the objects from the last | |
1502 | to be accessible. |
|
1521 | execution to be accessible. | |
|
1522 | ||||
|
1523 | Note: we can not allow the actual FakeModule instances to be deleted, | |||
|
1524 | because of how Python tears down modules (it hard-sets all their | |||
|
1525 | references to None without regard for reference counts). This method | |||
|
1526 | must therefore make a *copy* of the given namespace, to allow the | |||
|
1527 | original module's __dict__ to be cleared and reused. | |||
1503 |
|
1528 | |||
|
1529 | ||||
1504 | Parameters |
|
1530 | Parameters | |
1505 | ---------- |
|
1531 | ---------- | |
1506 | mod : a module object |
|
1532 | ns : a namespace (a dict, typically) | |
|
1533 | ||||
|
1534 | fname : str | |||
|
1535 | Filename associated with the namespace. | |||
1507 |
|
1536 | |||
1508 | Examples |
|
1537 | Examples | |
1509 | -------- |
|
1538 | -------- | |
1510 |
|
1539 | |||
1511 | In [10]: import IPython |
|
1540 | In [10]: import IPython | |
1512 |
|
1541 | |||
1513 | In [11]: _ip.IP.cache_main_mod(IPython) |
|
1542 | In [11]: _ip.IP.cache_main_mod(IPython.__dict__,IPython.__file__) | |
1514 |
|
1543 | |||
1515 |
In [12]: IPython.__file__ in _ip.IP._ |
|
1544 | In [12]: IPython.__file__ in _ip.IP._main_ns_cache | |
1516 | Out[12]: True |
|
1545 | Out[12]: True | |
1517 | """ |
|
1546 | """ | |
1518 | if fname is None: |
|
1547 | self._main_ns_cache[os.path.abspath(fname)] = ns.copy() | |
1519 | fname = mod.__file__ |
|
|||
1520 | #print >> sys.stderr, 'CFNAME :', os.path.abspath(fname) # dbg |
|
|||
1521 | self._user_main_modules[os.path.abspath(fname)] = mod |
|
|||
1522 |
|
1548 | |||
1523 | def clear_main_mod_cache(self): |
|
1549 | def clear_main_mod_cache(self): | |
1524 | """Clear the cache of main modules. |
|
1550 | """Clear the cache of main modules. | |
@@ -1530,17 +1556,17 b' class InteractiveShell(object,Magic):' | |||||
1530 |
|
1556 | |||
1531 | In [15]: import IPython |
|
1557 | In [15]: import IPython | |
1532 |
|
1558 | |||
1533 | In [16]: _ip.IP.cache_main_mod(IPython) |
|
1559 | In [16]: _ip.IP.cache_main_mod(IPython.__dict__,IPython.__file__) | |
1534 |
|
1560 | |||
1535 |
In [17]: len(_ip.IP._ |
|
1561 | In [17]: len(_ip.IP._main_ns_cache) > 0 | |
1536 | Out[17]: True |
|
1562 | Out[17]: True | |
1537 |
|
1563 | |||
1538 | In [18]: _ip.IP.clear_main_mod_cache() |
|
1564 | In [18]: _ip.IP.clear_main_mod_cache() | |
1539 |
|
1565 | |||
1540 |
In [19]: len(_ip.IP._ |
|
1566 | In [19]: len(_ip.IP._main_ns_cache) == 0 | |
1541 | Out[19]: True |
|
1567 | Out[19]: True | |
1542 | """ |
|
1568 | """ | |
1543 |
self. |
|
1569 | self._main_ns_cache.clear() | |
1544 |
|
1570 | |||
1545 | def _should_recompile(self,e): |
|
1571 | def _should_recompile(self,e): | |
1546 | """Utility routine for edit_syntax_error""" |
|
1572 | """Utility routine for edit_syntax_error""" |
@@ -136,6 +136,8 b' def doctest_refbug():' | |||||
136 | """Very nasty problem with references held by multiple runs of a script. |
|
136 | """Very nasty problem with references held by multiple runs of a script. | |
137 | See: https://bugs.launchpad.net/ipython/+bug/269966 |
|
137 | See: https://bugs.launchpad.net/ipython/+bug/269966 | |
138 |
|
138 | |||
|
139 | In [1]: _ip.IP.clear_main_mod_cache() | |||
|
140 | ||||
139 | In [2]: run refbug |
|
141 | In [2]: run refbug | |
140 |
|
142 | |||
141 | In [3]: call_f() |
|
143 | In [3]: call_f() |
General Comments 0
You need to be logged in to leave comments.
Login now