##// END OF EJS Templates
Better support compiling cells with separate __future__ environments
Thomas Kluyver -
Show More

The requested changes are too big and content was truncated. Show full diff

@@ -1,144 +1,144 b''
1 """Compiler tools with improved interactive support.
1 """Compiler tools with improved interactive support.
2
2
3 Provides compilation machinery similar to codeop, but with caching support so
3 Provides compilation machinery similar to codeop, but with caching support so
4 we can provide interactive tracebacks.
4 we can provide interactive tracebacks.
5
5
6 Authors
6 Authors
7 -------
7 -------
8 * Robert Kern
8 * Robert Kern
9 * Fernando Perez
9 * Fernando Perez
10 * Thomas Kluyver
10 * Thomas Kluyver
11 """
11 """
12
12
13 # Note: though it might be more natural to name this module 'compiler', that
13 # Note: though it might be more natural to name this module 'compiler', that
14 # name is in the stdlib and name collisions with the stdlib tend to produce
14 # name is in the stdlib and name collisions with the stdlib tend to produce
15 # weird problems (often with third-party tools).
15 # weird problems (often with third-party tools).
16
16
17 #-----------------------------------------------------------------------------
17 #-----------------------------------------------------------------------------
18 # Copyright (C) 2010-2011 The IPython Development Team.
18 # Copyright (C) 2010-2011 The IPython Development Team.
19 #
19 #
20 # Distributed under the terms of the BSD License.
20 # Distributed under the terms of the BSD License.
21 #
21 #
22 # The full license is in the file COPYING.txt, distributed with this software.
22 # The full license is in the file COPYING.txt, distributed with this software.
23 #-----------------------------------------------------------------------------
23 #-----------------------------------------------------------------------------
24
24
25 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
26 # Imports
26 # Imports
27 #-----------------------------------------------------------------------------
27 #-----------------------------------------------------------------------------
28 from __future__ import print_function
28 from __future__ import print_function
29
29
30 # Stdlib imports
30 # Stdlib imports
31 import __future__
31 import __future__
32 from ast import PyCF_ONLY_AST
32 from ast import PyCF_ONLY_AST
33 import codeop
33 import codeop
34 import functools
34 import functools
35 import hashlib
35 import hashlib
36 import linecache
36 import linecache
37 import operator
37 import operator
38 import time
38 import time
39
39
40 #-----------------------------------------------------------------------------
40 #-----------------------------------------------------------------------------
41 # Constants
41 # Constants
42 #-----------------------------------------------------------------------------
42 #-----------------------------------------------------------------------------
43
43
44 # Roughtly equal to PyCF_MASK | PyCF_MASK_OBSOLETE as defined in pythonrun.h,
44 # Roughtly equal to PyCF_MASK | PyCF_MASK_OBSOLETE as defined in pythonrun.h,
45 # this is used as a bitmask to extract future-related code flags.
45 # this is used as a bitmask to extract future-related code flags.
46 PyCF_MASK = functools.reduce(operator.or_,
46 PyCF_MASK = functools.reduce(operator.or_,
47 (getattr(__future__, fname).compiler_flag
47 (getattr(__future__, fname).compiler_flag
48 for fname in __future__.all_feature_names))
48 for fname in __future__.all_feature_names))
49
49
50 #-----------------------------------------------------------------------------
50 #-----------------------------------------------------------------------------
51 # Local utilities
51 # Local utilities
52 #-----------------------------------------------------------------------------
52 #-----------------------------------------------------------------------------
53
53
54 def code_name(code, number=0):
54 def code_name(code, number=0):
55 """ Compute a (probably) unique name for code for caching.
55 """ Compute a (probably) unique name for code for caching.
56
56
57 This now expects code to be unicode.
57 This now expects code to be unicode.
58 """
58 """
59 hash_digest = hashlib.md5(code.encode("utf-8")).hexdigest()
59 hash_digest = hashlib.md5(code.encode("utf-8")).hexdigest()
60 # Include the number and 12 characters of the hash in the name. It's
60 # Include the number and 12 characters of the hash in the name. It's
61 # pretty much impossible that in a single session we'll have collisions
61 # pretty much impossible that in a single session we'll have collisions
62 # even with truncated hashes, and the full one makes tracebacks too long
62 # even with truncated hashes, and the full one makes tracebacks too long
63 return '<ipython-input-{0}-{1}>'.format(number, hash_digest[:12])
63 return '<ipython-input-{0}-{1}>'.format(number, hash_digest[:12])
64
64
65 #-----------------------------------------------------------------------------
65 #-----------------------------------------------------------------------------
66 # Classes and functions
66 # Classes and functions
67 #-----------------------------------------------------------------------------
67 #-----------------------------------------------------------------------------
68
68
69 class CachingCompiler(codeop.Compile):
69 class CachingCompiler(codeop.Compile):
70 """A compiler that caches code compiled from interactive statements.
70 """A compiler that caches code compiled from interactive statements.
71 """
71 """
72
72
73 def __init__(self):
73 def __init__(self):
74 codeop.Compile.__init__(self)
74 codeop.Compile.__init__(self)
75
75
76 # This is ugly, but it must be done this way to allow multiple
76 # This is ugly, but it must be done this way to allow multiple
77 # simultaneous ipython instances to coexist. Since Python itself
77 # simultaneous ipython instances to coexist. Since Python itself
78 # directly accesses the data structures in the linecache module, and
78 # directly accesses the data structures in the linecache module, and
79 # the cache therein is global, we must work with that data structure.
79 # the cache therein is global, we must work with that data structure.
80 # We must hold a reference to the original checkcache routine and call
80 # We must hold a reference to the original checkcache routine and call
81 # that in our own check_cache() below, but the special IPython cache
81 # that in our own check_cache() below, but the special IPython cache
82 # must also be shared by all IPython instances. If we were to hold
82 # must also be shared by all IPython instances. If we were to hold
83 # separate caches (one in each CachingCompiler instance), any call made
83 # separate caches (one in each CachingCompiler instance), any call made
84 # by Python itself to linecache.checkcache() would obliterate the
84 # by Python itself to linecache.checkcache() would obliterate the
85 # cached data from the other IPython instances.
85 # cached data from the other IPython instances.
86 if not hasattr(linecache, '_ipython_cache'):
86 if not hasattr(linecache, '_ipython_cache'):
87 linecache._ipython_cache = {}
87 linecache._ipython_cache = {}
88 if not hasattr(linecache, '_checkcache_ori'):
88 if not hasattr(linecache, '_checkcache_ori'):
89 linecache._checkcache_ori = linecache.checkcache
89 linecache._checkcache_ori = linecache.checkcache
90 # Now, we must monkeypatch the linecache directly so that parts of the
90 # Now, we must monkeypatch the linecache directly so that parts of the
91 # stdlib that call it outside our control go through our codepath
91 # stdlib that call it outside our control go through our codepath
92 # (otherwise we'd lose our tracebacks).
92 # (otherwise we'd lose our tracebacks).
93 linecache.checkcache = self.check_cache
93 linecache.checkcache = check_linecache_ipython
94
94
95 def ast_parse(self, source, filename='<unknown>', symbol='exec'):
95 def ast_parse(self, source, filename='<unknown>', symbol='exec'):
96 """Parse code to an AST with the current compiler flags active.
96 """Parse code to an AST with the current compiler flags active.
97
97
98 Arguments are exactly the same as ast.parse (in the standard library),
98 Arguments are exactly the same as ast.parse (in the standard library),
99 and are passed to the built-in compile function."""
99 and are passed to the built-in compile function."""
100 return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1)
100 return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1)
101
101
102 def reset_compiler_flags(self):
102 def reset_compiler_flags(self):
103 """Reset compiler flags to default state."""
103 """Reset compiler flags to default state."""
104 # This value is copied from codeop.Compile.__init__, so if that ever
104 # This value is copied from codeop.Compile.__init__, so if that ever
105 # changes, it will need to be updated.
105 # changes, it will need to be updated.
106 self.flags = codeop.PyCF_DONT_IMPLY_DEDENT
106 self.flags = codeop.PyCF_DONT_IMPLY_DEDENT
107
107
108 @property
108 @property
109 def compiler_flags(self):
109 def compiler_flags(self):
110 """Flags currently active in the compilation process.
110 """Flags currently active in the compilation process.
111 """
111 """
112 return self.flags
112 return self.flags
113
113
114 def cache(self, code, number=0):
114 def cache(self, code, number=0):
115 """Make a name for a block of code, and cache the code.
115 """Make a name for a block of code, and cache the code.
116
116
117 Parameters
117 Parameters
118 ----------
118 ----------
119 code : str
119 code : str
120 The Python source code to cache.
120 The Python source code to cache.
121 number : int
121 number : int
122 A number which forms part of the code's name. Used for the execution
122 A number which forms part of the code's name. Used for the execution
123 counter.
123 counter.
124
124
125 Returns
125 Returns
126 -------
126 -------
127 The name of the cached code (as a string). Pass this as the filename
127 The name of the cached code (as a string). Pass this as the filename
128 argument to compilation, so that tracebacks are correctly hooked up.
128 argument to compilation, so that tracebacks are correctly hooked up.
129 """
129 """
130 name = code_name(code, number)
130 name = code_name(code, number)
131 entry = (len(code), time.time(),
131 entry = (len(code), time.time(),
132 [line+'\n' for line in code.splitlines()], name)
132 [line+'\n' for line in code.splitlines()], name)
133 linecache.cache[name] = entry
133 linecache.cache[name] = entry
134 linecache._ipython_cache[name] = entry
134 linecache._ipython_cache[name] = entry
135 return name
135 return name
136
136
137 def check_cache(self, *args):
137 def check_linecache_ipython(*args):
138 """Call linecache.checkcache() safely protecting our cached values.
138 """Call linecache.checkcache() safely protecting our cached values.
139 """
139 """
140 # First call the orignal checkcache as intended
140 # First call the orignal checkcache as intended
141 linecache._checkcache_ori(*args)
141 linecache._checkcache_ori(*args)
142 # Then, update back the cache with our data, so that tracebacks related
142 # Then, update back the cache with our data, so that tracebacks related
143 # to our compiled codes can be produced.
143 # to our compiled codes can be produced.
144 linecache.cache.update(linecache._ipython_cache)
144 linecache.cache.update(linecache._ipython_cache)
1 NO CONTENT: modified file
NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
@@ -1,75 +1,75 b''
1 # coding: utf-8
1 # coding: utf-8
2 """Tests for the compilerop module.
2 """Tests for the compilerop module.
3 """
3 """
4 #-----------------------------------------------------------------------------
4 #-----------------------------------------------------------------------------
5 # Copyright (C) 2010-2011 The IPython Development Team.
5 # Copyright (C) 2010-2011 The IPython Development Team.
6 #
6 #
7 # Distributed under the terms of the BSD License.
7 # Distributed under the terms of the BSD License.
8 #
8 #
9 # The full license is in the file COPYING.txt, distributed with this software.
9 # The full license is in the file COPYING.txt, distributed with this software.
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11
11
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13 # Imports
13 # Imports
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 from __future__ import print_function
15 from __future__ import print_function
16
16
17 # Stdlib imports
17 # Stdlib imports
18 import linecache
18 import linecache
19 import sys
19 import sys
20
20
21 # Third-party imports
21 # Third-party imports
22 import nose.tools as nt
22 import nose.tools as nt
23
23
24 # Our own imports
24 # Our own imports
25 from IPython.core import compilerop
25 from IPython.core import compilerop
26 from IPython.utils import py3compat
26 from IPython.utils import py3compat
27
27
28 #-----------------------------------------------------------------------------
28 #-----------------------------------------------------------------------------
29 # Test functions
29 # Test functions
30 #-----------------------------------------------------------------------------
30 #-----------------------------------------------------------------------------
31
31
32 def test_code_name():
32 def test_code_name():
33 code = 'x=1'
33 code = 'x=1'
34 name = compilerop.code_name(code)
34 name = compilerop.code_name(code)
35 nt.assert_true(name.startswith('<ipython-input-0'))
35 nt.assert_true(name.startswith('<ipython-input-0'))
36
36
37
37
38 def test_code_name2():
38 def test_code_name2():
39 code = 'x=1'
39 code = 'x=1'
40 name = compilerop.code_name(code, 9)
40 name = compilerop.code_name(code, 9)
41 nt.assert_true(name.startswith('<ipython-input-9'))
41 nt.assert_true(name.startswith('<ipython-input-9'))
42
42
43
43
44 def test_cache():
44 def test_cache():
45 """Test the compiler correctly compiles and caches inputs
45 """Test the compiler correctly compiles and caches inputs
46 """
46 """
47 cp = compilerop.CachingCompiler()
47 cp = compilerop.CachingCompiler()
48 ncache = len(linecache.cache)
48 ncache = len(linecache.cache)
49 cp.cache('x=1')
49 cp.cache('x=1')
50 nt.assert_true(len(linecache.cache) > ncache)
50 nt.assert_true(len(linecache.cache) > ncache)
51
51
52 def setUp():
52 def setUp():
53 # Check we're in a proper Python 2 environment (some imports, such
53 # Check we're in a proper Python 2 environment (some imports, such
54 # as GTK, can change the default encoding, which can hide bugs.)
54 # as GTK, can change the default encoding, which can hide bugs.)
55 nt.assert_equal(sys.getdefaultencoding(), "utf-8" if py3compat.PY3 else "ascii")
55 nt.assert_equal(sys.getdefaultencoding(), "utf-8" if py3compat.PY3 else "ascii")
56
56
57 def test_cache_unicode():
57 def test_cache_unicode():
58 cp = compilerop.CachingCompiler()
58 cp = compilerop.CachingCompiler()
59 ncache = len(linecache.cache)
59 ncache = len(linecache.cache)
60 cp.cache(u"t = 'žćčšđ'")
60 cp.cache(u"t = 'žćčšđ'")
61 nt.assert_true(len(linecache.cache) > ncache)
61 nt.assert_true(len(linecache.cache) > ncache)
62
62
63 def test_compiler_check_cache():
63 def test_compiler_check_cache():
64 """Test the compiler properly manages the cache.
64 """Test the compiler properly manages the cache.
65 """
65 """
66 # Rather simple-minded tests that just exercise the API
66 # Rather simple-minded tests that just exercise the API
67 cp = compilerop.CachingCompiler()
67 cp = compilerop.CachingCompiler()
68 cp.cache('x=1', 99)
68 cp.cache('x=1', 99)
69 # Ensure now that after clearing the cache, our entries survive
69 # Ensure now that after clearing the cache, our entries survive
70 cp.check_cache()
70 linecache.checkcache()
71 for k in linecache.cache:
71 for k in linecache.cache:
72 if k.startswith('<ipython-input-99'):
72 if k.startswith('<ipython-input-99'):
73 break
73 break
74 else:
74 else:
75 raise AssertionError('Entry for input-99 missing from linecache')
75 raise AssertionError('Entry for input-99 missing from linecache')
General Comments 0
You need to be logged in to leave comments. Login now