Show More
@@ -1,391 +1,398 b'' | |||||
1 | # __init__.py - Startup and module loading logic for Mercurial. |
|
1 | # __init__.py - Startup and module loading logic for Mercurial. | |
2 | # |
|
2 | # | |
3 | # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> |
|
3 | # Copyright 2015 Gregory Szorc <gregory.szorc@gmail.com> | |
4 | # |
|
4 | # | |
5 | # This software may be used and distributed according to the terms of the |
|
5 | # This software may be used and distributed according to the terms of the | |
6 | # GNU General Public License version 2 or any later version. |
|
6 | # GNU General Public License version 2 or any later version. | |
7 |
|
7 | |||
8 | from __future__ import absolute_import |
|
8 | from __future__ import absolute_import | |
9 |
|
9 | |||
10 | import imp |
|
10 | import imp | |
11 | import os |
|
11 | import os | |
12 | import sys |
|
12 | import sys | |
13 | import zipimport |
|
13 | import zipimport | |
14 |
|
14 | |||
15 | from . import ( |
|
15 | from . import ( | |
16 | policy |
|
16 | policy | |
17 | ) |
|
17 | ) | |
18 |
|
18 | |||
19 | __all__ = [] |
|
19 | __all__ = [] | |
20 |
|
20 | |||
21 | modulepolicy = policy.policy |
|
21 | modulepolicy = policy.policy | |
22 |
|
22 | |||
23 | # Modules that have both Python and C implementations. See also the |
|
23 | # Modules that have both Python and C implementations. See also the | |
24 | # set of .py files under mercurial/pure/. |
|
24 | # set of .py files under mercurial/pure/. | |
25 | _dualmodules = set([ |
|
25 | _dualmodules = set([ | |
26 | 'mercurial.base85', |
|
26 | 'mercurial.base85', | |
27 | 'mercurial.bdiff', |
|
27 | 'mercurial.bdiff', | |
28 | 'mercurial.diffhelpers', |
|
28 | 'mercurial.diffhelpers', | |
29 | 'mercurial.mpatch', |
|
29 | 'mercurial.mpatch', | |
30 | 'mercurial.osutil', |
|
30 | 'mercurial.osutil', | |
31 | 'mercurial.parsers', |
|
31 | 'mercurial.parsers', | |
32 | ]) |
|
32 | ]) | |
33 |
|
33 | |||
34 | class hgimporter(object): |
|
34 | class hgimporter(object): | |
35 | """Object that conforms to import hook interface defined in PEP-302.""" |
|
35 | """Object that conforms to import hook interface defined in PEP-302.""" | |
36 | def find_module(self, name, path=None): |
|
36 | def find_module(self, name, path=None): | |
37 | # We only care about modules that have both C and pure implementations. |
|
37 | # We only care about modules that have both C and pure implementations. | |
38 | if name in _dualmodules: |
|
38 | if name in _dualmodules: | |
39 | return self |
|
39 | return self | |
40 | return None |
|
40 | return None | |
41 |
|
41 | |||
42 | def load_module(self, name): |
|
42 | def load_module(self, name): | |
43 | mod = sys.modules.get(name, None) |
|
43 | mod = sys.modules.get(name, None) | |
44 | if mod: |
|
44 | if mod: | |
45 | return mod |
|
45 | return mod | |
46 |
|
46 | |||
47 | mercurial = sys.modules['mercurial'] |
|
47 | mercurial = sys.modules['mercurial'] | |
48 |
|
48 | |||
49 | # The zip importer behaves sufficiently differently from the default |
|
49 | # The zip importer behaves sufficiently differently from the default | |
50 | # importer to warrant its own code path. |
|
50 | # importer to warrant its own code path. | |
51 | loader = getattr(mercurial, '__loader__', None) |
|
51 | loader = getattr(mercurial, '__loader__', None) | |
52 | if isinstance(loader, zipimport.zipimporter): |
|
52 | if isinstance(loader, zipimport.zipimporter): | |
53 | def ziploader(*paths): |
|
53 | def ziploader(*paths): | |
54 | """Obtain a zipimporter for a directory under the main zip.""" |
|
54 | """Obtain a zipimporter for a directory under the main zip.""" | |
55 | path = os.path.join(loader.archive, *paths) |
|
55 | path = os.path.join(loader.archive, *paths) | |
56 | zl = sys.path_importer_cache.get(path) |
|
56 | zl = sys.path_importer_cache.get(path) | |
57 | if not zl: |
|
57 | if not zl: | |
58 | zl = zipimport.zipimporter(path) |
|
58 | zl = zipimport.zipimporter(path) | |
59 | return zl |
|
59 | return zl | |
60 |
|
60 | |||
61 | try: |
|
61 | try: | |
62 | if modulepolicy in policy.policynoc: |
|
62 | if modulepolicy in policy.policynoc: | |
63 | raise ImportError() |
|
63 | raise ImportError() | |
64 |
|
64 | |||
65 | zl = ziploader('mercurial') |
|
65 | zl = ziploader('mercurial') | |
66 | mod = zl.load_module(name) |
|
66 | mod = zl.load_module(name) | |
67 | # Unlike imp, ziploader doesn't expose module metadata that |
|
67 | # Unlike imp, ziploader doesn't expose module metadata that | |
68 | # indicates the type of module. So just assume what we found |
|
68 | # indicates the type of module. So just assume what we found | |
69 | # is OK (even though it could be a pure Python module). |
|
69 | # is OK (even though it could be a pure Python module). | |
70 | except ImportError: |
|
70 | except ImportError: | |
71 | if modulepolicy == 'c': |
|
71 | if modulepolicy == 'c': | |
72 | raise |
|
72 | raise | |
73 | zl = ziploader('mercurial', 'pure') |
|
73 | zl = ziploader('mercurial', 'pure') | |
74 | mod = zl.load_module(name) |
|
74 | mod = zl.load_module(name) | |
75 |
|
75 | |||
76 | sys.modules[name] = mod |
|
76 | sys.modules[name] = mod | |
77 | return mod |
|
77 | return mod | |
78 |
|
78 | |||
79 | # Unlike the default importer which searches special locations and |
|
79 | # Unlike the default importer which searches special locations and | |
80 | # sys.path, we only look in the directory where "mercurial" was |
|
80 | # sys.path, we only look in the directory where "mercurial" was | |
81 | # imported from. |
|
81 | # imported from. | |
82 |
|
82 | |||
83 | # imp.find_module doesn't support submodules (modules with "."). |
|
83 | # imp.find_module doesn't support submodules (modules with "."). | |
84 | # Instead you have to pass the parent package's __path__ attribute |
|
84 | # Instead you have to pass the parent package's __path__ attribute | |
85 | # as the path argument. |
|
85 | # as the path argument. | |
86 | stem = name.split('.')[-1] |
|
86 | stem = name.split('.')[-1] | |
87 |
|
87 | |||
88 | try: |
|
88 | try: | |
89 | if modulepolicy in policy.policynoc: |
|
89 | if modulepolicy in policy.policynoc: | |
90 | raise ImportError() |
|
90 | raise ImportError() | |
91 |
|
91 | |||
92 | modinfo = imp.find_module(stem, mercurial.__path__) |
|
92 | modinfo = imp.find_module(stem, mercurial.__path__) | |
93 |
|
93 | |||
94 | # The Mercurial installer used to copy files from |
|
94 | # The Mercurial installer used to copy files from | |
95 | # mercurial/pure/*.py to mercurial/*.py. Therefore, it's possible |
|
95 | # mercurial/pure/*.py to mercurial/*.py. Therefore, it's possible | |
96 | # for some installations to have .py files under mercurial/*. |
|
96 | # for some installations to have .py files under mercurial/*. | |
97 | # Loading Python modules when we expected C versions could result |
|
97 | # Loading Python modules when we expected C versions could result | |
98 | # in a) poor performance b) loading a version from a previous |
|
98 | # in a) poor performance b) loading a version from a previous | |
99 | # Mercurial version, potentially leading to incompatibility. Either |
|
99 | # Mercurial version, potentially leading to incompatibility. Either | |
100 | # scenario is bad. So we verify that modules loaded from |
|
100 | # scenario is bad. So we verify that modules loaded from | |
101 | # mercurial/* are C extensions. If the current policy allows the |
|
101 | # mercurial/* are C extensions. If the current policy allows the | |
102 | # loading of .py modules, the module will be re-imported from |
|
102 | # loading of .py modules, the module will be re-imported from | |
103 | # mercurial/pure/* below. |
|
103 | # mercurial/pure/* below. | |
104 | if modinfo[2][2] != imp.C_EXTENSION: |
|
104 | if modinfo[2][2] != imp.C_EXTENSION: | |
105 | raise ImportError('.py version of %s found where C ' |
|
105 | raise ImportError('.py version of %s found where C ' | |
106 | 'version should exist' % name) |
|
106 | 'version should exist' % name) | |
107 |
|
107 | |||
108 | except ImportError: |
|
108 | except ImportError: | |
109 | if modulepolicy == 'c': |
|
109 | if modulepolicy == 'c': | |
110 | raise |
|
110 | raise | |
111 |
|
111 | |||
112 | # Could not load the C extension and pure Python is allowed. So |
|
112 | # Could not load the C extension and pure Python is allowed. So | |
113 | # try to load them. |
|
113 | # try to load them. | |
114 | from . import pure |
|
114 | from . import pure | |
115 | modinfo = imp.find_module(stem, pure.__path__) |
|
115 | modinfo = imp.find_module(stem, pure.__path__) | |
116 | if not modinfo: |
|
116 | if not modinfo: | |
117 | raise ImportError('could not find mercurial module %s' % |
|
117 | raise ImportError('could not find mercurial module %s' % | |
118 | name) |
|
118 | name) | |
119 |
|
119 | |||
120 | mod = imp.load_module(name, *modinfo) |
|
120 | mod = imp.load_module(name, *modinfo) | |
121 | sys.modules[name] = mod |
|
121 | sys.modules[name] = mod | |
122 | return mod |
|
122 | return mod | |
123 |
|
123 | |||
124 | # Python 3 uses a custom module loader that transforms source code between |
|
124 | # Python 3 uses a custom module loader that transforms source code between | |
125 | # source file reading and compilation. This is done by registering a custom |
|
125 | # source file reading and compilation. This is done by registering a custom | |
126 | # finder that changes the spec for Mercurial modules to use a custom loader. |
|
126 | # finder that changes the spec for Mercurial modules to use a custom loader. | |
127 | if sys.version_info[0] >= 3: |
|
127 | if sys.version_info[0] >= 3: | |
128 | from . import pure |
|
128 | from . import pure | |
129 | import importlib |
|
129 | import importlib | |
130 | import io |
|
130 | import io | |
131 | import token |
|
131 | import token | |
132 | import tokenize |
|
132 | import tokenize | |
133 |
|
133 | |||
134 | class hgpathentryfinder(importlib.abc.MetaPathFinder): |
|
134 | class hgpathentryfinder(importlib.abc.MetaPathFinder): | |
135 | """A sys.meta_path finder that uses a custom module loader.""" |
|
135 | """A sys.meta_path finder that uses a custom module loader.""" | |
136 | def find_spec(self, fullname, path, target=None): |
|
136 | def find_spec(self, fullname, path, target=None): | |
137 | # Only handle Mercurial-related modules. |
|
137 | # Only handle Mercurial-related modules. | |
138 | if not fullname.startswith(('mercurial.', 'hgext.', 'hgext3rd.')): |
|
138 | if not fullname.startswith(('mercurial.', 'hgext.', 'hgext3rd.')): | |
139 | return None |
|
139 | return None | |
140 |
|
140 | |||
141 | # This assumes Python 3 doesn't support loading C modules. |
|
141 | # This assumes Python 3 doesn't support loading C modules. | |
142 | if fullname in _dualmodules: |
|
142 | if fullname in _dualmodules: | |
143 | stem = fullname.split('.')[-1] |
|
143 | stem = fullname.split('.')[-1] | |
144 | fullname = 'mercurial.pure.%s' % stem |
|
144 | fullname = 'mercurial.pure.%s' % stem | |
145 | target = pure |
|
145 | target = pure | |
146 | assert len(path) == 1 |
|
146 | assert len(path) == 1 | |
147 | path = [os.path.join(path[0], 'pure')] |
|
147 | path = [os.path.join(path[0], 'pure')] | |
148 |
|
148 | |||
149 | # Try to find the module using other registered finders. |
|
149 | # Try to find the module using other registered finders. | |
150 | spec = None |
|
150 | spec = None | |
151 | for finder in sys.meta_path: |
|
151 | for finder in sys.meta_path: | |
152 | if finder == self: |
|
152 | if finder == self: | |
153 | continue |
|
153 | continue | |
154 |
|
154 | |||
155 | spec = finder.find_spec(fullname, path, target=target) |
|
155 | spec = finder.find_spec(fullname, path, target=target) | |
156 | if spec: |
|
156 | if spec: | |
157 | break |
|
157 | break | |
158 |
|
158 | |||
159 | # This is a Mercurial-related module but we couldn't find it |
|
159 | # This is a Mercurial-related module but we couldn't find it | |
160 | # using the previously-registered finders. This likely means |
|
160 | # using the previously-registered finders. This likely means | |
161 | # the module doesn't exist. |
|
161 | # the module doesn't exist. | |
162 | if not spec: |
|
162 | if not spec: | |
163 | return None |
|
163 | return None | |
164 |
|
164 | |||
165 | if fullname.startswith('mercurial.pure.'): |
|
165 | if fullname.startswith('mercurial.pure.'): | |
166 | spec.name = spec.name.replace('.pure.', '.') |
|
166 | spec.name = spec.name.replace('.pure.', '.') | |
167 |
|
167 | |||
168 | # TODO need to support loaders from alternate specs, like zip |
|
168 | # TODO need to support loaders from alternate specs, like zip | |
169 | # loaders. |
|
169 | # loaders. | |
170 | spec.loader = hgloader(spec.name, spec.origin) |
|
170 | spec.loader = hgloader(spec.name, spec.origin) | |
171 | return spec |
|
171 | return spec | |
172 |
|
172 | |||
173 | def replacetokens(tokens, fullname): |
|
173 | def replacetokens(tokens, fullname): | |
174 | """Transform a stream of tokens from raw to Python 3. |
|
174 | """Transform a stream of tokens from raw to Python 3. | |
175 |
|
175 | |||
176 | It is called by the custom module loading machinery to rewrite |
|
176 | It is called by the custom module loading machinery to rewrite | |
177 | source/tokens between source decoding and compilation. |
|
177 | source/tokens between source decoding and compilation. | |
178 |
|
178 | |||
179 | Returns a generator of possibly rewritten tokens. |
|
179 | Returns a generator of possibly rewritten tokens. | |
180 |
|
180 | |||
181 | The input token list may be mutated as part of processing. However, |
|
181 | The input token list may be mutated as part of processing. However, | |
182 | its changes do not necessarily match the output token stream. |
|
182 | its changes do not necessarily match the output token stream. | |
183 |
|
183 | |||
184 | REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION |
|
184 | REMEMBER TO CHANGE ``BYTECODEHEADER`` WHEN CHANGING THIS FUNCTION | |
185 | OR CACHED FILES WON'T GET INVALIDATED PROPERLY. |
|
185 | OR CACHED FILES WON'T GET INVALIDATED PROPERLY. | |
186 | """ |
|
186 | """ | |
187 | futureimpline = False |
|
187 | futureimpline = False | |
188 | for i, t in enumerate(tokens): |
|
188 | for i, t in enumerate(tokens): | |
189 | # Convert most string literals to byte literals. String literals |
|
189 | # Convert most string literals to byte literals. String literals | |
190 | # in Python 2 are bytes. String literals in Python 3 are unicode. |
|
190 | # in Python 2 are bytes. String literals in Python 3 are unicode. | |
191 | # Most strings in Mercurial are bytes and unicode strings are rare. |
|
191 | # Most strings in Mercurial are bytes and unicode strings are rare. | |
192 | # Rather than rewrite all string literals to use ``b''`` to indicate |
|
192 | # Rather than rewrite all string literals to use ``b''`` to indicate | |
193 | # byte strings, we apply this token transformer to insert the ``b`` |
|
193 | # byte strings, we apply this token transformer to insert the ``b`` | |
194 | # prefix nearly everywhere. |
|
194 | # prefix nearly everywhere. | |
195 | if t.type == token.STRING: |
|
195 | if t.type == token.STRING: | |
196 | s = t.string |
|
196 | s = t.string | |
197 |
|
197 | |||
198 | # Preserve docstrings as string literals. This is inconsistent |
|
198 | # Preserve docstrings as string literals. This is inconsistent | |
199 | # with regular unprefixed strings. However, the |
|
199 | # with regular unprefixed strings. However, the | |
200 | # "from __future__" parsing (which allows a module docstring to |
|
200 | # "from __future__" parsing (which allows a module docstring to | |
201 | # exist before it) doesn't properly handle the docstring if it |
|
201 | # exist before it) doesn't properly handle the docstring if it | |
202 | # is b''' prefixed, leading to a SyntaxError. We leave all |
|
202 | # is b''' prefixed, leading to a SyntaxError. We leave all | |
203 | # docstrings as unprefixed to avoid this. This means Mercurial |
|
203 | # docstrings as unprefixed to avoid this. This means Mercurial | |
204 | # components touching docstrings need to handle unicode, |
|
204 | # components touching docstrings need to handle unicode, | |
205 | # unfortunately. |
|
205 | # unfortunately. | |
206 | if s[0:3] in ("'''", '"""'): |
|
206 | if s[0:3] in ("'''", '"""'): | |
207 | yield t |
|
207 | yield t | |
208 | continue |
|
208 | continue | |
209 |
|
209 | |||
210 | # If the first character isn't a quote, it is likely a string |
|
210 | # If the first character isn't a quote, it is likely a string | |
211 | # prefixing character (such as 'b', 'u', or 'r'. Ignore. |
|
211 | # prefixing character (such as 'b', 'u', or 'r'. Ignore. | |
212 | if s[0] not in ("'", '"'): |
|
212 | if s[0] not in ("'", '"'): | |
213 | yield t |
|
213 | yield t | |
214 | continue |
|
214 | continue | |
215 |
|
215 | |||
216 | # String literal. Prefix to make a b'' string. |
|
216 | # String literal. Prefix to make a b'' string. | |
217 | yield tokenize.TokenInfo(t.type, 'b%s' % s, t.start, t.end, |
|
217 | yield tokenize.TokenInfo(t.type, 'b%s' % s, t.start, t.end, | |
218 | t.line) |
|
218 | t.line) | |
219 | continue |
|
219 | continue | |
220 |
|
220 | |||
221 | # Insert compatibility imports at "from __future__ import" line. |
|
221 | # Insert compatibility imports at "from __future__ import" line. | |
222 | # No '\n' should be added to preserve line numbers. |
|
222 | # No '\n' should be added to preserve line numbers. | |
223 | if (t.type == token.NAME and t.string == 'import' and |
|
223 | if (t.type == token.NAME and t.string == 'import' and | |
224 | all(u.type == token.NAME for u in tokens[i - 2:i]) and |
|
224 | all(u.type == token.NAME for u in tokens[i - 2:i]) and | |
225 | [u.string for u in tokens[i - 2:i]] == ['from', '__future__']): |
|
225 | [u.string for u in tokens[i - 2:i]] == ['from', '__future__']): | |
226 | futureimpline = True |
|
226 | futureimpline = True | |
227 | if t.type == token.NEWLINE and futureimpline: |
|
227 | if t.type == token.NEWLINE and futureimpline: | |
228 | futureimpline = False |
|
228 | futureimpline = False | |
229 | if fullname == 'mercurial.pycompat': |
|
229 | if fullname == 'mercurial.pycompat': | |
230 | yield t |
|
230 | yield t | |
231 | continue |
|
231 | continue | |
232 | r, c = t.start |
|
232 | r, c = t.start | |
233 | l = (b'; from mercurial.pycompat import ' |
|
233 | l = (b'; from mercurial.pycompat import ' | |
234 | b'delattr, getattr, hasattr, setattr, xrange\n') |
|
234 | b'delattr, getattr, hasattr, setattr, xrange\n') | |
235 | for u in tokenize.tokenize(io.BytesIO(l).readline): |
|
235 | for u in tokenize.tokenize(io.BytesIO(l).readline): | |
236 | if u.type in (tokenize.ENCODING, token.ENDMARKER): |
|
236 | if u.type in (tokenize.ENCODING, token.ENDMARKER): | |
237 | continue |
|
237 | continue | |
238 | yield tokenize.TokenInfo(u.type, u.string, |
|
238 | yield tokenize.TokenInfo(u.type, u.string, | |
239 | (r, c + u.start[1]), |
|
239 | (r, c + u.start[1]), | |
240 | (r, c + u.end[1]), |
|
240 | (r, c + u.end[1]), | |
241 | '') |
|
241 | '') | |
242 | continue |
|
242 | continue | |
243 |
|
243 | |||
244 | try: |
|
244 | try: | |
245 | nexttoken = tokens[i + 1] |
|
245 | nexttoken = tokens[i + 1] | |
246 | except IndexError: |
|
246 | except IndexError: | |
247 | nexttoken = None |
|
247 | nexttoken = None | |
248 |
|
248 | |||
249 | try: |
|
249 | try: | |
250 | prevtoken = tokens[i - 1] |
|
250 | prevtoken = tokens[i - 1] | |
251 | except IndexError: |
|
251 | except IndexError: | |
252 | prevtoken = None |
|
252 | prevtoken = None | |
253 |
|
253 | |||
254 | # This looks like a function call. |
|
254 | # This looks like a function call. | |
255 | if (t.type == token.NAME and nexttoken and |
|
255 | if (t.type == token.NAME and nexttoken and | |
256 | nexttoken.type == token.OP and nexttoken.string == '('): |
|
256 | nexttoken.type == token.OP and nexttoken.string == '('): | |
257 | fn = t.string |
|
257 | fn = t.string | |
258 |
|
258 | |||
259 | # *attr() builtins don't accept byte strings to 2nd argument. |
|
259 | # *attr() builtins don't accept byte strings to 2nd argument. | |
260 | # Rewrite the token to include the unicode literal prefix so |
|
260 | # Rewrite the token to include the unicode literal prefix so | |
261 | # the string transformer above doesn't add the byte prefix. |
|
261 | # the string transformer above doesn't add the byte prefix. | |
262 | if fn in ('getattr', 'setattr', 'hasattr', 'safehasattr'): |
|
262 | if fn in ('getattr', 'setattr', 'hasattr', 'safehasattr'): | |
263 | try: |
|
263 | try: | |
264 | # (NAME, 'getattr') |
|
264 | # (NAME, 'getattr') | |
265 | # (OP, '(') |
|
265 | # (OP, '(') | |
266 | # (NAME, 'foo') |
|
266 | # (NAME, 'foo') | |
267 | # (OP, ',') |
|
267 | # (OP, ',') | |
268 | # (NAME|STRING, foo) |
|
268 | # (NAME|STRING, foo) | |
269 | st = tokens[i + 4] |
|
269 | st = tokens[i + 4] | |
270 | if (st.type == token.STRING and |
|
270 | if (st.type == token.STRING and | |
271 | st.string[0] in ("'", '"')): |
|
271 | st.string[0] in ("'", '"')): | |
272 | rt = tokenize.TokenInfo(st.type, 'u%s' % st.string, |
|
272 | rt = tokenize.TokenInfo(st.type, 'u%s' % st.string, | |
273 | st.start, st.end, st.line) |
|
273 | st.start, st.end, st.line) | |
274 | tokens[i + 4] = rt |
|
274 | tokens[i + 4] = rt | |
275 | except IndexError: |
|
275 | except IndexError: | |
276 | pass |
|
276 | pass | |
277 |
|
277 | |||
278 | # .encode() and .decode() on str/bytes/unicode don't accept |
|
278 | # .encode() and .decode() on str/bytes/unicode don't accept | |
279 | # byte strings on Python 3. Rewrite the token to include the |
|
279 | # byte strings on Python 3. Rewrite the token to include the | |
280 | # unicode literal prefix so the string transformer above doesn't |
|
280 | # unicode literal prefix so the string transformer above doesn't | |
281 | # add the byte prefix. The loop helps in handling multiple |
|
281 | # add the byte prefix. The loop helps in handling multiple | |
282 | # arguments. |
|
282 | # arguments. | |
283 | if (fn in ('encode', 'decode') and |
|
283 | if (fn in ('encode', 'decode') and | |
284 | prevtoken.type == token.OP and prevtoken.string == '.'): |
|
284 | prevtoken.type == token.OP and prevtoken.string == '.'): | |
285 | # (OP, '.') |
|
285 | # (OP, '.') | |
286 | # (NAME, 'encode') |
|
286 | # (NAME, 'encode') | |
287 | # (OP, '(') |
|
287 | # (OP, '(') | |
288 | # [(VARIABLE, encoding)] |
|
288 | # [(VARIABLE, encoding)] | |
289 | # [(OP, '.')] |
|
289 | # [(OP, '.')] | |
290 | # [(VARIABLE, encoding)] |
|
290 | # [(VARIABLE, encoding)] | |
291 | # [(OP, ',')] |
|
291 | # [(OP, ',')] | |
292 | # (STRING, 'utf-8') |
|
292 | # (STRING, 'utf-8') | |
293 | # (OP, ')') |
|
293 | # (OP, ')') | |
294 | j = i |
|
294 | j = i | |
295 | try: |
|
295 | try: | |
296 | while (tokens[j + 1].string in ('(', ',', '.')): |
|
296 | while (tokens[j + 1].string in ('(', ',', '.')): | |
297 | st = tokens[j + 2] |
|
297 | st = tokens[j + 2] | |
298 | if (st.type == token.STRING and |
|
298 | if (st.type == token.STRING and | |
299 | st.string[0] in ("'", '"')): |
|
299 | st.string[0] in ("'", '"')): | |
300 | rt = tokenize.TokenInfo(st.type, |
|
300 | rt = tokenize.TokenInfo(st.type, | |
301 | 'u%s' % st.string, |
|
301 | 'u%s' % st.string, | |
302 | st.start, st.end, st.line) |
|
302 | st.start, st.end, st.line) | |
303 | tokens[j + 2] = rt |
|
303 | tokens[j + 2] = rt | |
304 | j = j + 2 |
|
304 | j = j + 2 | |
305 | except IndexError: |
|
305 | except IndexError: | |
306 | pass |
|
306 | pass | |
307 |
|
307 | |||
|
308 | # It changes iteritems to items as iteritems is not | |||
|
309 | # present in Python 3 world. | |||
|
310 | if fn == 'iteritems': | |||
|
311 | yield tokenize.TokenInfo(t.type, 'items', | |||
|
312 | t.start, t.end, t.line) | |||
|
313 | continue | |||
|
314 | ||||
308 | # Emit unmodified token. |
|
315 | # Emit unmodified token. | |
309 | yield t |
|
316 | yield t | |
310 |
|
317 | |||
311 | # Header to add to bytecode files. This MUST be changed when |
|
318 | # Header to add to bytecode files. This MUST be changed when | |
312 | # ``replacetoken`` or any mechanism that changes semantics of module |
|
319 | # ``replacetoken`` or any mechanism that changes semantics of module | |
313 | # loading is changed. Otherwise cached bytecode may get loaded without |
|
320 | # loading is changed. Otherwise cached bytecode may get loaded without | |
314 | # the new transformation mechanisms applied. |
|
321 | # the new transformation mechanisms applied. | |
315 |
BYTECODEHEADER = b'HG\x00\x0 |
|
322 | BYTECODEHEADER = b'HG\x00\x04' | |
316 |
|
323 | |||
317 | class hgloader(importlib.machinery.SourceFileLoader): |
|
324 | class hgloader(importlib.machinery.SourceFileLoader): | |
318 | """Custom module loader that transforms source code. |
|
325 | """Custom module loader that transforms source code. | |
319 |
|
326 | |||
320 | When the source code is converted to a code object, we transform |
|
327 | When the source code is converted to a code object, we transform | |
321 | certain patterns to be Python 3 compatible. This allows us to write code |
|
328 | certain patterns to be Python 3 compatible. This allows us to write code | |
322 | that is natively Python 2 and compatible with Python 3 without |
|
329 | that is natively Python 2 and compatible with Python 3 without | |
323 | making the code excessively ugly. |
|
330 | making the code excessively ugly. | |
324 |
|
331 | |||
325 | We do this by transforming the token stream between parse and compile. |
|
332 | We do this by transforming the token stream between parse and compile. | |
326 |
|
333 | |||
327 | Implementing transformations invalidates caching assumptions made |
|
334 | Implementing transformations invalidates caching assumptions made | |
328 | by the built-in importer. The built-in importer stores a header on |
|
335 | by the built-in importer. The built-in importer stores a header on | |
329 | saved bytecode files indicating the Python/bytecode version. If the |
|
336 | saved bytecode files indicating the Python/bytecode version. If the | |
330 | version changes, the cached bytecode is ignored. The Mercurial |
|
337 | version changes, the cached bytecode is ignored. The Mercurial | |
331 | transformations could change at any time. This means we need to check |
|
338 | transformations could change at any time. This means we need to check | |
332 | that cached bytecode was generated with the current transformation |
|
339 | that cached bytecode was generated with the current transformation | |
333 | code or there could be a mismatch between cached bytecode and what |
|
340 | code or there could be a mismatch between cached bytecode and what | |
334 | would be generated from this class. |
|
341 | would be generated from this class. | |
335 |
|
342 | |||
336 | We supplement the bytecode caching layer by wrapping ``get_data`` |
|
343 | We supplement the bytecode caching layer by wrapping ``get_data`` | |
337 | and ``set_data``. These functions are called when the |
|
344 | and ``set_data``. These functions are called when the | |
338 | ``SourceFileLoader`` retrieves and saves bytecode cache files, |
|
345 | ``SourceFileLoader`` retrieves and saves bytecode cache files, | |
339 | respectively. We simply add an additional header on the file. As |
|
346 | respectively. We simply add an additional header on the file. As | |
340 | long as the version in this file is changed when semantics change, |
|
347 | long as the version in this file is changed when semantics change, | |
341 | cached bytecode should be invalidated when transformations change. |
|
348 | cached bytecode should be invalidated when transformations change. | |
342 |
|
349 | |||
343 | The added header has the form ``HG<VERSION>``. That is a literal |
|
350 | The added header has the form ``HG<VERSION>``. That is a literal | |
344 | ``HG`` with 2 binary bytes indicating the transformation version. |
|
351 | ``HG`` with 2 binary bytes indicating the transformation version. | |
345 | """ |
|
352 | """ | |
346 | def get_data(self, path): |
|
353 | def get_data(self, path): | |
347 | data = super(hgloader, self).get_data(path) |
|
354 | data = super(hgloader, self).get_data(path) | |
348 |
|
355 | |||
349 | if not path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): |
|
356 | if not path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): | |
350 | return data |
|
357 | return data | |
351 |
|
358 | |||
352 | # There should be a header indicating the Mercurial transformation |
|
359 | # There should be a header indicating the Mercurial transformation | |
353 | # version. If it doesn't exist or doesn't match the current version, |
|
360 | # version. If it doesn't exist or doesn't match the current version, | |
354 | # we raise an OSError because that is what |
|
361 | # we raise an OSError because that is what | |
355 | # ``SourceFileLoader.get_code()`` expects when loading bytecode |
|
362 | # ``SourceFileLoader.get_code()`` expects when loading bytecode | |
356 | # paths to indicate the cached file is "bad." |
|
363 | # paths to indicate the cached file is "bad." | |
357 | if data[0:2] != b'HG': |
|
364 | if data[0:2] != b'HG': | |
358 | raise OSError('no hg header') |
|
365 | raise OSError('no hg header') | |
359 | if data[0:4] != BYTECODEHEADER: |
|
366 | if data[0:4] != BYTECODEHEADER: | |
360 | raise OSError('hg header version mismatch') |
|
367 | raise OSError('hg header version mismatch') | |
361 |
|
368 | |||
362 | return data[4:] |
|
369 | return data[4:] | |
363 |
|
370 | |||
364 | def set_data(self, path, data, *args, **kwargs): |
|
371 | def set_data(self, path, data, *args, **kwargs): | |
365 | if path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): |
|
372 | if path.endswith(tuple(importlib.machinery.BYTECODE_SUFFIXES)): | |
366 | data = BYTECODEHEADER + data |
|
373 | data = BYTECODEHEADER + data | |
367 |
|
374 | |||
368 | return super(hgloader, self).set_data(path, data, *args, **kwargs) |
|
375 | return super(hgloader, self).set_data(path, data, *args, **kwargs) | |
369 |
|
376 | |||
370 | def source_to_code(self, data, path): |
|
377 | def source_to_code(self, data, path): | |
371 | """Perform token transformation before compilation.""" |
|
378 | """Perform token transformation before compilation.""" | |
372 | buf = io.BytesIO(data) |
|
379 | buf = io.BytesIO(data) | |
373 | tokens = tokenize.tokenize(buf.readline) |
|
380 | tokens = tokenize.tokenize(buf.readline) | |
374 | data = tokenize.untokenize(replacetokens(list(tokens), self.name)) |
|
381 | data = tokenize.untokenize(replacetokens(list(tokens), self.name)) | |
375 | # Python's built-in importer strips frames from exceptions raised |
|
382 | # Python's built-in importer strips frames from exceptions raised | |
376 | # for this code. Unfortunately, that mechanism isn't extensible |
|
383 | # for this code. Unfortunately, that mechanism isn't extensible | |
377 | # and our frame will be blamed for the import failure. There |
|
384 | # and our frame will be blamed for the import failure. There | |
378 | # are extremely hacky ways to do frame stripping. We haven't |
|
385 | # are extremely hacky ways to do frame stripping. We haven't | |
379 | # implemented them because they are very ugly. |
|
386 | # implemented them because they are very ugly. | |
380 | return super(hgloader, self).source_to_code(data, path) |
|
387 | return super(hgloader, self).source_to_code(data, path) | |
381 |
|
388 | |||
382 | # We automagically register our custom importer as a side-effect of loading. |
|
389 | # We automagically register our custom importer as a side-effect of loading. | |
383 | # This is necessary to ensure that any entry points are able to import |
|
390 | # This is necessary to ensure that any entry points are able to import | |
384 | # mercurial.* modules without having to perform this registration themselves. |
|
391 | # mercurial.* modules without having to perform this registration themselves. | |
385 | if sys.version_info[0] >= 3: |
|
392 | if sys.version_info[0] >= 3: | |
386 | _importercls = hgpathentryfinder |
|
393 | _importercls = hgpathentryfinder | |
387 | else: |
|
394 | else: | |
388 | _importercls = hgimporter |
|
395 | _importercls = hgimporter | |
389 | if not any(isinstance(x, _importercls) for x in sys.meta_path): |
|
396 | if not any(isinstance(x, _importercls) for x in sys.meta_path): | |
390 | # meta_path is used before any implicit finders and before sys.path. |
|
397 | # meta_path is used before any implicit finders and before sys.path. | |
391 | sys.meta_path.insert(0, _importercls()) |
|
398 | sys.meta_path.insert(0, _importercls()) |
General Comments 0
You need to be logged in to leave comments.
Login now