##// END OF EJS Templates
convert: remove old ElementTree import cruft from darcs...
Ian Moody -
r50265:1572f790 stable
parent child Browse files
Show More
@@ -1,770 +1,771 b''
1 #!/usr/bin/env python3
1 #!/usr/bin/env python3
2
2
3
3
4 import ast
4 import ast
5 import collections
5 import collections
6 import io
6 import io
7 import os
7 import os
8 import sys
8 import sys
9
9
10 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
10 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
11 # to work when run from a virtualenv. The modules were chosen empirically
11 # to work when run from a virtualenv. The modules were chosen empirically
12 # so that the return value matches the return value without virtualenv.
12 # so that the return value matches the return value without virtualenv.
13 if True: # disable lexical sorting checks
13 if True: # disable lexical sorting checks
14 try:
14 try:
15 import BaseHTTPServer as basehttpserver
15 import BaseHTTPServer as basehttpserver
16 except ImportError:
16 except ImportError:
17 basehttpserver = None
17 basehttpserver = None
18 import zlib
18 import zlib
19
19
20 import testparseutil
20 import testparseutil
21
21
22 # Allow list of modules that symbols can be directly imported from.
22 # Allow list of modules that symbols can be directly imported from.
23 allowsymbolimports = (
23 allowsymbolimports = (
24 '__future__',
24 '__future__',
25 'breezy',
25 'breezy',
26 'concurrent',
26 'concurrent',
27 'hgclient',
27 'hgclient',
28 'mercurial',
28 'mercurial',
29 'mercurial.hgweb.common',
29 'mercurial.hgweb.common',
30 'mercurial.hgweb.request',
30 'mercurial.hgweb.request',
31 'mercurial.i18n',
31 'mercurial.i18n',
32 'mercurial.interfaces',
32 'mercurial.interfaces',
33 'mercurial.node',
33 'mercurial.node',
34 'mercurial.pycompat',
34 'mercurial.pycompat',
35 # for revlog to re-export constant to extensions
35 # for revlog to re-export constant to extensions
36 'mercurial.revlogutils.constants',
36 'mercurial.revlogutils.constants',
37 'mercurial.revlogutils.flagutil',
37 'mercurial.revlogutils.flagutil',
38 # for cffi modules to re-export pure functions
38 # for cffi modules to re-export pure functions
39 'mercurial.pure.base85',
39 'mercurial.pure.base85',
40 'mercurial.pure.bdiff',
40 'mercurial.pure.bdiff',
41 'mercurial.pure.mpatch',
41 'mercurial.pure.mpatch',
42 'mercurial.pure.osutil',
42 'mercurial.pure.osutil',
43 'mercurial.pure.parsers',
43 'mercurial.pure.parsers',
44 # third-party imports should be directly imported
44 # third-party imports should be directly imported
45 'mercurial.thirdparty',
45 'mercurial.thirdparty',
46 'mercurial.thirdparty.attr',
46 'mercurial.thirdparty.attr',
47 'mercurial.thirdparty.zope',
47 'mercurial.thirdparty.zope',
48 'mercurial.thirdparty.zope.interface',
48 'mercurial.thirdparty.zope.interface',
49 'typing',
49 'typing',
50 'xml.etree.ElementTree',
50 )
51 )
51
52
52 # Allow list of symbols that can be directly imported.
53 # Allow list of symbols that can be directly imported.
53 directsymbols = ('demandimport',)
54 directsymbols = ('demandimport',)
54
55
55 # Modules that must be aliased because they are commonly confused with
56 # Modules that must be aliased because they are commonly confused with
56 # common variables and can create aliasing and readability issues.
57 # common variables and can create aliasing and readability issues.
57 requirealias = {
58 requirealias = {
58 'ui': 'uimod',
59 'ui': 'uimod',
59 }
60 }
60
61
61
62
62 def walklocal(root):
63 def walklocal(root):
63 """Recursively yield all descendant nodes but not in a different scope"""
64 """Recursively yield all descendant nodes but not in a different scope"""
64 todo = collections.deque(ast.iter_child_nodes(root))
65 todo = collections.deque(ast.iter_child_nodes(root))
65 yield root, False
66 yield root, False
66 while todo:
67 while todo:
67 node = todo.popleft()
68 node = todo.popleft()
68 newscope = isinstance(node, ast.FunctionDef)
69 newscope = isinstance(node, ast.FunctionDef)
69 if not newscope:
70 if not newscope:
70 todo.extend(ast.iter_child_nodes(node))
71 todo.extend(ast.iter_child_nodes(node))
71 yield node, newscope
72 yield node, newscope
72
73
73
74
74 def dotted_name_of_path(path):
75 def dotted_name_of_path(path):
75 """Given a relative path to a source file, return its dotted module name.
76 """Given a relative path to a source file, return its dotted module name.
76
77
77 >>> dotted_name_of_path('mercurial/error.py')
78 >>> dotted_name_of_path('mercurial/error.py')
78 'mercurial.error'
79 'mercurial.error'
79 >>> dotted_name_of_path('zlibmodule.so')
80 >>> dotted_name_of_path('zlibmodule.so')
80 'zlib'
81 'zlib'
81 """
82 """
82 parts = path.replace(os.sep, '/').split('/')
83 parts = path.replace(os.sep, '/').split('/')
83 parts[-1] = parts[-1].split('.', 1)[0] # remove .py and .so and .ARCH.so
84 parts[-1] = parts[-1].split('.', 1)[0] # remove .py and .so and .ARCH.so
84 if parts[-1].endswith('module'):
85 if parts[-1].endswith('module'):
85 parts[-1] = parts[-1][:-6]
86 parts[-1] = parts[-1][:-6]
86 return '.'.join(parts)
87 return '.'.join(parts)
87
88
88
89
89 def fromlocalfunc(modulename, localmods):
90 def fromlocalfunc(modulename, localmods):
90 """Get a function to examine which locally defined module the
91 """Get a function to examine which locally defined module the
91 target source imports via a specified name.
92 target source imports via a specified name.
92
93
93 `modulename` is an `dotted_name_of_path()`-ed source file path,
94 `modulename` is an `dotted_name_of_path()`-ed source file path,
94 which may have `.__init__` at the end of it, of the target source.
95 which may have `.__init__` at the end of it, of the target source.
95
96
96 `localmods` is a set of absolute `dotted_name_of_path()`-ed source file
97 `localmods` is a set of absolute `dotted_name_of_path()`-ed source file
97 paths of locally defined (= Mercurial specific) modules.
98 paths of locally defined (= Mercurial specific) modules.
98
99
99 This function assumes that module names not existing in
100 This function assumes that module names not existing in
100 `localmods` are from the Python standard library.
101 `localmods` are from the Python standard library.
101
102
102 This function returns the function, which takes `name` argument,
103 This function returns the function, which takes `name` argument,
103 and returns `(absname, dottedpath, hassubmod)` tuple if `name`
104 and returns `(absname, dottedpath, hassubmod)` tuple if `name`
104 matches against locally defined module. Otherwise, it returns
105 matches against locally defined module. Otherwise, it returns
105 False.
106 False.
106
107
107 It is assumed that `name` doesn't have `.__init__`.
108 It is assumed that `name` doesn't have `.__init__`.
108
109
109 `absname` is an absolute module name of specified `name`
110 `absname` is an absolute module name of specified `name`
110 (e.g. "hgext.convert"). This can be used to compose prefix for sub
111 (e.g. "hgext.convert"). This can be used to compose prefix for sub
111 modules or so.
112 modules or so.
112
113
113 `dottedpath` is a `dotted_name_of_path()`-ed source file path
114 `dottedpath` is a `dotted_name_of_path()`-ed source file path
114 (e.g. "hgext.convert.__init__") of `name`. This is used to look
115 (e.g. "hgext.convert.__init__") of `name`. This is used to look
115 module up in `localmods` again.
116 module up in `localmods` again.
116
117
117 `hassubmod` is whether it may have sub modules under it (for
118 `hassubmod` is whether it may have sub modules under it (for
118 convenient, even though this is also equivalent to "absname !=
119 convenient, even though this is also equivalent to "absname !=
119 dottednpath")
120 dottednpath")
120
121
121 >>> localmods = {'foo.__init__', 'foo.foo1',
122 >>> localmods = {'foo.__init__', 'foo.foo1',
122 ... 'foo.bar.__init__', 'foo.bar.bar1',
123 ... 'foo.bar.__init__', 'foo.bar.bar1',
123 ... 'baz.__init__', 'baz.baz1'}
124 ... 'baz.__init__', 'baz.baz1'}
124 >>> fromlocal = fromlocalfunc('foo.xxx', localmods)
125 >>> fromlocal = fromlocalfunc('foo.xxx', localmods)
125 >>> # relative
126 >>> # relative
126 >>> fromlocal('foo1')
127 >>> fromlocal('foo1')
127 ('foo.foo1', 'foo.foo1', False)
128 ('foo.foo1', 'foo.foo1', False)
128 >>> fromlocal('bar')
129 >>> fromlocal('bar')
129 ('foo.bar', 'foo.bar.__init__', True)
130 ('foo.bar', 'foo.bar.__init__', True)
130 >>> fromlocal('bar.bar1')
131 >>> fromlocal('bar.bar1')
131 ('foo.bar.bar1', 'foo.bar.bar1', False)
132 ('foo.bar.bar1', 'foo.bar.bar1', False)
132 >>> # absolute
133 >>> # absolute
133 >>> fromlocal('baz')
134 >>> fromlocal('baz')
134 ('baz', 'baz.__init__', True)
135 ('baz', 'baz.__init__', True)
135 >>> fromlocal('baz.baz1')
136 >>> fromlocal('baz.baz1')
136 ('baz.baz1', 'baz.baz1', False)
137 ('baz.baz1', 'baz.baz1', False)
137 >>> # unknown = maybe standard library
138 >>> # unknown = maybe standard library
138 >>> fromlocal('os')
139 >>> fromlocal('os')
139 False
140 False
140 >>> fromlocal(None, 1)
141 >>> fromlocal(None, 1)
141 ('foo', 'foo.__init__', True)
142 ('foo', 'foo.__init__', True)
142 >>> fromlocal('foo1', 1)
143 >>> fromlocal('foo1', 1)
143 ('foo.foo1', 'foo.foo1', False)
144 ('foo.foo1', 'foo.foo1', False)
144 >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
145 >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
145 >>> fromlocal2(None, 2)
146 >>> fromlocal2(None, 2)
146 ('foo', 'foo.__init__', True)
147 ('foo', 'foo.__init__', True)
147 >>> fromlocal2('bar2', 1)
148 >>> fromlocal2('bar2', 1)
148 False
149 False
149 >>> fromlocal2('bar', 2)
150 >>> fromlocal2('bar', 2)
150 ('foo.bar', 'foo.bar.__init__', True)
151 ('foo.bar', 'foo.bar.__init__', True)
151 """
152 """
152 if not isinstance(modulename, str):
153 if not isinstance(modulename, str):
153 modulename = modulename.decode('ascii')
154 modulename = modulename.decode('ascii')
154 prefix = '.'.join(modulename.split('.')[:-1])
155 prefix = '.'.join(modulename.split('.')[:-1])
155 if prefix:
156 if prefix:
156 prefix += '.'
157 prefix += '.'
157
158
158 def fromlocal(name, level=0):
159 def fromlocal(name, level=0):
159 # name is false value when relative imports are used.
160 # name is false value when relative imports are used.
160 if not name:
161 if not name:
161 # If relative imports are used, level must not be absolute.
162 # If relative imports are used, level must not be absolute.
162 assert level > 0
163 assert level > 0
163 candidates = ['.'.join(modulename.split('.')[:-level])]
164 candidates = ['.'.join(modulename.split('.')[:-level])]
164 else:
165 else:
165 if not level:
166 if not level:
166 # Check relative name first.
167 # Check relative name first.
167 candidates = [prefix + name, name]
168 candidates = [prefix + name, name]
168 else:
169 else:
169 candidates = [
170 candidates = [
170 '.'.join(modulename.split('.')[:-level]) + '.' + name
171 '.'.join(modulename.split('.')[:-level]) + '.' + name
171 ]
172 ]
172
173
173 for n in candidates:
174 for n in candidates:
174 if n in localmods:
175 if n in localmods:
175 return (n, n, False)
176 return (n, n, False)
176 dottedpath = n + '.__init__'
177 dottedpath = n + '.__init__'
177 if dottedpath in localmods:
178 if dottedpath in localmods:
178 return (n, dottedpath, True)
179 return (n, dottedpath, True)
179 return False
180 return False
180
181
181 return fromlocal
182 return fromlocal
182
183
183
184
184 def populateextmods(localmods):
185 def populateextmods(localmods):
185 """Populate C extension modules based on pure modules"""
186 """Populate C extension modules based on pure modules"""
186 newlocalmods = set(localmods)
187 newlocalmods = set(localmods)
187 for n in localmods:
188 for n in localmods:
188 if n.startswith('mercurial.pure.'):
189 if n.startswith('mercurial.pure.'):
189 m = n[len('mercurial.pure.') :]
190 m = n[len('mercurial.pure.') :]
190 newlocalmods.add('mercurial.cext.' + m)
191 newlocalmods.add('mercurial.cext.' + m)
191 newlocalmods.add('mercurial.cffi._' + m)
192 newlocalmods.add('mercurial.cffi._' + m)
192 return newlocalmods
193 return newlocalmods
193
194
194
195
195 def list_stdlib_modules():
196 def list_stdlib_modules():
196 """List the modules present in the stdlib.
197 """List the modules present in the stdlib.
197
198
198 >>> py3 = sys.version_info[0] >= 3
199 >>> py3 = sys.version_info[0] >= 3
199 >>> mods = set(list_stdlib_modules())
200 >>> mods = set(list_stdlib_modules())
200 >>> 'BaseHTTPServer' in mods or py3
201 >>> 'BaseHTTPServer' in mods or py3
201 True
202 True
202
203
203 os.path isn't really a module, so it's missing:
204 os.path isn't really a module, so it's missing:
204
205
205 >>> 'os.path' in mods
206 >>> 'os.path' in mods
206 False
207 False
207
208
208 sys requires special treatment, because it's baked into the
209 sys requires special treatment, because it's baked into the
209 interpreter, but it should still appear:
210 interpreter, but it should still appear:
210
211
211 >>> 'sys' in mods
212 >>> 'sys' in mods
212 True
213 True
213
214
214 >>> 'collections' in mods
215 >>> 'collections' in mods
215 True
216 True
216
217
217 >>> 'cStringIO' in mods or py3
218 >>> 'cStringIO' in mods or py3
218 True
219 True
219
220
220 >>> 'cffi' in mods
221 >>> 'cffi' in mods
221 True
222 True
222 """
223 """
223 for m in sys.builtin_module_names:
224 for m in sys.builtin_module_names:
224 yield m
225 yield m
225 # These modules only exist on windows, but we should always
226 # These modules only exist on windows, but we should always
226 # consider them stdlib.
227 # consider them stdlib.
227 for m in ['msvcrt', '_winreg']:
228 for m in ['msvcrt', '_winreg']:
228 yield m
229 yield m
229 yield '__builtin__'
230 yield '__builtin__'
230 yield 'builtins' # python3 only
231 yield 'builtins' # python3 only
231 yield 'importlib.abc' # python3 only
232 yield 'importlib.abc' # python3 only
232 yield 'importlib.machinery' # python3 only
233 yield 'importlib.machinery' # python3 only
233 yield 'importlib.util' # python3 only
234 yield 'importlib.util' # python3 only
234 for m in 'fcntl', 'grp', 'pwd', 'termios': # Unix only
235 for m in 'fcntl', 'grp', 'pwd', 'termios': # Unix only
235 yield m
236 yield m
236 for m in 'cPickle', 'datetime': # in Python (not C) on PyPy
237 for m in 'cPickle', 'datetime': # in Python (not C) on PyPy
237 yield m
238 yield m
238 for m in ['cffi']:
239 for m in ['cffi']:
239 yield m
240 yield m
240 stdlib_prefixes = {sys.prefix, sys.exec_prefix}
241 stdlib_prefixes = {sys.prefix, sys.exec_prefix}
241 # We need to supplement the list of prefixes for the search to work
242 # We need to supplement the list of prefixes for the search to work
242 # when run from within a virtualenv.
243 # when run from within a virtualenv.
243 for mod in (basehttpserver, zlib):
244 for mod in (basehttpserver, zlib):
244 if mod is None:
245 if mod is None:
245 continue
246 continue
246 try:
247 try:
247 # Not all module objects have a __file__ attribute.
248 # Not all module objects have a __file__ attribute.
248 filename = mod.__file__
249 filename = mod.__file__
249 except AttributeError:
250 except AttributeError:
250 continue
251 continue
251 dirname = os.path.dirname(filename)
252 dirname = os.path.dirname(filename)
252 for prefix in stdlib_prefixes:
253 for prefix in stdlib_prefixes:
253 if dirname.startswith(prefix):
254 if dirname.startswith(prefix):
254 # Then this directory is redundant.
255 # Then this directory is redundant.
255 break
256 break
256 else:
257 else:
257 stdlib_prefixes.add(dirname)
258 stdlib_prefixes.add(dirname)
258 sourceroot = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
259 sourceroot = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
259 for libpath in sys.path:
260 for libpath in sys.path:
260 # We want to walk everything in sys.path that starts with something in
261 # We want to walk everything in sys.path that starts with something in
261 # stdlib_prefixes, but not directories from the hg sources.
262 # stdlib_prefixes, but not directories from the hg sources.
262 if os.path.abspath(libpath).startswith(sourceroot) or not any(
263 if os.path.abspath(libpath).startswith(sourceroot) or not any(
263 libpath.startswith(p) for p in stdlib_prefixes
264 libpath.startswith(p) for p in stdlib_prefixes
264 ):
265 ):
265 continue
266 continue
266 for top, dirs, files in os.walk(libpath):
267 for top, dirs, files in os.walk(libpath):
267 if 'dist-packages' in top.split(os.path.sep):
268 if 'dist-packages' in top.split(os.path.sep):
268 continue
269 continue
269 for i, d in reversed(list(enumerate(dirs))):
270 for i, d in reversed(list(enumerate(dirs))):
270 if (
271 if (
271 not os.path.exists(os.path.join(top, d, '__init__.py'))
272 not os.path.exists(os.path.join(top, d, '__init__.py'))
272 or top == libpath
273 or top == libpath
273 and d in ('hgdemandimport', 'hgext', 'mercurial')
274 and d in ('hgdemandimport', 'hgext', 'mercurial')
274 ):
275 ):
275 del dirs[i]
276 del dirs[i]
276 for name in files:
277 for name in files:
277 if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')):
278 if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')):
278 continue
279 continue
279 if name.startswith('__init__.py'):
280 if name.startswith('__init__.py'):
280 full_path = top
281 full_path = top
281 else:
282 else:
282 full_path = os.path.join(top, name)
283 full_path = os.path.join(top, name)
283 rel_path = full_path[len(libpath) + 1 :]
284 rel_path = full_path[len(libpath) + 1 :]
284 mod = dotted_name_of_path(rel_path)
285 mod = dotted_name_of_path(rel_path)
285 yield mod
286 yield mod
286
287
287
288
288 stdlib_modules = set(list_stdlib_modules())
289 stdlib_modules = set(list_stdlib_modules())
289
290
290
291
291 def imported_modules(source, modulename, f, localmods, ignore_nested=False):
292 def imported_modules(source, modulename, f, localmods, ignore_nested=False):
292 """Given the source of a file as a string, yield the names
293 """Given the source of a file as a string, yield the names
293 imported by that file.
294 imported by that file.
294
295
295 Args:
296 Args:
296 source: The python source to examine as a string.
297 source: The python source to examine as a string.
297 modulename: of specified python source (may have `__init__`)
298 modulename: of specified python source (may have `__init__`)
298 localmods: set of locally defined module names (may have `__init__`)
299 localmods: set of locally defined module names (may have `__init__`)
299 ignore_nested: If true, import statements that do not start in
300 ignore_nested: If true, import statements that do not start in
300 column zero will be ignored.
301 column zero will be ignored.
301
302
302 Returns:
303 Returns:
303 A list of absolute module names imported by the given source.
304 A list of absolute module names imported by the given source.
304
305
305 >>> f = 'foo/xxx.py'
306 >>> f = 'foo/xxx.py'
306 >>> modulename = 'foo.xxx'
307 >>> modulename = 'foo.xxx'
307 >>> localmods = {'foo.__init__': True,
308 >>> localmods = {'foo.__init__': True,
308 ... 'foo.foo1': True, 'foo.foo2': True,
309 ... 'foo.foo1': True, 'foo.foo2': True,
309 ... 'foo.bar.__init__': True, 'foo.bar.bar1': True,
310 ... 'foo.bar.__init__': True, 'foo.bar.bar1': True,
310 ... 'baz.__init__': True, 'baz.baz1': True }
311 ... 'baz.__init__': True, 'baz.baz1': True }
311 >>> # standard library (= not locally defined ones)
312 >>> # standard library (= not locally defined ones)
312 >>> sorted(imported_modules(
313 >>> sorted(imported_modules(
313 ... 'from stdlib1 import foo, bar; import stdlib2',
314 ... 'from stdlib1 import foo, bar; import stdlib2',
314 ... modulename, f, localmods))
315 ... modulename, f, localmods))
315 []
316 []
316 >>> # relative importing
317 >>> # relative importing
317 >>> sorted(imported_modules(
318 >>> sorted(imported_modules(
318 ... 'import foo1; from bar import bar1',
319 ... 'import foo1; from bar import bar1',
319 ... modulename, f, localmods))
320 ... modulename, f, localmods))
320 ['foo.bar.bar1', 'foo.foo1']
321 ['foo.bar.bar1', 'foo.foo1']
321 >>> sorted(imported_modules(
322 >>> sorted(imported_modules(
322 ... 'from bar.bar1 import name1, name2, name3',
323 ... 'from bar.bar1 import name1, name2, name3',
323 ... modulename, f, localmods))
324 ... modulename, f, localmods))
324 ['foo.bar.bar1']
325 ['foo.bar.bar1']
325 >>> # absolute importing
326 >>> # absolute importing
326 >>> sorted(imported_modules(
327 >>> sorted(imported_modules(
327 ... 'from baz import baz1, name1',
328 ... 'from baz import baz1, name1',
328 ... modulename, f, localmods))
329 ... modulename, f, localmods))
329 ['baz.__init__', 'baz.baz1']
330 ['baz.__init__', 'baz.baz1']
330 >>> # mixed importing, even though it shouldn't be recommended
331 >>> # mixed importing, even though it shouldn't be recommended
331 >>> sorted(imported_modules(
332 >>> sorted(imported_modules(
332 ... 'import stdlib, foo1, baz',
333 ... 'import stdlib, foo1, baz',
333 ... modulename, f, localmods))
334 ... modulename, f, localmods))
334 ['baz.__init__', 'foo.foo1']
335 ['baz.__init__', 'foo.foo1']
335 >>> # ignore_nested
336 >>> # ignore_nested
336 >>> sorted(imported_modules(
337 >>> sorted(imported_modules(
337 ... '''import foo
338 ... '''import foo
338 ... def wat():
339 ... def wat():
339 ... import bar
340 ... import bar
340 ... ''', modulename, f, localmods))
341 ... ''', modulename, f, localmods))
341 ['foo.__init__', 'foo.bar.__init__']
342 ['foo.__init__', 'foo.bar.__init__']
342 >>> sorted(imported_modules(
343 >>> sorted(imported_modules(
343 ... '''import foo
344 ... '''import foo
344 ... def wat():
345 ... def wat():
345 ... import bar
346 ... import bar
346 ... ''', modulename, f, localmods, ignore_nested=True))
347 ... ''', modulename, f, localmods, ignore_nested=True))
347 ['foo.__init__']
348 ['foo.__init__']
348 """
349 """
349 fromlocal = fromlocalfunc(modulename, localmods)
350 fromlocal = fromlocalfunc(modulename, localmods)
350 for node in ast.walk(ast.parse(source, f)):
351 for node in ast.walk(ast.parse(source, f)):
351 if ignore_nested and getattr(node, 'col_offset', 0) > 0:
352 if ignore_nested and getattr(node, 'col_offset', 0) > 0:
352 continue
353 continue
353 if isinstance(node, ast.Import):
354 if isinstance(node, ast.Import):
354 for n in node.names:
355 for n in node.names:
355 found = fromlocal(n.name)
356 found = fromlocal(n.name)
356 if not found:
357 if not found:
357 # this should import standard library
358 # this should import standard library
358 continue
359 continue
359 yield found[1]
360 yield found[1]
360 elif isinstance(node, ast.ImportFrom):
361 elif isinstance(node, ast.ImportFrom):
361 found = fromlocal(node.module, node.level)
362 found = fromlocal(node.module, node.level)
362 if not found:
363 if not found:
363 # this should import standard library
364 # this should import standard library
364 continue
365 continue
365
366
366 absname, dottedpath, hassubmod = found
367 absname, dottedpath, hassubmod = found
367 if not hassubmod:
368 if not hassubmod:
368 # "dottedpath" is not a package; must be imported
369 # "dottedpath" is not a package; must be imported
369 yield dottedpath
370 yield dottedpath
370 # examination of "node.names" should be redundant
371 # examination of "node.names" should be redundant
371 # e.g.: from mercurial.node import nullid, nullrev
372 # e.g.: from mercurial.node import nullid, nullrev
372 continue
373 continue
373
374
374 modnotfound = False
375 modnotfound = False
375 prefix = absname + '.'
376 prefix = absname + '.'
376 for n in node.names:
377 for n in node.names:
377 found = fromlocal(prefix + n.name)
378 found = fromlocal(prefix + n.name)
378 if not found:
379 if not found:
379 # this should be a function or a property of "node.module"
380 # this should be a function or a property of "node.module"
380 modnotfound = True
381 modnotfound = True
381 continue
382 continue
382 yield found[1]
383 yield found[1]
383 if modnotfound and dottedpath != modulename:
384 if modnotfound and dottedpath != modulename:
384 # "dottedpath" is a package, but imported because of non-module
385 # "dottedpath" is a package, but imported because of non-module
385 # lookup
386 # lookup
386 # specifically allow "from . import foo" from __init__.py
387 # specifically allow "from . import foo" from __init__.py
387 yield dottedpath
388 yield dottedpath
388
389
389
390
390 def verify_import_convention(module, source, localmods):
391 def verify_import_convention(module, source, localmods):
391 """Verify imports match our established coding convention."""
392 """Verify imports match our established coding convention."""
392 root = ast.parse(source)
393 root = ast.parse(source)
393
394
394 return verify_modern_convention(module, root, localmods)
395 return verify_modern_convention(module, root, localmods)
395
396
396
397
397 def verify_modern_convention(module, root, localmods, root_col_offset=0):
398 def verify_modern_convention(module, root, localmods, root_col_offset=0):
398 """Verify a file conforms to the modern import convention rules.
399 """Verify a file conforms to the modern import convention rules.
399
400
400 The rules of the modern convention are:
401 The rules of the modern convention are:
401
402
402 * Ordering is stdlib followed by local imports. Each group is lexically
403 * Ordering is stdlib followed by local imports. Each group is lexically
403 sorted.
404 sorted.
404 * Importing multiple modules via "import X, Y" is not allowed: use
405 * Importing multiple modules via "import X, Y" is not allowed: use
405 separate import statements.
406 separate import statements.
406 * Importing multiple modules via "from X import ..." is allowed if using
407 * Importing multiple modules via "from X import ..." is allowed if using
407 parenthesis and one entry per line.
408 parenthesis and one entry per line.
408 * Only 1 relative import statement per import level ("from .", "from ..")
409 * Only 1 relative import statement per import level ("from .", "from ..")
409 is allowed.
410 is allowed.
410 * Relative imports from higher levels must occur before lower levels. e.g.
411 * Relative imports from higher levels must occur before lower levels. e.g.
411 "from .." must be before "from .".
412 "from .." must be before "from .".
412 * Imports from peer packages should use relative import (e.g. do not
413 * Imports from peer packages should use relative import (e.g. do not
413 "import mercurial.foo" from a "mercurial.*" module).
414 "import mercurial.foo" from a "mercurial.*" module).
414 * Symbols can only be imported from specific modules (see
415 * Symbols can only be imported from specific modules (see
415 `allowsymbolimports`). For other modules, first import the module then
416 `allowsymbolimports`). For other modules, first import the module then
416 assign the symbol to a module-level variable. In addition, these imports
417 assign the symbol to a module-level variable. In addition, these imports
417 must be performed before other local imports. This rule only
418 must be performed before other local imports. This rule only
418 applies to import statements outside of any blocks.
419 applies to import statements outside of any blocks.
419 * Relative imports from the standard library are not allowed, unless that
420 * Relative imports from the standard library are not allowed, unless that
420 library is also a local module.
421 library is also a local module.
421 * Certain modules must be aliased to alternate names to avoid aliasing
422 * Certain modules must be aliased to alternate names to avoid aliasing
422 and readability problems. See `requirealias`.
423 and readability problems. See `requirealias`.
423 """
424 """
424 if not isinstance(module, str):
425 if not isinstance(module, str):
425 module = module.decode('ascii')
426 module = module.decode('ascii')
426 topmodule = module.split('.')[0]
427 topmodule = module.split('.')[0]
427 fromlocal = fromlocalfunc(module, localmods)
428 fromlocal = fromlocalfunc(module, localmods)
428
429
429 # Whether a local/non-stdlib import has been performed.
430 # Whether a local/non-stdlib import has been performed.
430 seenlocal = None
431 seenlocal = None
431 # Whether a local/non-stdlib, non-symbol import has been seen.
432 # Whether a local/non-stdlib, non-symbol import has been seen.
432 seennonsymbollocal = False
433 seennonsymbollocal = False
433 # The last name to be imported (for sorting).
434 # The last name to be imported (for sorting).
434 lastname = None
435 lastname = None
435 laststdlib = None
436 laststdlib = None
436 # Relative import levels encountered so far.
437 # Relative import levels encountered so far.
437 seenlevels = set()
438 seenlevels = set()
438
439
439 for node, newscope in walklocal(root):
440 for node, newscope in walklocal(root):
440
441
441 def msg(fmt, *args):
442 def msg(fmt, *args):
442 return (fmt % args, node.lineno)
443 return (fmt % args, node.lineno)
443
444
444 if newscope:
445 if newscope:
445 # Check for local imports in function
446 # Check for local imports in function
446 for r in verify_modern_convention(
447 for r in verify_modern_convention(
447 module, node, localmods, node.col_offset + 4
448 module, node, localmods, node.col_offset + 4
448 ):
449 ):
449 yield r
450 yield r
450 elif isinstance(node, ast.Import):
451 elif isinstance(node, ast.Import):
451 # Disallow "import foo, bar" and require separate imports
452 # Disallow "import foo, bar" and require separate imports
452 # for each module.
453 # for each module.
453 if len(node.names) > 1:
454 if len(node.names) > 1:
454 yield msg(
455 yield msg(
455 'multiple imported names: %s',
456 'multiple imported names: %s',
456 ', '.join(n.name for n in node.names),
457 ', '.join(n.name for n in node.names),
457 )
458 )
458
459
459 name = node.names[0].name
460 name = node.names[0].name
460 asname = node.names[0].asname
461 asname = node.names[0].asname
461
462
462 stdlib = name in stdlib_modules
463 stdlib = name in stdlib_modules
463
464
464 # Ignore sorting rules on imports inside blocks.
465 # Ignore sorting rules on imports inside blocks.
465 if node.col_offset == root_col_offset:
466 if node.col_offset == root_col_offset:
466 if lastname and name < lastname and laststdlib == stdlib:
467 if lastname and name < lastname and laststdlib == stdlib:
467 yield msg(
468 yield msg(
468 'imports not lexically sorted: %s < %s', name, lastname
469 'imports not lexically sorted: %s < %s', name, lastname
469 )
470 )
470
471
471 lastname = name
472 lastname = name
472 laststdlib = stdlib
473 laststdlib = stdlib
473
474
474 # stdlib imports should be before local imports.
475 # stdlib imports should be before local imports.
475 if stdlib and seenlocal and node.col_offset == root_col_offset:
476 if stdlib and seenlocal and node.col_offset == root_col_offset:
476 yield msg(
477 yield msg(
477 'stdlib import "%s" follows local import: %s',
478 'stdlib import "%s" follows local import: %s',
478 name,
479 name,
479 seenlocal,
480 seenlocal,
480 )
481 )
481
482
482 if not stdlib:
483 if not stdlib:
483 seenlocal = name
484 seenlocal = name
484
485
485 # Import of sibling modules should use relative imports.
486 # Import of sibling modules should use relative imports.
486 topname = name.split('.')[0]
487 topname = name.split('.')[0]
487 if topname == topmodule:
488 if topname == topmodule:
488 yield msg('import should be relative: %s', name)
489 yield msg('import should be relative: %s', name)
489
490
490 if name in requirealias and asname != requirealias[name]:
491 if name in requirealias and asname != requirealias[name]:
491 yield msg(
492 yield msg(
492 '%s module must be "as" aliased to %s',
493 '%s module must be "as" aliased to %s',
493 name,
494 name,
494 requirealias[name],
495 requirealias[name],
495 )
496 )
496
497
497 elif isinstance(node, ast.ImportFrom):
498 elif isinstance(node, ast.ImportFrom):
498 # Resolve the full imported module name.
499 # Resolve the full imported module name.
499 if node.level > 0:
500 if node.level > 0:
500 fullname = '.'.join(module.split('.')[: -node.level])
501 fullname = '.'.join(module.split('.')[: -node.level])
501 if node.module:
502 if node.module:
502 fullname += '.%s' % node.module
503 fullname += '.%s' % node.module
503 else:
504 else:
504 assert node.module
505 assert node.module
505 fullname = node.module
506 fullname = node.module
506
507
507 topname = fullname.split('.')[0]
508 topname = fullname.split('.')[0]
508 if topname == topmodule:
509 if topname == topmodule:
509 yield msg('import should be relative: %s', fullname)
510 yield msg('import should be relative: %s', fullname)
510
511
511 # __future__ is special since it needs to come first and use
512 # __future__ is special since it needs to come first and use
512 # symbol import.
513 # symbol import.
513 if fullname != '__future__':
514 if fullname != '__future__':
514 if not fullname or (
515 if not fullname or (
515 fullname in stdlib_modules
516 fullname in stdlib_modules
516 # allow standard 'from typing import ...' style
517 # allow standard 'from typing import ...' style
517 and fullname.startswith('.')
518 and fullname.startswith('.')
518 and fullname not in localmods
519 and fullname not in localmods
519 and fullname + '.__init__' not in localmods
520 and fullname + '.__init__' not in localmods
520 ):
521 ):
521 yield msg('relative import of stdlib module')
522 yield msg('relative import of stdlib module')
522 else:
523 else:
523 seenlocal = fullname
524 seenlocal = fullname
524
525
525 # Direct symbol import is only allowed from certain modules and
526 # Direct symbol import is only allowed from certain modules and
526 # must occur before non-symbol imports.
527 # must occur before non-symbol imports.
527 found = fromlocal(node.module, node.level)
528 found = fromlocal(node.module, node.level)
528 if found and found[2]: # node.module is a package
529 if found and found[2]: # node.module is a package
529 prefix = found[0] + '.'
530 prefix = found[0] + '.'
530 symbols = (
531 symbols = (
531 n.name for n in node.names if not fromlocal(prefix + n.name)
532 n.name for n in node.names if not fromlocal(prefix + n.name)
532 )
533 )
533 else:
534 else:
534 symbols = (n.name for n in node.names)
535 symbols = (n.name for n in node.names)
535 symbols = [sym for sym in symbols if sym not in directsymbols]
536 symbols = [sym for sym in symbols if sym not in directsymbols]
536 if node.module and node.col_offset == root_col_offset:
537 if node.module and node.col_offset == root_col_offset:
537 if symbols and fullname not in allowsymbolimports:
538 if symbols and fullname not in allowsymbolimports:
538 yield msg(
539 yield msg(
539 'direct symbol import %s from %s',
540 'direct symbol import %s from %s',
540 ', '.join(symbols),
541 ', '.join(symbols),
541 fullname,
542 fullname,
542 )
543 )
543
544
544 if symbols and seennonsymbollocal:
545 if symbols and seennonsymbollocal:
545 yield msg(
546 yield msg(
546 'symbol import follows non-symbol import: %s', fullname
547 'symbol import follows non-symbol import: %s', fullname
547 )
548 )
548 if not symbols and fullname not in stdlib_modules:
549 if not symbols and fullname not in stdlib_modules:
549 seennonsymbollocal = True
550 seennonsymbollocal = True
550
551
551 if not node.module:
552 if not node.module:
552 assert node.level
553 assert node.level
553
554
554 # Only allow 1 group per level.
555 # Only allow 1 group per level.
555 if (
556 if (
556 node.level in seenlevels
557 node.level in seenlevels
557 and node.col_offset == root_col_offset
558 and node.col_offset == root_col_offset
558 ):
559 ):
559 yield msg(
560 yield msg(
560 'multiple "from %s import" statements', '.' * node.level
561 'multiple "from %s import" statements', '.' * node.level
561 )
562 )
562
563
563 # Higher-level groups come before lower-level groups.
564 # Higher-level groups come before lower-level groups.
564 if any(node.level > l for l in seenlevels):
565 if any(node.level > l for l in seenlevels):
565 yield msg(
566 yield msg(
566 'higher-level import should come first: %s', fullname
567 'higher-level import should come first: %s', fullname
567 )
568 )
568
569
569 seenlevels.add(node.level)
570 seenlevels.add(node.level)
570
571
571 # Entries in "from .X import ( ... )" lists must be lexically
572 # Entries in "from .X import ( ... )" lists must be lexically
572 # sorted.
573 # sorted.
573 lastentryname = None
574 lastentryname = None
574
575
575 for n in node.names:
576 for n in node.names:
576 if lastentryname and n.name < lastentryname:
577 if lastentryname and n.name < lastentryname:
577 yield msg(
578 yield msg(
578 'imports from %s not lexically sorted: %s < %s',
579 'imports from %s not lexically sorted: %s < %s',
579 fullname,
580 fullname,
580 n.name,
581 n.name,
581 lastentryname,
582 lastentryname,
582 )
583 )
583
584
584 lastentryname = n.name
585 lastentryname = n.name
585
586
586 if n.name in requirealias and n.asname != requirealias[n.name]:
587 if n.name in requirealias and n.asname != requirealias[n.name]:
587 yield msg(
588 yield msg(
588 '%s from %s must be "as" aliased to %s',
589 '%s from %s must be "as" aliased to %s',
589 n.name,
590 n.name,
590 fullname,
591 fullname,
591 requirealias[n.name],
592 requirealias[n.name],
592 )
593 )
593
594
594
595
595 class CircularImport(Exception):
596 class CircularImport(Exception):
596 pass
597 pass
597
598
598
599
599 def checkmod(mod, imports):
600 def checkmod(mod, imports):
600 shortest = {}
601 shortest = {}
601 visit = [[mod]]
602 visit = [[mod]]
602 while visit:
603 while visit:
603 path = visit.pop(0)
604 path = visit.pop(0)
604 for i in sorted(imports.get(path[-1], [])):
605 for i in sorted(imports.get(path[-1], [])):
605 if len(path) < shortest.get(i, 1000):
606 if len(path) < shortest.get(i, 1000):
606 shortest[i] = len(path)
607 shortest[i] = len(path)
607 if i in path:
608 if i in path:
608 if i == path[0]:
609 if i == path[0]:
609 raise CircularImport(path)
610 raise CircularImport(path)
610 continue
611 continue
611 visit.append(path + [i])
612 visit.append(path + [i])
612
613
613
614
614 def rotatecycle(cycle):
615 def rotatecycle(cycle):
615 """arrange a cycle so that the lexicographically first module listed first
616 """arrange a cycle so that the lexicographically first module listed first
616
617
617 >>> rotatecycle(['foo', 'bar'])
618 >>> rotatecycle(['foo', 'bar'])
618 ['bar', 'foo', 'bar']
619 ['bar', 'foo', 'bar']
619 """
620 """
620 lowest = min(cycle)
621 lowest = min(cycle)
621 idx = cycle.index(lowest)
622 idx = cycle.index(lowest)
622 return cycle[idx:] + cycle[:idx] + [lowest]
623 return cycle[idx:] + cycle[:idx] + [lowest]
623
624
624
625
625 def find_cycles(imports):
626 def find_cycles(imports):
626 """Find cycles in an already-loaded import graph.
627 """Find cycles in an already-loaded import graph.
627
628
628 All module names recorded in `imports` should be absolute one.
629 All module names recorded in `imports` should be absolute one.
629
630
630 >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'],
631 >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'],
631 ... 'top.bar': ['top.baz', 'sys'],
632 ... 'top.bar': ['top.baz', 'sys'],
632 ... 'top.baz': ['top.foo'],
633 ... 'top.baz': ['top.foo'],
633 ... 'top.qux': ['top.foo']}
634 ... 'top.qux': ['top.foo']}
634 >>> print('\\n'.join(sorted(find_cycles(imports))))
635 >>> print('\\n'.join(sorted(find_cycles(imports))))
635 top.bar -> top.baz -> top.foo -> top.bar
636 top.bar -> top.baz -> top.foo -> top.bar
636 top.foo -> top.qux -> top.foo
637 top.foo -> top.qux -> top.foo
637 """
638 """
638 cycles = set()
639 cycles = set()
639 for mod in sorted(imports.keys()):
640 for mod in sorted(imports.keys()):
640 try:
641 try:
641 checkmod(mod, imports)
642 checkmod(mod, imports)
642 except CircularImport as e:
643 except CircularImport as e:
643 cycle = e.args[0]
644 cycle = e.args[0]
644 cycles.add(" -> ".join(rotatecycle(cycle)))
645 cycles.add(" -> ".join(rotatecycle(cycle)))
645 return cycles
646 return cycles
646
647
647
648
648 def _cycle_sortkey(c):
649 def _cycle_sortkey(c):
649 return len(c), c
650 return len(c), c
650
651
651
652
652 def embedded(f, modname, src):
653 def embedded(f, modname, src):
653 """Extract embedded python code
654 """Extract embedded python code
654
655
655 >>> def _forcestr(thing):
656 >>> def _forcestr(thing):
656 ... if not isinstance(thing, str):
657 ... if not isinstance(thing, str):
657 ... return thing.decode('ascii')
658 ... return thing.decode('ascii')
658 ... return thing
659 ... return thing
659 >>> def test(fn, lines):
660 >>> def test(fn, lines):
660 ... for s, m, f, l in embedded(fn, b"example", lines):
661 ... for s, m, f, l in embedded(fn, b"example", lines):
661 ... print("%s %s %d" % (_forcestr(m), _forcestr(f), l))
662 ... print("%s %s %d" % (_forcestr(m), _forcestr(f), l))
662 ... print(repr(_forcestr(s)))
663 ... print(repr(_forcestr(s)))
663 >>> lines = [
664 >>> lines = [
664 ... 'comment',
665 ... 'comment',
665 ... ' >>> from __future__ import print_function',
666 ... ' >>> from __future__ import print_function',
666 ... " >>> ' multiline",
667 ... " >>> ' multiline",
667 ... " ... string'",
668 ... " ... string'",
668 ... ' ',
669 ... ' ',
669 ... 'comment',
670 ... 'comment',
670 ... ' $ cat > foo.py <<EOF',
671 ... ' $ cat > foo.py <<EOF',
671 ... ' > from __future__ import print_function',
672 ... ' > from __future__ import print_function',
672 ... ' > EOF',
673 ... ' > EOF',
673 ... ]
674 ... ]
674 >>> test(b"example.t", lines)
675 >>> test(b"example.t", lines)
675 example[2] doctest.py 1
676 example[2] doctest.py 1
676 "from __future__ import print_function\\n' multiline\\nstring'\\n\\n"
677 "from __future__ import print_function\\n' multiline\\nstring'\\n\\n"
677 example[8] foo.py 7
678 example[8] foo.py 7
678 'from __future__ import print_function\\n'
679 'from __future__ import print_function\\n'
679 """
680 """
680 errors = []
681 errors = []
681 for name, starts, ends, code in testparseutil.pyembedded(f, src, errors):
682 for name, starts, ends, code in testparseutil.pyembedded(f, src, errors):
682 if not name:
683 if not name:
683 # use 'doctest.py', in order to make already existing
684 # use 'doctest.py', in order to make already existing
684 # doctest above pass instantly
685 # doctest above pass instantly
685 name = 'doctest.py'
686 name = 'doctest.py'
686 # "starts" is "line number" (1-origin), but embedded() is
687 # "starts" is "line number" (1-origin), but embedded() is
687 # expected to return "line offset" (0-origin). Therefore, this
688 # expected to return "line offset" (0-origin). Therefore, this
688 # yields "starts - 1".
689 # yields "starts - 1".
689 if not isinstance(modname, str):
690 if not isinstance(modname, str):
690 modname = modname.decode('utf8')
691 modname = modname.decode('utf8')
691 yield code, "%s[%d]" % (modname, starts), name, starts - 1
692 yield code, "%s[%d]" % (modname, starts), name, starts - 1
692
693
693
694
694 def sources(f, modname):
695 def sources(f, modname):
695 """Yields possibly multiple sources from a filepath
696 """Yields possibly multiple sources from a filepath
696
697
697 input: filepath, modulename
698 input: filepath, modulename
698 yields: script(string), modulename, filepath, linenumber
699 yields: script(string), modulename, filepath, linenumber
699
700
700 For embedded scripts, the modulename and filepath will be different
701 For embedded scripts, the modulename and filepath will be different
701 from the function arguments. linenumber is an offset relative to
702 from the function arguments. linenumber is an offset relative to
702 the input file.
703 the input file.
703 """
704 """
704 py = False
705 py = False
705 if not f.endswith('.t'):
706 if not f.endswith('.t'):
706 with open(f, 'rb') as src:
707 with open(f, 'rb') as src:
707 yield src.read(), modname, f, 0
708 yield src.read(), modname, f, 0
708 py = True
709 py = True
709 if py or f.endswith('.t'):
710 if py or f.endswith('.t'):
710 # Strictly speaking we should sniff for the magic header that denotes
711 # Strictly speaking we should sniff for the magic header that denotes
711 # Python source file encoding. But in reality we don't use anything
712 # Python source file encoding. But in reality we don't use anything
712 # other than ASCII (mainly) and UTF-8 (in a few exceptions), so
713 # other than ASCII (mainly) and UTF-8 (in a few exceptions), so
713 # simplicity is fine.
714 # simplicity is fine.
714 with io.open(f, 'r', encoding='utf-8') as src:
715 with io.open(f, 'r', encoding='utf-8') as src:
715 for script, modname, t, line in embedded(f, modname, src):
716 for script, modname, t, line in embedded(f, modname, src):
716 yield script, modname.encode('utf8'), t, line
717 yield script, modname.encode('utf8'), t, line
717
718
718
719
719 def main(argv):
720 def main(argv):
720 if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):
721 if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):
721 print('Usage: %s {-|file [file] [file] ...}')
722 print('Usage: %s {-|file [file] [file] ...}')
722 return 1
723 return 1
723 if argv[1] == '-':
724 if argv[1] == '-':
724 argv = argv[:1]
725 argv = argv[:1]
725 argv.extend(l.rstrip() for l in sys.stdin.readlines())
726 argv.extend(l.rstrip() for l in sys.stdin.readlines())
726 localmodpaths = {}
727 localmodpaths = {}
727 used_imports = {}
728 used_imports = {}
728 any_errors = False
729 any_errors = False
729 for source_path in argv[1:]:
730 for source_path in argv[1:]:
730 modname = dotted_name_of_path(source_path)
731 modname = dotted_name_of_path(source_path)
731 localmodpaths[modname] = source_path
732 localmodpaths[modname] = source_path
732 localmods = populateextmods(localmodpaths)
733 localmods = populateextmods(localmodpaths)
733 for localmodname, source_path in sorted(localmodpaths.items()):
734 for localmodname, source_path in sorted(localmodpaths.items()):
734 if not isinstance(localmodname, bytes):
735 if not isinstance(localmodname, bytes):
735 # This is only safe because all hg's files are ascii
736 # This is only safe because all hg's files are ascii
736 localmodname = localmodname.encode('ascii')
737 localmodname = localmodname.encode('ascii')
737 for src, modname, name, line in sources(source_path, localmodname):
738 for src, modname, name, line in sources(source_path, localmodname):
738 try:
739 try:
739 used_imports[modname] = sorted(
740 used_imports[modname] = sorted(
740 imported_modules(
741 imported_modules(
741 src, modname, name, localmods, ignore_nested=True
742 src, modname, name, localmods, ignore_nested=True
742 )
743 )
743 )
744 )
744 for error, lineno in verify_import_convention(
745 for error, lineno in verify_import_convention(
745 modname, src, localmods
746 modname, src, localmods
746 ):
747 ):
747 any_errors = True
748 any_errors = True
748 print('%s:%d: %s' % (source_path, lineno + line, error))
749 print('%s:%d: %s' % (source_path, lineno + line, error))
749 except SyntaxError as e:
750 except SyntaxError as e:
750 print(
751 print(
751 '%s:%d: SyntaxError: %s' % (source_path, e.lineno + line, e)
752 '%s:%d: SyntaxError: %s' % (source_path, e.lineno + line, e)
752 )
753 )
753 cycles = find_cycles(used_imports)
754 cycles = find_cycles(used_imports)
754 if cycles:
755 if cycles:
755 firstmods = set()
756 firstmods = set()
756 for c in sorted(cycles, key=_cycle_sortkey):
757 for c in sorted(cycles, key=_cycle_sortkey):
757 first = c.split()[0]
758 first = c.split()[0]
758 # As a rough cut, ignore any cycle that starts with the
759 # As a rough cut, ignore any cycle that starts with the
759 # same module as some other cycle. Otherwise we see lots
760 # same module as some other cycle. Otherwise we see lots
760 # of cycles that are effectively duplicates.
761 # of cycles that are effectively duplicates.
761 if first in firstmods:
762 if first in firstmods:
762 continue
763 continue
763 print('Import cycle:', c)
764 print('Import cycle:', c)
764 firstmods.add(first)
765 firstmods.add(first)
765 any_errors = True
766 any_errors = True
766 return any_errors != 0
767 return any_errors != 0
767
768
768
769
769 if __name__ == '__main__':
770 if __name__ == '__main__':
770 sys.exit(int(main(sys.argv)))
771 sys.exit(int(main(sys.argv)))
@@ -1,238 +1,222 b''
1 # darcs.py - darcs support for the convert extension
1 # darcs.py - darcs support for the convert extension
2 #
2 #
3 # Copyright 2007-2009 Olivia Mackall <olivia@selenic.com> and others
3 # Copyright 2007-2009 Olivia Mackall <olivia@selenic.com> and others
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 import os
8 import os
9 import re
9 import re
10 import shutil
10 import shutil
11 from xml.etree.ElementTree import (
12 ElementTree,
13 XMLParser,
14 )
11
15
12 from mercurial.i18n import _
16 from mercurial.i18n import _
13 from mercurial import (
17 from mercurial import (
14 error,
18 error,
15 pycompat,
19 pycompat,
16 util,
20 util,
17 )
21 )
18 from mercurial.utils import dateutil
22 from mercurial.utils import dateutil
19 from . import common
23 from . import common
20
24
21 NoRepo = common.NoRepo
25 NoRepo = common.NoRepo
22
26
23 # The naming drift of ElementTree is fun!
24
25 try:
26 import xml.etree.cElementTree.ElementTree as ElementTree
27 import xml.etree.cElementTree.XMLParser as XMLParser
28 except ImportError:
29 try:
30 import xml.etree.ElementTree.ElementTree as ElementTree
31 import xml.etree.ElementTree.XMLParser as XMLParser
32 except ImportError:
33 try:
34 import elementtree.cElementTree.ElementTree as ElementTree
35 import elementtree.cElementTree.XMLParser as XMLParser
36 except ImportError:
37 try:
38 import elementtree.ElementTree.ElementTree as ElementTree
39 import elementtree.ElementTree.XMLParser as XMLParser
40 except ImportError:
41 pass
42
43
27
44 class darcs_source(common.converter_source, common.commandline):
28 class darcs_source(common.converter_source, common.commandline):
45 def __init__(self, ui, repotype, path, revs=None):
29 def __init__(self, ui, repotype, path, revs=None):
46 common.converter_source.__init__(self, ui, repotype, path, revs=revs)
30 common.converter_source.__init__(self, ui, repotype, path, revs=revs)
47 common.commandline.__init__(self, ui, b'darcs')
31 common.commandline.__init__(self, ui, b'darcs')
48
32
49 # check for _darcs, ElementTree so that we can easily skip
33 # check for _darcs, ElementTree so that we can easily skip
50 # test-convert-darcs if ElementTree is not around
34 # test-convert-darcs if ElementTree is not around
51 if not os.path.exists(os.path.join(path, b'_darcs')):
35 if not os.path.exists(os.path.join(path, b'_darcs')):
52 raise NoRepo(_(b"%s does not look like a darcs repository") % path)
36 raise NoRepo(_(b"%s does not look like a darcs repository") % path)
53
37
54 common.checktool(b'darcs')
38 common.checktool(b'darcs')
55 version = self.run0(b'--version').splitlines()[0].strip()
39 version = self.run0(b'--version').splitlines()[0].strip()
56 if version < b'2.1':
40 if version < b'2.1':
57 raise error.Abort(
41 raise error.Abort(
58 _(b'darcs version 2.1 or newer needed (found %r)') % version
42 _(b'darcs version 2.1 or newer needed (found %r)') % version
59 )
43 )
60
44
61 if b"ElementTree" not in globals():
45 if b"ElementTree" not in globals():
62 raise error.Abort(_(b"Python ElementTree module is not available"))
46 raise error.Abort(_(b"Python ElementTree module is not available"))
63
47
64 self.path = os.path.realpath(path)
48 self.path = os.path.realpath(path)
65
49
66 self.lastrev = None
50 self.lastrev = None
67 self.changes = {}
51 self.changes = {}
68 self.parents = {}
52 self.parents = {}
69 self.tags = {}
53 self.tags = {}
70
54
71 # Check darcs repository format
55 # Check darcs repository format
72 format = self.format()
56 format = self.format()
73 if format:
57 if format:
74 if format in (b'darcs-1.0', b'hashed'):
58 if format in (b'darcs-1.0', b'hashed'):
75 raise NoRepo(
59 raise NoRepo(
76 _(
60 _(
77 b"%s repository format is unsupported, "
61 b"%s repository format is unsupported, "
78 b"please upgrade"
62 b"please upgrade"
79 )
63 )
80 % format
64 % format
81 )
65 )
82 else:
66 else:
83 self.ui.warn(_(b'failed to detect repository format!'))
67 self.ui.warn(_(b'failed to detect repository format!'))
84
68
85 def before(self):
69 def before(self):
86 self.tmppath = pycompat.mkdtemp(
70 self.tmppath = pycompat.mkdtemp(
87 prefix=b'convert-' + os.path.basename(self.path) + b'-'
71 prefix=b'convert-' + os.path.basename(self.path) + b'-'
88 )
72 )
89 output, status = self.run(b'init', repodir=self.tmppath)
73 output, status = self.run(b'init', repodir=self.tmppath)
90 self.checkexit(status)
74 self.checkexit(status)
91
75
92 tree = self.xml(
76 tree = self.xml(
93 b'changes', xml_output=True, summary=True, repodir=self.path
77 b'changes', xml_output=True, summary=True, repodir=self.path
94 )
78 )
95 tagname = None
79 tagname = None
96 child = None
80 child = None
97 for elt in tree.findall(b'patch'):
81 for elt in tree.findall(b'patch'):
98 node = elt.get(b'hash')
82 node = elt.get(b'hash')
99 name = elt.findtext(b'name', b'')
83 name = elt.findtext(b'name', b'')
100 if name.startswith(b'TAG '):
84 if name.startswith(b'TAG '):
101 tagname = name[4:].strip()
85 tagname = name[4:].strip()
102 elif tagname is not None:
86 elif tagname is not None:
103 self.tags[tagname] = node
87 self.tags[tagname] = node
104 tagname = None
88 tagname = None
105 self.changes[node] = elt
89 self.changes[node] = elt
106 self.parents[child] = [node]
90 self.parents[child] = [node]
107 child = node
91 child = node
108 self.parents[child] = []
92 self.parents[child] = []
109
93
110 def after(self):
94 def after(self):
111 self.ui.debug(b'cleaning up %s\n' % self.tmppath)
95 self.ui.debug(b'cleaning up %s\n' % self.tmppath)
112 shutil.rmtree(self.tmppath, ignore_errors=True)
96 shutil.rmtree(self.tmppath, ignore_errors=True)
113
97
114 def recode(self, s, encoding=None):
98 def recode(self, s, encoding=None):
115 if isinstance(s, str):
99 if isinstance(s, str):
116 # XMLParser returns unicode objects for anything it can't
100 # XMLParser returns unicode objects for anything it can't
117 # encode into ASCII. We convert them back to str to get
101 # encode into ASCII. We convert them back to str to get
118 # recode's normal conversion behavior.
102 # recode's normal conversion behavior.
119 s = s.encode('latin-1')
103 s = s.encode('latin-1')
120 return super(darcs_source, self).recode(s, encoding)
104 return super(darcs_source, self).recode(s, encoding)
121
105
122 def xml(self, cmd, **kwargs):
106 def xml(self, cmd, **kwargs):
123 # NOTE: darcs is currently encoding agnostic and will print
107 # NOTE: darcs is currently encoding agnostic and will print
124 # patch metadata byte-for-byte, even in the XML changelog.
108 # patch metadata byte-for-byte, even in the XML changelog.
125 etree = ElementTree()
109 etree = ElementTree()
126 # While we are decoding the XML as latin-1 to be as liberal as
110 # While we are decoding the XML as latin-1 to be as liberal as
127 # possible, etree will still raise an exception if any
111 # possible, etree will still raise an exception if any
128 # non-printable characters are in the XML changelog.
112 # non-printable characters are in the XML changelog.
129 parser = XMLParser(encoding=b'latin-1')
113 parser = XMLParser(encoding=b'latin-1')
130 p = self._run(cmd, **kwargs)
114 p = self._run(cmd, **kwargs)
131 etree.parse(p.stdout, parser=parser)
115 etree.parse(p.stdout, parser=parser)
132 p.wait()
116 p.wait()
133 self.checkexit(p.returncode)
117 self.checkexit(p.returncode)
134 return etree.getroot()
118 return etree.getroot()
135
119
136 def format(self):
120 def format(self):
137 output, status = self.run(b'show', b'repo', repodir=self.path)
121 output, status = self.run(b'show', b'repo', repodir=self.path)
138 self.checkexit(status)
122 self.checkexit(status)
139 m = re.search(r'^\s*Format:\s*(.*)$', output, re.MULTILINE)
123 m = re.search(r'^\s*Format:\s*(.*)$', output, re.MULTILINE)
140 if not m:
124 if not m:
141 return None
125 return None
142 return b','.join(sorted(f.strip() for f in m.group(1).split(b',')))
126 return b','.join(sorted(f.strip() for f in m.group(1).split(b',')))
143
127
144 def manifest(self):
128 def manifest(self):
145 man = []
129 man = []
146 output, status = self.run(
130 output, status = self.run(
147 b'show', b'files', no_directories=True, repodir=self.tmppath
131 b'show', b'files', no_directories=True, repodir=self.tmppath
148 )
132 )
149 self.checkexit(status)
133 self.checkexit(status)
150 for line in output.split(b'\n'):
134 for line in output.split(b'\n'):
151 path = line[2:]
135 path = line[2:]
152 if path:
136 if path:
153 man.append(path)
137 man.append(path)
154 return man
138 return man
155
139
156 def getheads(self):
140 def getheads(self):
157 return self.parents[None]
141 return self.parents[None]
158
142
159 def getcommit(self, rev):
143 def getcommit(self, rev):
160 elt = self.changes[rev]
144 elt = self.changes[rev]
161 dateformat = b'%a %b %d %H:%M:%S %Z %Y'
145 dateformat = b'%a %b %d %H:%M:%S %Z %Y'
162 date = dateutil.strdate(elt.get(b'local_date'), dateformat)
146 date = dateutil.strdate(elt.get(b'local_date'), dateformat)
163 desc = elt.findtext(b'name') + b'\n' + elt.findtext(b'comment', b'')
147 desc = elt.findtext(b'name') + b'\n' + elt.findtext(b'comment', b'')
164 # etree can return unicode objects for name, comment, and author,
148 # etree can return unicode objects for name, comment, and author,
165 # so recode() is used to ensure str objects are emitted.
149 # so recode() is used to ensure str objects are emitted.
166 newdateformat = b'%Y-%m-%d %H:%M:%S %1%2'
150 newdateformat = b'%Y-%m-%d %H:%M:%S %1%2'
167 return common.commit(
151 return common.commit(
168 author=self.recode(elt.get(b'author')),
152 author=self.recode(elt.get(b'author')),
169 date=dateutil.datestr(date, newdateformat),
153 date=dateutil.datestr(date, newdateformat),
170 desc=self.recode(desc).strip(),
154 desc=self.recode(desc).strip(),
171 parents=self.parents[rev],
155 parents=self.parents[rev],
172 )
156 )
173
157
174 def pull(self, rev):
158 def pull(self, rev):
175 output, status = self.run(
159 output, status = self.run(
176 b'pull',
160 b'pull',
177 self.path,
161 self.path,
178 all=True,
162 all=True,
179 match=b'hash %s' % rev,
163 match=b'hash %s' % rev,
180 no_test=True,
164 no_test=True,
181 no_posthook=True,
165 no_posthook=True,
182 external_merge=b'/bin/false',
166 external_merge=b'/bin/false',
183 repodir=self.tmppath,
167 repodir=self.tmppath,
184 )
168 )
185 if status:
169 if status:
186 if output.find(b'We have conflicts in') == -1:
170 if output.find(b'We have conflicts in') == -1:
187 self.checkexit(status, output)
171 self.checkexit(status, output)
188 output, status = self.run(b'revert', all=True, repodir=self.tmppath)
172 output, status = self.run(b'revert', all=True, repodir=self.tmppath)
189 self.checkexit(status, output)
173 self.checkexit(status, output)
190
174
191 def getchanges(self, rev, full):
175 def getchanges(self, rev, full):
192 if full:
176 if full:
193 raise error.Abort(_(b"convert from darcs does not support --full"))
177 raise error.Abort(_(b"convert from darcs does not support --full"))
194 copies = {}
178 copies = {}
195 changes = []
179 changes = []
196 man = None
180 man = None
197 for elt in self.changes[rev].find(b'summary').getchildren():
181 for elt in self.changes[rev].find(b'summary').getchildren():
198 if elt.tag in (b'add_directory', b'remove_directory'):
182 if elt.tag in (b'add_directory', b'remove_directory'):
199 continue
183 continue
200 if elt.tag == b'move':
184 if elt.tag == b'move':
201 if man is None:
185 if man is None:
202 man = self.manifest()
186 man = self.manifest()
203 source, dest = elt.get(b'from'), elt.get(b'to')
187 source, dest = elt.get(b'from'), elt.get(b'to')
204 if source in man:
188 if source in man:
205 # File move
189 # File move
206 changes.append((source, rev))
190 changes.append((source, rev))
207 changes.append((dest, rev))
191 changes.append((dest, rev))
208 copies[dest] = source
192 copies[dest] = source
209 else:
193 else:
210 # Directory move, deduce file moves from manifest
194 # Directory move, deduce file moves from manifest
211 source = source + b'/'
195 source = source + b'/'
212 for f in man:
196 for f in man:
213 if not f.startswith(source):
197 if not f.startswith(source):
214 continue
198 continue
215 fdest = dest + b'/' + f[len(source) :]
199 fdest = dest + b'/' + f[len(source) :]
216 changes.append((f, rev))
200 changes.append((f, rev))
217 changes.append((fdest, rev))
201 changes.append((fdest, rev))
218 copies[fdest] = f
202 copies[fdest] = f
219 else:
203 else:
220 changes.append((elt.text.strip(), rev))
204 changes.append((elt.text.strip(), rev))
221 self.pull(rev)
205 self.pull(rev)
222 self.lastrev = rev
206 self.lastrev = rev
223 return sorted(changes), copies, set()
207 return sorted(changes), copies, set()
224
208
225 def getfile(self, name, rev):
209 def getfile(self, name, rev):
226 if rev != self.lastrev:
210 if rev != self.lastrev:
227 raise error.Abort(_(b'internal calling inconsistency'))
211 raise error.Abort(_(b'internal calling inconsistency'))
228 path = os.path.join(self.tmppath, name)
212 path = os.path.join(self.tmppath, name)
229 try:
213 try:
230 data = util.readfile(path)
214 data = util.readfile(path)
231 mode = os.lstat(path).st_mode
215 mode = os.lstat(path).st_mode
232 except FileNotFoundError:
216 except FileNotFoundError:
233 return None, None
217 return None, None
234 mode = (mode & 0o111) and b'x' or b''
218 mode = (mode & 0o111) and b'x' or b''
235 return data, mode
219 return data, mode
236
220
237 def gettags(self):
221 def gettags(self):
238 return self.tags
222 return self.tags
General Comments 0
You need to be logged in to leave comments. Login now