##// END OF EJS Templates
import-checker: make stdlib path detection work in virtual environments...
Manuel Jacob -
r52264:cf1bee12 default
parent child Browse files
Show More
@@ -1,774 +1,771 b''
1 1 #!/usr/bin/env python3
2 2
3 3
4 4 import ast
5 5 import collections
6 6 import io
7 7 import os
8 8 import sys
9 9
10 10 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
11 11 # to work when run from a virtualenv. The modules were chosen empirically
12 12 # so that the return value matches the return value without virtualenv.
13 13 if True: # disable lexical sorting checks
14 try:
15 import BaseHTTPServer as basehttpserver
16 except ImportError:
17 basehttpserver = None
14 import argparse
18 15 import zlib
19 16
20 17 import testparseutil
21 18
22 19 # Allow list of modules that symbols can be directly imported from.
23 20 allowsymbolimports = (
24 21 '__future__',
25 22 'breezy',
26 23 'concurrent',
27 24 'hgclient',
28 25 'mercurial',
29 26 'mercurial.hgweb.common',
30 27 'mercurial.hgweb.request',
31 28 'mercurial.i18n',
32 29 'mercurial.interfaces',
33 30 'mercurial.node',
34 31 'mercurial.pycompat',
35 32 # for revlog to re-export constant to extensions
36 33 'mercurial.revlogutils.constants',
37 34 'mercurial.revlogutils.flagutil',
38 35 # for cffi modules to re-export pure functions
39 36 'mercurial.pure.base85',
40 37 'mercurial.pure.bdiff',
41 38 'mercurial.pure.mpatch',
42 39 'mercurial.pure.osutil',
43 40 'mercurial.pure.parsers',
44 41 # third-party imports should be directly imported
45 42 'mercurial.thirdparty',
46 43 'mercurial.thirdparty.attr',
47 44 'mercurial.thirdparty.jaraco.collections',
48 45 'mercurial.thirdparty.tomli',
49 46 'mercurial.thirdparty.zope',
50 47 'mercurial.thirdparty.zope.interface',
51 48 'typing',
52 49 'xml.etree.ElementTree',
53 50 )
54 51
55 52 # Allow list of symbols that can be directly imported.
56 53 directsymbols = ('demandimport',)
57 54
58 55 # Modules that must be aliased because they are commonly confused with
59 56 # common variables and can create aliasing and readability issues.
60 57 requirealias = {
61 58 'ui': 'uimod',
62 59 }
63 60
64 61
65 62 def walklocal(root):
66 63 """Recursively yield all descendant nodes but not in a different scope"""
67 64 todo = collections.deque(ast.iter_child_nodes(root))
68 65 yield root, False
69 66 while todo:
70 67 node = todo.popleft()
71 68 newscope = isinstance(node, ast.FunctionDef)
72 69 if not newscope:
73 70 todo.extend(ast.iter_child_nodes(node))
74 71 yield node, newscope
75 72
76 73
77 74 def dotted_name_of_path(path):
78 75 """Given a relative path to a source file, return its dotted module name.
79 76
80 77 >>> dotted_name_of_path('mercurial/error.py')
81 78 'mercurial.error'
82 79 >>> dotted_name_of_path('zlibmodule.so')
83 80 'zlib'
84 81 """
85 82 parts = path.replace(os.sep, '/').split('/')
86 83 parts[-1] = parts[-1].split('.', 1)[0] # remove .py and .so and .ARCH.so
87 84 if parts[-1].endswith('module'):
88 85 parts[-1] = parts[-1][:-6]
89 86 return '.'.join(parts)
90 87
91 88
92 89 def fromlocalfunc(modulename, localmods):
93 90 """Get a function to examine which locally defined module the
94 91 target source imports via a specified name.
95 92
96 93 `modulename` is an `dotted_name_of_path()`-ed source file path,
97 94 which may have `.__init__` at the end of it, of the target source.
98 95
99 96 `localmods` is a set of absolute `dotted_name_of_path()`-ed source file
100 97 paths of locally defined (= Mercurial specific) modules.
101 98
102 99 This function assumes that module names not existing in
103 100 `localmods` are from the Python standard library.
104 101
105 102 This function returns the function, which takes `name` argument,
106 103 and returns `(absname, dottedpath, hassubmod)` tuple if `name`
107 104 matches against locally defined module. Otherwise, it returns
108 105 False.
109 106
110 107 It is assumed that `name` doesn't have `.__init__`.
111 108
112 109 `absname` is an absolute module name of specified `name`
113 110 (e.g. "hgext.convert"). This can be used to compose prefix for sub
114 111 modules or so.
115 112
116 113 `dottedpath` is a `dotted_name_of_path()`-ed source file path
117 114 (e.g. "hgext.convert.__init__") of `name`. This is used to look
118 115 module up in `localmods` again.
119 116
120 117 `hassubmod` is whether it may have sub modules under it (for
121 118 convenient, even though this is also equivalent to "absname !=
122 119 dottednpath")
123 120
124 121 >>> localmods = {'foo.__init__', 'foo.foo1',
125 122 ... 'foo.bar.__init__', 'foo.bar.bar1',
126 123 ... 'baz.__init__', 'baz.baz1'}
127 124 >>> fromlocal = fromlocalfunc('foo.xxx', localmods)
128 125 >>> # relative
129 126 >>> fromlocal('foo1')
130 127 ('foo.foo1', 'foo.foo1', False)
131 128 >>> fromlocal('bar')
132 129 ('foo.bar', 'foo.bar.__init__', True)
133 130 >>> fromlocal('bar.bar1')
134 131 ('foo.bar.bar1', 'foo.bar.bar1', False)
135 132 >>> # absolute
136 133 >>> fromlocal('baz')
137 134 ('baz', 'baz.__init__', True)
138 135 >>> fromlocal('baz.baz1')
139 136 ('baz.baz1', 'baz.baz1', False)
140 137 >>> # unknown = maybe standard library
141 138 >>> fromlocal('os')
142 139 False
143 140 >>> fromlocal(None, 1)
144 141 ('foo', 'foo.__init__', True)
145 142 >>> fromlocal('foo1', 1)
146 143 ('foo.foo1', 'foo.foo1', False)
147 144 >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
148 145 >>> fromlocal2(None, 2)
149 146 ('foo', 'foo.__init__', True)
150 147 >>> fromlocal2('bar2', 1)
151 148 False
152 149 >>> fromlocal2('bar', 2)
153 150 ('foo.bar', 'foo.bar.__init__', True)
154 151 """
155 152 if not isinstance(modulename, str):
156 153 modulename = modulename.decode('ascii')
157 154 prefix = '.'.join(modulename.split('.')[:-1])
158 155 if prefix:
159 156 prefix += '.'
160 157
161 158 def fromlocal(name, level=0):
162 159 # name is false value when relative imports are used.
163 160 if not name:
164 161 # If relative imports are used, level must not be absolute.
165 162 assert level > 0
166 163 candidates = ['.'.join(modulename.split('.')[:-level])]
167 164 else:
168 165 if not level:
169 166 # Check relative name first.
170 167 candidates = [prefix + name, name]
171 168 else:
172 169 candidates = [
173 170 '.'.join(modulename.split('.')[:-level]) + '.' + name
174 171 ]
175 172
176 173 for n in candidates:
177 174 if n in localmods:
178 175 return (n, n, False)
179 176 dottedpath = n + '.__init__'
180 177 if dottedpath in localmods:
181 178 return (n, dottedpath, True)
182 179 return False
183 180
184 181 return fromlocal
185 182
186 183
187 184 def populateextmods(localmods):
188 185 """Populate C extension modules based on pure modules"""
189 186 newlocalmods = set(localmods)
190 187 for n in localmods:
191 188 if n.startswith('mercurial.pure.'):
192 189 m = n[len('mercurial.pure.') :]
193 190 newlocalmods.add('mercurial.cext.' + m)
194 191 newlocalmods.add('mercurial.cffi._' + m)
195 192 return newlocalmods
196 193
197 194
198 195 def list_stdlib_modules():
199 196 """List the modules present in the stdlib.
200 197
201 198 >>> py3 = sys.version_info[0] >= 3
202 199 >>> mods = set(list_stdlib_modules())
203 200 >>> 'BaseHTTPServer' in mods or py3
204 201 True
205 202
206 203 os.path isn't really a module, so it's missing:
207 204
208 205 >>> 'os.path' in mods
209 206 False
210 207
211 208 sys requires special treatment, because it's baked into the
212 209 interpreter, but it should still appear:
213 210
214 211 >>> 'sys' in mods
215 212 True
216 213
217 214 >>> 'collections' in mods
218 215 True
219 216
220 217 >>> 'cStringIO' in mods or py3
221 218 True
222 219
223 220 >>> 'cffi' in mods
224 221 True
225 222 """
226 223 for m in sys.builtin_module_names:
227 224 yield m
228 225 # These modules only exist on windows, but we should always
229 226 # consider them stdlib.
230 227 for m in ['msvcrt', '_winreg']:
231 228 yield m
232 229 yield '__builtin__'
233 230 yield 'builtins' # python3 only
234 231 yield 'importlib.abc' # python3 only
235 232 yield 'importlib.machinery' # python3 only
236 233 yield 'importlib.util' # python3 only
237 234 yield 'packaging.version'
238 235 for m in 'fcntl', 'grp', 'pwd', 'termios': # Unix only
239 236 yield m
240 237 for m in 'cPickle', 'datetime': # in Python (not C) on PyPy
241 238 yield m
242 239 for m in ['cffi']:
243 240 yield m
244 241 stdlib_prefixes = {sys.prefix, sys.exec_prefix}
245 242 # We need to supplement the list of prefixes for the search to work
246 243 # when run from within a virtualenv.
247 for mod in (basehttpserver, zlib):
244 for mod in (argparse, zlib):
248 245 if mod is None:
249 246 continue
250 247 try:
251 248 # Not all module objects have a __file__ attribute.
252 249 filename = mod.__file__
253 250 except AttributeError:
254 251 continue
255 252 dirname = os.path.dirname(filename)
256 253 for prefix in stdlib_prefixes:
257 254 if dirname.startswith(prefix):
258 255 # Then this directory is redundant.
259 256 break
260 257 else:
261 258 stdlib_prefixes.add(dirname)
262 259 sourceroot = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
263 260 for libpath in sys.path:
264 261 # We want to walk everything in sys.path that starts with something in
265 262 # stdlib_prefixes, but not directories from the hg sources.
266 263 if os.path.abspath(libpath).startswith(sourceroot) or not any(
267 264 libpath.startswith(p) for p in stdlib_prefixes
268 265 ):
269 266 continue
270 267 for top, dirs, files in os.walk(libpath):
271 268 if 'dist-packages' in top.split(os.path.sep):
272 269 continue
273 270 for i, d in reversed(list(enumerate(dirs))):
274 271 if (
275 272 not os.path.exists(os.path.join(top, d, '__init__.py'))
276 273 or top == libpath
277 274 and d in ('hgdemandimport', 'hgext', 'mercurial')
278 275 ):
279 276 del dirs[i]
280 277 for name in files:
281 278 if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')):
282 279 continue
283 280 if name.startswith('__init__.py'):
284 281 full_path = top
285 282 else:
286 283 full_path = os.path.join(top, name)
287 284 rel_path = full_path[len(libpath) + 1 :]
288 285 mod = dotted_name_of_path(rel_path)
289 286 yield mod
290 287
291 288
292 289 stdlib_modules = set(list_stdlib_modules())
293 290
294 291
295 292 def imported_modules(source, modulename, f, localmods, ignore_nested=False):
296 293 """Given the source of a file as a string, yield the names
297 294 imported by that file.
298 295
299 296 Args:
300 297 source: The python source to examine as a string.
301 298 modulename: of specified python source (may have `__init__`)
302 299 localmods: set of locally defined module names (may have `__init__`)
303 300 ignore_nested: If true, import statements that do not start in
304 301 column zero will be ignored.
305 302
306 303 Returns:
307 304 A list of absolute module names imported by the given source.
308 305
309 306 >>> f = 'foo/xxx.py'
310 307 >>> modulename = 'foo.xxx'
311 308 >>> localmods = {'foo.__init__': True,
312 309 ... 'foo.foo1': True, 'foo.foo2': True,
313 310 ... 'foo.bar.__init__': True, 'foo.bar.bar1': True,
314 311 ... 'baz.__init__': True, 'baz.baz1': True }
315 312 >>> # standard library (= not locally defined ones)
316 313 >>> sorted(imported_modules(
317 314 ... 'from stdlib1 import foo, bar; import stdlib2',
318 315 ... modulename, f, localmods))
319 316 []
320 317 >>> # relative importing
321 318 >>> sorted(imported_modules(
322 319 ... 'import foo1; from bar import bar1',
323 320 ... modulename, f, localmods))
324 321 ['foo.bar.bar1', 'foo.foo1']
325 322 >>> sorted(imported_modules(
326 323 ... 'from bar.bar1 import name1, name2, name3',
327 324 ... modulename, f, localmods))
328 325 ['foo.bar.bar1']
329 326 >>> # absolute importing
330 327 >>> sorted(imported_modules(
331 328 ... 'from baz import baz1, name1',
332 329 ... modulename, f, localmods))
333 330 ['baz.__init__', 'baz.baz1']
334 331 >>> # mixed importing, even though it shouldn't be recommended
335 332 >>> sorted(imported_modules(
336 333 ... 'import stdlib, foo1, baz',
337 334 ... modulename, f, localmods))
338 335 ['baz.__init__', 'foo.foo1']
339 336 >>> # ignore_nested
340 337 >>> sorted(imported_modules(
341 338 ... '''import foo
342 339 ... def wat():
343 340 ... import bar
344 341 ... ''', modulename, f, localmods))
345 342 ['foo.__init__', 'foo.bar.__init__']
346 343 >>> sorted(imported_modules(
347 344 ... '''import foo
348 345 ... def wat():
349 346 ... import bar
350 347 ... ''', modulename, f, localmods, ignore_nested=True))
351 348 ['foo.__init__']
352 349 """
353 350 fromlocal = fromlocalfunc(modulename, localmods)
354 351 for node in ast.walk(ast.parse(source, f)):
355 352 if ignore_nested and getattr(node, 'col_offset', 0) > 0:
356 353 continue
357 354 if isinstance(node, ast.Import):
358 355 for n in node.names:
359 356 found = fromlocal(n.name)
360 357 if not found:
361 358 # this should import standard library
362 359 continue
363 360 yield found[1]
364 361 elif isinstance(node, ast.ImportFrom):
365 362 found = fromlocal(node.module, node.level)
366 363 if not found:
367 364 # this should import standard library
368 365 continue
369 366
370 367 absname, dottedpath, hassubmod = found
371 368 if not hassubmod:
372 369 # "dottedpath" is not a package; must be imported
373 370 yield dottedpath
374 371 # examination of "node.names" should be redundant
375 372 # e.g.: from mercurial.node import nullid, nullrev
376 373 continue
377 374
378 375 modnotfound = False
379 376 prefix = absname + '.'
380 377 for n in node.names:
381 378 found = fromlocal(prefix + n.name)
382 379 if not found:
383 380 # this should be a function or a property of "node.module"
384 381 modnotfound = True
385 382 continue
386 383 yield found[1]
387 384 if modnotfound and dottedpath != modulename:
388 385 # "dottedpath" is a package, but imported because of non-module
389 386 # lookup
390 387 # specifically allow "from . import foo" from __init__.py
391 388 yield dottedpath
392 389
393 390
394 391 def verify_import_convention(module, source, localmods):
395 392 """Verify imports match our established coding convention."""
396 393 root = ast.parse(source)
397 394
398 395 return verify_modern_convention(module, root, localmods)
399 396
400 397
401 398 def verify_modern_convention(module, root, localmods, root_col_offset=0):
402 399 """Verify a file conforms to the modern import convention rules.
403 400
404 401 The rules of the modern convention are:
405 402
406 403 * Ordering is stdlib followed by local imports. Each group is lexically
407 404 sorted.
408 405 * Importing multiple modules via "import X, Y" is not allowed: use
409 406 separate import statements.
410 407 * Importing multiple modules via "from X import ..." is allowed if using
411 408 parenthesis and one entry per line.
412 409 * Only 1 relative import statement per import level ("from .", "from ..")
413 410 is allowed.
414 411 * Relative imports from higher levels must occur before lower levels. e.g.
415 412 "from .." must be before "from .".
416 413 * Imports from peer packages should use relative import (e.g. do not
417 414 "import mercurial.foo" from a "mercurial.*" module).
418 415 * Symbols can only be imported from specific modules (see
419 416 `allowsymbolimports`). For other modules, first import the module then
420 417 assign the symbol to a module-level variable. In addition, these imports
421 418 must be performed before other local imports. This rule only
422 419 applies to import statements outside of any blocks.
423 420 * Relative imports from the standard library are not allowed, unless that
424 421 library is also a local module.
425 422 * Certain modules must be aliased to alternate names to avoid aliasing
426 423 and readability problems. See `requirealias`.
427 424 """
428 425 if not isinstance(module, str):
429 426 module = module.decode('ascii')
430 427 topmodule = module.split('.')[0]
431 428 fromlocal = fromlocalfunc(module, localmods)
432 429
433 430 # Whether a local/non-stdlib import has been performed.
434 431 seenlocal = None
435 432 # Whether a local/non-stdlib, non-symbol import has been seen.
436 433 seennonsymbollocal = False
437 434 # The last name to be imported (for sorting).
438 435 lastname = None
439 436 laststdlib = None
440 437 # Relative import levels encountered so far.
441 438 seenlevels = set()
442 439
443 440 for node, newscope in walklocal(root):
444 441
445 442 def msg(fmt, *args):
446 443 return (fmt % args, node.lineno)
447 444
448 445 if newscope:
449 446 # Check for local imports in function
450 447 for r in verify_modern_convention(
451 448 module, node, localmods, node.col_offset + 4
452 449 ):
453 450 yield r
454 451 elif isinstance(node, ast.Import):
455 452 # Disallow "import foo, bar" and require separate imports
456 453 # for each module.
457 454 if len(node.names) > 1:
458 455 yield msg(
459 456 'multiple imported names: %s',
460 457 ', '.join(n.name for n in node.names),
461 458 )
462 459
463 460 name = node.names[0].name
464 461 asname = node.names[0].asname
465 462
466 463 stdlib = name in stdlib_modules
467 464
468 465 # Ignore sorting rules on imports inside blocks.
469 466 if node.col_offset == root_col_offset:
470 467 if lastname and name < lastname and laststdlib == stdlib:
471 468 yield msg(
472 469 'imports not lexically sorted: %s < %s', name, lastname
473 470 )
474 471
475 472 lastname = name
476 473 laststdlib = stdlib
477 474
478 475 # stdlib imports should be before local imports.
479 476 if stdlib and seenlocal and node.col_offset == root_col_offset:
480 477 yield msg(
481 478 'stdlib import "%s" follows local import: %s',
482 479 name,
483 480 seenlocal,
484 481 )
485 482
486 483 if not stdlib:
487 484 seenlocal = name
488 485
489 486 # Import of sibling modules should use relative imports.
490 487 topname = name.split('.')[0]
491 488 if topname == topmodule:
492 489 yield msg('import should be relative: %s', name)
493 490
494 491 if name in requirealias and asname != requirealias[name]:
495 492 yield msg(
496 493 '%s module must be "as" aliased to %s',
497 494 name,
498 495 requirealias[name],
499 496 )
500 497
501 498 elif isinstance(node, ast.ImportFrom):
502 499 # Resolve the full imported module name.
503 500 if node.level > 0:
504 501 fullname = '.'.join(module.split('.')[: -node.level])
505 502 if node.module:
506 503 fullname += '.%s' % node.module
507 504 else:
508 505 assert node.module
509 506 fullname = node.module
510 507
511 508 topname = fullname.split('.')[0]
512 509 if topname == topmodule:
513 510 yield msg('import should be relative: %s', fullname)
514 511
515 512 # __future__ is special since it needs to come first and use
516 513 # symbol import.
517 514 if fullname != '__future__':
518 515 if not fullname or (
519 516 fullname in stdlib_modules
520 517 # allow standard 'from typing import ...' style
521 518 and fullname.startswith('.')
522 519 and fullname not in localmods
523 520 and fullname + '.__init__' not in localmods
524 521 ):
525 522 yield msg('relative import of stdlib module')
526 523 else:
527 524 seenlocal = fullname
528 525
529 526 # Direct symbol import is only allowed from certain modules and
530 527 # must occur before non-symbol imports.
531 528 found = fromlocal(node.module, node.level)
532 529 if found and found[2]: # node.module is a package
533 530 prefix = found[0] + '.'
534 531 symbols = (
535 532 n.name for n in node.names if not fromlocal(prefix + n.name)
536 533 )
537 534 else:
538 535 symbols = (n.name for n in node.names)
539 536 symbols = [sym for sym in symbols if sym not in directsymbols]
540 537 if node.module and node.col_offset == root_col_offset:
541 538 if symbols and fullname not in allowsymbolimports:
542 539 yield msg(
543 540 'direct symbol import %s from %s',
544 541 ', '.join(symbols),
545 542 fullname,
546 543 )
547 544
548 545 if symbols and seennonsymbollocal:
549 546 yield msg(
550 547 'symbol import follows non-symbol import: %s', fullname
551 548 )
552 549 if not symbols and fullname not in stdlib_modules:
553 550 seennonsymbollocal = True
554 551
555 552 if not node.module:
556 553 assert node.level
557 554
558 555 # Only allow 1 group per level.
559 556 if (
560 557 node.level in seenlevels
561 558 and node.col_offset == root_col_offset
562 559 ):
563 560 yield msg(
564 561 'multiple "from %s import" statements', '.' * node.level
565 562 )
566 563
567 564 # Higher-level groups come before lower-level groups.
568 565 if any(node.level > l for l in seenlevels):
569 566 yield msg(
570 567 'higher-level import should come first: %s', fullname
571 568 )
572 569
573 570 seenlevels.add(node.level)
574 571
575 572 # Entries in "from .X import ( ... )" lists must be lexically
576 573 # sorted.
577 574 lastentryname = None
578 575
579 576 for n in node.names:
580 577 if lastentryname and n.name < lastentryname:
581 578 yield msg(
582 579 'imports from %s not lexically sorted: %s < %s',
583 580 fullname,
584 581 n.name,
585 582 lastentryname,
586 583 )
587 584
588 585 lastentryname = n.name
589 586
590 587 if n.name in requirealias and n.asname != requirealias[n.name]:
591 588 yield msg(
592 589 '%s from %s must be "as" aliased to %s',
593 590 n.name,
594 591 fullname,
595 592 requirealias[n.name],
596 593 )
597 594
598 595
599 596 class CircularImport(Exception):
600 597 pass
601 598
602 599
603 600 def checkmod(mod, imports):
604 601 shortest = {}
605 602 visit = [[mod]]
606 603 while visit:
607 604 path = visit.pop(0)
608 605 for i in sorted(imports.get(path[-1], [])):
609 606 if len(path) < shortest.get(i, 1000):
610 607 shortest[i] = len(path)
611 608 if i in path:
612 609 if i == path[0]:
613 610 raise CircularImport(path)
614 611 continue
615 612 visit.append(path + [i])
616 613
617 614
618 615 def rotatecycle(cycle):
619 616 """arrange a cycle so that the lexicographically first module listed first
620 617
621 618 >>> rotatecycle(['foo', 'bar'])
622 619 ['bar', 'foo', 'bar']
623 620 """
624 621 lowest = min(cycle)
625 622 idx = cycle.index(lowest)
626 623 return cycle[idx:] + cycle[:idx] + [lowest]
627 624
628 625
629 626 def find_cycles(imports):
630 627 """Find cycles in an already-loaded import graph.
631 628
632 629 All module names recorded in `imports` should be absolute one.
633 630
634 631 >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'],
635 632 ... 'top.bar': ['top.baz', 'sys'],
636 633 ... 'top.baz': ['top.foo'],
637 634 ... 'top.qux': ['top.foo']}
638 635 >>> print('\\n'.join(sorted(find_cycles(imports))))
639 636 top.bar -> top.baz -> top.foo -> top.bar
640 637 top.foo -> top.qux -> top.foo
641 638 """
642 639 cycles = set()
643 640 for mod in sorted(imports.keys()):
644 641 try:
645 642 checkmod(mod, imports)
646 643 except CircularImport as e:
647 644 cycle = e.args[0]
648 645 cycles.add(" -> ".join(rotatecycle(cycle)))
649 646 return cycles
650 647
651 648
652 649 def _cycle_sortkey(c):
653 650 return len(c), c
654 651
655 652
656 653 def embedded(f, modname, src):
657 654 """Extract embedded python code
658 655
659 656 >>> def _forcestr(thing):
660 657 ... if not isinstance(thing, str):
661 658 ... return thing.decode('ascii')
662 659 ... return thing
663 660 >>> def test(fn, lines):
664 661 ... for s, m, f, l in embedded(fn, b"example", lines):
665 662 ... print("%s %s %d" % (_forcestr(m), _forcestr(f), l))
666 663 ... print(repr(_forcestr(s)))
667 664 >>> lines = [
668 665 ... 'comment',
669 666 ... ' >>> from __future__ import print_function',
670 667 ... " >>> ' multiline",
671 668 ... " ... string'",
672 669 ... ' ',
673 670 ... 'comment',
674 671 ... ' $ cat > foo.py <<EOF',
675 672 ... ' > from __future__ import print_function',
676 673 ... ' > EOF',
677 674 ... ]
678 675 >>> test(b"example.t", lines)
679 676 example[2] doctest.py 1
680 677 "from __future__ import print_function\\n' multiline\\nstring'\\n\\n"
681 678 example[8] foo.py 7
682 679 'from __future__ import print_function\\n'
683 680 """
684 681 errors = []
685 682 for name, starts, ends, code in testparseutil.pyembedded(f, src, errors):
686 683 if not name:
687 684 # use 'doctest.py', in order to make already existing
688 685 # doctest above pass instantly
689 686 name = 'doctest.py'
690 687 # "starts" is "line number" (1-origin), but embedded() is
691 688 # expected to return "line offset" (0-origin). Therefore, this
692 689 # yields "starts - 1".
693 690 if not isinstance(modname, str):
694 691 modname = modname.decode('utf8')
695 692 yield code, "%s[%d]" % (modname, starts), name, starts - 1
696 693
697 694
698 695 def sources(f, modname):
699 696 """Yields possibly multiple sources from a filepath
700 697
701 698 input: filepath, modulename
702 699 yields: script(string), modulename, filepath, linenumber
703 700
704 701 For embedded scripts, the modulename and filepath will be different
705 702 from the function arguments. linenumber is an offset relative to
706 703 the input file.
707 704 """
708 705 py = False
709 706 if not f.endswith('.t'):
710 707 with open(f, 'rb') as src:
711 708 yield src.read(), modname, f, 0
712 709 py = True
713 710 if py or f.endswith('.t'):
714 711 # Strictly speaking we should sniff for the magic header that denotes
715 712 # Python source file encoding. But in reality we don't use anything
716 713 # other than ASCII (mainly) and UTF-8 (in a few exceptions), so
717 714 # simplicity is fine.
718 715 with io.open(f, 'r', encoding='utf-8') as src:
719 716 for script, modname, t, line in embedded(f, modname, src):
720 717 yield script, modname.encode('utf8'), t, line
721 718
722 719
723 720 def main(argv):
724 721 if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):
725 722 print('Usage: %s {-|file [file] [file] ...}')
726 723 return 1
727 724 if argv[1] == '-':
728 725 argv = argv[:1]
729 726 argv.extend(l.rstrip() for l in sys.stdin.readlines())
730 727 localmodpaths = {}
731 728 used_imports = {}
732 729 any_errors = False
733 730 for source_path in argv[1:]:
734 731 modname = dotted_name_of_path(source_path)
735 732 localmodpaths[modname] = source_path
736 733 localmods = populateextmods(localmodpaths)
737 734 for localmodname, source_path in sorted(localmodpaths.items()):
738 735 if not isinstance(localmodname, bytes):
739 736 # This is only safe because all hg's files are ascii
740 737 localmodname = localmodname.encode('ascii')
741 738 for src, modname, name, line in sources(source_path, localmodname):
742 739 try:
743 740 used_imports[modname] = sorted(
744 741 imported_modules(
745 742 src, modname, name, localmods, ignore_nested=True
746 743 )
747 744 )
748 745 for error, lineno in verify_import_convention(
749 746 modname, src, localmods
750 747 ):
751 748 any_errors = True
752 749 print('%s:%d: %s' % (source_path, lineno + line, error))
753 750 except SyntaxError as e:
754 751 print(
755 752 '%s:%d: SyntaxError: %s' % (source_path, e.lineno + line, e)
756 753 )
757 754 cycles = find_cycles(used_imports)
758 755 if cycles:
759 756 firstmods = set()
760 757 for c in sorted(cycles, key=_cycle_sortkey):
761 758 first = c.split()[0]
762 759 # As a rough cut, ignore any cycle that starts with the
763 760 # same module as some other cycle. Otherwise we see lots
764 761 # of cycles that are effectively duplicates.
765 762 if first in firstmods:
766 763 continue
767 764 print('Import cycle:', c)
768 765 firstmods.add(first)
769 766 any_errors = True
770 767 return any_errors != 0
771 768
772 769
773 770 if __name__ == '__main__':
774 771 sys.exit(int(main(sys.argv)))
General Comments 0
You need to be logged in to leave comments. Login now