##// END OF EJS Templates
tests: treat `select` as a built-in module on Windows...
Matt Harbison -
r53080:3ba998d7 stable
parent child Browse files
Show More
@@ -1,772 +1,772
1 1 #!/usr/bin/env python3
2 2
3 3
4 4 import ast
5 5 import collections
6 6 import io
7 7 import os
8 8 import sys
9 9
10 10 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
11 11 # to work when run from a virtualenv. The modules were chosen empirically
12 12 # so that the return value matches the return value without virtualenv.
13 13 if True: # disable lexical sorting checks
14 14 import argparse
15 15 import zlib
16 16
17 17 import testparseutil
18 18
19 19 # Allow list of modules that symbols can be directly imported from.
20 20 allowsymbolimports = (
21 21 '__future__',
22 22 'breezy',
23 23 'concurrent',
24 24 'hgclient',
25 25 'mercurial',
26 26 'mercurial.hgweb.common',
27 27 'mercurial.hgweb.request',
28 28 'mercurial.i18n',
29 29 'mercurial.interfaces',
30 30 'mercurial.node',
31 31 'mercurial.pycompat',
32 32 # for revlog to re-export constant to extensions
33 33 'mercurial.revlogutils.constants',
34 34 'mercurial.revlogutils.flagutil',
35 35 # for cffi modules to re-export pure functions
36 36 'mercurial.pure.base85',
37 37 'mercurial.pure.bdiff',
38 38 'mercurial.pure.mpatch',
39 39 'mercurial.pure.osutil',
40 40 'mercurial.pure.parsers',
41 41 # third-party imports should be directly imported
42 42 'mercurial.thirdparty',
43 43 'mercurial.thirdparty.attr',
44 44 'mercurial.thirdparty.jaraco.collections',
45 45 'mercurial.thirdparty.tomli',
46 46 'mercurial.thirdparty.zope',
47 47 'mercurial.thirdparty.zope.interface',
48 48 'typing',
49 49 'xml.etree.ElementTree',
50 50 )
51 51
52 52 # Allow list of symbols that can be directly imported.
53 53 directsymbols = ('demandimport',)
54 54
55 55 # Modules that must be aliased because they are commonly confused with
56 56 # common variables and can create aliasing and readability issues.
57 57 requirealias = {
58 58 'ui': 'uimod',
59 59 }
60 60
61 61
62 62 def walklocal(root):
63 63 """Recursively yield all descendant nodes but not in a different scope"""
64 64 todo = collections.deque(ast.iter_child_nodes(root))
65 65 yield root, False
66 66 while todo:
67 67 node = todo.popleft()
68 68 newscope = isinstance(node, ast.FunctionDef)
69 69 if not newscope:
70 70 todo.extend(ast.iter_child_nodes(node))
71 71 yield node, newscope
72 72
73 73
74 74 def dotted_name_of_path(path):
75 75 """Given a relative path to a source file, return its dotted module name.
76 76
77 77 >>> dotted_name_of_path('mercurial/error.py')
78 78 'mercurial.error'
79 79 >>> dotted_name_of_path('zlibmodule.so')
80 80 'zlib'
81 81 """
82 82 parts = path.replace(os.sep, '/').split('/')
83 83 parts[-1] = parts[-1].split('.', 1)[0] # remove .py and .so and .ARCH.so
84 84 if parts[-1].endswith('module'):
85 85 parts[-1] = parts[-1][:-6]
86 86 return '.'.join(parts)
87 87
88 88
89 89 def fromlocalfunc(modulename, localmods):
90 90 """Get a function to examine which locally defined module the
91 91 target source imports via a specified name.
92 92
93 93 `modulename` is an `dotted_name_of_path()`-ed source file path,
94 94 which may have `.__init__` at the end of it, of the target source.
95 95
96 96 `localmods` is a set of absolute `dotted_name_of_path()`-ed source file
97 97 paths of locally defined (= Mercurial specific) modules.
98 98
99 99 This function assumes that module names not existing in
100 100 `localmods` are from the Python standard library.
101 101
102 102 This function returns the function, which takes `name` argument,
103 103 and returns `(absname, dottedpath, hassubmod)` tuple if `name`
104 104 matches against locally defined module. Otherwise, it returns
105 105 False.
106 106
107 107 It is assumed that `name` doesn't have `.__init__`.
108 108
109 109 `absname` is an absolute module name of specified `name`
110 110 (e.g. "hgext.convert"). This can be used to compose prefix for sub
111 111 modules or so.
112 112
113 113 `dottedpath` is a `dotted_name_of_path()`-ed source file path
114 114 (e.g. "hgext.convert.__init__") of `name`. This is used to look
115 115 module up in `localmods` again.
116 116
117 117 `hassubmod` is whether it may have sub modules under it (for
118 118 convenient, even though this is also equivalent to "absname !=
119 119 dottednpath")
120 120
121 121 >>> localmods = {'foo.__init__', 'foo.foo1',
122 122 ... 'foo.bar.__init__', 'foo.bar.bar1',
123 123 ... 'baz.__init__', 'baz.baz1'}
124 124 >>> fromlocal = fromlocalfunc('foo.xxx', localmods)
125 125 >>> # relative
126 126 >>> fromlocal('foo1')
127 127 ('foo.foo1', 'foo.foo1', False)
128 128 >>> fromlocal('bar')
129 129 ('foo.bar', 'foo.bar.__init__', True)
130 130 >>> fromlocal('bar.bar1')
131 131 ('foo.bar.bar1', 'foo.bar.bar1', False)
132 132 >>> # absolute
133 133 >>> fromlocal('baz')
134 134 ('baz', 'baz.__init__', True)
135 135 >>> fromlocal('baz.baz1')
136 136 ('baz.baz1', 'baz.baz1', False)
137 137 >>> # unknown = maybe standard library
138 138 >>> fromlocal('os')
139 139 False
140 140 >>> fromlocal(None, 1)
141 141 ('foo', 'foo.__init__', True)
142 142 >>> fromlocal('foo1', 1)
143 143 ('foo.foo1', 'foo.foo1', False)
144 144 >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
145 145 >>> fromlocal2(None, 2)
146 146 ('foo', 'foo.__init__', True)
147 147 >>> fromlocal2('bar2', 1)
148 148 False
149 149 >>> fromlocal2('bar', 2)
150 150 ('foo.bar', 'foo.bar.__init__', True)
151 151 """
152 152 if not isinstance(modulename, str):
153 153 modulename = modulename.decode('ascii')
154 154 prefix = '.'.join(modulename.split('.')[:-1])
155 155 if prefix:
156 156 prefix += '.'
157 157
158 158 def fromlocal(name, level=0):
159 159 # name is false value when relative imports are used.
160 160 if not name:
161 161 # If relative imports are used, level must not be absolute.
162 162 assert level > 0
163 163 candidates = ['.'.join(modulename.split('.')[:-level])]
164 164 else:
165 165 if not level:
166 166 # Check relative name first.
167 167 candidates = [prefix + name, name]
168 168 else:
169 169 candidates = [
170 170 '.'.join(modulename.split('.')[:-level]) + '.' + name
171 171 ]
172 172
173 173 for n in candidates:
174 174 if n in localmods:
175 175 return (n, n, False)
176 176 dottedpath = n + '.__init__'
177 177 if dottedpath in localmods:
178 178 return (n, dottedpath, True)
179 179 return False
180 180
181 181 return fromlocal
182 182
183 183
184 184 def populateextmods(localmods):
185 185 """Populate C extension modules based on pure modules"""
186 186 newlocalmods = set(localmods)
187 187 for n in localmods:
188 188 if n.startswith('mercurial.pure.'):
189 189 m = n[len('mercurial.pure.') :]
190 190 newlocalmods.add('mercurial.cext.' + m)
191 191 newlocalmods.add('mercurial.cffi._' + m)
192 192 return newlocalmods
193 193
194 194
195 195 def list_stdlib_modules():
196 196 """List the modules present in the stdlib.
197 197
198 198 >>> mods = set(list_stdlib_modules())
199 199 >>> 'http' in mods
200 200 True
201 201
202 202 os.path isn't really a module, so it's missing:
203 203
204 204 >>> 'os.path' in mods
205 205 False
206 206
207 207 sys requires special treatment, because it's baked into the
208 208 interpreter, but it should still appear:
209 209
210 210 >>> 'sys' in mods
211 211 True
212 212
213 213 >>> 'collections' in mods
214 214 True
215 215
216 216 >>> 'array' in mods
217 217 True
218 218
219 219 >>> 'cffi' in mods
220 220 True
221 221 """
222 222 for m in sys.builtin_module_names:
223 223 yield m
224 224 # These modules only exist on windows, but we should always
225 225 # consider them stdlib.
226 226 for m in ['msvcrt', '_winreg']:
227 227 yield m
228 228 yield '__builtin__'
229 229 yield 'builtins' # python3 only
230 230 yield 'importlib.abc' # python3 only
231 231 yield 'importlib.machinery' # python3 only
232 232 yield 'importlib.util' # python3 only
233 233 yield 'packaging.version'
234 for m in 'fcntl', 'grp', 'pwd', 'termios': # Unix only
234 for m in 'fcntl', 'grp', 'pwd', 'select', 'termios': # Unix only
235 235 yield m
236 236 for m in 'cPickle', 'datetime': # in Python (not C) on PyPy
237 237 yield m
238 238 for m in ['cffi']:
239 239 yield m
240 240 yield 'distutils' # in Python < 3.12
241 241 yield 'distutils.version' # in Python < 3.12
242 242 stdlib_prefixes = {sys.prefix, sys.exec_prefix}
243 243 # We need to supplement the list of prefixes for the search to work
244 244 # when run from within a virtualenv.
245 245 for mod in (argparse, zlib):
246 246 if mod is None:
247 247 continue
248 248 try:
249 249 # Not all module objects have a __file__ attribute.
250 250 filename = mod.__file__
251 251 except AttributeError:
252 252 continue
253 253 dirname = os.path.dirname(filename)
254 254 for prefix in stdlib_prefixes:
255 255 if dirname.startswith(prefix):
256 256 # Then this directory is redundant.
257 257 break
258 258 else:
259 259 stdlib_prefixes.add(dirname)
260 260 sourceroot = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
261 261 for libpath in sys.path:
262 262 # We want to walk everything in sys.path that starts with something in
263 263 # stdlib_prefixes, but not directories from the hg sources.
264 264 if os.path.abspath(libpath).startswith(sourceroot) or not any(
265 265 libpath.startswith(p) for p in stdlib_prefixes
266 266 ):
267 267 continue
268 268 for top, dirs, files in os.walk(libpath):
269 269 if 'dist-packages' in top.split(os.path.sep):
270 270 continue
271 271 for i, d in reversed(list(enumerate(dirs))):
272 272 if (
273 273 not os.path.exists(os.path.join(top, d, '__init__.py'))
274 274 or top == libpath
275 275 and d in ('hgdemandimport', 'hgext', 'mercurial')
276 276 ):
277 277 del dirs[i]
278 278 for name in files:
279 279 if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')):
280 280 continue
281 281 if name.startswith('__init__.py'):
282 282 full_path = top
283 283 else:
284 284 full_path = os.path.join(top, name)
285 285 rel_path = full_path[len(libpath) + 1 :]
286 286 mod = dotted_name_of_path(rel_path)
287 287 yield mod
288 288
289 289
290 290 stdlib_modules = set(list_stdlib_modules())
291 291
292 292
293 293 def imported_modules(source, modulename, f, localmods, ignore_nested=False):
294 294 """Given the source of a file as a string, yield the names
295 295 imported by that file.
296 296
297 297 Args:
298 298 source: The python source to examine as a string.
299 299 modulename: of specified python source (may have `__init__`)
300 300 localmods: set of locally defined module names (may have `__init__`)
301 301 ignore_nested: If true, import statements that do not start in
302 302 column zero will be ignored.
303 303
304 304 Returns:
305 305 A list of absolute module names imported by the given source.
306 306
307 307 >>> f = 'foo/xxx.py'
308 308 >>> modulename = 'foo.xxx'
309 309 >>> localmods = {'foo.__init__': True,
310 310 ... 'foo.foo1': True, 'foo.foo2': True,
311 311 ... 'foo.bar.__init__': True, 'foo.bar.bar1': True,
312 312 ... 'baz.__init__': True, 'baz.baz1': True }
313 313 >>> # standard library (= not locally defined ones)
314 314 >>> sorted(imported_modules(
315 315 ... 'from stdlib1 import foo, bar; import stdlib2',
316 316 ... modulename, f, localmods))
317 317 []
318 318 >>> # relative importing
319 319 >>> sorted(imported_modules(
320 320 ... 'import foo1; from bar import bar1',
321 321 ... modulename, f, localmods))
322 322 ['foo.bar.bar1', 'foo.foo1']
323 323 >>> sorted(imported_modules(
324 324 ... 'from bar.bar1 import name1, name2, name3',
325 325 ... modulename, f, localmods))
326 326 ['foo.bar.bar1']
327 327 >>> # absolute importing
328 328 >>> sorted(imported_modules(
329 329 ... 'from baz import baz1, name1',
330 330 ... modulename, f, localmods))
331 331 ['baz.__init__', 'baz.baz1']
332 332 >>> # mixed importing, even though it shouldn't be recommended
333 333 >>> sorted(imported_modules(
334 334 ... 'import stdlib, foo1, baz',
335 335 ... modulename, f, localmods))
336 336 ['baz.__init__', 'foo.foo1']
337 337 >>> # ignore_nested
338 338 >>> sorted(imported_modules(
339 339 ... '''import foo
340 340 ... def wat():
341 341 ... import bar
342 342 ... ''', modulename, f, localmods))
343 343 ['foo.__init__', 'foo.bar.__init__']
344 344 >>> sorted(imported_modules(
345 345 ... '''import foo
346 346 ... def wat():
347 347 ... import bar
348 348 ... ''', modulename, f, localmods, ignore_nested=True))
349 349 ['foo.__init__']
350 350 """
351 351 fromlocal = fromlocalfunc(modulename, localmods)
352 352 for node in ast.walk(ast.parse(source, f)):
353 353 if ignore_nested and getattr(node, 'col_offset', 0) > 0:
354 354 continue
355 355 if isinstance(node, ast.Import):
356 356 for n in node.names:
357 357 found = fromlocal(n.name)
358 358 if not found:
359 359 # this should import standard library
360 360 continue
361 361 yield found[1]
362 362 elif isinstance(node, ast.ImportFrom):
363 363 found = fromlocal(node.module, node.level)
364 364 if not found:
365 365 # this should import standard library
366 366 continue
367 367
368 368 absname, dottedpath, hassubmod = found
369 369 if not hassubmod:
370 370 # "dottedpath" is not a package; must be imported
371 371 yield dottedpath
372 372 # examination of "node.names" should be redundant
373 373 # e.g.: from mercurial.node import nullid, nullrev
374 374 continue
375 375
376 376 modnotfound = False
377 377 prefix = absname + '.'
378 378 for n in node.names:
379 379 found = fromlocal(prefix + n.name)
380 380 if not found:
381 381 # this should be a function or a property of "node.module"
382 382 modnotfound = True
383 383 continue
384 384 yield found[1]
385 385 if modnotfound and dottedpath != modulename:
386 386 # "dottedpath" is a package, but imported because of non-module
387 387 # lookup
388 388 # specifically allow "from . import foo" from __init__.py
389 389 yield dottedpath
390 390
391 391
392 392 def verify_import_convention(module, source, localmods):
393 393 """Verify imports match our established coding convention."""
394 394 root = ast.parse(source)
395 395
396 396 return verify_modern_convention(module, root, localmods)
397 397
398 398
399 399 def verify_modern_convention(module, root, localmods, root_col_offset=0):
400 400 """Verify a file conforms to the modern import convention rules.
401 401
402 402 The rules of the modern convention are:
403 403
404 404 * Ordering is stdlib followed by local imports. Each group is lexically
405 405 sorted.
406 406 * Importing multiple modules via "import X, Y" is not allowed: use
407 407 separate import statements.
408 408 * Importing multiple modules via "from X import ..." is allowed if using
409 409 parenthesis and one entry per line.
410 410 * Only 1 relative import statement per import level ("from .", "from ..")
411 411 is allowed.
412 412 * Relative imports from higher levels must occur before lower levels. e.g.
413 413 "from .." must be before "from .".
414 414 * Imports from peer packages should use relative import (e.g. do not
415 415 "import mercurial.foo" from a "mercurial.*" module).
416 416 * Symbols can only be imported from specific modules (see
417 417 `allowsymbolimports`). For other modules, first import the module then
418 418 assign the symbol to a module-level variable. In addition, these imports
419 419 must be performed before other local imports. This rule only
420 420 applies to import statements outside of any blocks.
421 421 * Relative imports from the standard library are not allowed, unless that
422 422 library is also a local module.
423 423 * Certain modules must be aliased to alternate names to avoid aliasing
424 424 and readability problems. See `requirealias`.
425 425 """
426 426 if not isinstance(module, str):
427 427 module = module.decode('ascii')
428 428 topmodule = module.split('.')[0]
429 429 fromlocal = fromlocalfunc(module, localmods)
430 430
431 431 # Whether a local/non-stdlib import has been performed.
432 432 seenlocal = None
433 433 # Whether a local/non-stdlib, non-symbol import has been seen.
434 434 seennonsymbollocal = False
435 435 # The last name to be imported (for sorting).
436 436 lastname = None
437 437 laststdlib = None
438 438 # Relative import levels encountered so far.
439 439 seenlevels = set()
440 440
441 441 for node, newscope in walklocal(root):
442 442
443 443 def msg(fmt, *args):
444 444 return (fmt % args, node.lineno)
445 445
446 446 if newscope:
447 447 # Check for local imports in function
448 448 for r in verify_modern_convention(
449 449 module, node, localmods, node.col_offset + 4
450 450 ):
451 451 yield r
452 452 elif isinstance(node, ast.Import):
453 453 # Disallow "import foo, bar" and require separate imports
454 454 # for each module.
455 455 if len(node.names) > 1:
456 456 yield msg(
457 457 'multiple imported names: %s',
458 458 ', '.join(n.name for n in node.names),
459 459 )
460 460
461 461 name = node.names[0].name
462 462 asname = node.names[0].asname
463 463
464 464 stdlib = name in stdlib_modules
465 465
466 466 # Ignore sorting rules on imports inside blocks.
467 467 if node.col_offset == root_col_offset:
468 468 if lastname and name < lastname and laststdlib == stdlib:
469 469 yield msg(
470 470 'imports not lexically sorted: %s < %s', name, lastname
471 471 )
472 472
473 473 lastname = name
474 474 laststdlib = stdlib
475 475
476 476 # stdlib imports should be before local imports.
477 477 if stdlib and seenlocal and node.col_offset == root_col_offset:
478 478 yield msg(
479 479 'stdlib import "%s" follows local import: %s',
480 480 name,
481 481 seenlocal,
482 482 )
483 483
484 484 if not stdlib:
485 485 seenlocal = name
486 486
487 487 # Import of sibling modules should use relative imports.
488 488 topname = name.split('.')[0]
489 489 if topname == topmodule:
490 490 yield msg('import should be relative: %s', name)
491 491
492 492 if name in requirealias and asname != requirealias[name]:
493 493 yield msg(
494 494 '%s module must be "as" aliased to %s',
495 495 name,
496 496 requirealias[name],
497 497 )
498 498
499 499 elif isinstance(node, ast.ImportFrom):
500 500 # Resolve the full imported module name.
501 501 if node.level > 0:
502 502 fullname = '.'.join(module.split('.')[: -node.level])
503 503 if node.module:
504 504 fullname += '.%s' % node.module
505 505 else:
506 506 assert node.module
507 507 fullname = node.module
508 508
509 509 topname = fullname.split('.')[0]
510 510 if topname == topmodule:
511 511 yield msg('import should be relative: %s', fullname)
512 512
513 513 # __future__ is special since it needs to come first and use
514 514 # symbol import.
515 515 if fullname != '__future__':
516 516 if not fullname or (
517 517 fullname in stdlib_modules
518 518 # allow standard 'from typing import ...' style
519 519 and fullname.startswith('.')
520 520 and fullname not in localmods
521 521 and fullname + '.__init__' not in localmods
522 522 ):
523 523 yield msg('relative import of stdlib module')
524 524 else:
525 525 seenlocal = fullname
526 526
527 527 # Direct symbol import is only allowed from certain modules and
528 528 # must occur before non-symbol imports.
529 529 found = fromlocal(node.module, node.level)
530 530 if found and found[2]: # node.module is a package
531 531 prefix = found[0] + '.'
532 532 symbols = (
533 533 n.name for n in node.names if not fromlocal(prefix + n.name)
534 534 )
535 535 else:
536 536 symbols = (n.name for n in node.names)
537 537 symbols = [sym for sym in symbols if sym not in directsymbols]
538 538 if node.module and node.col_offset == root_col_offset:
539 539 if symbols and fullname not in allowsymbolimports:
540 540 yield msg(
541 541 'direct symbol import %s from %s',
542 542 ', '.join(symbols),
543 543 fullname,
544 544 )
545 545
546 546 if symbols and seennonsymbollocal:
547 547 yield msg(
548 548 'symbol import follows non-symbol import: %s', fullname
549 549 )
550 550 if not symbols and fullname not in stdlib_modules:
551 551 seennonsymbollocal = True
552 552
553 553 if not node.module:
554 554 assert node.level
555 555
556 556 # Only allow 1 group per level.
557 557 if (
558 558 node.level in seenlevels
559 559 and node.col_offset == root_col_offset
560 560 ):
561 561 yield msg(
562 562 'multiple "from %s import" statements', '.' * node.level
563 563 )
564 564
565 565 # Higher-level groups come before lower-level groups.
566 566 if any(node.level > l for l in seenlevels):
567 567 yield msg(
568 568 'higher-level import should come first: %s', fullname
569 569 )
570 570
571 571 seenlevels.add(node.level)
572 572
573 573 # Entries in "from .X import ( ... )" lists must be lexically
574 574 # sorted.
575 575 lastentryname = None
576 576
577 577 for n in node.names:
578 578 if lastentryname and n.name < lastentryname:
579 579 yield msg(
580 580 'imports from %s not lexically sorted: %s < %s',
581 581 fullname,
582 582 n.name,
583 583 lastentryname,
584 584 )
585 585
586 586 lastentryname = n.name
587 587
588 588 if n.name in requirealias and n.asname != requirealias[n.name]:
589 589 yield msg(
590 590 '%s from %s must be "as" aliased to %s',
591 591 n.name,
592 592 fullname,
593 593 requirealias[n.name],
594 594 )
595 595
596 596
597 597 class CircularImport(Exception):
598 598 pass
599 599
600 600
601 601 def checkmod(mod, imports):
602 602 shortest = {}
603 603 visit = [[mod]]
604 604 while visit:
605 605 path = visit.pop(0)
606 606 for i in sorted(imports.get(path[-1], [])):
607 607 if len(path) < shortest.get(i, 1000):
608 608 shortest[i] = len(path)
609 609 if i in path:
610 610 if i == path[0]:
611 611 raise CircularImport(path)
612 612 continue
613 613 visit.append(path + [i])
614 614
615 615
616 616 def rotatecycle(cycle):
617 617 """arrange a cycle so that the lexicographically first module listed first
618 618
619 619 >>> rotatecycle(['foo', 'bar'])
620 620 ['bar', 'foo', 'bar']
621 621 """
622 622 lowest = min(cycle)
623 623 idx = cycle.index(lowest)
624 624 return cycle[idx:] + cycle[:idx] + [lowest]
625 625
626 626
627 627 def find_cycles(imports):
628 628 """Find cycles in an already-loaded import graph.
629 629
630 630 All module names recorded in `imports` should be absolute one.
631 631
632 632 >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'],
633 633 ... 'top.bar': ['top.baz', 'sys'],
634 634 ... 'top.baz': ['top.foo'],
635 635 ... 'top.qux': ['top.foo']}
636 636 >>> print('\\n'.join(sorted(find_cycles(imports))))
637 637 top.bar -> top.baz -> top.foo -> top.bar
638 638 top.foo -> top.qux -> top.foo
639 639 """
640 640 cycles = set()
641 641 for mod in sorted(imports.keys()):
642 642 try:
643 643 checkmod(mod, imports)
644 644 except CircularImport as e:
645 645 cycle = e.args[0]
646 646 cycles.add(" -> ".join(rotatecycle(cycle)))
647 647 return cycles
648 648
649 649
650 650 def _cycle_sortkey(c):
651 651 return len(c), c
652 652
653 653
654 654 def embedded(f, modname, src):
655 655 """Extract embedded python code
656 656
657 657 >>> def _forcestr(thing):
658 658 ... if not isinstance(thing, str):
659 659 ... return thing.decode('ascii')
660 660 ... return thing
661 661 >>> def test(fn, lines):
662 662 ... for s, m, f, l in embedded(fn, b"example", lines):
663 663 ... print("%s %s %d" % (_forcestr(m), _forcestr(f), l))
664 664 ... print(repr(_forcestr(s)))
665 665 >>> lines = [
666 666 ... 'comment',
667 667 ... ' >>> from __future__ import print_function',
668 668 ... " >>> ' multiline",
669 669 ... " ... string'",
670 670 ... ' ',
671 671 ... 'comment',
672 672 ... ' $ cat > foo.py <<EOF',
673 673 ... ' > from __future__ import print_function',
674 674 ... ' > EOF',
675 675 ... ]
676 676 >>> test(b"example.t", lines)
677 677 example[2] doctest.py 1
678 678 "from __future__ import print_function\\n' multiline\\nstring'\\n\\n"
679 679 example[8] foo.py 7
680 680 'from __future__ import print_function\\n'
681 681 """
682 682 errors = []
683 683 for name, starts, ends, code in testparseutil.pyembedded(f, src, errors):
684 684 if not name:
685 685 # use 'doctest.py', in order to make already existing
686 686 # doctest above pass instantly
687 687 name = 'doctest.py'
688 688 # "starts" is "line number" (1-origin), but embedded() is
689 689 # expected to return "line offset" (0-origin). Therefore, this
690 690 # yields "starts - 1".
691 691 if not isinstance(modname, str):
692 692 modname = modname.decode('utf8')
693 693 yield code, "%s[%d]" % (modname, starts), name, starts - 1
694 694
695 695
696 696 def sources(f, modname):
697 697 """Yields possibly multiple sources from a filepath
698 698
699 699 input: filepath, modulename
700 700 yields: script(string), modulename, filepath, linenumber
701 701
702 702 For embedded scripts, the modulename and filepath will be different
703 703 from the function arguments. linenumber is an offset relative to
704 704 the input file.
705 705 """
706 706 py = False
707 707 if not f.endswith('.t'):
708 708 with open(f, 'rb') as src:
709 709 yield src.read(), modname, f, 0
710 710 py = True
711 711 if py or f.endswith('.t'):
712 712 # Strictly speaking we should sniff for the magic header that denotes
713 713 # Python source file encoding. But in reality we don't use anything
714 714 # other than ASCII (mainly) and UTF-8 (in a few exceptions), so
715 715 # simplicity is fine.
716 716 with io.open(f, 'r', encoding='utf-8') as src:
717 717 for script, modname, t, line in embedded(f, modname, src):
718 718 yield script, modname.encode('utf8'), t, line
719 719
720 720
721 721 def main(argv):
722 722 if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):
723 723 print('Usage: %s {-|file [file] [file] ...}')
724 724 return 1
725 725 if argv[1] == '-':
726 726 argv = argv[:1]
727 727 argv.extend(l.rstrip() for l in sys.stdin.readlines())
728 728 localmodpaths = {}
729 729 used_imports = {}
730 730 any_errors = False
731 731 for source_path in argv[1:]:
732 732 modname = dotted_name_of_path(source_path)
733 733 localmodpaths[modname] = source_path
734 734 localmods = populateextmods(localmodpaths)
735 735 for localmodname, source_path in sorted(localmodpaths.items()):
736 736 if not isinstance(localmodname, bytes):
737 737 # This is only safe because all hg's files are ascii
738 738 localmodname = localmodname.encode('ascii')
739 739 for src, modname, name, line in sources(source_path, localmodname):
740 740 try:
741 741 used_imports[modname] = sorted(
742 742 imported_modules(
743 743 src, modname, name, localmods, ignore_nested=True
744 744 )
745 745 )
746 746 for error, lineno in verify_import_convention(
747 747 modname, src, localmods
748 748 ):
749 749 any_errors = True
750 750 print('%s:%d: %s' % (source_path, lineno + line, error))
751 751 except SyntaxError as e:
752 752 print(
753 753 '%s:%d: SyntaxError: %s' % (source_path, e.lineno + line, e)
754 754 )
755 755 cycles = find_cycles(used_imports)
756 756 if cycles:
757 757 firstmods = set()
758 758 for c in sorted(cycles, key=_cycle_sortkey):
759 759 first = c.split()[0]
760 760 # As a rough cut, ignore any cycle that starts with the
761 761 # same module as some other cycle. Otherwise we see lots
762 762 # of cycles that are effectively duplicates.
763 763 if first in firstmods:
764 764 continue
765 765 print('Import cycle:', c)
766 766 firstmods.add(first)
767 767 any_errors = True
768 768 return any_errors != 0
769 769
770 770
771 771 if __name__ == '__main__':
772 772 sys.exit(int(main(sys.argv)))
General Comments 0
You need to be logged in to leave comments. Login now