##// END OF EJS Templates
contrib: allow importing "symbols" from mercurial
Augie Fackler -
r33894:c9cf69d0 default
parent child Browse files
Show More
@@ -1,762 +1,763
1 1 #!/usr/bin/env python
2 2
3 3 from __future__ import absolute_import, print_function
4 4
5 5 import ast
6 6 import collections
7 7 import os
8 8 import re
9 9 import sys
10 10
11 11 # Import a minimal set of stdlib modules needed for list_stdlib_modules()
12 12 # to work when run from a virtualenv. The modules were chosen empirically
13 13 # so that the return value matches the return value without virtualenv.
14 14 if True: # disable lexical sorting checks
15 15 try:
16 16 import BaseHTTPServer as basehttpserver
17 17 except ImportError:
18 18 basehttpserver = None
19 19 import zlib
20 20
21 21 # Whitelist of modules that symbols can be directly imported from.
22 22 allowsymbolimports = (
23 23 '__future__',
24 'mercurial',
24 25 'mercurial.hgweb.common',
25 26 'mercurial.hgweb.request',
26 27 'mercurial.i18n',
27 28 'mercurial.node',
28 29 # for cffi modules to re-export pure functions
29 30 'mercurial.pure.base85',
30 31 'mercurial.pure.bdiff',
31 32 'mercurial.pure.diffhelpers',
32 33 'mercurial.pure.mpatch',
33 34 'mercurial.pure.osutil',
34 35 'mercurial.pure.parsers',
35 36 )
36 37
37 38 # Whitelist of symbols that can be directly imported.
38 39 directsymbols = (
39 40 'demandimport',
40 41 )
41 42
42 43 # Modules that must be aliased because they are commonly confused with
43 44 # common variables and can create aliasing and readability issues.
44 45 requirealias = {
45 46 'ui': 'uimod',
46 47 }
47 48
48 49 def usingabsolute(root):
49 50 """Whether absolute imports are being used."""
50 51 if sys.version_info[0] >= 3:
51 52 return True
52 53
53 54 for node in ast.walk(root):
54 55 if isinstance(node, ast.ImportFrom):
55 56 if node.module == '__future__':
56 57 for n in node.names:
57 58 if n.name == 'absolute_import':
58 59 return True
59 60
60 61 return False
61 62
62 63 def walklocal(root):
63 64 """Recursively yield all descendant nodes but not in a different scope"""
64 65 todo = collections.deque(ast.iter_child_nodes(root))
65 66 yield root, False
66 67 while todo:
67 68 node = todo.popleft()
68 69 newscope = isinstance(node, ast.FunctionDef)
69 70 if not newscope:
70 71 todo.extend(ast.iter_child_nodes(node))
71 72 yield node, newscope
72 73
73 74 def dotted_name_of_path(path):
74 75 """Given a relative path to a source file, return its dotted module name.
75 76
76 77 >>> dotted_name_of_path('mercurial/error.py')
77 78 'mercurial.error'
78 79 >>> dotted_name_of_path('zlibmodule.so')
79 80 'zlib'
80 81 """
81 82 parts = path.replace(os.sep, '/').split('/')
82 83 parts[-1] = parts[-1].split('.', 1)[0] # remove .py and .so and .ARCH.so
83 84 if parts[-1].endswith('module'):
84 85 parts[-1] = parts[-1][:-6]
85 86 return '.'.join(parts)
86 87
87 88 def fromlocalfunc(modulename, localmods):
88 89 """Get a function to examine which locally defined module the
89 90 target source imports via a specified name.
90 91
91 92 `modulename` is an `dotted_name_of_path()`-ed source file path,
92 93 which may have `.__init__` at the end of it, of the target source.
93 94
94 95 `localmods` is a set of absolute `dotted_name_of_path()`-ed source file
95 96 paths of locally defined (= Mercurial specific) modules.
96 97
97 98 This function assumes that module names not existing in
98 99 `localmods` are from the Python standard library.
99 100
100 101 This function returns the function, which takes `name` argument,
101 102 and returns `(absname, dottedpath, hassubmod)` tuple if `name`
102 103 matches against locally defined module. Otherwise, it returns
103 104 False.
104 105
105 106 It is assumed that `name` doesn't have `.__init__`.
106 107
107 108 `absname` is an absolute module name of specified `name`
108 109 (e.g. "hgext.convert"). This can be used to compose prefix for sub
109 110 modules or so.
110 111
111 112 `dottedpath` is a `dotted_name_of_path()`-ed source file path
112 113 (e.g. "hgext.convert.__init__") of `name`. This is used to look
113 114 module up in `localmods` again.
114 115
115 116 `hassubmod` is whether it may have sub modules under it (for
116 117 convenient, even though this is also equivalent to "absname !=
117 118 dottednpath")
118 119
119 120 >>> localmods = {'foo.__init__', 'foo.foo1',
120 121 ... 'foo.bar.__init__', 'foo.bar.bar1',
121 122 ... 'baz.__init__', 'baz.baz1'}
122 123 >>> fromlocal = fromlocalfunc('foo.xxx', localmods)
123 124 >>> # relative
124 125 >>> fromlocal('foo1')
125 126 ('foo.foo1', 'foo.foo1', False)
126 127 >>> fromlocal('bar')
127 128 ('foo.bar', 'foo.bar.__init__', True)
128 129 >>> fromlocal('bar.bar1')
129 130 ('foo.bar.bar1', 'foo.bar.bar1', False)
130 131 >>> # absolute
131 132 >>> fromlocal('baz')
132 133 ('baz', 'baz.__init__', True)
133 134 >>> fromlocal('baz.baz1')
134 135 ('baz.baz1', 'baz.baz1', False)
135 136 >>> # unknown = maybe standard library
136 137 >>> fromlocal('os')
137 138 False
138 139 >>> fromlocal(None, 1)
139 140 ('foo', 'foo.__init__', True)
140 141 >>> fromlocal('foo1', 1)
141 142 ('foo.foo1', 'foo.foo1', False)
142 143 >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods)
143 144 >>> fromlocal2(None, 2)
144 145 ('foo', 'foo.__init__', True)
145 146 >>> fromlocal2('bar2', 1)
146 147 False
147 148 >>> fromlocal2('bar', 2)
148 149 ('foo.bar', 'foo.bar.__init__', True)
149 150 """
150 151 if not isinstance(modulename, str):
151 152 modulename = modulename.decode('ascii')
152 153 prefix = '.'.join(modulename.split('.')[:-1])
153 154 if prefix:
154 155 prefix += '.'
155 156 def fromlocal(name, level=0):
156 157 # name is false value when relative imports are used.
157 158 if not name:
158 159 # If relative imports are used, level must not be absolute.
159 160 assert level > 0
160 161 candidates = ['.'.join(modulename.split('.')[:-level])]
161 162 else:
162 163 if not level:
163 164 # Check relative name first.
164 165 candidates = [prefix + name, name]
165 166 else:
166 167 candidates = ['.'.join(modulename.split('.')[:-level]) +
167 168 '.' + name]
168 169
169 170 for n in candidates:
170 171 if n in localmods:
171 172 return (n, n, False)
172 173 dottedpath = n + '.__init__'
173 174 if dottedpath in localmods:
174 175 return (n, dottedpath, True)
175 176 return False
176 177 return fromlocal
177 178
178 179 def populateextmods(localmods):
179 180 """Populate C extension modules based on pure modules"""
180 181 newlocalmods = set(localmods)
181 182 for n in localmods:
182 183 if n.startswith('mercurial.pure.'):
183 184 m = n[len('mercurial.pure.'):]
184 185 newlocalmods.add('mercurial.cext.' + m)
185 186 newlocalmods.add('mercurial.cffi._' + m)
186 187 return newlocalmods
187 188
188 189 def list_stdlib_modules():
189 190 """List the modules present in the stdlib.
190 191
191 192 >>> py3 = sys.version_info[0] >= 3
192 193 >>> mods = set(list_stdlib_modules())
193 194 >>> 'BaseHTTPServer' in mods or py3
194 195 True
195 196
196 197 os.path isn't really a module, so it's missing:
197 198
198 199 >>> 'os.path' in mods
199 200 False
200 201
201 202 sys requires special treatment, because it's baked into the
202 203 interpreter, but it should still appear:
203 204
204 205 >>> 'sys' in mods
205 206 True
206 207
207 208 >>> 'collections' in mods
208 209 True
209 210
210 211 >>> 'cStringIO' in mods or py3
211 212 True
212 213
213 214 >>> 'cffi' in mods
214 215 True
215 216 """
216 217 for m in sys.builtin_module_names:
217 218 yield m
218 219 # These modules only exist on windows, but we should always
219 220 # consider them stdlib.
220 221 for m in ['msvcrt', '_winreg']:
221 222 yield m
222 223 yield 'builtins' # python3 only
223 224 for m in 'fcntl', 'grp', 'pwd', 'termios': # Unix only
224 225 yield m
225 226 for m in 'cPickle', 'datetime': # in Python (not C) on PyPy
226 227 yield m
227 228 for m in ['cffi']:
228 229 yield m
229 230 stdlib_prefixes = {sys.prefix, sys.exec_prefix}
230 231 # We need to supplement the list of prefixes for the search to work
231 232 # when run from within a virtualenv.
232 233 for mod in (basehttpserver, zlib):
233 234 if mod is None:
234 235 continue
235 236 try:
236 237 # Not all module objects have a __file__ attribute.
237 238 filename = mod.__file__
238 239 except AttributeError:
239 240 continue
240 241 dirname = os.path.dirname(filename)
241 242 for prefix in stdlib_prefixes:
242 243 if dirname.startswith(prefix):
243 244 # Then this directory is redundant.
244 245 break
245 246 else:
246 247 stdlib_prefixes.add(dirname)
247 248 for libpath in sys.path:
248 249 # We want to walk everything in sys.path that starts with
249 250 # something in stdlib_prefixes.
250 251 if not any(libpath.startswith(p) for p in stdlib_prefixes):
251 252 continue
252 253 for top, dirs, files in os.walk(libpath):
253 254 for i, d in reversed(list(enumerate(dirs))):
254 255 if (not os.path.exists(os.path.join(top, d, '__init__.py'))
255 256 or top == libpath and d in ('hgdemandimport', 'hgext',
256 257 'mercurial')):
257 258 del dirs[i]
258 259 for name in files:
259 260 if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')):
260 261 continue
261 262 if name.startswith('__init__.py'):
262 263 full_path = top
263 264 else:
264 265 full_path = os.path.join(top, name)
265 266 rel_path = full_path[len(libpath) + 1:]
266 267 mod = dotted_name_of_path(rel_path)
267 268 yield mod
268 269
269 270 stdlib_modules = set(list_stdlib_modules())
270 271
271 272 def imported_modules(source, modulename, f, localmods, ignore_nested=False):
272 273 """Given the source of a file as a string, yield the names
273 274 imported by that file.
274 275
275 276 Args:
276 277 source: The python source to examine as a string.
277 278 modulename: of specified python source (may have `__init__`)
278 279 localmods: set of locally defined module names (may have `__init__`)
279 280 ignore_nested: If true, import statements that do not start in
280 281 column zero will be ignored.
281 282
282 283 Returns:
283 284 A list of absolute module names imported by the given source.
284 285
285 286 >>> f = 'foo/xxx.py'
286 287 >>> modulename = 'foo.xxx'
287 288 >>> localmods = {'foo.__init__': True,
288 289 ... 'foo.foo1': True, 'foo.foo2': True,
289 290 ... 'foo.bar.__init__': True, 'foo.bar.bar1': True,
290 291 ... 'baz.__init__': True, 'baz.baz1': True }
291 292 >>> # standard library (= not locally defined ones)
292 293 >>> sorted(imported_modules(
293 294 ... 'from stdlib1 import foo, bar; import stdlib2',
294 295 ... modulename, f, localmods))
295 296 []
296 297 >>> # relative importing
297 298 >>> sorted(imported_modules(
298 299 ... 'import foo1; from bar import bar1',
299 300 ... modulename, f, localmods))
300 301 ['foo.bar.bar1', 'foo.foo1']
301 302 >>> sorted(imported_modules(
302 303 ... 'from bar.bar1 import name1, name2, name3',
303 304 ... modulename, f, localmods))
304 305 ['foo.bar.bar1']
305 306 >>> # absolute importing
306 307 >>> sorted(imported_modules(
307 308 ... 'from baz import baz1, name1',
308 309 ... modulename, f, localmods))
309 310 ['baz.__init__', 'baz.baz1']
310 311 >>> # mixed importing, even though it shouldn't be recommended
311 312 >>> sorted(imported_modules(
312 313 ... 'import stdlib, foo1, baz',
313 314 ... modulename, f, localmods))
314 315 ['baz.__init__', 'foo.foo1']
315 316 >>> # ignore_nested
316 317 >>> sorted(imported_modules(
317 318 ... '''import foo
318 319 ... def wat():
319 320 ... import bar
320 321 ... ''', modulename, f, localmods))
321 322 ['foo.__init__', 'foo.bar.__init__']
322 323 >>> sorted(imported_modules(
323 324 ... '''import foo
324 325 ... def wat():
325 326 ... import bar
326 327 ... ''', modulename, f, localmods, ignore_nested=True))
327 328 ['foo.__init__']
328 329 """
329 330 fromlocal = fromlocalfunc(modulename, localmods)
330 331 for node in ast.walk(ast.parse(source, f)):
331 332 if ignore_nested and getattr(node, 'col_offset', 0) > 0:
332 333 continue
333 334 if isinstance(node, ast.Import):
334 335 for n in node.names:
335 336 found = fromlocal(n.name)
336 337 if not found:
337 338 # this should import standard library
338 339 continue
339 340 yield found[1]
340 341 elif isinstance(node, ast.ImportFrom):
341 342 found = fromlocal(node.module, node.level)
342 343 if not found:
343 344 # this should import standard library
344 345 continue
345 346
346 347 absname, dottedpath, hassubmod = found
347 348 if not hassubmod:
348 349 # "dottedpath" is not a package; must be imported
349 350 yield dottedpath
350 351 # examination of "node.names" should be redundant
351 352 # e.g.: from mercurial.node import nullid, nullrev
352 353 continue
353 354
354 355 modnotfound = False
355 356 prefix = absname + '.'
356 357 for n in node.names:
357 358 found = fromlocal(prefix + n.name)
358 359 if not found:
359 360 # this should be a function or a property of "node.module"
360 361 modnotfound = True
361 362 continue
362 363 yield found[1]
363 364 if modnotfound:
364 365 # "dottedpath" is a package, but imported because of non-module
365 366 # lookup
366 367 yield dottedpath
367 368
368 369 def verify_import_convention(module, source, localmods):
369 370 """Verify imports match our established coding convention.
370 371
371 372 We have 2 conventions: legacy and modern. The modern convention is in
372 373 effect when using absolute imports.
373 374
374 375 The legacy convention only looks for mixed imports. The modern convention
375 376 is much more thorough.
376 377 """
377 378 root = ast.parse(source)
378 379 absolute = usingabsolute(root)
379 380
380 381 if absolute:
381 382 return verify_modern_convention(module, root, localmods)
382 383 else:
383 384 return verify_stdlib_on_own_line(root)
384 385
385 386 def verify_modern_convention(module, root, localmods, root_col_offset=0):
386 387 """Verify a file conforms to the modern import convention rules.
387 388
388 389 The rules of the modern convention are:
389 390
390 391 * Ordering is stdlib followed by local imports. Each group is lexically
391 392 sorted.
392 393 * Importing multiple modules via "import X, Y" is not allowed: use
393 394 separate import statements.
394 395 * Importing multiple modules via "from X import ..." is allowed if using
395 396 parenthesis and one entry per line.
396 397 * Only 1 relative import statement per import level ("from .", "from ..")
397 398 is allowed.
398 399 * Relative imports from higher levels must occur before lower levels. e.g.
399 400 "from .." must be before "from .".
400 401 * Imports from peer packages should use relative import (e.g. do not
401 402 "import mercurial.foo" from a "mercurial.*" module).
402 403 * Symbols can only be imported from specific modules (see
403 404 `allowsymbolimports`). For other modules, first import the module then
404 405 assign the symbol to a module-level variable. In addition, these imports
405 406 must be performed before other local imports. This rule only
406 407 applies to import statements outside of any blocks.
407 408 * Relative imports from the standard library are not allowed.
408 409 * Certain modules must be aliased to alternate names to avoid aliasing
409 410 and readability problems. See `requirealias`.
410 411 """
411 412 if not isinstance(module, str):
412 413 module = module.decode('ascii')
413 414 topmodule = module.split('.')[0]
414 415 fromlocal = fromlocalfunc(module, localmods)
415 416
416 417 # Whether a local/non-stdlib import has been performed.
417 418 seenlocal = None
418 419 # Whether a local/non-stdlib, non-symbol import has been seen.
419 420 seennonsymbollocal = False
420 421 # The last name to be imported (for sorting).
421 422 lastname = None
422 423 laststdlib = None
423 424 # Relative import levels encountered so far.
424 425 seenlevels = set()
425 426
426 427 for node, newscope in walklocal(root):
427 428 def msg(fmt, *args):
428 429 return (fmt % args, node.lineno)
429 430 if newscope:
430 431 # Check for local imports in function
431 432 for r in verify_modern_convention(module, node, localmods,
432 433 node.col_offset + 4):
433 434 yield r
434 435 elif isinstance(node, ast.Import):
435 436 # Disallow "import foo, bar" and require separate imports
436 437 # for each module.
437 438 if len(node.names) > 1:
438 439 yield msg('multiple imported names: %s',
439 440 ', '.join(n.name for n in node.names))
440 441
441 442 name = node.names[0].name
442 443 asname = node.names[0].asname
443 444
444 445 stdlib = name in stdlib_modules
445 446
446 447 # Ignore sorting rules on imports inside blocks.
447 448 if node.col_offset == root_col_offset:
448 449 if lastname and name < lastname and laststdlib == stdlib:
449 450 yield msg('imports not lexically sorted: %s < %s',
450 451 name, lastname)
451 452
452 453 lastname = name
453 454 laststdlib = stdlib
454 455
455 456 # stdlib imports should be before local imports.
456 457 if stdlib and seenlocal and node.col_offset == root_col_offset:
457 458 yield msg('stdlib import "%s" follows local import: %s',
458 459 name, seenlocal)
459 460
460 461 if not stdlib:
461 462 seenlocal = name
462 463
463 464 # Import of sibling modules should use relative imports.
464 465 topname = name.split('.')[0]
465 466 if topname == topmodule:
466 467 yield msg('import should be relative: %s', name)
467 468
468 469 if name in requirealias and asname != requirealias[name]:
469 470 yield msg('%s module must be "as" aliased to %s',
470 471 name, requirealias[name])
471 472
472 473 elif isinstance(node, ast.ImportFrom):
473 474 # Resolve the full imported module name.
474 475 if node.level > 0:
475 476 fullname = '.'.join(module.split('.')[:-node.level])
476 477 if node.module:
477 478 fullname += '.%s' % node.module
478 479 else:
479 480 assert node.module
480 481 fullname = node.module
481 482
482 483 topname = fullname.split('.')[0]
483 484 if topname == topmodule:
484 485 yield msg('import should be relative: %s', fullname)
485 486
486 487 # __future__ is special since it needs to come first and use
487 488 # symbol import.
488 489 if fullname != '__future__':
489 490 if not fullname or fullname in stdlib_modules:
490 491 yield msg('relative import of stdlib module')
491 492 else:
492 493 seenlocal = fullname
493 494
494 495 # Direct symbol import is only allowed from certain modules and
495 496 # must occur before non-symbol imports.
496 497 found = fromlocal(node.module, node.level)
497 498 if found and found[2]: # node.module is a package
498 499 prefix = found[0] + '.'
499 500 symbols = (n.name for n in node.names
500 501 if not fromlocal(prefix + n.name))
501 502 else:
502 503 symbols = (n.name for n in node.names)
503 504 symbols = [sym for sym in symbols if sym not in directsymbols]
504 505 if node.module and node.col_offset == root_col_offset:
505 506 if symbols and fullname not in allowsymbolimports:
506 507 yield msg('direct symbol import %s from %s',
507 508 ', '.join(symbols), fullname)
508 509
509 510 if symbols and seennonsymbollocal:
510 511 yield msg('symbol import follows non-symbol import: %s',
511 512 fullname)
512 513 if not symbols and fullname not in stdlib_modules:
513 514 seennonsymbollocal = True
514 515
515 516 if not node.module:
516 517 assert node.level
517 518
518 519 # Only allow 1 group per level.
519 520 if (node.level in seenlevels
520 521 and node.col_offset == root_col_offset):
521 522 yield msg('multiple "from %s import" statements',
522 523 '.' * node.level)
523 524
524 525 # Higher-level groups come before lower-level groups.
525 526 if any(node.level > l for l in seenlevels):
526 527 yield msg('higher-level import should come first: %s',
527 528 fullname)
528 529
529 530 seenlevels.add(node.level)
530 531
531 532 # Entries in "from .X import ( ... )" lists must be lexically
532 533 # sorted.
533 534 lastentryname = None
534 535
535 536 for n in node.names:
536 537 if lastentryname and n.name < lastentryname:
537 538 yield msg('imports from %s not lexically sorted: %s < %s',
538 539 fullname, n.name, lastentryname)
539 540
540 541 lastentryname = n.name
541 542
542 543 if n.name in requirealias and n.asname != requirealias[n.name]:
543 544 yield msg('%s from %s must be "as" aliased to %s',
544 545 n.name, fullname, requirealias[n.name])
545 546
546 547 def verify_stdlib_on_own_line(root):
547 548 """Given some python source, verify that stdlib imports are done
548 549 in separate statements from relative local module imports.
549 550
550 551 >>> list(verify_stdlib_on_own_line(ast.parse('import sys, foo')))
551 552 [('mixed imports\\n stdlib: sys\\n relative: foo', 1)]
552 553 >>> list(verify_stdlib_on_own_line(ast.parse('import sys, os')))
553 554 []
554 555 >>> list(verify_stdlib_on_own_line(ast.parse('import foo, bar')))
555 556 []
556 557 """
557 558 for node in ast.walk(root):
558 559 if isinstance(node, ast.Import):
559 560 from_stdlib = {False: [], True: []}
560 561 for n in node.names:
561 562 from_stdlib[n.name in stdlib_modules].append(n.name)
562 563 if from_stdlib[True] and from_stdlib[False]:
563 564 yield ('mixed imports\n stdlib: %s\n relative: %s' %
564 565 (', '.join(sorted(from_stdlib[True])),
565 566 ', '.join(sorted(from_stdlib[False]))), node.lineno)
566 567
567 568 class CircularImport(Exception):
568 569 pass
569 570
570 571 def checkmod(mod, imports):
571 572 shortest = {}
572 573 visit = [[mod]]
573 574 while visit:
574 575 path = visit.pop(0)
575 576 for i in sorted(imports.get(path[-1], [])):
576 577 if len(path) < shortest.get(i, 1000):
577 578 shortest[i] = len(path)
578 579 if i in path:
579 580 if i == path[0]:
580 581 raise CircularImport(path)
581 582 continue
582 583 visit.append(path + [i])
583 584
584 585 def rotatecycle(cycle):
585 586 """arrange a cycle so that the lexicographically first module listed first
586 587
587 588 >>> rotatecycle(['foo', 'bar'])
588 589 ['bar', 'foo', 'bar']
589 590 """
590 591 lowest = min(cycle)
591 592 idx = cycle.index(lowest)
592 593 return cycle[idx:] + cycle[:idx] + [lowest]
593 594
594 595 def find_cycles(imports):
595 596 """Find cycles in an already-loaded import graph.
596 597
597 598 All module names recorded in `imports` should be absolute one.
598 599
599 600 >>> from __future__ import print_function
600 601 >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'],
601 602 ... 'top.bar': ['top.baz', 'sys'],
602 603 ... 'top.baz': ['top.foo'],
603 604 ... 'top.qux': ['top.foo']}
604 605 >>> print('\\n'.join(sorted(find_cycles(imports))))
605 606 top.bar -> top.baz -> top.foo -> top.bar
606 607 top.foo -> top.qux -> top.foo
607 608 """
608 609 cycles = set()
609 610 for mod in sorted(imports.keys()):
610 611 try:
611 612 checkmod(mod, imports)
612 613 except CircularImport as e:
613 614 cycle = e.args[0]
614 615 cycles.add(" -> ".join(rotatecycle(cycle)))
615 616 return cycles
616 617
617 618 def _cycle_sortkey(c):
618 619 return len(c), c
619 620
620 621 def embedded(f, modname, src):
621 622 """Extract embedded python code
622 623
623 624 >>> def _forcestr(thing):
624 625 ... if not isinstance(thing, str):
625 626 ... return thing.decode('ascii')
626 627 ... return thing
627 628 >>> def test(fn, lines):
628 629 ... for s, m, f, l in embedded(fn, b"example", lines):
629 630 ... print("%s %s %d" % (_forcestr(m), _forcestr(f), l))
630 631 ... print(repr(_forcestr(s)))
631 632 >>> lines = [
632 633 ... b'comment',
633 634 ... b' >>> from __future__ import print_function',
634 635 ... b" >>> ' multiline",
635 636 ... b" ... string'",
636 637 ... b' ',
637 638 ... b'comment',
638 639 ... b' $ cat > foo.py <<EOF',
639 640 ... b' > from __future__ import print_function',
640 641 ... b' > EOF',
641 642 ... ]
642 643 >>> test(b"example.t", lines)
643 644 example[2] doctest.py 2
644 645 "from __future__ import print_function\\n' multiline\\nstring'\\n"
645 646 example[7] foo.py 7
646 647 'from __future__ import print_function\\n'
647 648 """
648 649 inlinepython = 0
649 650 shpython = 0
650 651 script = []
651 652 prefix = 6
652 653 t = ''
653 654 n = 0
654 655 for l in src:
655 656 n += 1
656 657 if not l.endswith(b'\n'):
657 658 l += b'\n'
658 659 if l.startswith(b' >>> '): # python inlines
659 660 if shpython:
660 661 print("%s:%d: Parse Error" % (f, n))
661 662 if not inlinepython:
662 663 # We've just entered a Python block.
663 664 inlinepython = n
664 665 t = b'doctest.py'
665 666 script.append(l[prefix:])
666 667 continue
667 668 if l.startswith(b' ... '): # python inlines
668 669 script.append(l[prefix:])
669 670 continue
670 671 cat = re.search(br"\$ \s*cat\s*>\s*(\S+\.py)\s*<<\s*EOF", l)
671 672 if cat:
672 673 if inlinepython:
673 674 yield b''.join(script), (b"%s[%d]" %
674 675 (modname, inlinepython)), t, inlinepython
675 676 script = []
676 677 inlinepython = 0
677 678 shpython = n
678 679 t = cat.group(1)
679 680 continue
680 681 if shpython and l.startswith(b' > '): # sh continuation
681 682 if l == b' > EOF\n':
682 683 yield b''.join(script), (b"%s[%d]" %
683 684 (modname, shpython)), t, shpython
684 685 script = []
685 686 shpython = 0
686 687 else:
687 688 script.append(l[4:])
688 689 continue
689 690 if inlinepython and l == b' \n':
690 691 yield b''.join(script), (b"%s[%d]" %
691 692 (modname, inlinepython)), t, inlinepython
692 693 script = []
693 694 inlinepython = 0
694 695 continue
695 696
696 697 def sources(f, modname):
697 698 """Yields possibly multiple sources from a filepath
698 699
699 700 input: filepath, modulename
700 701 yields: script(string), modulename, filepath, linenumber
701 702
702 703 For embedded scripts, the modulename and filepath will be different
703 704 from the function arguments. linenumber is an offset relative to
704 705 the input file.
705 706 """
706 707 py = False
707 708 if not f.endswith('.t'):
708 709 with open(f, 'rb') as src:
709 710 yield src.read(), modname, f, 0
710 711 py = True
711 712 if py or f.endswith('.t'):
712 713 with open(f, 'rb') as src:
713 714 for script, modname, t, line in embedded(f, modname, src):
714 715 yield script, modname, t, line
715 716
716 717 def main(argv):
717 718 if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2):
718 719 print('Usage: %s {-|file [file] [file] ...}')
719 720 return 1
720 721 if argv[1] == '-':
721 722 argv = argv[:1]
722 723 argv.extend(l.rstrip() for l in sys.stdin.readlines())
723 724 localmodpaths = {}
724 725 used_imports = {}
725 726 any_errors = False
726 727 for source_path in argv[1:]:
727 728 modname = dotted_name_of_path(source_path)
728 729 localmodpaths[modname] = source_path
729 730 localmods = populateextmods(localmodpaths)
730 731 for localmodname, source_path in sorted(localmodpaths.items()):
731 732 if not isinstance(localmodname, bytes):
732 733 # This is only safe because all hg's files are ascii
733 734 localmodname = localmodname.encode('ascii')
734 735 for src, modname, name, line in sources(source_path, localmodname):
735 736 try:
736 737 used_imports[modname] = sorted(
737 738 imported_modules(src, modname, name, localmods,
738 739 ignore_nested=True))
739 740 for error, lineno in verify_import_convention(modname, src,
740 741 localmods):
741 742 any_errors = True
742 743 print('%s:%d: %s' % (source_path, lineno + line, error))
743 744 except SyntaxError as e:
744 745 print('%s:%d: SyntaxError: %s' %
745 746 (source_path, e.lineno + line, e))
746 747 cycles = find_cycles(used_imports)
747 748 if cycles:
748 749 firstmods = set()
749 750 for c in sorted(cycles, key=_cycle_sortkey):
750 751 first = c.split()[0]
751 752 # As a rough cut, ignore any cycle that starts with the
752 753 # same module as some other cycle. Otherwise we see lots
753 754 # of cycles that are effectively duplicates.
754 755 if first in firstmods:
755 756 continue
756 757 print('Import cycle:', c)
757 758 firstmods.add(first)
758 759 any_errors = True
759 760 return any_errors != 0
760 761
761 762 if __name__ == '__main__':
762 763 sys.exit(int(main(sys.argv)))
General Comments 0
You need to be logged in to leave comments. Login now