import-checker.py
780 lines
| 28.0 KiB
| text/x-python
|
PythonLexer
/ contrib / import-checker.py
Yuya Nishihara
|
r26954 | #!/usr/bin/env python | ||
timeless
|
r28703 | from __future__ import absolute_import, print_function | ||
timeless
|
r28702 | |||
Augie Fackler
|
r20036 | import ast | ||
Yuya Nishihara
|
r26965 | import collections | ||
Augie Fackler
|
r20036 | import os | ||
timeless
|
r28922 | import re | ||
Augie Fackler
|
r20036 | import sys | ||
Chris Jerdonek
|
r20198 | # Import a minimal set of stdlib modules needed for list_stdlib_modules() | ||
# to work when run from a virtualenv. The modules were chosen empirically | ||||
# so that the return value matches the return value without virtualenv. | ||||
Yuya Nishihara
|
r29211 | if True: # disable lexical sorting checks | ||
Augie Fackler
|
r33878 | try: | ||
import BaseHTTPServer as basehttpserver | ||||
except ImportError: | ||||
basehttpserver = None | ||||
Yuya Nishihara
|
r29211 | import zlib | ||
Chris Jerdonek
|
r20198 | |||
Gregory Szorc
|
r25703 | # Whitelist of modules that symbols can be directly imported from. | ||
allowsymbolimports = ( | ||||
'__future__', | ||||
Augie Fackler
|
r33920 | 'bzrlib', | ||
Augie Fackler
|
r33916 | 'hgclient', | ||
Augie Fackler
|
r33894 | 'mercurial', | ||
Yuya Nishihara
|
r27018 | 'mercurial.hgweb.common', | ||
'mercurial.hgweb.request', | ||||
Gregory Szorc
|
r25703 | 'mercurial.i18n', | ||
'mercurial.node', | ||||
Yuya Nishihara
|
r32507 | # for cffi modules to re-export pure functions | ||
'mercurial.pure.base85', | ||||
'mercurial.pure.bdiff', | ||||
'mercurial.pure.diffhelpers', | ||||
'mercurial.pure.mpatch', | ||||
'mercurial.pure.osutil', | ||||
'mercurial.pure.parsers', | ||||
Siddharth Agarwal
|
r34396 | # third-party imports should be directly imported | ||
'mercurial.thirdparty', | ||||
Gregory Szorc
|
r37197 | 'mercurial.thirdparty.zope', | ||
'mercurial.thirdparty.zope.interface', | ||||
Gregory Szorc
|
r25703 | ) | ||
Siddharth Agarwal
|
r32419 | # Whitelist of symbols that can be directly imported. | ||
Siddharth Agarwal
|
r32420 | directsymbols = ( | ||
'demandimport', | ||||
) | ||||
Siddharth Agarwal
|
r32419 | |||
Gregory Szorc
|
r25703 | # Modules that must be aliased because they are commonly confused with | ||
# common variables and can create aliasing and readability issues. | ||||
requirealias = { | ||||
'ui': 'uimod', | ||||
} | ||||
def usingabsolute(root): | ||||
"""Whether absolute imports are being used.""" | ||||
if sys.version_info[0] >= 3: | ||||
return True | ||||
for node in ast.walk(root): | ||||
if isinstance(node, ast.ImportFrom): | ||||
if node.module == '__future__': | ||||
for n in node.names: | ||||
if n.name == 'absolute_import': | ||||
return True | ||||
return False | ||||
Yuya Nishihara
|
r26965 | def walklocal(root): | ||
"""Recursively yield all descendant nodes but not in a different scope""" | ||||
todo = collections.deque(ast.iter_child_nodes(root)) | ||||
yield root, False | ||||
while todo: | ||||
node = todo.popleft() | ||||
newscope = isinstance(node, ast.FunctionDef) | ||||
if not newscope: | ||||
todo.extend(ast.iter_child_nodes(node)) | ||||
yield node, newscope | ||||
Yuya Nishihara
|
r32374 | def dotted_name_of_path(path): | ||
Augie Fackler
|
r20036 | """Given a relative path to a source file, return its dotted module name. | ||
>>> dotted_name_of_path('mercurial/error.py') | ||||
'mercurial.error' | ||||
Mads Kiilerich
|
r20383 | >>> dotted_name_of_path('zlibmodule.so') | ||
'zlib' | ||||
Augie Fackler
|
r20036 | """ | ||
Yuya Nishihara
|
r27620 | parts = path.replace(os.sep, '/').split('/') | ||
Mads Kiilerich
|
r20391 | parts[-1] = parts[-1].split('.', 1)[0] # remove .py and .so and .ARCH.so | ||
Mads Kiilerich
|
r20383 | if parts[-1].endswith('module'): | ||
parts[-1] = parts[-1][:-6] | ||||
Augie Fackler
|
r20036 | return '.'.join(parts) | ||
FUJIWARA Katsunori
|
r25173 | def fromlocalfunc(modulename, localmods): | ||
"""Get a function to examine which locally defined module the | ||||
target source imports via a specified name. | ||||
`modulename` is an `dotted_name_of_path()`-ed source file path, | ||||
which may have `.__init__` at the end of it, of the target source. | ||||
Yuya Nishihara
|
r32508 | `localmods` is a set of absolute `dotted_name_of_path()`-ed source file | ||
paths of locally defined (= Mercurial specific) modules. | ||||
FUJIWARA Katsunori
|
r25173 | |||
This function assumes that module names not existing in | ||||
Mads Kiilerich
|
r26781 | `localmods` are from the Python standard library. | ||
FUJIWARA Katsunori
|
r25173 | |||
This function returns the function, which takes `name` argument, | ||||
and returns `(absname, dottedpath, hassubmod)` tuple if `name` | ||||
matches against locally defined module. Otherwise, it returns | ||||
False. | ||||
It is assumed that `name` doesn't have `.__init__`. | ||||
`absname` is an absolute module name of specified `name` | ||||
(e.g. "hgext.convert"). This can be used to compose prefix for sub | ||||
modules or so. | ||||
`dottedpath` is a `dotted_name_of_path()`-ed source file path | ||||
(e.g. "hgext.convert.__init__") of `name`. This is used to look | ||||
module up in `localmods` again. | ||||
`hassubmod` is whether it may have sub modules under it (for | ||||
convenient, even though this is also equivalent to "absname != | ||||
dottednpath") | ||||
Yuya Nishihara
|
r32508 | >>> localmods = {'foo.__init__', 'foo.foo1', | ||
... 'foo.bar.__init__', 'foo.bar.bar1', | ||||
... 'baz.__init__', 'baz.baz1'} | ||||
FUJIWARA Katsunori
|
r25173 | >>> fromlocal = fromlocalfunc('foo.xxx', localmods) | ||
>>> # relative | ||||
>>> fromlocal('foo1') | ||||
('foo.foo1', 'foo.foo1', False) | ||||
>>> fromlocal('bar') | ||||
('foo.bar', 'foo.bar.__init__', True) | ||||
>>> fromlocal('bar.bar1') | ||||
('foo.bar.bar1', 'foo.bar.bar1', False) | ||||
>>> # absolute | ||||
>>> fromlocal('baz') | ||||
('baz', 'baz.__init__', True) | ||||
>>> fromlocal('baz.baz1') | ||||
('baz.baz1', 'baz.baz1', False) | ||||
>>> # unknown = maybe standard library | ||||
>>> fromlocal('os') | ||||
False | ||||
Gregory Szorc
|
r25701 | >>> fromlocal(None, 1) | ||
('foo', 'foo.__init__', True) | ||||
liscju
|
r29122 | >>> fromlocal('foo1', 1) | ||
('foo.foo1', 'foo.foo1', False) | ||||
Gregory Szorc
|
r25701 | >>> fromlocal2 = fromlocalfunc('foo.xxx.yyy', localmods) | ||
>>> fromlocal2(None, 2) | ||||
('foo', 'foo.__init__', True) | ||||
liscju
|
r29122 | >>> fromlocal2('bar2', 1) | ||
False | ||||
>>> fromlocal2('bar', 2) | ||||
('foo.bar', 'foo.bar.__init__', True) | ||||
FUJIWARA Katsunori
|
r25173 | """ | ||
Augie Fackler
|
r33891 | if not isinstance(modulename, str): | ||
modulename = modulename.decode('ascii') | ||||
FUJIWARA Katsunori
|
r25173 | prefix = '.'.join(modulename.split('.')[:-1]) | ||
if prefix: | ||||
prefix += '.' | ||||
Gregory Szorc
|
r25701 | def fromlocal(name, level=0): | ||
FUJIWARA Katsunori
|
r29374 | # name is false value when relative imports are used. | ||
if not name: | ||||
Gregory Szorc
|
r25701 | # If relative imports are used, level must not be absolute. | ||
assert level > 0 | ||||
candidates = ['.'.join(modulename.split('.')[:-level])] | ||||
else: | ||||
liscju
|
r29122 | if not level: | ||
# Check relative name first. | ||||
candidates = [prefix + name, name] | ||||
else: | ||||
candidates = ['.'.join(modulename.split('.')[:-level]) + | ||||
'.' + name] | ||||
Gregory Szorc
|
r25701 | |||
for n in candidates: | ||||
FUJIWARA Katsunori
|
r25173 | if n in localmods: | ||
return (n, n, False) | ||||
dottedpath = n + '.__init__' | ||||
if dottedpath in localmods: | ||||
return (n, dottedpath, True) | ||||
return False | ||||
return fromlocal | ||||
Augie Fackler
|
r20036 | |||
Yuya Nishihara
|
r32509 | def populateextmods(localmods): | ||
"""Populate C extension modules based on pure modules""" | ||||
newlocalmods = set(localmods) | ||||
for n in localmods: | ||||
if n.startswith('mercurial.pure.'): | ||||
m = n[len('mercurial.pure.'):] | ||||
newlocalmods.add('mercurial.cext.' + m) | ||||
newlocalmods.add('mercurial.cffi._' + m) | ||||
return newlocalmods | ||||
Augie Fackler
|
r20036 | def list_stdlib_modules(): | ||
"""List the modules present in the stdlib. | ||||
Augie Fackler
|
r33878 | >>> py3 = sys.version_info[0] >= 3 | ||
Augie Fackler
|
r20036 | >>> mods = set(list_stdlib_modules()) | ||
Augie Fackler
|
r33878 | >>> 'BaseHTTPServer' in mods or py3 | ||
Augie Fackler
|
r20036 | True | ||
os.path isn't really a module, so it's missing: | ||||
>>> 'os.path' in mods | ||||
False | ||||
sys requires special treatment, because it's baked into the | ||||
interpreter, but it should still appear: | ||||
>>> 'sys' in mods | ||||
True | ||||
>>> 'collections' in mods | ||||
True | ||||
Augie Fackler
|
r33878 | >>> 'cStringIO' in mods or py3 | ||
Augie Fackler
|
r20036 | True | ||
Augie Fackler
|
r29395 | |||
>>> 'cffi' in mods | ||||
True | ||||
Augie Fackler
|
r20036 | """ | ||
for m in sys.builtin_module_names: | ||||
yield m | ||||
# These modules only exist on windows, but we should always | ||||
# consider them stdlib. | ||||
for m in ['msvcrt', '_winreg']: | ||||
yield m | ||||
Augie Fackler
|
r33895 | yield '__builtin__' | ||
Augie Fackler
|
r20036 | yield 'builtins' # python3 only | ||
Augie Fackler
|
r33898 | yield 'importlib.abc' # python3 only | ||
yield 'importlib.machinery' # python3 only | ||||
yield 'importlib.util' # python3 only | ||||
Matt Harbison
|
r24669 | for m in 'fcntl', 'grp', 'pwd', 'termios': # Unix only | ||
yield m | ||||
Maciej Fijalkowski
|
r28713 | for m in 'cPickle', 'datetime': # in Python (not C) on PyPy | ||
yield m | ||||
Augie Fackler
|
r29395 | for m in ['cffi']: | ||
yield m | ||||
Martin von Zweigbergk
|
r32291 | stdlib_prefixes = {sys.prefix, sys.exec_prefix} | ||
Chris Jerdonek
|
r20198 | # We need to supplement the list of prefixes for the search to work | ||
# when run from within a virtualenv. | ||||
Augie Fackler
|
r33878 | for mod in (basehttpserver, zlib): | ||
if mod is None: | ||||
continue | ||||
Chris Jerdonek
|
r20198 | try: | ||
# Not all module objects have a __file__ attribute. | ||||
filename = mod.__file__ | ||||
except AttributeError: | ||||
continue | ||||
dirname = os.path.dirname(filename) | ||||
for prefix in stdlib_prefixes: | ||||
if dirname.startswith(prefix): | ||||
# Then this directory is redundant. | ||||
break | ||||
else: | ||||
stdlib_prefixes.add(dirname) | ||||
Augie Fackler
|
r20036 | for libpath in sys.path: | ||
Augie Fackler
|
r20201 | # We want to walk everything in sys.path that starts with | ||
timeless
|
r28700 | # something in stdlib_prefixes. | ||
if not any(libpath.startswith(p) for p in stdlib_prefixes): | ||||
Augie Fackler
|
r20036 | continue | ||
for top, dirs, files in os.walk(libpath): | ||||
Yuya Nishihara
|
r25733 | for i, d in reversed(list(enumerate(dirs))): | ||
Yuya Nishihara
|
r25734 | if (not os.path.exists(os.path.join(top, d, '__init__.py')) | ||
Siddharth Agarwal
|
r32590 | or top == libpath and d in ('hgdemandimport', 'hgext', | ||
'mercurial')): | ||||
Yuya Nishihara
|
r25733 | del dirs[i] | ||
Augie Fackler
|
r20036 | for name in files: | ||
Augie Fackler
|
r26221 | if not name.endswith(('.py', '.so', '.pyc', '.pyo', '.pyd')): | ||
Augie Fackler
|
r20036 | continue | ||
Yuya Nishihara
|
r27621 | if name.startswith('__init__.py'): | ||
full_path = top | ||||
else: | ||||
full_path = os.path.join(top, name) | ||||
Augie Fackler
|
r20036 | rel_path = full_path[len(libpath) + 1:] | ||
mod = dotted_name_of_path(rel_path) | ||||
yield mod | ||||
stdlib_modules = set(list_stdlib_modules()) | ||||
timeless
|
r28921 | def imported_modules(source, modulename, f, localmods, ignore_nested=False): | ||
Augie Fackler
|
r20036 | """Given the source of a file as a string, yield the names | ||
imported by that file. | ||||
Augie Fackler
|
r20037 | Args: | ||
source: The python source to examine as a string. | ||||
FUJIWARA Katsunori
|
r25174 | modulename: of specified python source (may have `__init__`) | ||
Yuya Nishihara
|
r32508 | localmods: set of locally defined module names (may have `__init__`) | ||
Augie Fackler
|
r20037 | ignore_nested: If true, import statements that do not start in | ||
column zero will be ignored. | ||||
Returns: | ||||
FUJIWARA Katsunori
|
r25174 | A list of absolute module names imported by the given source. | ||
Augie Fackler
|
r20037 | |||
timeless
|
r28921 | >>> f = 'foo/xxx.py' | ||
FUJIWARA Katsunori
|
r25174 | >>> modulename = 'foo.xxx' | ||
>>> localmods = {'foo.__init__': True, | ||||
... 'foo.foo1': True, 'foo.foo2': True, | ||||
... 'foo.bar.__init__': True, 'foo.bar.bar1': True, | ||||
... 'baz.__init__': True, 'baz.baz1': True } | ||||
>>> # standard library (= not locally defined ones) | ||||
>>> sorted(imported_modules( | ||||
... 'from stdlib1 import foo, bar; import stdlib2', | ||||
timeless
|
r28921 | ... modulename, f, localmods)) | ||
FUJIWARA Katsunori
|
r25174 | [] | ||
>>> # relative importing | ||||
Augie Fackler
|
r20037 | >>> sorted(imported_modules( | ||
FUJIWARA Katsunori
|
r25174 | ... 'import foo1; from bar import bar1', | ||
timeless
|
r28921 | ... modulename, f, localmods)) | ||
Yuya Nishihara
|
r26964 | ['foo.bar.bar1', 'foo.foo1'] | ||
FUJIWARA Katsunori
|
r25174 | >>> sorted(imported_modules( | ||
... 'from bar.bar1 import name1, name2, name3', | ||||
timeless
|
r28921 | ... modulename, f, localmods)) | ||
FUJIWARA Katsunori
|
r25174 | ['foo.bar.bar1'] | ||
>>> # absolute importing | ||||
>>> sorted(imported_modules( | ||||
... 'from baz import baz1, name1', | ||||
timeless
|
r28921 | ... modulename, f, localmods)) | ||
FUJIWARA Katsunori
|
r25174 | ['baz.__init__', 'baz.baz1'] | ||
>>> # mixed importing, even though it shouldn't be recommended | ||||
>>> sorted(imported_modules( | ||||
... 'import stdlib, foo1, baz', | ||||
timeless
|
r28921 | ... modulename, f, localmods)) | ||
FUJIWARA Katsunori
|
r25174 | ['baz.__init__', 'foo.foo1'] | ||
>>> # ignore_nested | ||||
Augie Fackler
|
r20037 | >>> sorted(imported_modules( | ||
... '''import foo | ||||
... def wat(): | ||||
... import bar | ||||
timeless
|
r28921 | ... ''', modulename, f, localmods)) | ||
FUJIWARA Katsunori
|
r25174 | ['foo.__init__', 'foo.bar.__init__'] | ||
>>> sorted(imported_modules( | ||||
... '''import foo | ||||
... def wat(): | ||||
... import bar | ||||
timeless
|
r28921 | ... ''', modulename, f, localmods, ignore_nested=True)) | ||
FUJIWARA Katsunori
|
r25174 | ['foo.__init__'] | ||
Augie Fackler
|
r20036 | """ | ||
FUJIWARA Katsunori
|
r25174 | fromlocal = fromlocalfunc(modulename, localmods) | ||
timeless
|
r28921 | for node in ast.walk(ast.parse(source, f)): | ||
Augie Fackler
|
r20037 | if ignore_nested and getattr(node, 'col_offset', 0) > 0: | ||
continue | ||||
Augie Fackler
|
r20036 | if isinstance(node, ast.Import): | ||
for n in node.names: | ||||
FUJIWARA Katsunori
|
r25174 | found = fromlocal(n.name) | ||
if not found: | ||||
# this should import standard library | ||||
continue | ||||
yield found[1] | ||||
Augie Fackler
|
r20036 | elif isinstance(node, ast.ImportFrom): | ||
Gregory Szorc
|
r25701 | found = fromlocal(node.module, node.level) | ||
FUJIWARA Katsunori
|
r25174 | if not found: | ||
# this should import standard library | ||||
continue | ||||
absname, dottedpath, hassubmod = found | ||||
if not hassubmod: | ||||
Yuya Nishihara
|
r26964 | # "dottedpath" is not a package; must be imported | ||
yield dottedpath | ||||
FUJIWARA Katsunori
|
r25174 | # examination of "node.names" should be redundant | ||
# e.g.: from mercurial.node import nullid, nullrev | ||||
continue | ||||
Yuya Nishihara
|
r26964 | modnotfound = False | ||
FUJIWARA Katsunori
|
r25174 | prefix = absname + '.' | ||
Augie Fackler
|
r20036 | for n in node.names: | ||
FUJIWARA Katsunori
|
r25174 | found = fromlocal(prefix + n.name) | ||
if not found: | ||||
# this should be a function or a property of "node.module" | ||||
Yuya Nishihara
|
r26964 | modnotfound = True | ||
FUJIWARA Katsunori
|
r25174 | continue | ||
yield found[1] | ||||
Yuya Nishihara
|
r26964 | if modnotfound: | ||
# "dottedpath" is a package, but imported because of non-module | ||||
# lookup | ||||
yield dottedpath | ||||
Augie Fackler
|
r20036 | |||
Yuya Nishihara
|
r27272 | def verify_import_convention(module, source, localmods): | ||
Gregory Szorc
|
r25703 | """Verify imports match our established coding convention. | ||
We have 2 conventions: legacy and modern. The modern convention is in | ||||
effect when using absolute imports. | ||||
The legacy convention only looks for mixed imports. The modern convention | ||||
is much more thorough. | ||||
""" | ||||
Gregory Szorc
|
r25702 | root = ast.parse(source) | ||
Gregory Szorc
|
r25703 | absolute = usingabsolute(root) | ||
Gregory Szorc
|
r25702 | |||
Gregory Szorc
|
r25703 | if absolute: | ||
Yuya Nishihara
|
r27272 | return verify_modern_convention(module, root, localmods) | ||
Gregory Szorc
|
r25703 | else: | ||
return verify_stdlib_on_own_line(root) | ||||
Yuya Nishihara
|
r27272 | def verify_modern_convention(module, root, localmods, root_col_offset=0): | ||
Gregory Szorc
|
r25703 | """Verify a file conforms to the modern import convention rules. | ||
The rules of the modern convention are: | ||||
* Ordering is stdlib followed by local imports. Each group is lexically | ||||
sorted. | ||||
* Importing multiple modules via "import X, Y" is not allowed: use | ||||
separate import statements. | ||||
* Importing multiple modules via "from X import ..." is allowed if using | ||||
parenthesis and one entry per line. | ||||
* Only 1 relative import statement per import level ("from .", "from ..") | ||||
is allowed. | ||||
* Relative imports from higher levels must occur before lower levels. e.g. | ||||
"from .." must be before "from .". | ||||
* Imports from peer packages should use relative import (e.g. do not | ||||
"import mercurial.foo" from a "mercurial.*" module). | ||||
* Symbols can only be imported from specific modules (see | ||||
`allowsymbolimports`). For other modules, first import the module then | ||||
assign the symbol to a module-level variable. In addition, these imports | ||||
Yuya Nishihara
|
r29208 | must be performed before other local imports. This rule only | ||
Gregory Szorc
|
r25703 | applies to import statements outside of any blocks. | ||
Jun Wu
|
r34040 | * Relative imports from the standard library are not allowed, unless that | ||
library is also a local module. | ||||
Gregory Szorc
|
r25703 | * Certain modules must be aliased to alternate names to avoid aliasing | ||
and readability problems. See `requirealias`. | ||||
""" | ||||
Augie Fackler
|
r33891 | if not isinstance(module, str): | ||
module = module.decode('ascii') | ||||
Gregory Szorc
|
r25703 | topmodule = module.split('.')[0] | ||
Yuya Nishihara
|
r27272 | fromlocal = fromlocalfunc(module, localmods) | ||
Gregory Szorc
|
r25703 | |||
# Whether a local/non-stdlib import has been performed. | ||||
timeless
|
r28330 | seenlocal = None | ||
Yuya Nishihara
|
r29208 | # Whether a local/non-stdlib, non-symbol import has been seen. | ||
seennonsymbollocal = False | ||||
Gregory Szorc
|
r25703 | # The last name to be imported (for sorting). | ||
lastname = None | ||||
Pierre-Yves David
|
r30590 | laststdlib = None | ||
Gregory Szorc
|
r25703 | # Relative import levels encountered so far. | ||
seenlevels = set() | ||||
Yuya Nishihara
|
r26965 | for node, newscope in walklocal(root): | ||
Yuya Nishihara
|
r26955 | def msg(fmt, *args): | ||
Yuya Nishihara
|
r26956 | return (fmt % args, node.lineno) | ||
Yuya Nishihara
|
r26965 | if newscope: | ||
# Check for local imports in function | ||||
Yuya Nishihara
|
r27272 | for r in verify_modern_convention(module, node, localmods, | ||
Yuya Nishihara
|
r26965 | node.col_offset + 4): | ||
yield r | ||||
elif isinstance(node, ast.Import): | ||||
Gregory Szorc
|
r25703 | # Disallow "import foo, bar" and require separate imports | ||
# for each module. | ||||
if len(node.names) > 1: | ||||
Yuya Nishihara
|
r26955 | yield msg('multiple imported names: %s', | ||
', '.join(n.name for n in node.names)) | ||||
Gregory Szorc
|
r25703 | |||
name = node.names[0].name | ||||
asname = node.names[0].asname | ||||
Pierre-Yves David
|
r30590 | stdlib = name in stdlib_modules | ||
Gregory Szorc
|
r25703 | # Ignore sorting rules on imports inside blocks. | ||
Yuya Nishihara
|
r26965 | if node.col_offset == root_col_offset: | ||
Pierre-Yves David
|
r30590 | if lastname and name < lastname and laststdlib == stdlib: | ||
Yuya Nishihara
|
r26955 | yield msg('imports not lexically sorted: %s < %s', | ||
name, lastname) | ||||
Gregory Szorc
|
r25703 | |||
Pierre-Yves David
|
r30590 | lastname = name | ||
laststdlib = stdlib | ||||
Gregory Szorc
|
r25703 | |||
# stdlib imports should be before local imports. | ||||
Yuya Nishihara
|
r26965 | if stdlib and seenlocal and node.col_offset == root_col_offset: | ||
timeless
|
r28330 | yield msg('stdlib import "%s" follows local import: %s', | ||
name, seenlocal) | ||||
Gregory Szorc
|
r25703 | |||
if not stdlib: | ||||
timeless
|
r28330 | seenlocal = name | ||
Gregory Szorc
|
r25703 | |||
# Import of sibling modules should use relative imports. | ||||
topname = name.split('.')[0] | ||||
if topname == topmodule: | ||||
Yuya Nishihara
|
r26955 | yield msg('import should be relative: %s', name) | ||
Gregory Szorc
|
r25703 | |||
if name in requirealias and asname != requirealias[name]: | ||||
Yuya Nishihara
|
r26955 | yield msg('%s module must be "as" aliased to %s', | ||
name, requirealias[name]) | ||||
Gregory Szorc
|
r25703 | |||
elif isinstance(node, ast.ImportFrom): | ||||
# Resolve the full imported module name. | ||||
if node.level > 0: | ||||
fullname = '.'.join(module.split('.')[:-node.level]) | ||||
if node.module: | ||||
fullname += '.%s' % node.module | ||||
else: | ||||
assert node.module | ||||
fullname = node.module | ||||
topname = fullname.split('.')[0] | ||||
if topname == topmodule: | ||||
Yuya Nishihara
|
r26955 | yield msg('import should be relative: %s', fullname) | ||
Gregory Szorc
|
r25703 | |||
# __future__ is special since it needs to come first and use | ||||
# symbol import. | ||||
if fullname != '__future__': | ||||
Jun Wu
|
r34040 | if not fullname or ( | ||
fullname in stdlib_modules | ||||
and fullname not in localmods | ||||
and fullname + '.__init__' not in localmods): | ||||
Yuya Nishihara
|
r26955 | yield msg('relative import of stdlib module') | ||
Gregory Szorc
|
r25703 | else: | ||
timeless
|
r28330 | seenlocal = fullname | ||
Gregory Szorc
|
r25703 | |||
# Direct symbol import is only allowed from certain modules and | ||||
# must occur before non-symbol imports. | ||||
Yuya Nishihara
|
r29207 | found = fromlocal(node.module, node.level) | ||
if found and found[2]: # node.module is a package | ||||
prefix = found[0] + '.' | ||||
Siddharth Agarwal
|
r32419 | symbols = (n.name for n in node.names | ||
if not fromlocal(prefix + n.name)) | ||||
Yuya Nishihara
|
r29207 | else: | ||
Siddharth Agarwal
|
r32419 | symbols = (n.name for n in node.names) | ||
symbols = [sym for sym in symbols if sym not in directsymbols] | ||||
Yuya Nishihara
|
r26965 | if node.module and node.col_offset == root_col_offset: | ||
Yuya Nishihara
|
r27272 | if symbols and fullname not in allowsymbolimports: | ||
Yuya Nishihara
|
r27273 | yield msg('direct symbol import %s from %s', | ||
', '.join(symbols), fullname) | ||||
Gregory Szorc
|
r25703 | |||
Yuya Nishihara
|
r29208 | if symbols and seennonsymbollocal: | ||
Yuya Nishihara
|
r26955 | yield msg('symbol import follows non-symbol import: %s', | ||
fullname) | ||||
Yuya Nishihara
|
r29208 | if not symbols and fullname not in stdlib_modules: | ||
seennonsymbollocal = True | ||||
Gregory Szorc
|
r25703 | |||
if not node.module: | ||||
assert node.level | ||||
# Only allow 1 group per level. | ||||
Yuya Nishihara
|
r26965 | if (node.level in seenlevels | ||
and node.col_offset == root_col_offset): | ||||
Yuya Nishihara
|
r26955 | yield msg('multiple "from %s import" statements', | ||
'.' * node.level) | ||||
Gregory Szorc
|
r25703 | |||
# Higher-level groups come before lower-level groups. | ||||
if any(node.level > l for l in seenlevels): | ||||
Yuya Nishihara
|
r26955 | yield msg('higher-level import should come first: %s', | ||
fullname) | ||||
Gregory Szorc
|
r25703 | |||
seenlevels.add(node.level) | ||||
# Entries in "from .X import ( ... )" lists must be lexically | ||||
# sorted. | ||||
lastentryname = None | ||||
for n in node.names: | ||||
if lastentryname and n.name < lastentryname: | ||||
Yuya Nishihara
|
r26955 | yield msg('imports from %s not lexically sorted: %s < %s', | ||
fullname, n.name, lastentryname) | ||||
Gregory Szorc
|
r25703 | |||
lastentryname = n.name | ||||
if n.name in requirealias and n.asname != requirealias[n.name]: | ||||
Yuya Nishihara
|
r26955 | yield msg('%s from %s must be "as" aliased to %s', | ||
n.name, fullname, requirealias[n.name]) | ||||
Gregory Szorc
|
r25702 | |||
def verify_stdlib_on_own_line(root): | ||||
Augie Fackler
|
r20036 | """Given some python source, verify that stdlib imports are done | ||
in separate statements from relative local module imports. | ||||
Gregory Szorc
|
r25702 | >>> list(verify_stdlib_on_own_line(ast.parse('import sys, foo'))) | ||
Yuya Nishihara
|
r26956 | [('mixed imports\\n stdlib: sys\\n relative: foo', 1)] | ||
Gregory Szorc
|
r25702 | >>> list(verify_stdlib_on_own_line(ast.parse('import sys, os'))) | ||
Augie Fackler
|
r20036 | [] | ||
Gregory Szorc
|
r25702 | >>> list(verify_stdlib_on_own_line(ast.parse('import foo, bar'))) | ||
Augie Fackler
|
r20036 | [] | ||
""" | ||||
Gregory Szorc
|
r25702 | for node in ast.walk(root): | ||
Augie Fackler
|
r20036 | if isinstance(node, ast.Import): | ||
Mads Kiilerich
|
r20386 | from_stdlib = {False: [], True: []} | ||
Augie Fackler
|
r20036 | for n in node.names: | ||
Mads Kiilerich
|
r20386 | from_stdlib[n.name in stdlib_modules].append(n.name) | ||
if from_stdlib[True] and from_stdlib[False]: | ||||
yield ('mixed imports\n stdlib: %s\n relative: %s' % | ||||
(', '.join(sorted(from_stdlib[True])), | ||||
Yuya Nishihara
|
r26956 | ', '.join(sorted(from_stdlib[False]))), node.lineno) | ||
Augie Fackler
|
r20036 | |||
class CircularImport(Exception): | ||||
pass | ||||
Matt Mackall
|
r24490 | def checkmod(mod, imports): | ||
shortest = {} | ||||
visit = [[mod]] | ||||
while visit: | ||||
path = visit.pop(0) | ||||
for i in sorted(imports.get(path[-1], [])): | ||||
if len(path) < shortest.get(i, 1000): | ||||
shortest[i] = len(path) | ||||
if i in path: | ||||
if i == path[0]: | ||||
raise CircularImport(path) | ||||
continue | ||||
visit.append(path + [i]) | ||||
Augie Fackler
|
r20036 | |||
Augie Fackler
|
r20038 | def rotatecycle(cycle): | ||
"""arrange a cycle so that the lexicographically first module listed first | ||||
Matt Mackall
|
r24488 | >>> rotatecycle(['foo', 'bar']) | ||
Augie Fackler
|
r20038 | ['bar', 'foo', 'bar'] | ||
""" | ||||
lowest = min(cycle) | ||||
idx = cycle.index(lowest) | ||||
Matt Mackall
|
r24488 | return cycle[idx:] + cycle[:idx] + [lowest] | ||
Augie Fackler
|
r20036 | |||
def find_cycles(imports): | ||||
"""Find cycles in an already-loaded import graph. | ||||
FUJIWARA Katsunori
|
r25175 | All module names recorded in `imports` should be absolute one. | ||
timeless
|
r28703 | >>> from __future__ import print_function | ||
FUJIWARA Katsunori
|
r25175 | >>> imports = {'top.foo': ['top.bar', 'os.path', 'top.qux'], | ||
... 'top.bar': ['top.baz', 'sys'], | ||||
... 'top.baz': ['top.foo'], | ||||
... 'top.qux': ['top.foo']} | ||||
timeless
|
r28703 | >>> print('\\n'.join(sorted(find_cycles(imports)))) | ||
Matt Mackall
|
r24487 | top.bar -> top.baz -> top.foo -> top.bar | ||
top.foo -> top.qux -> top.foo | ||||
Augie Fackler
|
r20036 | """ | ||
Matt Mackall
|
r24491 | cycles = set() | ||
timeless
|
r28704 | for mod in sorted(imports.keys()): | ||
Augie Fackler
|
r20036 | try: | ||
Matt Mackall
|
r24490 | checkmod(mod, imports) | ||
Gregory Szorc
|
r25660 | except CircularImport as e: | ||
Augie Fackler
|
r20036 | cycle = e.args[0] | ||
Matt Mackall
|
r24491 | cycles.add(" -> ".join(rotatecycle(cycle))) | ||
return cycles | ||||
Augie Fackler
|
r20036 | |||
def _cycle_sortkey(c): | ||||
return len(c), c | ||||
timeless
|
r28922 | def embedded(f, modname, src): | ||
"""Extract embedded python code | ||||
Augie Fackler
|
r33879 | >>> def _forcestr(thing): | ||
... if not isinstance(thing, str): | ||||
... return thing.decode('ascii') | ||||
... return thing | ||||
timeless
|
r28922 | >>> def test(fn, lines): | ||
Augie Fackler
|
r33879 | ... for s, m, f, l in embedded(fn, b"example", lines): | ||
... print("%s %s %d" % (_forcestr(m), _forcestr(f), l)) | ||||
... print(repr(_forcestr(s))) | ||||
timeless
|
r28922 | >>> lines = [ | ||
Augie Fackler
|
r33879 | ... b'comment', | ||
... b' >>> from __future__ import print_function', | ||||
... b" >>> ' multiline", | ||||
... b" ... string'", | ||||
... b' ', | ||||
... b'comment', | ||||
... b' $ cat > foo.py <<EOF', | ||||
... b' > from __future__ import print_function', | ||||
... b' > EOF', | ||||
timeless
|
r28922 | ... ] | ||
Augie Fackler
|
r33879 | >>> test(b"example.t", lines) | ||
timeless
|
r28922 | example[2] doctest.py 2 | ||
"from __future__ import print_function\\n' multiline\\nstring'\\n" | ||||
example[7] foo.py 7 | ||||
'from __future__ import print_function\\n' | ||||
""" | ||||
inlinepython = 0 | ||||
shpython = 0 | ||||
script = [] | ||||
prefix = 6 | ||||
t = '' | ||||
n = 0 | ||||
for l in src: | ||||
n += 1 | ||||
if not l.endswith(b'\n'): | ||||
l += b'\n' | ||||
if l.startswith(b' >>> '): # python inlines | ||||
if shpython: | ||||
print("%s:%d: Parse Error" % (f, n)) | ||||
if not inlinepython: | ||||
# We've just entered a Python block. | ||||
inlinepython = n | ||||
Augie Fackler
|
r33879 | t = b'doctest.py' | ||
timeless
|
r28922 | script.append(l[prefix:]) | ||
continue | ||||
if l.startswith(b' ... '): # python inlines | ||||
script.append(l[prefix:]) | ||||
continue | ||||
Augie Fackler
|
r33879 | cat = re.search(br"\$ \s*cat\s*>\s*(\S+\.py)\s*<<\s*EOF", l) | ||
timeless
|
r28922 | if cat: | ||
if inlinepython: | ||||
Augie Fackler
|
r33879 | yield b''.join(script), (b"%s[%d]" % | ||
timeless
|
r28922 | (modname, inlinepython)), t, inlinepython | ||
script = [] | ||||
inlinepython = 0 | ||||
shpython = n | ||||
t = cat.group(1) | ||||
continue | ||||
if shpython and l.startswith(b' > '): # sh continuation | ||||
if l == b' > EOF\n': | ||||
Augie Fackler
|
r33879 | yield b''.join(script), (b"%s[%d]" % | ||
timeless
|
r28922 | (modname, shpython)), t, shpython | ||
script = [] | ||||
shpython = 0 | ||||
else: | ||||
script.append(l[4:]) | ||||
continue | ||||
Augie Fackler
|
r33921 | # If we have an empty line or a command for sh, we end the | ||
# inline script. | ||||
if inlinepython and (l == b' \n' | ||||
or l.startswith(b' $ ')): | ||||
Augie Fackler
|
r33879 | yield b''.join(script), (b"%s[%d]" % | ||
timeless
|
r28922 | (modname, inlinepython)), t, inlinepython | ||
script = [] | ||||
inlinepython = 0 | ||||
continue | ||||
timeless
|
r28919 | def sources(f, modname): | ||
timeless
|
r28922 | """Yields possibly multiple sources from a filepath | ||
input: filepath, modulename | ||||
yields: script(string), modulename, filepath, linenumber | ||||
For embedded scripts, the modulename and filepath will be different | ||||
from the function arguments. linenumber is an offset relative to | ||||
the input file. | ||||
""" | ||||
py = False | ||||
Yuya Nishihara
|
r29234 | if not f.endswith('.t'): | ||
Augie Fackler
|
r33879 | with open(f, 'rb') as src: | ||
timeless
|
r28922 | yield src.read(), modname, f, 0 | ||
py = True | ||||
if py or f.endswith('.t'): | ||||
Augie Fackler
|
r33879 | with open(f, 'rb') as src: | ||
timeless
|
r28922 | for script, modname, t, line in embedded(f, modname, src): | ||
yield script, modname, t, line | ||||
timeless
|
r28919 | |||
Augie Fackler
|
r20036 | def main(argv): | ||
FUJIWARA Katsunori
|
r25063 | if len(argv) < 2 or (argv[1] == '-' and len(argv) > 2): | ||
timeless
|
r28703 | print('Usage: %s {-|file [file] [file] ...}') | ||
Augie Fackler
|
r20036 | return 1 | ||
FUJIWARA Katsunori
|
r25063 | if argv[1] == '-': | ||
argv = argv[:1] | ||||
argv.extend(l.rstrip() for l in sys.stdin.readlines()) | ||||
Yuya Nishihara
|
r32508 | localmodpaths = {} | ||
Augie Fackler
|
r20036 | used_imports = {} | ||
any_errors = False | ||||
for source_path in argv[1:]: | ||||
Yuya Nishihara
|
r32374 | modname = dotted_name_of_path(source_path) | ||
Yuya Nishihara
|
r32508 | localmodpaths[modname] = source_path | ||
Yuya Nishihara
|
r32509 | localmods = populateextmods(localmodpaths) | ||
Yuya Nishihara
|
r32508 | for localmodname, source_path in sorted(localmodpaths.items()): | ||
Augie Fackler
|
r33891 | if not isinstance(localmodname, bytes): | ||
# This is only safe because all hg's files are ascii | ||||
localmodname = localmodname.encode('ascii') | ||||
timeless
|
r28922 | for src, modname, name, line in sources(source_path, localmodname): | ||
timeless
|
r28920 | try: | ||
used_imports[modname] = sorted( | ||||
timeless
|
r28922 | imported_modules(src, modname, name, localmods, | ||
timeless
|
r28920 | ignore_nested=True)) | ||
for error, lineno in verify_import_convention(modname, src, | ||||
localmods): | ||||
any_errors = True | ||||
timeless
|
r28922 | print('%s:%d: %s' % (source_path, lineno + line, error)) | ||
timeless
|
r28920 | except SyntaxError as e: | ||
print('%s:%d: SyntaxError: %s' % | ||||
timeless
|
r28922 | (source_path, e.lineno + line, e)) | ||
Augie Fackler
|
r20036 | cycles = find_cycles(used_imports) | ||
if cycles: | ||||
firstmods = set() | ||||
for c in sorted(cycles, key=_cycle_sortkey): | ||||
first = c.split()[0] | ||||
# As a rough cut, ignore any cycle that starts with the | ||||
# same module as some other cycle. Otherwise we see lots | ||||
# of cycles that are effectively duplicates. | ||||
if first in firstmods: | ||||
continue | ||||
timeless
|
r28703 | print('Import cycle:', c) | ||
Augie Fackler
|
r20036 | firstmods.add(first) | ||
any_errors = True | ||||
FUJIWARA Katsunori
|
r25731 | return any_errors != 0 | ||
Augie Fackler
|
r20036 | |||
if __name__ == '__main__': | ||||
sys.exit(int(main(sys.argv))) | ||||