deps.py
294 lines
| 6.9 KiB
| text/x-python
|
PythonLexer
/ scripts / deps.py
Mads Kiilerich
|
r8550 | #!/usr/bin/env python3 | ||
import re | ||||
import sys | ||||
ignored_modules = set(''' | ||||
argparse | ||||
base64 | ||||
bcrypt | ||||
binascii | ||||
bleach | ||||
calendar | ||||
celery | ||||
celery | ||||
chardet | ||||
click | ||||
collections | ||||
configparser | ||||
copy | ||||
csv | ||||
ctypes | ||||
datetime | ||||
dateutil | ||||
decimal | ||||
decorator | ||||
difflib | ||||
distutils | ||||
docutils | ||||
errno | ||||
fileinput | ||||
functools | ||||
getpass | ||||
grp | ||||
hashlib | ||||
hmac | ||||
html | ||||
http | ||||
imp | ||||
importlib | ||||
inspect | ||||
io | ||||
ipaddr | ||||
IPython | ||||
isapi_wsgi | ||||
itertools | ||||
json | ||||
kajiki | ||||
ldap | ||||
logging | ||||
mako | ||||
markdown | ||||
mimetypes | ||||
mock | ||||
msvcrt | ||||
multiprocessing | ||||
operator | ||||
os | ||||
paginate | ||||
paginate_sqlalchemy | ||||
pam | ||||
paste | ||||
pkg_resources | ||||
platform | ||||
posixpath | ||||
pprint | ||||
pwd | ||||
pyflakes | ||||
pytest | ||||
pytest_localserver | ||||
random | ||||
re | ||||
routes | ||||
setuptools | ||||
shlex | ||||
shutil | ||||
smtplib | ||||
socket | ||||
ssl | ||||
stat | ||||
string | ||||
struct | ||||
subprocess | ||||
sys | ||||
tarfile | ||||
tempfile | ||||
textwrap | ||||
tgext | ||||
threading | ||||
time | ||||
traceback | ||||
traitlets | ||||
types | ||||
Mads Kiilerich
|
r8635 | typing | ||
Mads Kiilerich
|
r8550 | urllib | ||
urlobject | ||||
uuid | ||||
warnings | ||||
webhelpers2 | ||||
webob | ||||
webtest | ||||
whoosh | ||||
win32traceutil | ||||
zipfile | ||||
'''.split()) | ||||
top_modules = set(''' | ||||
kallithea.alembic | ||||
kallithea.bin | ||||
kallithea.config | ||||
kallithea.controllers | ||||
kallithea.templates.py | ||||
scripts | ||||
'''.split()) | ||||
bottom_external_modules = set(''' | ||||
tg | ||||
mercurial | ||||
sqlalchemy | ||||
alembic | ||||
formencode | ||||
pygments | ||||
dulwich | ||||
beaker | ||||
psycopg2 | ||||
docs | ||||
setup | ||||
conftest | ||||
Mads Kiilerich
|
r8778 | packaging | ||
Mads Kiilerich
|
r8550 | '''.split()) | ||
normal_modules = set(''' | ||||
kallithea | ||||
Mads Kiilerich
|
r8590 | kallithea.controllers.base | ||
Mads Kiilerich
|
r8550 | kallithea.lib | ||
kallithea.lib.auth | ||||
kallithea.lib.auth_modules | ||||
kallithea.lib.celerylib | ||||
kallithea.lib.db_manage | ||||
kallithea.lib.helpers | ||||
kallithea.lib.hooks | ||||
kallithea.lib.indexers | ||||
kallithea.lib.utils | ||||
kallithea.lib.utils2 | ||||
kallithea.lib.vcs | ||||
kallithea.lib.webutils | ||||
kallithea.model | ||||
Mads Kiilerich
|
r8556 | kallithea.model.async_tasks | ||
Mads Kiilerich
|
r8550 | kallithea.model.scm | ||
kallithea.templates.py | ||||
'''.split()) | ||||
shown_modules = normal_modules | top_modules | ||||
# break the chains somehow - this is a cleanup TODO list | ||||
Mads Kiilerich
|
r8600 | known_violations = set([ | ||
Mads Kiilerich
|
r8550 | ('kallithea.lib.auth_modules', 'kallithea.lib.auth'), # needs base&facade | ||
('kallithea.lib.utils', 'kallithea.model'), # clean up utils | ||||
('kallithea.lib.utils', 'kallithea.model.db'), | ||||
('kallithea.lib.utils', 'kallithea.model.scm'), | ||||
('kallithea.model', 'kallithea.lib.auth'), # auth.HasXXX | ||||
('kallithea.model', 'kallithea.lib.auth_modules'), # validators | ||||
('kallithea.model', 'kallithea.lib.hooks'), # clean up hooks | ||||
('kallithea.model', 'kallithea.model.scm'), | ||||
('kallithea.model.scm', 'kallithea.lib.hooks'), | ||||
Mads Kiilerich
|
r8600 | ]) | ||
Mads Kiilerich
|
r8550 | |||
extra_edges = [ | ||||
('kallithea.config', 'kallithea.controllers'), # through TG | ||||
('kallithea.lib.auth', 'kallithea.lib.auth_modules'), # custom loader | ||||
] | ||||
def normalize(s): | ||||
"""Given a string with dot path, return the string it should be shown as.""" | ||||
parts = s.replace('.__init__', '').split('.') | ||||
short_2 = '.'.join(parts[:2]) | ||||
short_3 = '.'.join(parts[:3]) | ||||
short_4 = '.'.join(parts[:4]) | ||||
if parts[0] in ['scripts', 'contributor_data', 'i18n_utils']: | ||||
return 'scripts' | ||||
if short_3 == 'kallithea.model.meta': | ||||
return 'kallithea.model.db' | ||||
if parts[:4] == ['kallithea', 'lib', 'vcs', 'ssh']: | ||||
return 'kallithea.lib.vcs.ssh' | ||||
if short_4 in shown_modules: | ||||
return short_4 | ||||
if short_3 in shown_modules: | ||||
return short_3 | ||||
if short_2 in shown_modules: | ||||
return short_2 | ||||
if short_2 == 'kallithea.tests': | ||||
return None | ||||
if parts[0] in ignored_modules: | ||||
return None | ||||
assert parts[0] in bottom_external_modules, parts | ||||
return parts[0] | ||||
def main(filenames): | ||||
if not filenames or filenames[0].startswith('-'): | ||||
print('''\ | ||||
Usage: | ||||
hg files 'set:!binary()&grep("^#!.*python")' 'set:**.py' | xargs scripts/deps.py | ||||
dot -Tsvg deps.dot > deps.svg | ||||
''') | ||||
raise SystemExit(1) | ||||
files_imports = dict() # map filenames to its imports | ||||
import_deps = set() # set of tuples with module name and its imports | ||||
for fn in filenames: | ||||
with open(fn) as f: | ||||
s = f.read() | ||||
dot_name = (fn[:-3] if fn.endswith('.py') else fn).replace('/', '.') | ||||
file_imports = set() | ||||
for m in re.finditer(r'^ *(?:from ([^ ]*) import (?:([a-zA-Z].*)|\(([^)]*)\))|import (.*))$', s, re.MULTILINE): | ||||
m_from, m_from_import, m_from_import2, m_import = m.groups() | ||||
if m_from: | ||||
pre = m_from + '.' | ||||
if pre.startswith('.'): | ||||
pre = dot_name.rsplit('.', 1)[0] + pre | ||||
importlist = m_from_import or m_from_import2 | ||||
else: | ||||
pre = '' | ||||
importlist = m_import | ||||
for imp in importlist.split('#', 1)[0].split(','): | ||||
full_imp = pre + imp.strip().split(' as ', 1)[0] | ||||
file_imports.add(full_imp) | ||||
import_deps.add((dot_name, full_imp)) | ||||
files_imports[fn] = file_imports | ||||
# dump out all deps for debugging and analysis | ||||
with open('deps.txt', 'w') as f: | ||||
for fn, file_imports in sorted(files_imports.items()): | ||||
for file_import in sorted(file_imports): | ||||
if file_import.split('.', 1)[0] in ignored_modules: | ||||
continue | ||||
f.write('%s: %s\n' % (fn, file_import)) | ||||
# find leafs that haven't been ignored - they are the important external dependencies and shown in the bottom row | ||||
only_imported = set( | ||||
set(normalize(b) for a, b in import_deps) - | ||||
set(normalize(a) for a, b in import_deps) - | ||||
set([None, 'kallithea']) | ||||
) | ||||
normalized_dep_edges = set() | ||||
for dot_name, full_imp in import_deps: | ||||
a = normalize(dot_name) | ||||
b = normalize(full_imp) | ||||
if a is None or b is None or a == b: | ||||
continue | ||||
normalized_dep_edges.add((a, b)) | ||||
#print((dot_name, full_imp, a, b)) | ||||
normalized_dep_edges.update(extra_edges) | ||||
unseen_shown_modules = shown_modules.difference(a for a, b in normalized_dep_edges).difference(b for a, b in normalized_dep_edges) | ||||
assert not unseen_shown_modules, unseen_shown_modules | ||||
with open('deps.dot', 'w') as f: | ||||
f.write('digraph {\n') | ||||
f.write('subgraph { rank = same; %s}\n' % ''.join('"%s"; ' % s for s in sorted(top_modules))) | ||||
f.write('subgraph { rank = same; %s}\n' % ''.join('"%s"; ' % s for s in sorted(only_imported))) | ||||
for a, b in sorted(normalized_dep_edges): | ||||
f.write(' "%s" -> "%s"%s\n' % (a, b, ' [color=red]' if (a, b) in known_violations else ' [color=green]' if (a, b) in extra_edges else '')) | ||||
f.write('}\n') | ||||
# verify dependencies by untangling dependency chain bottom-up: | ||||
todo = set(normalized_dep_edges) | ||||
Mads Kiilerich
|
r8600 | unseen_violations = known_violations.difference(todo) | ||
assert not unseen_violations, unseen_violations | ||||
Mads Kiilerich
|
r8550 | for x in known_violations: | ||
todo.remove(x) | ||||
while todo: | ||||
depending = set(a for a, b in todo) | ||||
depended = set(b for a, b in todo) | ||||
drop = depended - depending | ||||
if not drop: | ||||
print('ERROR: cycles:', len(todo)) | ||||
for x in sorted(todo): | ||||
print('%s,' % (x,)) | ||||
raise SystemExit(1) | ||||
#for do_b in sorted(drop): | ||||
# print('Picking', do_b, '- unblocks:', ' '.join(a for a, b in sorted((todo)) if b == do_b)) | ||||
todo = set((a, b) for a, b in todo if b in depending) | ||||
#print() | ||||
if __name__ == '__main__': | ||||
main(sys.argv[1:]) | ||||