##// END OF EJS Templates
docs: describe, visualize, and verify internal code structure and layering...
Mads Kiilerich -
r8550:52816813 default
parent child Browse files
Show More
@@ -0,0 +1,295 b''
1 #!/usr/bin/env python3
2
3
4 import re
5 import sys
6
7
8 ignored_modules = set('''
9 argparse
10 base64
11 bcrypt
12 binascii
13 bleach
14 calendar
15 celery
16 celery
17 chardet
18 click
19 collections
20 configparser
21 copy
22 csv
23 ctypes
24 datetime
25 dateutil
26 decimal
27 decorator
28 difflib
29 distutils
30 docutils
31 email
32 errno
33 fileinput
34 functools
35 getpass
36 grp
37 hashlib
38 hmac
39 html
40 http
41 imp
42 importlib
43 inspect
44 io
45 ipaddr
46 IPython
47 isapi_wsgi
48 itertools
49 json
50 kajiki
51 ldap
52 logging
53 mako
54 markdown
55 mimetypes
56 mock
57 msvcrt
58 multiprocessing
59 operator
60 os
61 paginate
62 paginate_sqlalchemy
63 pam
64 paste
65 pkg_resources
66 platform
67 posixpath
68 pprint
69 pwd
70 pyflakes
71 pytest
72 pytest_localserver
73 random
74 re
75 routes
76 setuptools
77 shlex
78 shutil
79 smtplib
80 socket
81 ssl
82 stat
83 string
84 struct
85 subprocess
86 sys
87 tarfile
88 tempfile
89 textwrap
90 tgext
91 threading
92 time
93 traceback
94 traitlets
95 types
96 urllib
97 urlobject
98 uuid
99 warnings
100 webhelpers2
101 webob
102 webtest
103 whoosh
104 win32traceutil
105 zipfile
106 '''.split())
107
108 top_modules = set('''
109 kallithea.alembic
110 kallithea.bin
111 kallithea.config
112 kallithea.controllers
113 kallithea.templates.py
114 scripts
115 '''.split())
116
117 bottom_external_modules = set('''
118 tg
119 mercurial
120 sqlalchemy
121 alembic
122 formencode
123 pygments
124 dulwich
125 beaker
126 psycopg2
127 docs
128 setup
129 conftest
130 '''.split())
131
132 normal_modules = set('''
133 kallithea
134 kallithea.lib.celerylib.tasks
135 kallithea.lib
136 kallithea.lib.auth
137 kallithea.lib.auth_modules
138 kallithea.lib.base
139 kallithea.lib.celerylib
140 kallithea.lib.db_manage
141 kallithea.lib.helpers
142 kallithea.lib.hooks
143 kallithea.lib.indexers
144 kallithea.lib.utils
145 kallithea.lib.utils2
146 kallithea.lib.vcs
147 kallithea.lib.webutils
148 kallithea.model
149 kallithea.model.scm
150 kallithea.templates.py
151 '''.split())
152
153 shown_modules = normal_modules | top_modules
154
155 # break the chains somehow - this is a cleanup TODO list
156 known_violations = [
157 ('kallithea.lib.auth_modules', 'kallithea.lib.auth'), # needs base&facade
158 ('kallithea.lib.utils', 'kallithea.model'), # clean up utils
159 ('kallithea.lib.utils', 'kallithea.model.db'),
160 ('kallithea.lib.utils', 'kallithea.model.scm'),
161 ('kallithea.lib.celerylib.tasks', 'kallithea.lib.helpers'),
162 ('kallithea.lib.celerylib.tasks', 'kallithea.lib.hooks'),
163 ('kallithea.lib.celerylib.tasks', 'kallithea.lib.indexers'),
164 ('kallithea.lib.celerylib.tasks', 'kallithea.model'),
165 ('kallithea.model', 'kallithea.lib.auth'), # auth.HasXXX
166 ('kallithea.model', 'kallithea.lib.auth_modules'), # validators
167 ('kallithea.model', 'kallithea.lib.helpers'),
168 ('kallithea.model', 'kallithea.lib.hooks'), # clean up hooks
169 ('kallithea.model', 'kallithea.model.scm'),
170 ('kallithea.model.scm', 'kallithea.lib.hooks'),
171 ]
172
173 extra_edges = [
174 ('kallithea.config', 'kallithea.controllers'), # through TG
175 ('kallithea.lib.auth', 'kallithea.lib.auth_modules'), # custom loader
176 ]
177
178
179 def normalize(s):
180 """Given a string with dot path, return the string it should be shown as."""
181 parts = s.replace('.__init__', '').split('.')
182 short_2 = '.'.join(parts[:2])
183 short_3 = '.'.join(parts[:3])
184 short_4 = '.'.join(parts[:4])
185 if parts[0] in ['scripts', 'contributor_data', 'i18n_utils']:
186 return 'scripts'
187 if short_3 == 'kallithea.model.meta':
188 return 'kallithea.model.db'
189 if parts[:4] == ['kallithea', 'lib', 'vcs', 'ssh']:
190 return 'kallithea.lib.vcs.ssh'
191 if short_4 in shown_modules:
192 return short_4
193 if short_3 in shown_modules:
194 return short_3
195 if short_2 in shown_modules:
196 return short_2
197 if short_2 == 'kallithea.tests':
198 return None
199 if parts[0] in ignored_modules:
200 return None
201 assert parts[0] in bottom_external_modules, parts
202 return parts[0]
203
204
205 def main(filenames):
206 if not filenames or filenames[0].startswith('-'):
207 print('''\
208 Usage:
209 hg files 'set:!binary()&grep("^#!.*python")' 'set:**.py' | xargs scripts/deps.py
210 dot -Tsvg deps.dot > deps.svg
211 ''')
212 raise SystemExit(1)
213
214 files_imports = dict() # map filenames to its imports
215 import_deps = set() # set of tuples with module name and its imports
216 for fn in filenames:
217 with open(fn) as f:
218 s = f.read()
219
220 dot_name = (fn[:-3] if fn.endswith('.py') else fn).replace('/', '.')
221 file_imports = set()
222 for m in re.finditer(r'^ *(?:from ([^ ]*) import (?:([a-zA-Z].*)|\(([^)]*)\))|import (.*))$', s, re.MULTILINE):
223 m_from, m_from_import, m_from_import2, m_import = m.groups()
224 if m_from:
225 pre = m_from + '.'
226 if pre.startswith('.'):
227 pre = dot_name.rsplit('.', 1)[0] + pre
228 importlist = m_from_import or m_from_import2
229 else:
230 pre = ''
231 importlist = m_import
232 for imp in importlist.split('#', 1)[0].split(','):
233 full_imp = pre + imp.strip().split(' as ', 1)[0]
234 file_imports.add(full_imp)
235 import_deps.add((dot_name, full_imp))
236 files_imports[fn] = file_imports
237
238 # dump out all deps for debugging and analysis
239 with open('deps.txt', 'w') as f:
240 for fn, file_imports in sorted(files_imports.items()):
241 for file_import in sorted(file_imports):
242 if file_import.split('.', 1)[0] in ignored_modules:
243 continue
244 f.write('%s: %s\n' % (fn, file_import))
245
246 # find leafs that haven't been ignored - they are the important external dependencies and shown in the bottom row
247 only_imported = set(
248 set(normalize(b) for a, b in import_deps) -
249 set(normalize(a) for a, b in import_deps) -
250 set([None, 'kallithea'])
251 )
252
253 normalized_dep_edges = set()
254 for dot_name, full_imp in import_deps:
255 a = normalize(dot_name)
256 b = normalize(full_imp)
257 if a is None or b is None or a == b:
258 continue
259 normalized_dep_edges.add((a, b))
260 #print((dot_name, full_imp, a, b))
261 normalized_dep_edges.update(extra_edges)
262
263 unseen_shown_modules = shown_modules.difference(a for a, b in normalized_dep_edges).difference(b for a, b in normalized_dep_edges)
264 assert not unseen_shown_modules, unseen_shown_modules
265
266 with open('deps.dot', 'w') as f:
267 f.write('digraph {\n')
268 f.write('subgraph { rank = same; %s}\n' % ''.join('"%s"; ' % s for s in sorted(top_modules)))
269 f.write('subgraph { rank = same; %s}\n' % ''.join('"%s"; ' % s for s in sorted(only_imported)))
270 for a, b in sorted(normalized_dep_edges):
271 f.write(' "%s" -> "%s"%s\n' % (a, b, ' [color=red]' if (a, b) in known_violations else ' [color=green]' if (a, b) in extra_edges else ''))
272 f.write('}\n')
273
274 # verify dependencies by untangling dependency chain bottom-up:
275 todo = set(normalized_dep_edges)
276 for x in known_violations:
277 todo.remove(x)
278
279 while todo:
280 depending = set(a for a, b in todo)
281 depended = set(b for a, b in todo)
282 drop = depended - depending
283 if not drop:
284 print('ERROR: cycles:', len(todo))
285 for x in sorted(todo):
286 print('%s,' % (x,))
287 raise SystemExit(1)
288 #for do_b in sorted(drop):
289 # print('Picking', do_b, '- unblocks:', ' '.join(a for a, b in sorted((todo)) if b == do_b))
290 todo = set((a, b) for a, b in todo if b in depending)
291 #print()
292
293
294 if __name__ == '__main__':
295 main(sys.argv[1:])
@@ -53,3 +53,6 b' syntax: regexp'
53 ^\.pytest_cache$
53 ^\.pytest_cache$
54 ^venv$
54 ^venv$
55 /__pycache__$
55 /__pycache__$
56 ^deps\.dot$
57 ^deps\.svg$
58 ^deps\.txt$
@@ -72,6 +72,94 b' review and inclusion, via the mailing li'
72 .. _contributing-tests:
72 .. _contributing-tests:
73
73
74
74
75 Internal dependencies
76 ---------------------
77
78 We try to keep the code base clean and modular and avoid circular dependencies.
79 Code should only invoke code in layers below itself.
80
81 Imports should import whole modules ``from`` their parent module, perhaps
82 ``as`` a shortened name. Avoid imports ``from`` modules.
83
84 To avoid cycles and partially initialized modules, ``__init__.py`` should *not*
85 contain any non-trivial imports. The top level of a module should *not* be a
86 facade for the module functionality.
87
88 Common code for a module is often in ``base.py``.
89
90 The important part of the dependency graph is approximately linear. In the
91 following list, modules may only depend on modules below them:
92
93 ``tests``
94 Just get the job done - anything goes.
95
96 ``bin/`` & ``config/`` & ``alembic/``
97 The main entry points, defined in ``setup.py``. Note: The TurboGears template
98 use ``config`` for the high WSGI application - this is not for low level
99 configuration.
100
101 ``controllers/``
102 The top level web application, with TurboGears using the ``root`` controller
103 as entry point, and ``routing`` dispatching to other controllers.
104
105 ``templates/**.html``
106 The "view", rendering to HTML. Invoked by controllers which can pass them
107 anything from lower layers - especially ``helpers`` available as ``h`` will
108 cut through all layers, and ``c`` gives access to global variables.
109
110 ``lib/helpers.py``
111 High level helpers, exposing everything to templates as ``h``. It depends on
112 everything and has a huge dependency chain, so it should not be used for
113 anything else. TODO.
114
115 ``controlles/base.py``
116 The base class of controllers, with lots of model knowledge.
117
118 ``lib/auth.py``
119 All things related to authentication. TODO.
120
121 ``lib/utils.py``
122 High level utils with lots of model knowledge. TODO.
123
124 ``lib/hooks.py``
125 Hooks into "everything" to give centralized logging to database, cache
126 invalidation, and extension handling. TODO.
127
128 ``model/``
129 Convenience business logic wrappers around database models.
130
131 ``model/db.py``
132 Defines the database schema and provides some additional logic.
133
134 ``model/scm.py``
135 All things related to anything. TODO.
136
137 SQLAlchemy
138 Database session and transaction in thread-local variables.
139
140 ``lib/utils2.py``
141 Low level utils specific to Kallithea.
142
143 ``lib/webutils.py``
144 Low level generic utils with awareness of the TurboGears environment.
145
146 TurboGears
147 Request, response and state like i18n gettext in thread-local variables.
148 External dependency with global state - usage should be minimized.
149
150 ``lib/vcs/``
151 Previously an independent library. No awareness of web, database, or state.
152
153 ``lib/*``
154 Various "pure" functionality not depending on anything else.
155
156 ``__init__``
157 Very basic Kallithea constants - some of them are set very early based on ``.ini``.
158
159 This is not exactly how it is right now, but we aim for something like that.
160 Especially the areas marked as TODO have some problems that need untangling.
161
162
75 Running tests
163 Running tests
76 -------------
164 -------------
77
165
@@ -5,6 +5,9 b''
5 set -e
5 set -e
6 set -x
6 set -x
7
7
8 hg files 'set:!binary()&grep("^#!.*python")' 'set:**.py' | xargs scripts/deps.py
9 dot -Tsvg deps.dot > deps.svg
10
8 scripts/docs-headings.py
11 scripts/docs-headings.py
9 scripts/generate-ini.py
12 scripts/generate-ini.py
10 scripts/whitespacecleanup.sh
13 scripts/whitespacecleanup.sh
General Comments 0
You need to be logged in to leave comments. Login now