##// END OF EJS Templates
lib: use packaging.version.Version instead of dropped distutils.version.StrictVersion...
Mads Kiilerich -
r8778:a5d15a75 stable
parent child Browse files
Show More
@@ -1,545 +1,545 b''
1 1 # -*- coding: utf-8 -*-
2 2 # This program is free software: you can redistribute it and/or modify
3 3 # it under the terms of the GNU General Public License as published by
4 4 # the Free Software Foundation, either version 3 of the License, or
5 5 # (at your option) any later version.
6 6 #
7 7 # This program is distributed in the hope that it will be useful,
8 8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 10 # GNU General Public License for more details.
11 11 #
12 12 # You should have received a copy of the GNU General Public License
13 13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 14 """
15 15 kallithea.lib.utils2
16 16 ~~~~~~~~~~~~~~~~~~~~
17 17
18 18 Some simple helper functions.
19 19 Note: all these functions should be independent of Kallithea classes, i.e.
20 20 models, controllers, etc. to prevent import cycles.
21 21
22 22 This file was forked by the Kallithea project in July 2014.
23 23 Original author and date, and relevant copyright and licensing information is below:
24 24 :created_on: Jan 5, 2011
25 25 :author: marcink
26 26 :copyright: (c) 2013 RhodeCode GmbH, and others.
27 27 :license: GPLv3, see LICENSE.md for more details.
28 28 """
29 29
30 30 import binascii
31 31 import datetime
32 32 import hashlib
33 33 import json
34 34 import logging
35 35 import os
36 36 import re
37 37 import string
38 38 import sys
39 39 import time
40 40 import urllib.parse
41 from distutils.version import StrictVersion
42 41
43 42 import bcrypt
44 43 import urlobject
44 from packaging.version import Version
45 45 from sqlalchemy.engine import url as sa_url
46 46 from sqlalchemy.exc import ArgumentError
47 47 from tg import tmpl_context
48 48 from tg.support.converters import asbool, aslist
49 49 from webhelpers2.text import collapse, remove_formatting, strip_tags
50 50
51 51 import kallithea
52 52 from kallithea.lib import webutils
53 53 from kallithea.lib.vcs.backends.base import BaseRepository, EmptyChangeset
54 54 from kallithea.lib.vcs.backends.git.repository import GitRepository
55 55 from kallithea.lib.vcs.conf import settings
56 56 from kallithea.lib.vcs.exceptions import RepositoryError
57 57 from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, safe_bytes, safe_str # re-export
58 58 from kallithea.lib.vcs.utils.lazy import LazyProperty
59 59
60 60
61 61 try:
62 62 import pwd
63 63 except ImportError:
64 64 pass
65 65
66 66
67 67 log = logging.getLogger(__name__)
68 68
69 69
70 70 # mute pyflakes "imported but unused"
71 71 assert asbool
72 72 assert aslist
73 73 assert ascii_bytes
74 74 assert ascii_str
75 75 assert safe_bytes
76 76 assert safe_str
77 77 assert LazyProperty
78 78
79 79
80 80 # get current umask value without changing it
81 81 umask = os.umask(0)
82 82 os.umask(umask)
83 83
84 84
85 85 def convert_line_endings(line, mode):
86 86 """
87 87 Converts a given line "line end" according to given mode
88 88
89 89 Available modes are::
90 90 0 - Unix
91 91 1 - Mac
92 92 2 - DOS
93 93
94 94 :param line: given line to convert
95 95 :param mode: mode to convert to
96 96 :rtype: str
97 97 :return: converted line according to mode
98 98 """
99 99 if mode == 0:
100 100 line = line.replace('\r\n', '\n')
101 101 line = line.replace('\r', '\n')
102 102 elif mode == 1:
103 103 line = line.replace('\r\n', '\r')
104 104 line = line.replace('\n', '\r')
105 105 elif mode == 2:
106 106 line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
107 107 return line
108 108
109 109
110 110 def detect_mode(line, default):
111 111 """
112 112 Detects line break for given line, if line break couldn't be found
113 113 given default value is returned
114 114
115 115 :param line: str line
116 116 :param default: default
117 117 :rtype: int
118 118 :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
119 119 """
120 120 if line.endswith('\r\n'):
121 121 return 2
122 122 elif line.endswith('\n'):
123 123 return 0
124 124 elif line.endswith('\r'):
125 125 return 1
126 126 else:
127 127 return default
128 128
129 129
130 130 def generate_api_key():
131 131 """
132 132 Generates a random (presumably unique) API key.
133 133
134 134 This value is used in URLs and "Bearer" HTTP Authorization headers,
135 135 which in practice means it should only contain URL-safe characters
136 136 (RFC 3986):
137 137
138 138 unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
139 139 """
140 140 # Hexadecimal certainly qualifies as URL-safe.
141 141 return ascii_str(binascii.hexlify(os.urandom(20)))
142 142
143 143
144 144 def safe_int(val, default=None):
145 145 """
146 146 Returns int() of val if val is not convertable to int use default
147 147 instead
148 148
149 149 :param val:
150 150 :param default:
151 151 """
152 152 try:
153 153 val = int(val)
154 154 except (ValueError, TypeError):
155 155 val = default
156 156 return val
157 157
158 158
159 159 def remove_suffix(s, suffix):
160 160 if s.endswith(suffix):
161 161 s = s[:-1 * len(suffix)]
162 162 return s
163 163
164 164
165 165 def remove_prefix(s, prefix):
166 166 if s.startswith(prefix):
167 167 s = s[len(prefix):]
168 168 return s
169 169
170 170
171 171 def uri_filter(uri):
172 172 """
173 173 Removes user:password from given url string
174 174
175 175 :param uri:
176 176 :rtype: str
177 177 :returns: filtered list of strings
178 178 """
179 179 if not uri:
180 180 return []
181 181
182 182 proto = ''
183 183
184 184 for pat in ('https://', 'http://', 'git://'):
185 185 if uri.startswith(pat):
186 186 uri = uri[len(pat):]
187 187 proto = pat
188 188 break
189 189
190 190 # remove passwords and username
191 191 uri = uri[uri.find('@') + 1:]
192 192
193 193 # get the port
194 194 cred_pos = uri.find(':')
195 195 if cred_pos == -1:
196 196 host, port = uri, None
197 197 else:
198 198 host, port = uri[:cred_pos], uri[cred_pos + 1:]
199 199
200 200 return [_f for _f in [proto, host, port] if _f]
201 201
202 202
203 203 def credentials_filter(uri):
204 204 """
205 205 Returns a url with removed credentials
206 206
207 207 :param uri:
208 208 """
209 209
210 210 uri = uri_filter(uri)
211 211 # check if we have port
212 212 if len(uri) > 2 and uri[2]:
213 213 uri[2] = ':' + uri[2]
214 214
215 215 return ''.join(uri)
216 216
217 217
218 218 def get_clone_url(clone_uri_tmpl, prefix_url, repo_name, repo_id, username=None):
219 219 parsed_url = urlobject.URLObject(prefix_url)
220 220 prefix = urllib.parse.unquote(parsed_url.path.rstrip('/'))
221 221 try:
222 222 system_user = pwd.getpwuid(os.getuid()).pw_name
223 223 except NameError: # TODO: support all systems - especially Windows
224 224 system_user = 'kallithea' # hardcoded default value ...
225 225 args = {
226 226 'scheme': parsed_url.scheme,
227 227 'user': urllib.parse.quote(username or ''),
228 228 'netloc': parsed_url.netloc + prefix, # like "hostname:port/prefix" (with optional ":port" and "/prefix")
229 229 'prefix': prefix, # undocumented, empty or starting with /
230 230 'repo': repo_name,
231 231 'repoid': str(repo_id),
232 232 'system_user': system_user,
233 233 'hostname': parsed_url.hostname,
234 234 }
235 235 url = re.sub('{([^{}]+)}', lambda m: args.get(m.group(1), m.group(0)), clone_uri_tmpl)
236 236
237 237 # remove leading @ sign if it's present. Case of empty user
238 238 url_obj = urlobject.URLObject(url)
239 239 if not url_obj.username:
240 240 url_obj = url_obj.with_username(None)
241 241
242 242 return str(url_obj)
243 243
244 244
245 245 def short_ref_name(ref_type, ref_name):
246 246 """Return short description of PR ref - revs will be truncated"""
247 247 if ref_type == 'rev':
248 248 return ref_name[:12]
249 249 return ref_name
250 250
251 251
252 252 def link_to_ref(repo_name, ref_type, ref_name, rev=None):
253 253 """
254 254 Return full markup for a PR ref to changeset_home for a changeset.
255 255 If ref_type is 'branch', it will link to changelog.
256 256 ref_name is shortened if ref_type is 'rev'.
257 257 if rev is specified, show it too, explicitly linking to that revision.
258 258 """
259 259 txt = short_ref_name(ref_type, ref_name)
260 260 if ref_type == 'branch':
261 261 u = webutils.url('changelog_home', repo_name=repo_name, branch=ref_name)
262 262 else:
263 263 u = webutils.url('changeset_home', repo_name=repo_name, revision=ref_name)
264 264 l = webutils.link_to(repo_name + '#' + txt, u)
265 265 if rev and ref_type != 'rev':
266 266 l = webutils.literal('%s (%s)' % (l, webutils.link_to(rev[:12], webutils.url('changeset_home', repo_name=repo_name, revision=rev))))
267 267 return l
268 268
269 269
270 270 def get_changeset_safe(repo, rev):
271 271 """
272 272 Safe version of get_changeset if this changeset doesn't exists for a
273 273 repo it returns a Dummy one instead
274 274
275 275 :param repo:
276 276 :param rev:
277 277 """
278 278 if not isinstance(repo, BaseRepository):
279 279 raise Exception('You must pass an Repository '
280 280 'object as first argument got %s' % type(repo))
281 281
282 282 try:
283 283 cs = repo.get_changeset(rev)
284 284 except (RepositoryError, LookupError):
285 285 cs = EmptyChangeset(requested_revision=rev)
286 286 return cs
287 287
288 288
289 289 def datetime_to_time(dt):
290 290 if dt:
291 291 return time.mktime(dt.timetuple())
292 292
293 293
294 294 def time_to_datetime(tm):
295 295 if tm:
296 296 if isinstance(tm, str):
297 297 try:
298 298 tm = float(tm)
299 299 except ValueError:
300 300 return
301 301 return datetime.datetime.fromtimestamp(tm)
302 302
303 303
304 304 class AttributeDict(dict):
305 305 def __getattr__(self, attr):
306 306 return self.get(attr, None)
307 307 __setattr__ = dict.__setitem__
308 308 __delattr__ = dict.__delitem__
309 309
310 310
311 311 def obfuscate_url_pw(engine):
312 312 try:
313 313 _url = sa_url.make_url(engine or '')
314 314 except ArgumentError:
315 315 return engine
316 316 if _url.password:
317 317 _url.password = 'XXXXX'
318 318 return str(_url)
319 319
320 320
321 321 class HookEnvironmentError(Exception): pass
322 322
323 323
324 324 def get_hook_environment():
325 325 """
326 326 Get hook context by deserializing the global KALLITHEA_EXTRAS environment
327 327 variable.
328 328
329 329 Called early in Git out-of-process hooks to get .ini config path so the
330 330 basic environment can be configured properly. Also used in all hooks to get
331 331 information about the action that triggered it.
332 332 """
333 333
334 334 try:
335 335 kallithea_extras = os.environ['KALLITHEA_EXTRAS']
336 336 except KeyError:
337 337 raise HookEnvironmentError("Environment variable KALLITHEA_EXTRAS not found")
338 338
339 339 extras = json.loads(kallithea_extras)
340 340 for k in ['username', 'repository', 'scm', 'action', 'ip', 'config']:
341 341 try:
342 342 extras[k]
343 343 except KeyError:
344 344 raise HookEnvironmentError('Missing key %s in KALLITHEA_EXTRAS %s' % (k, extras))
345 345
346 346 return AttributeDict(extras)
347 347
348 348
349 349 def set_hook_environment(username, ip_addr, repo_name, repo_alias, action=None):
350 350 """Prepare global context for running hooks by serializing data in the
351 351 global KALLITHEA_EXTRAS environment variable.
352 352
353 353 Most importantly, this allow Git hooks to do proper logging and updating of
354 354 caches after pushes.
355 355
356 356 Must always be called before anything with hooks are invoked.
357 357 """
358 358 extras = {
359 359 'ip': ip_addr, # used in action_logger
360 360 'username': username,
361 361 'action': action or 'push_local', # used in process_pushed_raw_ids action_logger
362 362 'repository': repo_name,
363 363 'scm': repo_alias,
364 364 'config': kallithea.CONFIG['__file__'], # used by git hook to read config
365 365 }
366 366 os.environ['KALLITHEA_EXTRAS'] = json.dumps(extras)
367 367
368 368
369 369 def get_current_authuser():
370 370 """
371 371 Gets kallithea user from threadlocal tmpl_context variable if it's
372 372 defined, else returns None.
373 373 """
374 374 try:
375 375 return getattr(tmpl_context, 'authuser', None)
376 376 except TypeError: # No object (name: context) has been registered for this thread
377 377 return None
378 378
379 379
380 380 def urlreadable(s, _cleanstringsub=re.compile('[^-a-zA-Z0-9./]+').sub):
381 381 return _cleanstringsub('_', s).rstrip('_')
382 382
383 383
384 384 def recursive_replace(str_, replace=' '):
385 385 """
386 386 Recursive replace of given sign to just one instance
387 387
388 388 :param str_: given string
389 389 :param replace: char to find and replace multiple instances
390 390
391 391 Examples::
392 392 >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')
393 393 'Mighty-Mighty-Bo-sstones'
394 394 """
395 395
396 396 if str_.find(replace * 2) == -1:
397 397 return str_
398 398 else:
399 399 str_ = str_.replace(replace * 2, replace)
400 400 return recursive_replace(str_, replace)
401 401
402 402
403 403 def repo_name_slug(value):
404 404 """
405 405 Return slug of name of repository
406 406 This function is called on each creation/modification
407 407 of repository to prevent bad names in repo
408 408 """
409 409
410 410 slug = remove_formatting(value)
411 411 slug = strip_tags(slug)
412 412
413 413 for c in r"""`?=[]\;'"<>,/~!@#$%^&*()+{}|: """:
414 414 slug = slug.replace(c, '-')
415 415 slug = recursive_replace(slug, '-')
416 416 slug = collapse(slug, '-')
417 417 return slug
418 418
419 419
420 420 def ask_ok(prompt, retries=4, complaint='Yes or no please!'):
421 421 while True:
422 422 ok = input(prompt)
423 423 if ok in ('y', 'ye', 'yes'):
424 424 return True
425 425 if ok in ('n', 'no', 'nop', 'nope'):
426 426 return False
427 427 retries = retries - 1
428 428 if retries < 0:
429 429 raise IOError
430 430 print(complaint)
431 431
432 432
433 433 class PasswordGenerator(object):
434 434 """
435 435 This is a simple class for generating password from different sets of
436 436 characters
437 437 usage::
438 438
439 439 passwd_gen = PasswordGenerator()
440 440 #print 8-letter password containing only big and small letters
441 441 of alphabet
442 442 passwd_gen.gen_password(8, passwd_gen.ALPHABETS_BIG_SMALL)
443 443 """
444 444 ALPHABETS_NUM = r'''1234567890'''
445 445 ALPHABETS_SMALL = r'''qwertyuiopasdfghjklzxcvbnm'''
446 446 ALPHABETS_BIG = r'''QWERTYUIOPASDFGHJKLZXCVBNM'''
447 447 ALPHABETS_SPECIAL = r'''`-=[]\;',./~!@#$%^&*()_+{}|:"<>?'''
448 448 ALPHABETS_FULL = ALPHABETS_BIG + ALPHABETS_SMALL \
449 449 + ALPHABETS_NUM + ALPHABETS_SPECIAL
450 450 ALPHABETS_ALPHANUM = ALPHABETS_BIG + ALPHABETS_SMALL + ALPHABETS_NUM
451 451 ALPHABETS_BIG_SMALL = ALPHABETS_BIG + ALPHABETS_SMALL
452 452 ALPHABETS_ALPHANUM_BIG = ALPHABETS_BIG + ALPHABETS_NUM
453 453 ALPHABETS_ALPHANUM_SMALL = ALPHABETS_SMALL + ALPHABETS_NUM
454 454
455 455 def gen_password(self, length, alphabet=ALPHABETS_FULL):
456 456 assert len(alphabet) <= 256, alphabet
457 457 l = []
458 458 while len(l) < length:
459 459 i = ord(os.urandom(1))
460 460 if i < len(alphabet):
461 461 l.append(alphabet[i])
462 462 return ''.join(l)
463 463
464 464
465 465 def get_crypt_password(password):
466 466 """
467 467 Cryptographic function used for bcrypt password hashing.
468 468
469 469 :param password: password to hash
470 470 """
471 471 return ascii_str(bcrypt.hashpw(safe_bytes(password), bcrypt.gensalt(10)))
472 472
473 473
474 474 def check_password(password, hashed):
475 475 """
476 476 Checks password match the hashed value using bcrypt.
477 477 Remains backwards compatible and accept plain sha256 hashes which used to
478 478 be used on Windows.
479 479
480 480 :param password: password
481 481 :param hashed: password in hashed form
482 482 """
483 483 # sha256 hashes will always be 64 hex chars
484 484 # bcrypt hashes will always contain $ (and be shorter)
485 485 if len(hashed) == 64 and all(x in string.hexdigits for x in hashed):
486 486 return hashlib.sha256(password).hexdigest() == hashed
487 487 try:
488 488 return bcrypt.checkpw(safe_bytes(password), ascii_bytes(hashed))
489 489 except ValueError as e:
490 490 # bcrypt will throw ValueError 'Invalid hashed_password salt' on all password errors
491 491 log.error('error from bcrypt checking password: %s', e)
492 492 return False
493 493 log.error('check_password failed - no method found for hash length %s', len(hashed))
494 494 return False
495 495
496 496
497 git_req_ver = StrictVersion('1.7.4')
497 git_req_ver = Version('1.7.4')
498 498
499 499 def check_git_version():
500 500 """
501 501 Checks what version of git is installed on the system, and raise a system exit
502 502 if it's too old for Kallithea to work properly.
503 503 """
504 504 if 'git' not in kallithea.BACKENDS:
505 505 return None
506 506
507 507 if not settings.GIT_EXECUTABLE_PATH:
508 508 log.warning('No git executable configured - check "git_path" in the ini file.')
509 509 return None
510 510
511 511 try:
512 512 stdout, stderr = GitRepository._run_git_command(['--version'])
513 513 except RepositoryError as e:
514 514 # message will already have been logged as error
515 515 log.warning('No working git executable found - check "git_path" in the ini file.')
516 516 return None
517 517
518 518 if stderr:
519 519 log.warning('Error/stderr from "%s --version":\n%s', settings.GIT_EXECUTABLE_PATH, safe_str(stderr))
520 520
521 521 if not stdout:
522 522 log.warning('No working git executable found - check "git_path" in the ini file.')
523 523 return None
524 524
525 525 output = safe_str(stdout).strip()
526 526 m = re.search(r"\d+.\d+.\d+", output)
527 527 if m:
528 ver = StrictVersion(m.group(0))
528 ver = Version(m.group(0))
529 529 log.debug('Git executable: "%s", version %s (parsed from: "%s")',
530 530 settings.GIT_EXECUTABLE_PATH, ver, output)
531 531 if ver < git_req_ver:
532 532 log.error('Kallithea detected %s version %s, which is too old '
533 533 'for the system to function properly. '
534 534 'Please upgrade to version %s or later. '
535 535 'If you strictly need Mercurial repositories, you can '
536 536 'clear the "git_path" setting in the ini file.',
537 537 settings.GIT_EXECUTABLE_PATH, ver, git_req_ver)
538 538 log.error("Terminating ...")
539 539 sys.exit(1)
540 540 else:
541 ver = StrictVersion('0.0.0')
541 ver = Version('0.0.0')
542 542 log.warning('Error finding version number in "%s --version" stdout:\n%s',
543 543 settings.GIT_EXECUTABLE_PATH, output)
544 544
545 545 return ver
@@ -1,293 +1,294 b''
1 1 #!/usr/bin/env python3
2 2
3 3
4 4 import re
5 5 import sys
6 6
7 7
8 8 ignored_modules = set('''
9 9 argparse
10 10 base64
11 11 bcrypt
12 12 binascii
13 13 bleach
14 14 calendar
15 15 celery
16 16 celery
17 17 chardet
18 18 click
19 19 collections
20 20 configparser
21 21 copy
22 22 csv
23 23 ctypes
24 24 datetime
25 25 dateutil
26 26 decimal
27 27 decorator
28 28 difflib
29 29 distutils
30 30 docutils
31 31 email
32 32 errno
33 33 fileinput
34 34 functools
35 35 getpass
36 36 grp
37 37 hashlib
38 38 hmac
39 39 html
40 40 http
41 41 imp
42 42 importlib
43 43 inspect
44 44 io
45 45 ipaddr
46 46 IPython
47 47 isapi_wsgi
48 48 itertools
49 49 json
50 50 kajiki
51 51 ldap
52 52 logging
53 53 mako
54 54 markdown
55 55 mimetypes
56 56 mock
57 57 msvcrt
58 58 multiprocessing
59 59 operator
60 60 os
61 61 paginate
62 62 paginate_sqlalchemy
63 63 pam
64 64 paste
65 65 pkg_resources
66 66 platform
67 67 posixpath
68 68 pprint
69 69 pwd
70 70 pyflakes
71 71 pytest
72 72 pytest_localserver
73 73 random
74 74 re
75 75 routes
76 76 setuptools
77 77 shlex
78 78 shutil
79 79 smtplib
80 80 socket
81 81 ssl
82 82 stat
83 83 string
84 84 struct
85 85 subprocess
86 86 sys
87 87 tarfile
88 88 tempfile
89 89 textwrap
90 90 tgext
91 91 threading
92 92 time
93 93 traceback
94 94 traitlets
95 95 types
96 96 typing
97 97 urllib
98 98 urlobject
99 99 uuid
100 100 warnings
101 101 webhelpers2
102 102 webob
103 103 webtest
104 104 whoosh
105 105 win32traceutil
106 106 zipfile
107 107 '''.split())
108 108
109 109 top_modules = set('''
110 110 kallithea.alembic
111 111 kallithea.bin
112 112 kallithea.config
113 113 kallithea.controllers
114 114 kallithea.templates.py
115 115 scripts
116 116 '''.split())
117 117
118 118 bottom_external_modules = set('''
119 119 tg
120 120 mercurial
121 121 sqlalchemy
122 122 alembic
123 123 formencode
124 124 pygments
125 125 dulwich
126 126 beaker
127 127 psycopg2
128 128 docs
129 129 setup
130 130 conftest
131 packaging
131 132 '''.split())
132 133
133 134 normal_modules = set('''
134 135 kallithea
135 136 kallithea.controllers.base
136 137 kallithea.lib
137 138 kallithea.lib.auth
138 139 kallithea.lib.auth_modules
139 140 kallithea.lib.celerylib
140 141 kallithea.lib.db_manage
141 142 kallithea.lib.helpers
142 143 kallithea.lib.hooks
143 144 kallithea.lib.indexers
144 145 kallithea.lib.utils
145 146 kallithea.lib.utils2
146 147 kallithea.lib.vcs
147 148 kallithea.lib.webutils
148 149 kallithea.model
149 150 kallithea.model.async_tasks
150 151 kallithea.model.scm
151 152 kallithea.templates.py
152 153 '''.split())
153 154
154 155 shown_modules = normal_modules | top_modules
155 156
156 157 # break the chains somehow - this is a cleanup TODO list
157 158 known_violations = set([
158 159 ('kallithea.lib.auth_modules', 'kallithea.lib.auth'), # needs base&facade
159 160 ('kallithea.lib.utils', 'kallithea.model'), # clean up utils
160 161 ('kallithea.lib.utils', 'kallithea.model.db'),
161 162 ('kallithea.lib.utils', 'kallithea.model.scm'),
162 163 ('kallithea.model', 'kallithea.lib.auth'), # auth.HasXXX
163 164 ('kallithea.model', 'kallithea.lib.auth_modules'), # validators
164 165 ('kallithea.model', 'kallithea.lib.hooks'), # clean up hooks
165 166 ('kallithea.model', 'kallithea.model.scm'),
166 167 ('kallithea.model.scm', 'kallithea.lib.hooks'),
167 168 ])
168 169
169 170 extra_edges = [
170 171 ('kallithea.config', 'kallithea.controllers'), # through TG
171 172 ('kallithea.lib.auth', 'kallithea.lib.auth_modules'), # custom loader
172 173 ]
173 174
174 175
175 176 def normalize(s):
176 177 """Given a string with dot path, return the string it should be shown as."""
177 178 parts = s.replace('.__init__', '').split('.')
178 179 short_2 = '.'.join(parts[:2])
179 180 short_3 = '.'.join(parts[:3])
180 181 short_4 = '.'.join(parts[:4])
181 182 if parts[0] in ['scripts', 'contributor_data', 'i18n_utils']:
182 183 return 'scripts'
183 184 if short_3 == 'kallithea.model.meta':
184 185 return 'kallithea.model.db'
185 186 if parts[:4] == ['kallithea', 'lib', 'vcs', 'ssh']:
186 187 return 'kallithea.lib.vcs.ssh'
187 188 if short_4 in shown_modules:
188 189 return short_4
189 190 if short_3 in shown_modules:
190 191 return short_3
191 192 if short_2 in shown_modules:
192 193 return short_2
193 194 if short_2 == 'kallithea.tests':
194 195 return None
195 196 if parts[0] in ignored_modules:
196 197 return None
197 198 assert parts[0] in bottom_external_modules, parts
198 199 return parts[0]
199 200
200 201
201 202 def main(filenames):
202 203 if not filenames or filenames[0].startswith('-'):
203 204 print('''\
204 205 Usage:
205 206 hg files 'set:!binary()&grep("^#!.*python")' 'set:**.py' | xargs scripts/deps.py
206 207 dot -Tsvg deps.dot > deps.svg
207 208 ''')
208 209 raise SystemExit(1)
209 210
210 211 files_imports = dict() # map filenames to its imports
211 212 import_deps = set() # set of tuples with module name and its imports
212 213 for fn in filenames:
213 214 with open(fn) as f:
214 215 s = f.read()
215 216
216 217 dot_name = (fn[:-3] if fn.endswith('.py') else fn).replace('/', '.')
217 218 file_imports = set()
218 219 for m in re.finditer(r'^ *(?:from ([^ ]*) import (?:([a-zA-Z].*)|\(([^)]*)\))|import (.*))$', s, re.MULTILINE):
219 220 m_from, m_from_import, m_from_import2, m_import = m.groups()
220 221 if m_from:
221 222 pre = m_from + '.'
222 223 if pre.startswith('.'):
223 224 pre = dot_name.rsplit('.', 1)[0] + pre
224 225 importlist = m_from_import or m_from_import2
225 226 else:
226 227 pre = ''
227 228 importlist = m_import
228 229 for imp in importlist.split('#', 1)[0].split(','):
229 230 full_imp = pre + imp.strip().split(' as ', 1)[0]
230 231 file_imports.add(full_imp)
231 232 import_deps.add((dot_name, full_imp))
232 233 files_imports[fn] = file_imports
233 234
234 235 # dump out all deps for debugging and analysis
235 236 with open('deps.txt', 'w') as f:
236 237 for fn, file_imports in sorted(files_imports.items()):
237 238 for file_import in sorted(file_imports):
238 239 if file_import.split('.', 1)[0] in ignored_modules:
239 240 continue
240 241 f.write('%s: %s\n' % (fn, file_import))
241 242
242 243 # find leafs that haven't been ignored - they are the important external dependencies and shown in the bottom row
243 244 only_imported = set(
244 245 set(normalize(b) for a, b in import_deps) -
245 246 set(normalize(a) for a, b in import_deps) -
246 247 set([None, 'kallithea'])
247 248 )
248 249
249 250 normalized_dep_edges = set()
250 251 for dot_name, full_imp in import_deps:
251 252 a = normalize(dot_name)
252 253 b = normalize(full_imp)
253 254 if a is None or b is None or a == b:
254 255 continue
255 256 normalized_dep_edges.add((a, b))
256 257 #print((dot_name, full_imp, a, b))
257 258 normalized_dep_edges.update(extra_edges)
258 259
259 260 unseen_shown_modules = shown_modules.difference(a for a, b in normalized_dep_edges).difference(b for a, b in normalized_dep_edges)
260 261 assert not unseen_shown_modules, unseen_shown_modules
261 262
262 263 with open('deps.dot', 'w') as f:
263 264 f.write('digraph {\n')
264 265 f.write('subgraph { rank = same; %s}\n' % ''.join('"%s"; ' % s for s in sorted(top_modules)))
265 266 f.write('subgraph { rank = same; %s}\n' % ''.join('"%s"; ' % s for s in sorted(only_imported)))
266 267 for a, b in sorted(normalized_dep_edges):
267 268 f.write(' "%s" -> "%s"%s\n' % (a, b, ' [color=red]' if (a, b) in known_violations else ' [color=green]' if (a, b) in extra_edges else ''))
268 269 f.write('}\n')
269 270
270 271 # verify dependencies by untangling dependency chain bottom-up:
271 272 todo = set(normalized_dep_edges)
272 273 unseen_violations = known_violations.difference(todo)
273 274 assert not unseen_violations, unseen_violations
274 275 for x in known_violations:
275 276 todo.remove(x)
276 277
277 278 while todo:
278 279 depending = set(a for a, b in todo)
279 280 depended = set(b for a, b in todo)
280 281 drop = depended - depending
281 282 if not drop:
282 283 print('ERROR: cycles:', len(todo))
283 284 for x in sorted(todo):
284 285 print('%s,' % (x,))
285 286 raise SystemExit(1)
286 287 #for do_b in sorted(drop):
287 288 # print('Picking', do_b, '- unblocks:', ' '.join(a for a, b in sorted((todo)) if b == do_b))
288 289 todo = set((a, b) for a, b in todo if b in depending)
289 290 #print()
290 291
291 292
292 293 if __name__ == '__main__':
293 294 main(sys.argv[1:])
General Comments 0
You need to be logged in to leave comments. Login now