##// END OF EJS Templates
lib: use packaging.version.Version instead of dropped distutils.version.StrictVersion...
Mads Kiilerich -
r8778:a5d15a75 stable
parent child Browse files
Show More
@@ -1,545 +1,545 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 # This program is free software: you can redistribute it and/or modify
2 # This program is free software: you can redistribute it and/or modify
3 # it under the terms of the GNU General Public License as published by
3 # it under the terms of the GNU General Public License as published by
4 # the Free Software Foundation, either version 3 of the License, or
4 # the Free Software Foundation, either version 3 of the License, or
5 # (at your option) any later version.
5 # (at your option) any later version.
6 #
6 #
7 # This program is distributed in the hope that it will be useful,
7 # This program is distributed in the hope that it will be useful,
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
8 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 # GNU General Public License for more details.
10 # GNU General Public License for more details.
11 #
11 #
12 # You should have received a copy of the GNU General Public License
12 # You should have received a copy of the GNU General Public License
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
13 # along with this program. If not, see <http://www.gnu.org/licenses/>.
14 """
14 """
15 kallithea.lib.utils2
15 kallithea.lib.utils2
16 ~~~~~~~~~~~~~~~~~~~~
16 ~~~~~~~~~~~~~~~~~~~~
17
17
18 Some simple helper functions.
18 Some simple helper functions.
19 Note: all these functions should be independent of Kallithea classes, i.e.
19 Note: all these functions should be independent of Kallithea classes, i.e.
20 models, controllers, etc. to prevent import cycles.
20 models, controllers, etc. to prevent import cycles.
21
21
22 This file was forked by the Kallithea project in July 2014.
22 This file was forked by the Kallithea project in July 2014.
23 Original author and date, and relevant copyright and licensing information is below:
23 Original author and date, and relevant copyright and licensing information is below:
24 :created_on: Jan 5, 2011
24 :created_on: Jan 5, 2011
25 :author: marcink
25 :author: marcink
26 :copyright: (c) 2013 RhodeCode GmbH, and others.
26 :copyright: (c) 2013 RhodeCode GmbH, and others.
27 :license: GPLv3, see LICENSE.md for more details.
27 :license: GPLv3, see LICENSE.md for more details.
28 """
28 """
29
29
30 import binascii
30 import binascii
31 import datetime
31 import datetime
32 import hashlib
32 import hashlib
33 import json
33 import json
34 import logging
34 import logging
35 import os
35 import os
36 import re
36 import re
37 import string
37 import string
38 import sys
38 import sys
39 import time
39 import time
40 import urllib.parse
40 import urllib.parse
41 from distutils.version import StrictVersion
42
41
43 import bcrypt
42 import bcrypt
44 import urlobject
43 import urlobject
44 from packaging.version import Version
45 from sqlalchemy.engine import url as sa_url
45 from sqlalchemy.engine import url as sa_url
46 from sqlalchemy.exc import ArgumentError
46 from sqlalchemy.exc import ArgumentError
47 from tg import tmpl_context
47 from tg import tmpl_context
48 from tg.support.converters import asbool, aslist
48 from tg.support.converters import asbool, aslist
49 from webhelpers2.text import collapse, remove_formatting, strip_tags
49 from webhelpers2.text import collapse, remove_formatting, strip_tags
50
50
51 import kallithea
51 import kallithea
52 from kallithea.lib import webutils
52 from kallithea.lib import webutils
53 from kallithea.lib.vcs.backends.base import BaseRepository, EmptyChangeset
53 from kallithea.lib.vcs.backends.base import BaseRepository, EmptyChangeset
54 from kallithea.lib.vcs.backends.git.repository import GitRepository
54 from kallithea.lib.vcs.backends.git.repository import GitRepository
55 from kallithea.lib.vcs.conf import settings
55 from kallithea.lib.vcs.conf import settings
56 from kallithea.lib.vcs.exceptions import RepositoryError
56 from kallithea.lib.vcs.exceptions import RepositoryError
57 from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, safe_bytes, safe_str # re-export
57 from kallithea.lib.vcs.utils import ascii_bytes, ascii_str, safe_bytes, safe_str # re-export
58 from kallithea.lib.vcs.utils.lazy import LazyProperty
58 from kallithea.lib.vcs.utils.lazy import LazyProperty
59
59
60
60
61 try:
61 try:
62 import pwd
62 import pwd
63 except ImportError:
63 except ImportError:
64 pass
64 pass
65
65
66
66
67 log = logging.getLogger(__name__)
67 log = logging.getLogger(__name__)
68
68
69
69
70 # mute pyflakes "imported but unused"
70 # mute pyflakes "imported but unused"
71 assert asbool
71 assert asbool
72 assert aslist
72 assert aslist
73 assert ascii_bytes
73 assert ascii_bytes
74 assert ascii_str
74 assert ascii_str
75 assert safe_bytes
75 assert safe_bytes
76 assert safe_str
76 assert safe_str
77 assert LazyProperty
77 assert LazyProperty
78
78
79
79
80 # get current umask value without changing it
80 # get current umask value without changing it
81 umask = os.umask(0)
81 umask = os.umask(0)
82 os.umask(umask)
82 os.umask(umask)
83
83
84
84
85 def convert_line_endings(line, mode):
85 def convert_line_endings(line, mode):
86 """
86 """
87 Converts a given line "line end" according to given mode
87 Converts a given line "line end" according to given mode
88
88
89 Available modes are::
89 Available modes are::
90 0 - Unix
90 0 - Unix
91 1 - Mac
91 1 - Mac
92 2 - DOS
92 2 - DOS
93
93
94 :param line: given line to convert
94 :param line: given line to convert
95 :param mode: mode to convert to
95 :param mode: mode to convert to
96 :rtype: str
96 :rtype: str
97 :return: converted line according to mode
97 :return: converted line according to mode
98 """
98 """
99 if mode == 0:
99 if mode == 0:
100 line = line.replace('\r\n', '\n')
100 line = line.replace('\r\n', '\n')
101 line = line.replace('\r', '\n')
101 line = line.replace('\r', '\n')
102 elif mode == 1:
102 elif mode == 1:
103 line = line.replace('\r\n', '\r')
103 line = line.replace('\r\n', '\r')
104 line = line.replace('\n', '\r')
104 line = line.replace('\n', '\r')
105 elif mode == 2:
105 elif mode == 2:
106 line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
106 line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line)
107 return line
107 return line
108
108
109
109
110 def detect_mode(line, default):
110 def detect_mode(line, default):
111 """
111 """
112 Detects line break for given line, if line break couldn't be found
112 Detects line break for given line, if line break couldn't be found
113 given default value is returned
113 given default value is returned
114
114
115 :param line: str line
115 :param line: str line
116 :param default: default
116 :param default: default
117 :rtype: int
117 :rtype: int
118 :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
118 :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
119 """
119 """
120 if line.endswith('\r\n'):
120 if line.endswith('\r\n'):
121 return 2
121 return 2
122 elif line.endswith('\n'):
122 elif line.endswith('\n'):
123 return 0
123 return 0
124 elif line.endswith('\r'):
124 elif line.endswith('\r'):
125 return 1
125 return 1
126 else:
126 else:
127 return default
127 return default
128
128
129
129
130 def generate_api_key():
130 def generate_api_key():
131 """
131 """
132 Generates a random (presumably unique) API key.
132 Generates a random (presumably unique) API key.
133
133
134 This value is used in URLs and "Bearer" HTTP Authorization headers,
134 This value is used in URLs and "Bearer" HTTP Authorization headers,
135 which in practice means it should only contain URL-safe characters
135 which in practice means it should only contain URL-safe characters
136 (RFC 3986):
136 (RFC 3986):
137
137
138 unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
138 unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~"
139 """
139 """
140 # Hexadecimal certainly qualifies as URL-safe.
140 # Hexadecimal certainly qualifies as URL-safe.
141 return ascii_str(binascii.hexlify(os.urandom(20)))
141 return ascii_str(binascii.hexlify(os.urandom(20)))
142
142
143
143
144 def safe_int(val, default=None):
144 def safe_int(val, default=None):
145 """
145 """
146 Returns int() of val if val is not convertable to int use default
146 Returns int() of val if val is not convertable to int use default
147 instead
147 instead
148
148
149 :param val:
149 :param val:
150 :param default:
150 :param default:
151 """
151 """
152 try:
152 try:
153 val = int(val)
153 val = int(val)
154 except (ValueError, TypeError):
154 except (ValueError, TypeError):
155 val = default
155 val = default
156 return val
156 return val
157
157
158
158
159 def remove_suffix(s, suffix):
159 def remove_suffix(s, suffix):
160 if s.endswith(suffix):
160 if s.endswith(suffix):
161 s = s[:-1 * len(suffix)]
161 s = s[:-1 * len(suffix)]
162 return s
162 return s
163
163
164
164
165 def remove_prefix(s, prefix):
165 def remove_prefix(s, prefix):
166 if s.startswith(prefix):
166 if s.startswith(prefix):
167 s = s[len(prefix):]
167 s = s[len(prefix):]
168 return s
168 return s
169
169
170
170
171 def uri_filter(uri):
171 def uri_filter(uri):
172 """
172 """
173 Removes user:password from given url string
173 Removes user:password from given url string
174
174
175 :param uri:
175 :param uri:
176 :rtype: str
176 :rtype: str
177 :returns: filtered list of strings
177 :returns: filtered list of strings
178 """
178 """
179 if not uri:
179 if not uri:
180 return []
180 return []
181
181
182 proto = ''
182 proto = ''
183
183
184 for pat in ('https://', 'http://', 'git://'):
184 for pat in ('https://', 'http://', 'git://'):
185 if uri.startswith(pat):
185 if uri.startswith(pat):
186 uri = uri[len(pat):]
186 uri = uri[len(pat):]
187 proto = pat
187 proto = pat
188 break
188 break
189
189
190 # remove passwords and username
190 # remove passwords and username
191 uri = uri[uri.find('@') + 1:]
191 uri = uri[uri.find('@') + 1:]
192
192
193 # get the port
193 # get the port
194 cred_pos = uri.find(':')
194 cred_pos = uri.find(':')
195 if cred_pos == -1:
195 if cred_pos == -1:
196 host, port = uri, None
196 host, port = uri, None
197 else:
197 else:
198 host, port = uri[:cred_pos], uri[cred_pos + 1:]
198 host, port = uri[:cred_pos], uri[cred_pos + 1:]
199
199
200 return [_f for _f in [proto, host, port] if _f]
200 return [_f for _f in [proto, host, port] if _f]
201
201
202
202
203 def credentials_filter(uri):
203 def credentials_filter(uri):
204 """
204 """
205 Returns a url with removed credentials
205 Returns a url with removed credentials
206
206
207 :param uri:
207 :param uri:
208 """
208 """
209
209
210 uri = uri_filter(uri)
210 uri = uri_filter(uri)
211 # check if we have port
211 # check if we have port
212 if len(uri) > 2 and uri[2]:
212 if len(uri) > 2 and uri[2]:
213 uri[2] = ':' + uri[2]
213 uri[2] = ':' + uri[2]
214
214
215 return ''.join(uri)
215 return ''.join(uri)
216
216
217
217
218 def get_clone_url(clone_uri_tmpl, prefix_url, repo_name, repo_id, username=None):
218 def get_clone_url(clone_uri_tmpl, prefix_url, repo_name, repo_id, username=None):
219 parsed_url = urlobject.URLObject(prefix_url)
219 parsed_url = urlobject.URLObject(prefix_url)
220 prefix = urllib.parse.unquote(parsed_url.path.rstrip('/'))
220 prefix = urllib.parse.unquote(parsed_url.path.rstrip('/'))
221 try:
221 try:
222 system_user = pwd.getpwuid(os.getuid()).pw_name
222 system_user = pwd.getpwuid(os.getuid()).pw_name
223 except NameError: # TODO: support all systems - especially Windows
223 except NameError: # TODO: support all systems - especially Windows
224 system_user = 'kallithea' # hardcoded default value ...
224 system_user = 'kallithea' # hardcoded default value ...
225 args = {
225 args = {
226 'scheme': parsed_url.scheme,
226 'scheme': parsed_url.scheme,
227 'user': urllib.parse.quote(username or ''),
227 'user': urllib.parse.quote(username or ''),
228 'netloc': parsed_url.netloc + prefix, # like "hostname:port/prefix" (with optional ":port" and "/prefix")
228 'netloc': parsed_url.netloc + prefix, # like "hostname:port/prefix" (with optional ":port" and "/prefix")
229 'prefix': prefix, # undocumented, empty or starting with /
229 'prefix': prefix, # undocumented, empty or starting with /
230 'repo': repo_name,
230 'repo': repo_name,
231 'repoid': str(repo_id),
231 'repoid': str(repo_id),
232 'system_user': system_user,
232 'system_user': system_user,
233 'hostname': parsed_url.hostname,
233 'hostname': parsed_url.hostname,
234 }
234 }
235 url = re.sub('{([^{}]+)}', lambda m: args.get(m.group(1), m.group(0)), clone_uri_tmpl)
235 url = re.sub('{([^{}]+)}', lambda m: args.get(m.group(1), m.group(0)), clone_uri_tmpl)
236
236
237 # remove leading @ sign if it's present. Case of empty user
237 # remove leading @ sign if it's present. Case of empty user
238 url_obj = urlobject.URLObject(url)
238 url_obj = urlobject.URLObject(url)
239 if not url_obj.username:
239 if not url_obj.username:
240 url_obj = url_obj.with_username(None)
240 url_obj = url_obj.with_username(None)
241
241
242 return str(url_obj)
242 return str(url_obj)
243
243
244
244
245 def short_ref_name(ref_type, ref_name):
245 def short_ref_name(ref_type, ref_name):
246 """Return short description of PR ref - revs will be truncated"""
246 """Return short description of PR ref - revs will be truncated"""
247 if ref_type == 'rev':
247 if ref_type == 'rev':
248 return ref_name[:12]
248 return ref_name[:12]
249 return ref_name
249 return ref_name
250
250
251
251
252 def link_to_ref(repo_name, ref_type, ref_name, rev=None):
252 def link_to_ref(repo_name, ref_type, ref_name, rev=None):
253 """
253 """
254 Return full markup for a PR ref to changeset_home for a changeset.
254 Return full markup for a PR ref to changeset_home for a changeset.
255 If ref_type is 'branch', it will link to changelog.
255 If ref_type is 'branch', it will link to changelog.
256 ref_name is shortened if ref_type is 'rev'.
256 ref_name is shortened if ref_type is 'rev'.
257 if rev is specified, show it too, explicitly linking to that revision.
257 if rev is specified, show it too, explicitly linking to that revision.
258 """
258 """
259 txt = short_ref_name(ref_type, ref_name)
259 txt = short_ref_name(ref_type, ref_name)
260 if ref_type == 'branch':
260 if ref_type == 'branch':
261 u = webutils.url('changelog_home', repo_name=repo_name, branch=ref_name)
261 u = webutils.url('changelog_home', repo_name=repo_name, branch=ref_name)
262 else:
262 else:
263 u = webutils.url('changeset_home', repo_name=repo_name, revision=ref_name)
263 u = webutils.url('changeset_home', repo_name=repo_name, revision=ref_name)
264 l = webutils.link_to(repo_name + '#' + txt, u)
264 l = webutils.link_to(repo_name + '#' + txt, u)
265 if rev and ref_type != 'rev':
265 if rev and ref_type != 'rev':
266 l = webutils.literal('%s (%s)' % (l, webutils.link_to(rev[:12], webutils.url('changeset_home', repo_name=repo_name, revision=rev))))
266 l = webutils.literal('%s (%s)' % (l, webutils.link_to(rev[:12], webutils.url('changeset_home', repo_name=repo_name, revision=rev))))
267 return l
267 return l
268
268
269
269
270 def get_changeset_safe(repo, rev):
270 def get_changeset_safe(repo, rev):
271 """
271 """
272 Safe version of get_changeset if this changeset doesn't exists for a
272 Safe version of get_changeset if this changeset doesn't exists for a
273 repo it returns a Dummy one instead
273 repo it returns a Dummy one instead
274
274
275 :param repo:
275 :param repo:
276 :param rev:
276 :param rev:
277 """
277 """
278 if not isinstance(repo, BaseRepository):
278 if not isinstance(repo, BaseRepository):
279 raise Exception('You must pass an Repository '
279 raise Exception('You must pass an Repository '
280 'object as first argument got %s' % type(repo))
280 'object as first argument got %s' % type(repo))
281
281
282 try:
282 try:
283 cs = repo.get_changeset(rev)
283 cs = repo.get_changeset(rev)
284 except (RepositoryError, LookupError):
284 except (RepositoryError, LookupError):
285 cs = EmptyChangeset(requested_revision=rev)
285 cs = EmptyChangeset(requested_revision=rev)
286 return cs
286 return cs
287
287
288
288
289 def datetime_to_time(dt):
289 def datetime_to_time(dt):
290 if dt:
290 if dt:
291 return time.mktime(dt.timetuple())
291 return time.mktime(dt.timetuple())
292
292
293
293
294 def time_to_datetime(tm):
294 def time_to_datetime(tm):
295 if tm:
295 if tm:
296 if isinstance(tm, str):
296 if isinstance(tm, str):
297 try:
297 try:
298 tm = float(tm)
298 tm = float(tm)
299 except ValueError:
299 except ValueError:
300 return
300 return
301 return datetime.datetime.fromtimestamp(tm)
301 return datetime.datetime.fromtimestamp(tm)
302
302
303
303
304 class AttributeDict(dict):
304 class AttributeDict(dict):
305 def __getattr__(self, attr):
305 def __getattr__(self, attr):
306 return self.get(attr, None)
306 return self.get(attr, None)
307 __setattr__ = dict.__setitem__
307 __setattr__ = dict.__setitem__
308 __delattr__ = dict.__delitem__
308 __delattr__ = dict.__delitem__
309
309
310
310
311 def obfuscate_url_pw(engine):
311 def obfuscate_url_pw(engine):
312 try:
312 try:
313 _url = sa_url.make_url(engine or '')
313 _url = sa_url.make_url(engine or '')
314 except ArgumentError:
314 except ArgumentError:
315 return engine
315 return engine
316 if _url.password:
316 if _url.password:
317 _url.password = 'XXXXX'
317 _url.password = 'XXXXX'
318 return str(_url)
318 return str(_url)
319
319
320
320
321 class HookEnvironmentError(Exception): pass
321 class HookEnvironmentError(Exception): pass
322
322
323
323
324 def get_hook_environment():
324 def get_hook_environment():
325 """
325 """
326 Get hook context by deserializing the global KALLITHEA_EXTRAS environment
326 Get hook context by deserializing the global KALLITHEA_EXTRAS environment
327 variable.
327 variable.
328
328
329 Called early in Git out-of-process hooks to get .ini config path so the
329 Called early in Git out-of-process hooks to get .ini config path so the
330 basic environment can be configured properly. Also used in all hooks to get
330 basic environment can be configured properly. Also used in all hooks to get
331 information about the action that triggered it.
331 information about the action that triggered it.
332 """
332 """
333
333
334 try:
334 try:
335 kallithea_extras = os.environ['KALLITHEA_EXTRAS']
335 kallithea_extras = os.environ['KALLITHEA_EXTRAS']
336 except KeyError:
336 except KeyError:
337 raise HookEnvironmentError("Environment variable KALLITHEA_EXTRAS not found")
337 raise HookEnvironmentError("Environment variable KALLITHEA_EXTRAS not found")
338
338
339 extras = json.loads(kallithea_extras)
339 extras = json.loads(kallithea_extras)
340 for k in ['username', 'repository', 'scm', 'action', 'ip', 'config']:
340 for k in ['username', 'repository', 'scm', 'action', 'ip', 'config']:
341 try:
341 try:
342 extras[k]
342 extras[k]
343 except KeyError:
343 except KeyError:
344 raise HookEnvironmentError('Missing key %s in KALLITHEA_EXTRAS %s' % (k, extras))
344 raise HookEnvironmentError('Missing key %s in KALLITHEA_EXTRAS %s' % (k, extras))
345
345
346 return AttributeDict(extras)
346 return AttributeDict(extras)
347
347
348
348
349 def set_hook_environment(username, ip_addr, repo_name, repo_alias, action=None):
349 def set_hook_environment(username, ip_addr, repo_name, repo_alias, action=None):
350 """Prepare global context for running hooks by serializing data in the
350 """Prepare global context for running hooks by serializing data in the
351 global KALLITHEA_EXTRAS environment variable.
351 global KALLITHEA_EXTRAS environment variable.
352
352
353 Most importantly, this allow Git hooks to do proper logging and updating of
353 Most importantly, this allow Git hooks to do proper logging and updating of
354 caches after pushes.
354 caches after pushes.
355
355
356 Must always be called before anything with hooks are invoked.
356 Must always be called before anything with hooks are invoked.
357 """
357 """
358 extras = {
358 extras = {
359 'ip': ip_addr, # used in action_logger
359 'ip': ip_addr, # used in action_logger
360 'username': username,
360 'username': username,
361 'action': action or 'push_local', # used in process_pushed_raw_ids action_logger
361 'action': action or 'push_local', # used in process_pushed_raw_ids action_logger
362 'repository': repo_name,
362 'repository': repo_name,
363 'scm': repo_alias,
363 'scm': repo_alias,
364 'config': kallithea.CONFIG['__file__'], # used by git hook to read config
364 'config': kallithea.CONFIG['__file__'], # used by git hook to read config
365 }
365 }
366 os.environ['KALLITHEA_EXTRAS'] = json.dumps(extras)
366 os.environ['KALLITHEA_EXTRAS'] = json.dumps(extras)
367
367
368
368
369 def get_current_authuser():
369 def get_current_authuser():
370 """
370 """
371 Gets kallithea user from threadlocal tmpl_context variable if it's
371 Gets kallithea user from threadlocal tmpl_context variable if it's
372 defined, else returns None.
372 defined, else returns None.
373 """
373 """
374 try:
374 try:
375 return getattr(tmpl_context, 'authuser', None)
375 return getattr(tmpl_context, 'authuser', None)
376 except TypeError: # No object (name: context) has been registered for this thread
376 except TypeError: # No object (name: context) has been registered for this thread
377 return None
377 return None
378
378
379
379
380 def urlreadable(s, _cleanstringsub=re.compile('[^-a-zA-Z0-9./]+').sub):
380 def urlreadable(s, _cleanstringsub=re.compile('[^-a-zA-Z0-9./]+').sub):
381 return _cleanstringsub('_', s).rstrip('_')
381 return _cleanstringsub('_', s).rstrip('_')
382
382
383
383
384 def recursive_replace(str_, replace=' '):
384 def recursive_replace(str_, replace=' '):
385 """
385 """
386 Recursive replace of given sign to just one instance
386 Recursive replace of given sign to just one instance
387
387
388 :param str_: given string
388 :param str_: given string
389 :param replace: char to find and replace multiple instances
389 :param replace: char to find and replace multiple instances
390
390
391 Examples::
391 Examples::
392 >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')
392 >>> recursive_replace("Mighty---Mighty-Bo--sstones",'-')
393 'Mighty-Mighty-Bo-sstones'
393 'Mighty-Mighty-Bo-sstones'
394 """
394 """
395
395
396 if str_.find(replace * 2) == -1:
396 if str_.find(replace * 2) == -1:
397 return str_
397 return str_
398 else:
398 else:
399 str_ = str_.replace(replace * 2, replace)
399 str_ = str_.replace(replace * 2, replace)
400 return recursive_replace(str_, replace)
400 return recursive_replace(str_, replace)
401
401
402
402
403 def repo_name_slug(value):
403 def repo_name_slug(value):
404 """
404 """
405 Return slug of name of repository
405 Return slug of name of repository
406 This function is called on each creation/modification
406 This function is called on each creation/modification
407 of repository to prevent bad names in repo
407 of repository to prevent bad names in repo
408 """
408 """
409
409
410 slug = remove_formatting(value)
410 slug = remove_formatting(value)
411 slug = strip_tags(slug)
411 slug = strip_tags(slug)
412
412
413 for c in r"""`?=[]\;'"<>,/~!@#$%^&*()+{}|: """:
413 for c in r"""`?=[]\;'"<>,/~!@#$%^&*()+{}|: """:
414 slug = slug.replace(c, '-')
414 slug = slug.replace(c, '-')
415 slug = recursive_replace(slug, '-')
415 slug = recursive_replace(slug, '-')
416 slug = collapse(slug, '-')
416 slug = collapse(slug, '-')
417 return slug
417 return slug
418
418
419
419
420 def ask_ok(prompt, retries=4, complaint='Yes or no please!'):
420 def ask_ok(prompt, retries=4, complaint='Yes or no please!'):
421 while True:
421 while True:
422 ok = input(prompt)
422 ok = input(prompt)
423 if ok in ('y', 'ye', 'yes'):
423 if ok in ('y', 'ye', 'yes'):
424 return True
424 return True
425 if ok in ('n', 'no', 'nop', 'nope'):
425 if ok in ('n', 'no', 'nop', 'nope'):
426 return False
426 return False
427 retries = retries - 1
427 retries = retries - 1
428 if retries < 0:
428 if retries < 0:
429 raise IOError
429 raise IOError
430 print(complaint)
430 print(complaint)
431
431
432
432
433 class PasswordGenerator(object):
433 class PasswordGenerator(object):
434 """
434 """
435 This is a simple class for generating password from different sets of
435 This is a simple class for generating password from different sets of
436 characters
436 characters
437 usage::
437 usage::
438
438
439 passwd_gen = PasswordGenerator()
439 passwd_gen = PasswordGenerator()
440 #print 8-letter password containing only big and small letters
440 #print 8-letter password containing only big and small letters
441 of alphabet
441 of alphabet
442 passwd_gen.gen_password(8, passwd_gen.ALPHABETS_BIG_SMALL)
442 passwd_gen.gen_password(8, passwd_gen.ALPHABETS_BIG_SMALL)
443 """
443 """
444 ALPHABETS_NUM = r'''1234567890'''
444 ALPHABETS_NUM = r'''1234567890'''
445 ALPHABETS_SMALL = r'''qwertyuiopasdfghjklzxcvbnm'''
445 ALPHABETS_SMALL = r'''qwertyuiopasdfghjklzxcvbnm'''
446 ALPHABETS_BIG = r'''QWERTYUIOPASDFGHJKLZXCVBNM'''
446 ALPHABETS_BIG = r'''QWERTYUIOPASDFGHJKLZXCVBNM'''
447 ALPHABETS_SPECIAL = r'''`-=[]\;',./~!@#$%^&*()_+{}|:"<>?'''
447 ALPHABETS_SPECIAL = r'''`-=[]\;',./~!@#$%^&*()_+{}|:"<>?'''
448 ALPHABETS_FULL = ALPHABETS_BIG + ALPHABETS_SMALL \
448 ALPHABETS_FULL = ALPHABETS_BIG + ALPHABETS_SMALL \
449 + ALPHABETS_NUM + ALPHABETS_SPECIAL
449 + ALPHABETS_NUM + ALPHABETS_SPECIAL
450 ALPHABETS_ALPHANUM = ALPHABETS_BIG + ALPHABETS_SMALL + ALPHABETS_NUM
450 ALPHABETS_ALPHANUM = ALPHABETS_BIG + ALPHABETS_SMALL + ALPHABETS_NUM
451 ALPHABETS_BIG_SMALL = ALPHABETS_BIG + ALPHABETS_SMALL
451 ALPHABETS_BIG_SMALL = ALPHABETS_BIG + ALPHABETS_SMALL
452 ALPHABETS_ALPHANUM_BIG = ALPHABETS_BIG + ALPHABETS_NUM
452 ALPHABETS_ALPHANUM_BIG = ALPHABETS_BIG + ALPHABETS_NUM
453 ALPHABETS_ALPHANUM_SMALL = ALPHABETS_SMALL + ALPHABETS_NUM
453 ALPHABETS_ALPHANUM_SMALL = ALPHABETS_SMALL + ALPHABETS_NUM
454
454
455 def gen_password(self, length, alphabet=ALPHABETS_FULL):
455 def gen_password(self, length, alphabet=ALPHABETS_FULL):
456 assert len(alphabet) <= 256, alphabet
456 assert len(alphabet) <= 256, alphabet
457 l = []
457 l = []
458 while len(l) < length:
458 while len(l) < length:
459 i = ord(os.urandom(1))
459 i = ord(os.urandom(1))
460 if i < len(alphabet):
460 if i < len(alphabet):
461 l.append(alphabet[i])
461 l.append(alphabet[i])
462 return ''.join(l)
462 return ''.join(l)
463
463
464
464
465 def get_crypt_password(password):
465 def get_crypt_password(password):
466 """
466 """
467 Cryptographic function used for bcrypt password hashing.
467 Cryptographic function used for bcrypt password hashing.
468
468
469 :param password: password to hash
469 :param password: password to hash
470 """
470 """
471 return ascii_str(bcrypt.hashpw(safe_bytes(password), bcrypt.gensalt(10)))
471 return ascii_str(bcrypt.hashpw(safe_bytes(password), bcrypt.gensalt(10)))
472
472
473
473
474 def check_password(password, hashed):
474 def check_password(password, hashed):
475 """
475 """
476 Checks password match the hashed value using bcrypt.
476 Checks password match the hashed value using bcrypt.
477 Remains backwards compatible and accept plain sha256 hashes which used to
477 Remains backwards compatible and accept plain sha256 hashes which used to
478 be used on Windows.
478 be used on Windows.
479
479
480 :param password: password
480 :param password: password
481 :param hashed: password in hashed form
481 :param hashed: password in hashed form
482 """
482 """
483 # sha256 hashes will always be 64 hex chars
483 # sha256 hashes will always be 64 hex chars
484 # bcrypt hashes will always contain $ (and be shorter)
484 # bcrypt hashes will always contain $ (and be shorter)
485 if len(hashed) == 64 and all(x in string.hexdigits for x in hashed):
485 if len(hashed) == 64 and all(x in string.hexdigits for x in hashed):
486 return hashlib.sha256(password).hexdigest() == hashed
486 return hashlib.sha256(password).hexdigest() == hashed
487 try:
487 try:
488 return bcrypt.checkpw(safe_bytes(password), ascii_bytes(hashed))
488 return bcrypt.checkpw(safe_bytes(password), ascii_bytes(hashed))
489 except ValueError as e:
489 except ValueError as e:
490 # bcrypt will throw ValueError 'Invalid hashed_password salt' on all password errors
490 # bcrypt will throw ValueError 'Invalid hashed_password salt' on all password errors
491 log.error('error from bcrypt checking password: %s', e)
491 log.error('error from bcrypt checking password: %s', e)
492 return False
492 return False
493 log.error('check_password failed - no method found for hash length %s', len(hashed))
493 log.error('check_password failed - no method found for hash length %s', len(hashed))
494 return False
494 return False
495
495
496
496
497 git_req_ver = StrictVersion('1.7.4')
497 git_req_ver = Version('1.7.4')
498
498
499 def check_git_version():
499 def check_git_version():
500 """
500 """
501 Checks what version of git is installed on the system, and raise a system exit
501 Checks what version of git is installed on the system, and raise a system exit
502 if it's too old for Kallithea to work properly.
502 if it's too old for Kallithea to work properly.
503 """
503 """
504 if 'git' not in kallithea.BACKENDS:
504 if 'git' not in kallithea.BACKENDS:
505 return None
505 return None
506
506
507 if not settings.GIT_EXECUTABLE_PATH:
507 if not settings.GIT_EXECUTABLE_PATH:
508 log.warning('No git executable configured - check "git_path" in the ini file.')
508 log.warning('No git executable configured - check "git_path" in the ini file.')
509 return None
509 return None
510
510
511 try:
511 try:
512 stdout, stderr = GitRepository._run_git_command(['--version'])
512 stdout, stderr = GitRepository._run_git_command(['--version'])
513 except RepositoryError as e:
513 except RepositoryError as e:
514 # message will already have been logged as error
514 # message will already have been logged as error
515 log.warning('No working git executable found - check "git_path" in the ini file.')
515 log.warning('No working git executable found - check "git_path" in the ini file.')
516 return None
516 return None
517
517
518 if stderr:
518 if stderr:
519 log.warning('Error/stderr from "%s --version":\n%s', settings.GIT_EXECUTABLE_PATH, safe_str(stderr))
519 log.warning('Error/stderr from "%s --version":\n%s', settings.GIT_EXECUTABLE_PATH, safe_str(stderr))
520
520
521 if not stdout:
521 if not stdout:
522 log.warning('No working git executable found - check "git_path" in the ini file.')
522 log.warning('No working git executable found - check "git_path" in the ini file.')
523 return None
523 return None
524
524
525 output = safe_str(stdout).strip()
525 output = safe_str(stdout).strip()
526 m = re.search(r"\d+.\d+.\d+", output)
526 m = re.search(r"\d+.\d+.\d+", output)
527 if m:
527 if m:
528 ver = StrictVersion(m.group(0))
528 ver = Version(m.group(0))
529 log.debug('Git executable: "%s", version %s (parsed from: "%s")',
529 log.debug('Git executable: "%s", version %s (parsed from: "%s")',
530 settings.GIT_EXECUTABLE_PATH, ver, output)
530 settings.GIT_EXECUTABLE_PATH, ver, output)
531 if ver < git_req_ver:
531 if ver < git_req_ver:
532 log.error('Kallithea detected %s version %s, which is too old '
532 log.error('Kallithea detected %s version %s, which is too old '
533 'for the system to function properly. '
533 'for the system to function properly. '
534 'Please upgrade to version %s or later. '
534 'Please upgrade to version %s or later. '
535 'If you strictly need Mercurial repositories, you can '
535 'If you strictly need Mercurial repositories, you can '
536 'clear the "git_path" setting in the ini file.',
536 'clear the "git_path" setting in the ini file.',
537 settings.GIT_EXECUTABLE_PATH, ver, git_req_ver)
537 settings.GIT_EXECUTABLE_PATH, ver, git_req_ver)
538 log.error("Terminating ...")
538 log.error("Terminating ...")
539 sys.exit(1)
539 sys.exit(1)
540 else:
540 else:
541 ver = StrictVersion('0.0.0')
541 ver = Version('0.0.0')
542 log.warning('Error finding version number in "%s --version" stdout:\n%s',
542 log.warning('Error finding version number in "%s --version" stdout:\n%s',
543 settings.GIT_EXECUTABLE_PATH, output)
543 settings.GIT_EXECUTABLE_PATH, output)
544
544
545 return ver
545 return ver
@@ -1,293 +1,294 b''
1 #!/usr/bin/env python3
1 #!/usr/bin/env python3
2
2
3
3
4 import re
4 import re
5 import sys
5 import sys
6
6
7
7
8 ignored_modules = set('''
8 ignored_modules = set('''
9 argparse
9 argparse
10 base64
10 base64
11 bcrypt
11 bcrypt
12 binascii
12 binascii
13 bleach
13 bleach
14 calendar
14 calendar
15 celery
15 celery
16 celery
16 celery
17 chardet
17 chardet
18 click
18 click
19 collections
19 collections
20 configparser
20 configparser
21 copy
21 copy
22 csv
22 csv
23 ctypes
23 ctypes
24 datetime
24 datetime
25 dateutil
25 dateutil
26 decimal
26 decimal
27 decorator
27 decorator
28 difflib
28 difflib
29 distutils
29 distutils
30 docutils
30 docutils
31 email
31 email
32 errno
32 errno
33 fileinput
33 fileinput
34 functools
34 functools
35 getpass
35 getpass
36 grp
36 grp
37 hashlib
37 hashlib
38 hmac
38 hmac
39 html
39 html
40 http
40 http
41 imp
41 imp
42 importlib
42 importlib
43 inspect
43 inspect
44 io
44 io
45 ipaddr
45 ipaddr
46 IPython
46 IPython
47 isapi_wsgi
47 isapi_wsgi
48 itertools
48 itertools
49 json
49 json
50 kajiki
50 kajiki
51 ldap
51 ldap
52 logging
52 logging
53 mako
53 mako
54 markdown
54 markdown
55 mimetypes
55 mimetypes
56 mock
56 mock
57 msvcrt
57 msvcrt
58 multiprocessing
58 multiprocessing
59 operator
59 operator
60 os
60 os
61 paginate
61 paginate
62 paginate_sqlalchemy
62 paginate_sqlalchemy
63 pam
63 pam
64 paste
64 paste
65 pkg_resources
65 pkg_resources
66 platform
66 platform
67 posixpath
67 posixpath
68 pprint
68 pprint
69 pwd
69 pwd
70 pyflakes
70 pyflakes
71 pytest
71 pytest
72 pytest_localserver
72 pytest_localserver
73 random
73 random
74 re
74 re
75 routes
75 routes
76 setuptools
76 setuptools
77 shlex
77 shlex
78 shutil
78 shutil
79 smtplib
79 smtplib
80 socket
80 socket
81 ssl
81 ssl
82 stat
82 stat
83 string
83 string
84 struct
84 struct
85 subprocess
85 subprocess
86 sys
86 sys
87 tarfile
87 tarfile
88 tempfile
88 tempfile
89 textwrap
89 textwrap
90 tgext
90 tgext
91 threading
91 threading
92 time
92 time
93 traceback
93 traceback
94 traitlets
94 traitlets
95 types
95 types
96 typing
96 typing
97 urllib
97 urllib
98 urlobject
98 urlobject
99 uuid
99 uuid
100 warnings
100 warnings
101 webhelpers2
101 webhelpers2
102 webob
102 webob
103 webtest
103 webtest
104 whoosh
104 whoosh
105 win32traceutil
105 win32traceutil
106 zipfile
106 zipfile
107 '''.split())
107 '''.split())
108
108
109 top_modules = set('''
109 top_modules = set('''
110 kallithea.alembic
110 kallithea.alembic
111 kallithea.bin
111 kallithea.bin
112 kallithea.config
112 kallithea.config
113 kallithea.controllers
113 kallithea.controllers
114 kallithea.templates.py
114 kallithea.templates.py
115 scripts
115 scripts
116 '''.split())
116 '''.split())
117
117
118 bottom_external_modules = set('''
118 bottom_external_modules = set('''
119 tg
119 tg
120 mercurial
120 mercurial
121 sqlalchemy
121 sqlalchemy
122 alembic
122 alembic
123 formencode
123 formencode
124 pygments
124 pygments
125 dulwich
125 dulwich
126 beaker
126 beaker
127 psycopg2
127 psycopg2
128 docs
128 docs
129 setup
129 setup
130 conftest
130 conftest
131 packaging
131 '''.split())
132 '''.split())
132
133
133 normal_modules = set('''
134 normal_modules = set('''
134 kallithea
135 kallithea
135 kallithea.controllers.base
136 kallithea.controllers.base
136 kallithea.lib
137 kallithea.lib
137 kallithea.lib.auth
138 kallithea.lib.auth
138 kallithea.lib.auth_modules
139 kallithea.lib.auth_modules
139 kallithea.lib.celerylib
140 kallithea.lib.celerylib
140 kallithea.lib.db_manage
141 kallithea.lib.db_manage
141 kallithea.lib.helpers
142 kallithea.lib.helpers
142 kallithea.lib.hooks
143 kallithea.lib.hooks
143 kallithea.lib.indexers
144 kallithea.lib.indexers
144 kallithea.lib.utils
145 kallithea.lib.utils
145 kallithea.lib.utils2
146 kallithea.lib.utils2
146 kallithea.lib.vcs
147 kallithea.lib.vcs
147 kallithea.lib.webutils
148 kallithea.lib.webutils
148 kallithea.model
149 kallithea.model
149 kallithea.model.async_tasks
150 kallithea.model.async_tasks
150 kallithea.model.scm
151 kallithea.model.scm
151 kallithea.templates.py
152 kallithea.templates.py
152 '''.split())
153 '''.split())
153
154
154 shown_modules = normal_modules | top_modules
155 shown_modules = normal_modules | top_modules
155
156
156 # break the chains somehow - this is a cleanup TODO list
157 # break the chains somehow - this is a cleanup TODO list
157 known_violations = set([
158 known_violations = set([
158 ('kallithea.lib.auth_modules', 'kallithea.lib.auth'), # needs base&facade
159 ('kallithea.lib.auth_modules', 'kallithea.lib.auth'), # needs base&facade
159 ('kallithea.lib.utils', 'kallithea.model'), # clean up utils
160 ('kallithea.lib.utils', 'kallithea.model'), # clean up utils
160 ('kallithea.lib.utils', 'kallithea.model.db'),
161 ('kallithea.lib.utils', 'kallithea.model.db'),
161 ('kallithea.lib.utils', 'kallithea.model.scm'),
162 ('kallithea.lib.utils', 'kallithea.model.scm'),
162 ('kallithea.model', 'kallithea.lib.auth'), # auth.HasXXX
163 ('kallithea.model', 'kallithea.lib.auth'), # auth.HasXXX
163 ('kallithea.model', 'kallithea.lib.auth_modules'), # validators
164 ('kallithea.model', 'kallithea.lib.auth_modules'), # validators
164 ('kallithea.model', 'kallithea.lib.hooks'), # clean up hooks
165 ('kallithea.model', 'kallithea.lib.hooks'), # clean up hooks
165 ('kallithea.model', 'kallithea.model.scm'),
166 ('kallithea.model', 'kallithea.model.scm'),
166 ('kallithea.model.scm', 'kallithea.lib.hooks'),
167 ('kallithea.model.scm', 'kallithea.lib.hooks'),
167 ])
168 ])
168
169
169 extra_edges = [
170 extra_edges = [
170 ('kallithea.config', 'kallithea.controllers'), # through TG
171 ('kallithea.config', 'kallithea.controllers'), # through TG
171 ('kallithea.lib.auth', 'kallithea.lib.auth_modules'), # custom loader
172 ('kallithea.lib.auth', 'kallithea.lib.auth_modules'), # custom loader
172 ]
173 ]
173
174
174
175
175 def normalize(s):
176 def normalize(s):
176 """Given a string with dot path, return the string it should be shown as."""
177 """Given a string with dot path, return the string it should be shown as."""
177 parts = s.replace('.__init__', '').split('.')
178 parts = s.replace('.__init__', '').split('.')
178 short_2 = '.'.join(parts[:2])
179 short_2 = '.'.join(parts[:2])
179 short_3 = '.'.join(parts[:3])
180 short_3 = '.'.join(parts[:3])
180 short_4 = '.'.join(parts[:4])
181 short_4 = '.'.join(parts[:4])
181 if parts[0] in ['scripts', 'contributor_data', 'i18n_utils']:
182 if parts[0] in ['scripts', 'contributor_data', 'i18n_utils']:
182 return 'scripts'
183 return 'scripts'
183 if short_3 == 'kallithea.model.meta':
184 if short_3 == 'kallithea.model.meta':
184 return 'kallithea.model.db'
185 return 'kallithea.model.db'
185 if parts[:4] == ['kallithea', 'lib', 'vcs', 'ssh']:
186 if parts[:4] == ['kallithea', 'lib', 'vcs', 'ssh']:
186 return 'kallithea.lib.vcs.ssh'
187 return 'kallithea.lib.vcs.ssh'
187 if short_4 in shown_modules:
188 if short_4 in shown_modules:
188 return short_4
189 return short_4
189 if short_3 in shown_modules:
190 if short_3 in shown_modules:
190 return short_3
191 return short_3
191 if short_2 in shown_modules:
192 if short_2 in shown_modules:
192 return short_2
193 return short_2
193 if short_2 == 'kallithea.tests':
194 if short_2 == 'kallithea.tests':
194 return None
195 return None
195 if parts[0] in ignored_modules:
196 if parts[0] in ignored_modules:
196 return None
197 return None
197 assert parts[0] in bottom_external_modules, parts
198 assert parts[0] in bottom_external_modules, parts
198 return parts[0]
199 return parts[0]
199
200
200
201
201 def main(filenames):
202 def main(filenames):
202 if not filenames or filenames[0].startswith('-'):
203 if not filenames or filenames[0].startswith('-'):
203 print('''\
204 print('''\
204 Usage:
205 Usage:
205 hg files 'set:!binary()&grep("^#!.*python")' 'set:**.py' | xargs scripts/deps.py
206 hg files 'set:!binary()&grep("^#!.*python")' 'set:**.py' | xargs scripts/deps.py
206 dot -Tsvg deps.dot > deps.svg
207 dot -Tsvg deps.dot > deps.svg
207 ''')
208 ''')
208 raise SystemExit(1)
209 raise SystemExit(1)
209
210
210 files_imports = dict() # map filenames to its imports
211 files_imports = dict() # map filenames to its imports
211 import_deps = set() # set of tuples with module name and its imports
212 import_deps = set() # set of tuples with module name and its imports
212 for fn in filenames:
213 for fn in filenames:
213 with open(fn) as f:
214 with open(fn) as f:
214 s = f.read()
215 s = f.read()
215
216
216 dot_name = (fn[:-3] if fn.endswith('.py') else fn).replace('/', '.')
217 dot_name = (fn[:-3] if fn.endswith('.py') else fn).replace('/', '.')
217 file_imports = set()
218 file_imports = set()
218 for m in re.finditer(r'^ *(?:from ([^ ]*) import (?:([a-zA-Z].*)|\(([^)]*)\))|import (.*))$', s, re.MULTILINE):
219 for m in re.finditer(r'^ *(?:from ([^ ]*) import (?:([a-zA-Z].*)|\(([^)]*)\))|import (.*))$', s, re.MULTILINE):
219 m_from, m_from_import, m_from_import2, m_import = m.groups()
220 m_from, m_from_import, m_from_import2, m_import = m.groups()
220 if m_from:
221 if m_from:
221 pre = m_from + '.'
222 pre = m_from + '.'
222 if pre.startswith('.'):
223 if pre.startswith('.'):
223 pre = dot_name.rsplit('.', 1)[0] + pre
224 pre = dot_name.rsplit('.', 1)[0] + pre
224 importlist = m_from_import or m_from_import2
225 importlist = m_from_import or m_from_import2
225 else:
226 else:
226 pre = ''
227 pre = ''
227 importlist = m_import
228 importlist = m_import
228 for imp in importlist.split('#', 1)[0].split(','):
229 for imp in importlist.split('#', 1)[0].split(','):
229 full_imp = pre + imp.strip().split(' as ', 1)[0]
230 full_imp = pre + imp.strip().split(' as ', 1)[0]
230 file_imports.add(full_imp)
231 file_imports.add(full_imp)
231 import_deps.add((dot_name, full_imp))
232 import_deps.add((dot_name, full_imp))
232 files_imports[fn] = file_imports
233 files_imports[fn] = file_imports
233
234
234 # dump out all deps for debugging and analysis
235 # dump out all deps for debugging and analysis
235 with open('deps.txt', 'w') as f:
236 with open('deps.txt', 'w') as f:
236 for fn, file_imports in sorted(files_imports.items()):
237 for fn, file_imports in sorted(files_imports.items()):
237 for file_import in sorted(file_imports):
238 for file_import in sorted(file_imports):
238 if file_import.split('.', 1)[0] in ignored_modules:
239 if file_import.split('.', 1)[0] in ignored_modules:
239 continue
240 continue
240 f.write('%s: %s\n' % (fn, file_import))
241 f.write('%s: %s\n' % (fn, file_import))
241
242
242 # find leafs that haven't been ignored - they are the important external dependencies and shown in the bottom row
243 # find leafs that haven't been ignored - they are the important external dependencies and shown in the bottom row
243 only_imported = set(
244 only_imported = set(
244 set(normalize(b) for a, b in import_deps) -
245 set(normalize(b) for a, b in import_deps) -
245 set(normalize(a) for a, b in import_deps) -
246 set(normalize(a) for a, b in import_deps) -
246 set([None, 'kallithea'])
247 set([None, 'kallithea'])
247 )
248 )
248
249
249 normalized_dep_edges = set()
250 normalized_dep_edges = set()
250 for dot_name, full_imp in import_deps:
251 for dot_name, full_imp in import_deps:
251 a = normalize(dot_name)
252 a = normalize(dot_name)
252 b = normalize(full_imp)
253 b = normalize(full_imp)
253 if a is None or b is None or a == b:
254 if a is None or b is None or a == b:
254 continue
255 continue
255 normalized_dep_edges.add((a, b))
256 normalized_dep_edges.add((a, b))
256 #print((dot_name, full_imp, a, b))
257 #print((dot_name, full_imp, a, b))
257 normalized_dep_edges.update(extra_edges)
258 normalized_dep_edges.update(extra_edges)
258
259
259 unseen_shown_modules = shown_modules.difference(a for a, b in normalized_dep_edges).difference(b for a, b in normalized_dep_edges)
260 unseen_shown_modules = shown_modules.difference(a for a, b in normalized_dep_edges).difference(b for a, b in normalized_dep_edges)
260 assert not unseen_shown_modules, unseen_shown_modules
261 assert not unseen_shown_modules, unseen_shown_modules
261
262
262 with open('deps.dot', 'w') as f:
263 with open('deps.dot', 'w') as f:
263 f.write('digraph {\n')
264 f.write('digraph {\n')
264 f.write('subgraph { rank = same; %s}\n' % ''.join('"%s"; ' % s for s in sorted(top_modules)))
265 f.write('subgraph { rank = same; %s}\n' % ''.join('"%s"; ' % s for s in sorted(top_modules)))
265 f.write('subgraph { rank = same; %s}\n' % ''.join('"%s"; ' % s for s in sorted(only_imported)))
266 f.write('subgraph { rank = same; %s}\n' % ''.join('"%s"; ' % s for s in sorted(only_imported)))
266 for a, b in sorted(normalized_dep_edges):
267 for a, b in sorted(normalized_dep_edges):
267 f.write(' "%s" -> "%s"%s\n' % (a, b, ' [color=red]' if (a, b) in known_violations else ' [color=green]' if (a, b) in extra_edges else ''))
268 f.write(' "%s" -> "%s"%s\n' % (a, b, ' [color=red]' if (a, b) in known_violations else ' [color=green]' if (a, b) in extra_edges else ''))
268 f.write('}\n')
269 f.write('}\n')
269
270
270 # verify dependencies by untangling dependency chain bottom-up:
271 # verify dependencies by untangling dependency chain bottom-up:
271 todo = set(normalized_dep_edges)
272 todo = set(normalized_dep_edges)
272 unseen_violations = known_violations.difference(todo)
273 unseen_violations = known_violations.difference(todo)
273 assert not unseen_violations, unseen_violations
274 assert not unseen_violations, unseen_violations
274 for x in known_violations:
275 for x in known_violations:
275 todo.remove(x)
276 todo.remove(x)
276
277
277 while todo:
278 while todo:
278 depending = set(a for a, b in todo)
279 depending = set(a for a, b in todo)
279 depended = set(b for a, b in todo)
280 depended = set(b for a, b in todo)
280 drop = depended - depending
281 drop = depended - depending
281 if not drop:
282 if not drop:
282 print('ERROR: cycles:', len(todo))
283 print('ERROR: cycles:', len(todo))
283 for x in sorted(todo):
284 for x in sorted(todo):
284 print('%s,' % (x,))
285 print('%s,' % (x,))
285 raise SystemExit(1)
286 raise SystemExit(1)
286 #for do_b in sorted(drop):
287 #for do_b in sorted(drop):
287 # print('Picking', do_b, '- unblocks:', ' '.join(a for a, b in sorted((todo)) if b == do_b))
288 # print('Picking', do_b, '- unblocks:', ' '.join(a for a, b in sorted((todo)) if b == do_b))
288 todo = set((a, b) for a, b in todo if b in depending)
289 todo = set((a, b) for a, b in todo if b in depending)
289 #print()
290 #print()
290
291
291
292
292 if __name__ == '__main__':
293 if __name__ == '__main__':
293 main(sys.argv[1:])
294 main(sys.argv[1:])
General Comments 0
You need to be logged in to leave comments. Login now