##// END OF EJS Templates
helpers: unicode/str add flag to control usage of chardet....
marcink -
r4109:19a6ab7e default
parent child Browse files
Show More
@@ -1,1090 +1,1095 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2011-2019 RhodeCode GmbH
3 # Copyright (C) 2011-2019 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21
21
22 """
22 """
23 Some simple helper functions
23 Some simple helper functions
24 """
24 """
25
25
26 import collections
26 import collections
27 import datetime
27 import datetime
28 import dateutil.relativedelta
28 import dateutil.relativedelta
29 import hashlib
29 import hashlib
30 import logging
30 import logging
31 import re
31 import re
32 import sys
32 import sys
33 import time
33 import time
34 import urllib
34 import urllib
35 import urlobject
35 import urlobject
36 import uuid
36 import uuid
37 import getpass
37 import getpass
38 from functools import update_wrapper, partial
38 from functools import update_wrapper, partial
39
39
40 import pygments.lexers
40 import pygments.lexers
41 import sqlalchemy
41 import sqlalchemy
42 import sqlalchemy.engine.url
42 import sqlalchemy.engine.url
43 import sqlalchemy.exc
43 import sqlalchemy.exc
44 import sqlalchemy.sql
44 import sqlalchemy.sql
45 import webob
45 import webob
46 import pyramid.threadlocal
46 import pyramid.threadlocal
47 from pyramid import compat
47 from pyramid import compat
48 from pyramid.settings import asbool
48 from pyramid.settings import asbool
49
49
50 import rhodecode
50 import rhodecode
51 from rhodecode.translation import _, _pluralize
51 from rhodecode.translation import _, _pluralize
52
52
53
53
54 def md5(s):
54 def md5(s):
55 return hashlib.md5(s).hexdigest()
55 return hashlib.md5(s).hexdigest()
56
56
57
57
58 def md5_safe(s):
58 def md5_safe(s):
59 return md5(safe_str(s))
59 return md5(safe_str(s))
60
60
61
61
62 def sha1(s):
62 def sha1(s):
63 return hashlib.sha1(s).hexdigest()
63 return hashlib.sha1(s).hexdigest()
64
64
65
65
66 def sha1_safe(s):
66 def sha1_safe(s):
67 return sha1(safe_str(s))
67 return sha1(safe_str(s))
68
68
69
69
70 def __get_lem(extra_mapping=None):
70 def __get_lem(extra_mapping=None):
71 """
71 """
72 Get language extension map based on what's inside pygments lexers
72 Get language extension map based on what's inside pygments lexers
73 """
73 """
74 d = collections.defaultdict(lambda: [])
74 d = collections.defaultdict(lambda: [])
75
75
76 def __clean(s):
76 def __clean(s):
77 s = s.lstrip('*')
77 s = s.lstrip('*')
78 s = s.lstrip('.')
78 s = s.lstrip('.')
79
79
80 if s.find('[') != -1:
80 if s.find('[') != -1:
81 exts = []
81 exts = []
82 start, stop = s.find('['), s.find(']')
82 start, stop = s.find('['), s.find(']')
83
83
84 for suffix in s[start + 1:stop]:
84 for suffix in s[start + 1:stop]:
85 exts.append(s[:s.find('[')] + suffix)
85 exts.append(s[:s.find('[')] + suffix)
86 return [e.lower() for e in exts]
86 return [e.lower() for e in exts]
87 else:
87 else:
88 return [s.lower()]
88 return [s.lower()]
89
89
90 for lx, t in sorted(pygments.lexers.LEXERS.items()):
90 for lx, t in sorted(pygments.lexers.LEXERS.items()):
91 m = map(__clean, t[-2])
91 m = map(__clean, t[-2])
92 if m:
92 if m:
93 m = reduce(lambda x, y: x + y, m)
93 m = reduce(lambda x, y: x + y, m)
94 for ext in m:
94 for ext in m:
95 desc = lx.replace('Lexer', '')
95 desc = lx.replace('Lexer', '')
96 d[ext].append(desc)
96 d[ext].append(desc)
97
97
98 data = dict(d)
98 data = dict(d)
99
99
100 extra_mapping = extra_mapping or {}
100 extra_mapping = extra_mapping or {}
101 if extra_mapping:
101 if extra_mapping:
102 for k, v in extra_mapping.items():
102 for k, v in extra_mapping.items():
103 if k not in data:
103 if k not in data:
104 # register new mapping2lexer
104 # register new mapping2lexer
105 data[k] = [v]
105 data[k] = [v]
106
106
107 return data
107 return data
108
108
109
109
110 def str2bool(_str):
110 def str2bool(_str):
111 """
111 """
112 returns True/False value from given string, it tries to translate the
112 returns True/False value from given string, it tries to translate the
113 string into boolean
113 string into boolean
114
114
115 :param _str: string value to translate into boolean
115 :param _str: string value to translate into boolean
116 :rtype: boolean
116 :rtype: boolean
117 :returns: boolean from given string
117 :returns: boolean from given string
118 """
118 """
119 if _str is None:
119 if _str is None:
120 return False
120 return False
121 if _str in (True, False):
121 if _str in (True, False):
122 return _str
122 return _str
123 _str = str(_str).strip().lower()
123 _str = str(_str).strip().lower()
124 return _str in ('t', 'true', 'y', 'yes', 'on', '1')
124 return _str in ('t', 'true', 'y', 'yes', 'on', '1')
125
125
126
126
127 def aslist(obj, sep=None, strip=True):
127 def aslist(obj, sep=None, strip=True):
128 """
128 """
129 Returns given string separated by sep as list
129 Returns given string separated by sep as list
130
130
131 :param obj:
131 :param obj:
132 :param sep:
132 :param sep:
133 :param strip:
133 :param strip:
134 """
134 """
135 if isinstance(obj, (basestring,)):
135 if isinstance(obj, (basestring,)):
136 lst = obj.split(sep)
136 lst = obj.split(sep)
137 if strip:
137 if strip:
138 lst = [v.strip() for v in lst]
138 lst = [v.strip() for v in lst]
139 return lst
139 return lst
140 elif isinstance(obj, (list, tuple)):
140 elif isinstance(obj, (list, tuple)):
141 return obj
141 return obj
142 elif obj is None:
142 elif obj is None:
143 return []
143 return []
144 else:
144 else:
145 return [obj]
145 return [obj]
146
146
147
147
148 def convert_line_endings(line, mode):
148 def convert_line_endings(line, mode):
149 """
149 """
150 Converts a given line "line end" accordingly to given mode
150 Converts a given line "line end" accordingly to given mode
151
151
152 Available modes are::
152 Available modes are::
153 0 - Unix
153 0 - Unix
154 1 - Mac
154 1 - Mac
155 2 - DOS
155 2 - DOS
156
156
157 :param line: given line to convert
157 :param line: given line to convert
158 :param mode: mode to convert to
158 :param mode: mode to convert to
159 :rtype: str
159 :rtype: str
160 :return: converted line according to mode
160 :return: converted line according to mode
161 """
161 """
162 if mode == 0:
162 if mode == 0:
163 line = line.replace('\r\n', '\n')
163 line = line.replace('\r\n', '\n')
164 line = line.replace('\r', '\n')
164 line = line.replace('\r', '\n')
165 elif mode == 1:
165 elif mode == 1:
166 line = line.replace('\r\n', '\r')
166 line = line.replace('\r\n', '\r')
167 line = line.replace('\n', '\r')
167 line = line.replace('\n', '\r')
168 elif mode == 2:
168 elif mode == 2:
169 line = re.sub('\r(?!\n)|(?<!\r)\n', '\r\n', line)
169 line = re.sub('\r(?!\n)|(?<!\r)\n', '\r\n', line)
170 return line
170 return line
171
171
172
172
173 def detect_mode(line, default):
173 def detect_mode(line, default):
174 """
174 """
175 Detects line break for given line, if line break couldn't be found
175 Detects line break for given line, if line break couldn't be found
176 given default value is returned
176 given default value is returned
177
177
178 :param line: str line
178 :param line: str line
179 :param default: default
179 :param default: default
180 :rtype: int
180 :rtype: int
181 :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
181 :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS
182 """
182 """
183 if line.endswith('\r\n'):
183 if line.endswith('\r\n'):
184 return 2
184 return 2
185 elif line.endswith('\n'):
185 elif line.endswith('\n'):
186 return 0
186 return 0
187 elif line.endswith('\r'):
187 elif line.endswith('\r'):
188 return 1
188 return 1
189 else:
189 else:
190 return default
190 return default
191
191
192
192
193 def safe_int(val, default=None):
193 def safe_int(val, default=None):
194 """
194 """
195 Returns int() of val if val is not convertable to int use default
195 Returns int() of val if val is not convertable to int use default
196 instead
196 instead
197
197
198 :param val:
198 :param val:
199 :param default:
199 :param default:
200 """
200 """
201
201
202 try:
202 try:
203 val = int(val)
203 val = int(val)
204 except (ValueError, TypeError):
204 except (ValueError, TypeError):
205 val = default
205 val = default
206
206
207 return val
207 return val
208
208
209
209
210 def safe_unicode(str_, from_encoding=None):
210 def safe_unicode(str_, from_encoding=None, use_chardet=False):
211 """
211 """
212 safe unicode function. Does few trick to turn str_ into unicode
212 safe unicode function. Does few trick to turn str_ into unicode
213
213
214 In case of UnicodeDecode error, we try to return it with encoding detected
214 In case of UnicodeDecode error, we try to return it with encoding detected
215 by chardet library if it fails fallback to unicode with errors replaced
215 by chardet library if it fails fallback to unicode with errors replaced
216
216
217 :param str_: string to decode
217 :param str_: string to decode
218 :rtype: unicode
218 :rtype: unicode
219 :returns: unicode object
219 :returns: unicode object
220 """
220 """
221 if isinstance(str_, unicode):
221 if isinstance(str_, unicode):
222 return str_
222 return str_
223
223
224 if not from_encoding:
224 if not from_encoding:
225 DEFAULT_ENCODINGS = aslist(rhodecode.CONFIG.get('default_encoding',
225 DEFAULT_ENCODINGS = aslist(rhodecode.CONFIG.get('default_encoding',
226 'utf8'), sep=',')
226 'utf8'), sep=',')
227 from_encoding = DEFAULT_ENCODINGS
227 from_encoding = DEFAULT_ENCODINGS
228
228
229 if not isinstance(from_encoding, (list, tuple)):
229 if not isinstance(from_encoding, (list, tuple)):
230 from_encoding = [from_encoding]
230 from_encoding = [from_encoding]
231
231
232 try:
232 try:
233 return unicode(str_)
233 return unicode(str_)
234 except UnicodeDecodeError:
234 except UnicodeDecodeError:
235 pass
235 pass
236
236
237 for enc in from_encoding:
237 for enc in from_encoding:
238 try:
238 try:
239 return unicode(str_, enc)
239 return unicode(str_, enc)
240 except UnicodeDecodeError:
240 except UnicodeDecodeError:
241 pass
241 pass
242
242
243 if use_chardet:
243 try:
244 try:
244 import chardet
245 import chardet
245 encoding = chardet.detect(str_)['encoding']
246 encoding = chardet.detect(str_)['encoding']
246 if encoding is None:
247 if encoding is None:
247 raise Exception()
248 raise Exception()
248 return str_.decode(encoding)
249 return str_.decode(encoding)
249 except (ImportError, UnicodeDecodeError, Exception):
250 except (ImportError, UnicodeDecodeError, Exception):
250 return unicode(str_, from_encoding[0], 'replace')
251 return unicode(str_, from_encoding[0], 'replace')
251
252 else:
253 return unicode(str_, from_encoding[0], 'replace')
252
254
253 def safe_str(unicode_, to_encoding=None):
255 def safe_str(unicode_, to_encoding=None, use_chardet=False):
254 """
256 """
255 safe str function. Does few trick to turn unicode_ into string
257 safe str function. Does few trick to turn unicode_ into string
256
258
257 In case of UnicodeEncodeError, we try to return it with encoding detected
259 In case of UnicodeEncodeError, we try to return it with encoding detected
258 by chardet library if it fails fallback to string with errors replaced
260 by chardet library if it fails fallback to string with errors replaced
259
261
260 :param unicode_: unicode to encode
262 :param unicode_: unicode to encode
261 :rtype: str
263 :rtype: str
262 :returns: str object
264 :returns: str object
263 """
265 """
264
266
265 # if it's not basestr cast to str
267 # if it's not basestr cast to str
266 if not isinstance(unicode_, compat.string_types):
268 if not isinstance(unicode_, compat.string_types):
267 return str(unicode_)
269 return str(unicode_)
268
270
269 if isinstance(unicode_, str):
271 if isinstance(unicode_, str):
270 return unicode_
272 return unicode_
271
273
272 if not to_encoding:
274 if not to_encoding:
273 DEFAULT_ENCODINGS = aslist(rhodecode.CONFIG.get('default_encoding',
275 DEFAULT_ENCODINGS = aslist(rhodecode.CONFIG.get('default_encoding',
274 'utf8'), sep=',')
276 'utf8'), sep=',')
275 to_encoding = DEFAULT_ENCODINGS
277 to_encoding = DEFAULT_ENCODINGS
276
278
277 if not isinstance(to_encoding, (list, tuple)):
279 if not isinstance(to_encoding, (list, tuple)):
278 to_encoding = [to_encoding]
280 to_encoding = [to_encoding]
279
281
280 for enc in to_encoding:
282 for enc in to_encoding:
281 try:
283 try:
282 return unicode_.encode(enc)
284 return unicode_.encode(enc)
283 except UnicodeEncodeError:
285 except UnicodeEncodeError:
284 pass
286 pass
285
287
288 if use_chardet:
286 try:
289 try:
287 import chardet
290 import chardet
288 encoding = chardet.detect(unicode_)['encoding']
291 encoding = chardet.detect(unicode_)['encoding']
289 if encoding is None:
292 if encoding is None:
290 raise UnicodeEncodeError()
293 raise UnicodeEncodeError()
291
294
292 return unicode_.encode(encoding)
295 return unicode_.encode(encoding)
293 except (ImportError, UnicodeEncodeError):
296 except (ImportError, UnicodeEncodeError):
294 return unicode_.encode(to_encoding[0], 'replace')
297 return unicode_.encode(to_encoding[0], 'replace')
298 else:
299 return unicode_.encode(to_encoding[0], 'replace')
295
300
296
301
297 def remove_suffix(s, suffix):
302 def remove_suffix(s, suffix):
298 if s.endswith(suffix):
303 if s.endswith(suffix):
299 s = s[:-1 * len(suffix)]
304 s = s[:-1 * len(suffix)]
300 return s
305 return s
301
306
302
307
303 def remove_prefix(s, prefix):
308 def remove_prefix(s, prefix):
304 if s.startswith(prefix):
309 if s.startswith(prefix):
305 s = s[len(prefix):]
310 s = s[len(prefix):]
306 return s
311 return s
307
312
308
313
309 def find_calling_context(ignore_modules=None):
314 def find_calling_context(ignore_modules=None):
310 """
315 """
311 Look through the calling stack and return the frame which called
316 Look through the calling stack and return the frame which called
312 this function and is part of core module ( ie. rhodecode.* )
317 this function and is part of core module ( ie. rhodecode.* )
313
318
314 :param ignore_modules: list of modules to ignore eg. ['rhodecode.lib']
319 :param ignore_modules: list of modules to ignore eg. ['rhodecode.lib']
315 """
320 """
316
321
317 ignore_modules = ignore_modules or []
322 ignore_modules = ignore_modules or []
318
323
319 f = sys._getframe(2)
324 f = sys._getframe(2)
320 while f.f_back is not None:
325 while f.f_back is not None:
321 name = f.f_globals.get('__name__')
326 name = f.f_globals.get('__name__')
322 if name and name.startswith(__name__.split('.')[0]):
327 if name and name.startswith(__name__.split('.')[0]):
323 if name not in ignore_modules:
328 if name not in ignore_modules:
324 return f
329 return f
325 f = f.f_back
330 f = f.f_back
326 return None
331 return None
327
332
328
333
329 def ping_connection(connection, branch):
334 def ping_connection(connection, branch):
330 if branch:
335 if branch:
331 # "branch" refers to a sub-connection of a connection,
336 # "branch" refers to a sub-connection of a connection,
332 # we don't want to bother pinging on these.
337 # we don't want to bother pinging on these.
333 return
338 return
334
339
335 # turn off "close with result". This flag is only used with
340 # turn off "close with result". This flag is only used with
336 # "connectionless" execution, otherwise will be False in any case
341 # "connectionless" execution, otherwise will be False in any case
337 save_should_close_with_result = connection.should_close_with_result
342 save_should_close_with_result = connection.should_close_with_result
338 connection.should_close_with_result = False
343 connection.should_close_with_result = False
339
344
340 try:
345 try:
341 # run a SELECT 1. use a core select() so that
346 # run a SELECT 1. use a core select() so that
342 # the SELECT of a scalar value without a table is
347 # the SELECT of a scalar value without a table is
343 # appropriately formatted for the backend
348 # appropriately formatted for the backend
344 connection.scalar(sqlalchemy.sql.select([1]))
349 connection.scalar(sqlalchemy.sql.select([1]))
345 except sqlalchemy.exc.DBAPIError as err:
350 except sqlalchemy.exc.DBAPIError as err:
346 # catch SQLAlchemy's DBAPIError, which is a wrapper
351 # catch SQLAlchemy's DBAPIError, which is a wrapper
347 # for the DBAPI's exception. It includes a .connection_invalidated
352 # for the DBAPI's exception. It includes a .connection_invalidated
348 # attribute which specifies if this connection is a "disconnect"
353 # attribute which specifies if this connection is a "disconnect"
349 # condition, which is based on inspection of the original exception
354 # condition, which is based on inspection of the original exception
350 # by the dialect in use.
355 # by the dialect in use.
351 if err.connection_invalidated:
356 if err.connection_invalidated:
352 # run the same SELECT again - the connection will re-validate
357 # run the same SELECT again - the connection will re-validate
353 # itself and establish a new connection. The disconnect detection
358 # itself and establish a new connection. The disconnect detection
354 # here also causes the whole connection pool to be invalidated
359 # here also causes the whole connection pool to be invalidated
355 # so that all stale connections are discarded.
360 # so that all stale connections are discarded.
356 connection.scalar(sqlalchemy.sql.select([1]))
361 connection.scalar(sqlalchemy.sql.select([1]))
357 else:
362 else:
358 raise
363 raise
359 finally:
364 finally:
360 # restore "close with result"
365 # restore "close with result"
361 connection.should_close_with_result = save_should_close_with_result
366 connection.should_close_with_result = save_should_close_with_result
362
367
363
368
364 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
369 def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs):
365 """Custom engine_from_config functions."""
370 """Custom engine_from_config functions."""
366 log = logging.getLogger('sqlalchemy.engine')
371 log = logging.getLogger('sqlalchemy.engine')
367 use_ping_connection = asbool(configuration.pop('sqlalchemy.db1.ping_connection', None))
372 use_ping_connection = asbool(configuration.pop('sqlalchemy.db1.ping_connection', None))
368 debug = asbool(configuration.get('debug'))
373 debug = asbool(configuration.get('debug'))
369
374
370 engine = sqlalchemy.engine_from_config(configuration, prefix, **kwargs)
375 engine = sqlalchemy.engine_from_config(configuration, prefix, **kwargs)
371
376
372 def color_sql(sql):
377 def color_sql(sql):
373 color_seq = '\033[1;33m' # This is yellow: code 33
378 color_seq = '\033[1;33m' # This is yellow: code 33
374 normal = '\x1b[0m'
379 normal = '\x1b[0m'
375 return ''.join([color_seq, sql, normal])
380 return ''.join([color_seq, sql, normal])
376
381
377 if use_ping_connection:
382 if use_ping_connection:
378 log.debug('Adding ping_connection on the engine config.')
383 log.debug('Adding ping_connection on the engine config.')
379 sqlalchemy.event.listen(engine, "engine_connect", ping_connection)
384 sqlalchemy.event.listen(engine, "engine_connect", ping_connection)
380
385
381 if debug:
386 if debug:
382 # attach events only for debug configuration
387 # attach events only for debug configuration
383 def before_cursor_execute(conn, cursor, statement,
388 def before_cursor_execute(conn, cursor, statement,
384 parameters, context, executemany):
389 parameters, context, executemany):
385 setattr(conn, 'query_start_time', time.time())
390 setattr(conn, 'query_start_time', time.time())
386 log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
391 log.info(color_sql(">>>>> STARTING QUERY >>>>>"))
387 calling_context = find_calling_context(ignore_modules=[
392 calling_context = find_calling_context(ignore_modules=[
388 'rhodecode.lib.caching_query',
393 'rhodecode.lib.caching_query',
389 'rhodecode.model.settings',
394 'rhodecode.model.settings',
390 ])
395 ])
391 if calling_context:
396 if calling_context:
392 log.info(color_sql('call context %s:%s' % (
397 log.info(color_sql('call context %s:%s' % (
393 calling_context.f_code.co_filename,
398 calling_context.f_code.co_filename,
394 calling_context.f_lineno,
399 calling_context.f_lineno,
395 )))
400 )))
396
401
397 def after_cursor_execute(conn, cursor, statement,
402 def after_cursor_execute(conn, cursor, statement,
398 parameters, context, executemany):
403 parameters, context, executemany):
399 delattr(conn, 'query_start_time')
404 delattr(conn, 'query_start_time')
400
405
401 sqlalchemy.event.listen(engine, "before_cursor_execute", before_cursor_execute)
406 sqlalchemy.event.listen(engine, "before_cursor_execute", before_cursor_execute)
402 sqlalchemy.event.listen(engine, "after_cursor_execute", after_cursor_execute)
407 sqlalchemy.event.listen(engine, "after_cursor_execute", after_cursor_execute)
403
408
404 return engine
409 return engine
405
410
406
411
407 def get_encryption_key(config):
412 def get_encryption_key(config):
408 secret = config.get('rhodecode.encrypted_values.secret')
413 secret = config.get('rhodecode.encrypted_values.secret')
409 default = config['beaker.session.secret']
414 default = config['beaker.session.secret']
410 return secret or default
415 return secret or default
411
416
412
417
413 def age(prevdate, now=None, show_short_version=False, show_suffix=True,
418 def age(prevdate, now=None, show_short_version=False, show_suffix=True,
414 short_format=False):
419 short_format=False):
415 """
420 """
416 Turns a datetime into an age string.
421 Turns a datetime into an age string.
417 If show_short_version is True, this generates a shorter string with
422 If show_short_version is True, this generates a shorter string with
418 an approximate age; ex. '1 day ago', rather than '1 day and 23 hours ago'.
423 an approximate age; ex. '1 day ago', rather than '1 day and 23 hours ago'.
419
424
420 * IMPORTANT*
425 * IMPORTANT*
421 Code of this function is written in special way so it's easier to
426 Code of this function is written in special way so it's easier to
422 backport it to javascript. If you mean to update it, please also update
427 backport it to javascript. If you mean to update it, please also update
423 `jquery.timeago-extension.js` file
428 `jquery.timeago-extension.js` file
424
429
425 :param prevdate: datetime object
430 :param prevdate: datetime object
426 :param now: get current time, if not define we use
431 :param now: get current time, if not define we use
427 `datetime.datetime.now()`
432 `datetime.datetime.now()`
428 :param show_short_version: if it should approximate the date and
433 :param show_short_version: if it should approximate the date and
429 return a shorter string
434 return a shorter string
430 :param show_suffix:
435 :param show_suffix:
431 :param short_format: show short format, eg 2D instead of 2 days
436 :param short_format: show short format, eg 2D instead of 2 days
432 :rtype: unicode
437 :rtype: unicode
433 :returns: unicode words describing age
438 :returns: unicode words describing age
434 """
439 """
435
440
436 def _get_relative_delta(now, prevdate):
441 def _get_relative_delta(now, prevdate):
437 base = dateutil.relativedelta.relativedelta(now, prevdate)
442 base = dateutil.relativedelta.relativedelta(now, prevdate)
438 return {
443 return {
439 'year': base.years,
444 'year': base.years,
440 'month': base.months,
445 'month': base.months,
441 'day': base.days,
446 'day': base.days,
442 'hour': base.hours,
447 'hour': base.hours,
443 'minute': base.minutes,
448 'minute': base.minutes,
444 'second': base.seconds,
449 'second': base.seconds,
445 }
450 }
446
451
447 def _is_leap_year(year):
452 def _is_leap_year(year):
448 return year % 4 == 0 and (year % 100 != 0 or year % 400 == 0)
453 return year % 4 == 0 and (year % 100 != 0 or year % 400 == 0)
449
454
450 def get_month(prevdate):
455 def get_month(prevdate):
451 return prevdate.month
456 return prevdate.month
452
457
453 def get_year(prevdate):
458 def get_year(prevdate):
454 return prevdate.year
459 return prevdate.year
455
460
456 now = now or datetime.datetime.now()
461 now = now or datetime.datetime.now()
457 order = ['year', 'month', 'day', 'hour', 'minute', 'second']
462 order = ['year', 'month', 'day', 'hour', 'minute', 'second']
458 deltas = {}
463 deltas = {}
459 future = False
464 future = False
460
465
461 if prevdate > now:
466 if prevdate > now:
462 now_old = now
467 now_old = now
463 now = prevdate
468 now = prevdate
464 prevdate = now_old
469 prevdate = now_old
465 future = True
470 future = True
466 if future:
471 if future:
467 prevdate = prevdate.replace(microsecond=0)
472 prevdate = prevdate.replace(microsecond=0)
468 # Get date parts deltas
473 # Get date parts deltas
469 for part in order:
474 for part in order:
470 rel_delta = _get_relative_delta(now, prevdate)
475 rel_delta = _get_relative_delta(now, prevdate)
471 deltas[part] = rel_delta[part]
476 deltas[part] = rel_delta[part]
472
477
473 # Fix negative offsets (there is 1 second between 10:59:59 and 11:00:00,
478 # Fix negative offsets (there is 1 second between 10:59:59 and 11:00:00,
474 # not 1 hour, -59 minutes and -59 seconds)
479 # not 1 hour, -59 minutes and -59 seconds)
475 offsets = [[5, 60], [4, 60], [3, 24]]
480 offsets = [[5, 60], [4, 60], [3, 24]]
476 for element in offsets: # seconds, minutes, hours
481 for element in offsets: # seconds, minutes, hours
477 num = element[0]
482 num = element[0]
478 length = element[1]
483 length = element[1]
479
484
480 part = order[num]
485 part = order[num]
481 carry_part = order[num - 1]
486 carry_part = order[num - 1]
482
487
483 if deltas[part] < 0:
488 if deltas[part] < 0:
484 deltas[part] += length
489 deltas[part] += length
485 deltas[carry_part] -= 1
490 deltas[carry_part] -= 1
486
491
487 # Same thing for days except that the increment depends on the (variable)
492 # Same thing for days except that the increment depends on the (variable)
488 # number of days in the month
493 # number of days in the month
489 month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
494 month_lengths = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
490 if deltas['day'] < 0:
495 if deltas['day'] < 0:
491 if get_month(prevdate) == 2 and _is_leap_year(get_year(prevdate)):
496 if get_month(prevdate) == 2 and _is_leap_year(get_year(prevdate)):
492 deltas['day'] += 29
497 deltas['day'] += 29
493 else:
498 else:
494 deltas['day'] += month_lengths[get_month(prevdate) - 1]
499 deltas['day'] += month_lengths[get_month(prevdate) - 1]
495
500
496 deltas['month'] -= 1
501 deltas['month'] -= 1
497
502
498 if deltas['month'] < 0:
503 if deltas['month'] < 0:
499 deltas['month'] += 12
504 deltas['month'] += 12
500 deltas['year'] -= 1
505 deltas['year'] -= 1
501
506
502 # Format the result
507 # Format the result
503 if short_format:
508 if short_format:
504 fmt_funcs = {
509 fmt_funcs = {
505 'year': lambda d: u'%dy' % d,
510 'year': lambda d: u'%dy' % d,
506 'month': lambda d: u'%dm' % d,
511 'month': lambda d: u'%dm' % d,
507 'day': lambda d: u'%dd' % d,
512 'day': lambda d: u'%dd' % d,
508 'hour': lambda d: u'%dh' % d,
513 'hour': lambda d: u'%dh' % d,
509 'minute': lambda d: u'%dmin' % d,
514 'minute': lambda d: u'%dmin' % d,
510 'second': lambda d: u'%dsec' % d,
515 'second': lambda d: u'%dsec' % d,
511 }
516 }
512 else:
517 else:
513 fmt_funcs = {
518 fmt_funcs = {
514 'year': lambda d: _pluralize(u'${num} year', u'${num} years', d, mapping={'num': d}).interpolate(),
519 'year': lambda d: _pluralize(u'${num} year', u'${num} years', d, mapping={'num': d}).interpolate(),
515 'month': lambda d: _pluralize(u'${num} month', u'${num} months', d, mapping={'num': d}).interpolate(),
520 'month': lambda d: _pluralize(u'${num} month', u'${num} months', d, mapping={'num': d}).interpolate(),
516 'day': lambda d: _pluralize(u'${num} day', u'${num} days', d, mapping={'num': d}).interpolate(),
521 'day': lambda d: _pluralize(u'${num} day', u'${num} days', d, mapping={'num': d}).interpolate(),
517 'hour': lambda d: _pluralize(u'${num} hour', u'${num} hours', d, mapping={'num': d}).interpolate(),
522 'hour': lambda d: _pluralize(u'${num} hour', u'${num} hours', d, mapping={'num': d}).interpolate(),
518 'minute': lambda d: _pluralize(u'${num} minute', u'${num} minutes', d, mapping={'num': d}).interpolate(),
523 'minute': lambda d: _pluralize(u'${num} minute', u'${num} minutes', d, mapping={'num': d}).interpolate(),
519 'second': lambda d: _pluralize(u'${num} second', u'${num} seconds', d, mapping={'num': d}).interpolate(),
524 'second': lambda d: _pluralize(u'${num} second', u'${num} seconds', d, mapping={'num': d}).interpolate(),
520 }
525 }
521
526
522 i = 0
527 i = 0
523 for part in order:
528 for part in order:
524 value = deltas[part]
529 value = deltas[part]
525 if value != 0:
530 if value != 0:
526
531
527 if i < 5:
532 if i < 5:
528 sub_part = order[i + 1]
533 sub_part = order[i + 1]
529 sub_value = deltas[sub_part]
534 sub_value = deltas[sub_part]
530 else:
535 else:
531 sub_value = 0
536 sub_value = 0
532
537
533 if sub_value == 0 or show_short_version:
538 if sub_value == 0 or show_short_version:
534 _val = fmt_funcs[part](value)
539 _val = fmt_funcs[part](value)
535 if future:
540 if future:
536 if show_suffix:
541 if show_suffix:
537 return _(u'in ${ago}', mapping={'ago': _val})
542 return _(u'in ${ago}', mapping={'ago': _val})
538 else:
543 else:
539 return _(_val)
544 return _(_val)
540
545
541 else:
546 else:
542 if show_suffix:
547 if show_suffix:
543 return _(u'${ago} ago', mapping={'ago': _val})
548 return _(u'${ago} ago', mapping={'ago': _val})
544 else:
549 else:
545 return _(_val)
550 return _(_val)
546
551
547 val = fmt_funcs[part](value)
552 val = fmt_funcs[part](value)
548 val_detail = fmt_funcs[sub_part](sub_value)
553 val_detail = fmt_funcs[sub_part](sub_value)
549 mapping = {'val': val, 'detail': val_detail}
554 mapping = {'val': val, 'detail': val_detail}
550
555
551 if short_format:
556 if short_format:
552 datetime_tmpl = _(u'${val}, ${detail}', mapping=mapping)
557 datetime_tmpl = _(u'${val}, ${detail}', mapping=mapping)
553 if show_suffix:
558 if show_suffix:
554 datetime_tmpl = _(u'${val}, ${detail} ago', mapping=mapping)
559 datetime_tmpl = _(u'${val}, ${detail} ago', mapping=mapping)
555 if future:
560 if future:
556 datetime_tmpl = _(u'in ${val}, ${detail}', mapping=mapping)
561 datetime_tmpl = _(u'in ${val}, ${detail}', mapping=mapping)
557 else:
562 else:
558 datetime_tmpl = _(u'${val} and ${detail}', mapping=mapping)
563 datetime_tmpl = _(u'${val} and ${detail}', mapping=mapping)
559 if show_suffix:
564 if show_suffix:
560 datetime_tmpl = _(u'${val} and ${detail} ago', mapping=mapping)
565 datetime_tmpl = _(u'${val} and ${detail} ago', mapping=mapping)
561 if future:
566 if future:
562 datetime_tmpl = _(u'in ${val} and ${detail}', mapping=mapping)
567 datetime_tmpl = _(u'in ${val} and ${detail}', mapping=mapping)
563
568
564 return datetime_tmpl
569 return datetime_tmpl
565 i += 1
570 i += 1
566 return _(u'just now')
571 return _(u'just now')
567
572
568
573
569 def age_from_seconds(seconds):
574 def age_from_seconds(seconds):
570 seconds = safe_int(seconds) or 0
575 seconds = safe_int(seconds) or 0
571 prevdate = time_to_datetime(time.time() + seconds)
576 prevdate = time_to_datetime(time.time() + seconds)
572 return age(prevdate, show_suffix=False, show_short_version=True)
577 return age(prevdate, show_suffix=False, show_short_version=True)
573
578
574
579
575 def cleaned_uri(uri):
580 def cleaned_uri(uri):
576 """
581 """
577 Quotes '[' and ']' from uri if there is only one of them.
582 Quotes '[' and ']' from uri if there is only one of them.
578 according to RFC3986 we cannot use such chars in uri
583 according to RFC3986 we cannot use such chars in uri
579 :param uri:
584 :param uri:
580 :return: uri without this chars
585 :return: uri without this chars
581 """
586 """
582 return urllib.quote(uri, safe='@$:/')
587 return urllib.quote(uri, safe='@$:/')
583
588
584
589
585 def uri_filter(uri):
590 def uri_filter(uri):
586 """
591 """
587 Removes user:password from given url string
592 Removes user:password from given url string
588
593
589 :param uri:
594 :param uri:
590 :rtype: unicode
595 :rtype: unicode
591 :returns: filtered list of strings
596 :returns: filtered list of strings
592 """
597 """
593 if not uri:
598 if not uri:
594 return ''
599 return ''
595
600
596 proto = ''
601 proto = ''
597
602
598 for pat in ('https://', 'http://'):
603 for pat in ('https://', 'http://'):
599 if uri.startswith(pat):
604 if uri.startswith(pat):
600 uri = uri[len(pat):]
605 uri = uri[len(pat):]
601 proto = pat
606 proto = pat
602 break
607 break
603
608
604 # remove passwords and username
609 # remove passwords and username
605 uri = uri[uri.find('@') + 1:]
610 uri = uri[uri.find('@') + 1:]
606
611
607 # get the port
612 # get the port
608 cred_pos = uri.find(':')
613 cred_pos = uri.find(':')
609 if cred_pos == -1:
614 if cred_pos == -1:
610 host, port = uri, None
615 host, port = uri, None
611 else:
616 else:
612 host, port = uri[:cred_pos], uri[cred_pos + 1:]
617 host, port = uri[:cred_pos], uri[cred_pos + 1:]
613
618
614 return filter(None, [proto, host, port])
619 return filter(None, [proto, host, port])
615
620
616
621
617 def credentials_filter(uri):
622 def credentials_filter(uri):
618 """
623 """
619 Returns a url with removed credentials
624 Returns a url with removed credentials
620
625
621 :param uri:
626 :param uri:
622 """
627 """
623
628
624 uri = uri_filter(uri)
629 uri = uri_filter(uri)
625 # check if we have port
630 # check if we have port
626 if len(uri) > 2 and uri[2]:
631 if len(uri) > 2 and uri[2]:
627 uri[2] = ':' + uri[2]
632 uri[2] = ':' + uri[2]
628
633
629 return ''.join(uri)
634 return ''.join(uri)
630
635
631
636
632 def get_host_info(request):
637 def get_host_info(request):
633 """
638 """
634 Generate host info, to obtain full url e.g https://server.com
639 Generate host info, to obtain full url e.g https://server.com
635 use this
640 use this
636 `{scheme}://{netloc}`
641 `{scheme}://{netloc}`
637 """
642 """
638 if not request:
643 if not request:
639 return {}
644 return {}
640
645
641 qualified_home_url = request.route_url('home')
646 qualified_home_url = request.route_url('home')
642 parsed_url = urlobject.URLObject(qualified_home_url)
647 parsed_url = urlobject.URLObject(qualified_home_url)
643 decoded_path = safe_unicode(urllib.unquote(parsed_url.path.rstrip('/')))
648 decoded_path = safe_unicode(urllib.unquote(parsed_url.path.rstrip('/')))
644
649
645 return {
650 return {
646 'scheme': parsed_url.scheme,
651 'scheme': parsed_url.scheme,
647 'netloc': parsed_url.netloc+decoded_path,
652 'netloc': parsed_url.netloc+decoded_path,
648 'hostname': parsed_url.hostname,
653 'hostname': parsed_url.hostname,
649 }
654 }
650
655
651
656
652 def get_clone_url(request, uri_tmpl, repo_name, repo_id, **override):
657 def get_clone_url(request, uri_tmpl, repo_name, repo_id, **override):
653 qualified_home_url = request.route_url('home')
658 qualified_home_url = request.route_url('home')
654 parsed_url = urlobject.URLObject(qualified_home_url)
659 parsed_url = urlobject.URLObject(qualified_home_url)
655 decoded_path = safe_unicode(urllib.unquote(parsed_url.path.rstrip('/')))
660 decoded_path = safe_unicode(urllib.unquote(parsed_url.path.rstrip('/')))
656
661
657 args = {
662 args = {
658 'scheme': parsed_url.scheme,
663 'scheme': parsed_url.scheme,
659 'user': '',
664 'user': '',
660 'sys_user': getpass.getuser(),
665 'sys_user': getpass.getuser(),
661 # path if we use proxy-prefix
666 # path if we use proxy-prefix
662 'netloc': parsed_url.netloc+decoded_path,
667 'netloc': parsed_url.netloc+decoded_path,
663 'hostname': parsed_url.hostname,
668 'hostname': parsed_url.hostname,
664 'prefix': decoded_path,
669 'prefix': decoded_path,
665 'repo': repo_name,
670 'repo': repo_name,
666 'repoid': str(repo_id)
671 'repoid': str(repo_id)
667 }
672 }
668 args.update(override)
673 args.update(override)
669 args['user'] = urllib.quote(safe_str(args['user']))
674 args['user'] = urllib.quote(safe_str(args['user']))
670
675
671 for k, v in args.items():
676 for k, v in args.items():
672 uri_tmpl = uri_tmpl.replace('{%s}' % k, v)
677 uri_tmpl = uri_tmpl.replace('{%s}' % k, v)
673
678
674 # remove leading @ sign if it's present. Case of empty user
679 # remove leading @ sign if it's present. Case of empty user
675 url_obj = urlobject.URLObject(uri_tmpl)
680 url_obj = urlobject.URLObject(uri_tmpl)
676 url = url_obj.with_netloc(url_obj.netloc.lstrip('@'))
681 url = url_obj.with_netloc(url_obj.netloc.lstrip('@'))
677
682
678 return safe_unicode(url)
683 return safe_unicode(url)
679
684
680
685
681 def get_commit_safe(repo, commit_id=None, commit_idx=None, pre_load=None):
686 def get_commit_safe(repo, commit_id=None, commit_idx=None, pre_load=None):
682 """
687 """
683 Safe version of get_commit if this commit doesn't exists for a
688 Safe version of get_commit if this commit doesn't exists for a
684 repository it returns a Dummy one instead
689 repository it returns a Dummy one instead
685
690
686 :param repo: repository instance
691 :param repo: repository instance
687 :param commit_id: commit id as str
692 :param commit_id: commit id as str
688 :param pre_load: optional list of commit attributes to load
693 :param pre_load: optional list of commit attributes to load
689 """
694 """
690 # TODO(skreft): remove these circular imports
695 # TODO(skreft): remove these circular imports
691 from rhodecode.lib.vcs.backends.base import BaseRepository, EmptyCommit
696 from rhodecode.lib.vcs.backends.base import BaseRepository, EmptyCommit
692 from rhodecode.lib.vcs.exceptions import RepositoryError
697 from rhodecode.lib.vcs.exceptions import RepositoryError
693 if not isinstance(repo, BaseRepository):
698 if not isinstance(repo, BaseRepository):
694 raise Exception('You must pass an Repository '
699 raise Exception('You must pass an Repository '
695 'object as first argument got %s', type(repo))
700 'object as first argument got %s', type(repo))
696
701
697 try:
702 try:
698 commit = repo.get_commit(
703 commit = repo.get_commit(
699 commit_id=commit_id, commit_idx=commit_idx, pre_load=pre_load)
704 commit_id=commit_id, commit_idx=commit_idx, pre_load=pre_load)
700 except (RepositoryError, LookupError):
705 except (RepositoryError, LookupError):
701 commit = EmptyCommit()
706 commit = EmptyCommit()
702 return commit
707 return commit
703
708
704
709
705 def datetime_to_time(dt):
710 def datetime_to_time(dt):
706 if dt:
711 if dt:
707 return time.mktime(dt.timetuple())
712 return time.mktime(dt.timetuple())
708
713
709
714
710 def time_to_datetime(tm):
715 def time_to_datetime(tm):
711 if tm:
716 if tm:
712 if isinstance(tm, compat.string_types):
717 if isinstance(tm, compat.string_types):
713 try:
718 try:
714 tm = float(tm)
719 tm = float(tm)
715 except ValueError:
720 except ValueError:
716 return
721 return
717 return datetime.datetime.fromtimestamp(tm)
722 return datetime.datetime.fromtimestamp(tm)
718
723
719
724
720 def time_to_utcdatetime(tm):
725 def time_to_utcdatetime(tm):
721 if tm:
726 if tm:
722 if isinstance(tm, compat.string_types):
727 if isinstance(tm, compat.string_types):
723 try:
728 try:
724 tm = float(tm)
729 tm = float(tm)
725 except ValueError:
730 except ValueError:
726 return
731 return
727 return datetime.datetime.utcfromtimestamp(tm)
732 return datetime.datetime.utcfromtimestamp(tm)
728
733
729
734
730 MENTIONS_REGEX = re.compile(
735 MENTIONS_REGEX = re.compile(
731 # ^@ or @ without any special chars in front
736 # ^@ or @ without any special chars in front
732 r'(?:^@|[^a-zA-Z0-9\-\_\.]@)'
737 r'(?:^@|[^a-zA-Z0-9\-\_\.]@)'
733 # main body starts with letter, then can be . - _
738 # main body starts with letter, then can be . - _
734 r'([a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+)',
739 r'([a-zA-Z0-9]{1}[a-zA-Z0-9\-\_\.]+)',
735 re.VERBOSE | re.MULTILINE)
740 re.VERBOSE | re.MULTILINE)
736
741
737
742
738 def extract_mentioned_users(s):
743 def extract_mentioned_users(s):
739 """
744 """
740 Returns unique usernames from given string s that have @mention
745 Returns unique usernames from given string s that have @mention
741
746
742 :param s: string to get mentions
747 :param s: string to get mentions
743 """
748 """
744 usrs = set()
749 usrs = set()
745 for username in MENTIONS_REGEX.findall(s):
750 for username in MENTIONS_REGEX.findall(s):
746 usrs.add(username)
751 usrs.add(username)
747
752
748 return sorted(list(usrs), key=lambda k: k.lower())
753 return sorted(list(usrs), key=lambda k: k.lower())
749
754
750
755
751 class AttributeDictBase(dict):
756 class AttributeDictBase(dict):
752 def __getstate__(self):
757 def __getstate__(self):
753 odict = self.__dict__ # get attribute dictionary
758 odict = self.__dict__ # get attribute dictionary
754 return odict
759 return odict
755
760
756 def __setstate__(self, dict):
761 def __setstate__(self, dict):
757 self.__dict__ = dict
762 self.__dict__ = dict
758
763
759 __setattr__ = dict.__setitem__
764 __setattr__ = dict.__setitem__
760 __delattr__ = dict.__delitem__
765 __delattr__ = dict.__delitem__
761
766
762
767
763 class StrictAttributeDict(AttributeDictBase):
768 class StrictAttributeDict(AttributeDictBase):
764 """
769 """
765 Strict Version of Attribute dict which raises an Attribute error when
770 Strict Version of Attribute dict which raises an Attribute error when
766 requested attribute is not set
771 requested attribute is not set
767 """
772 """
768 def __getattr__(self, attr):
773 def __getattr__(self, attr):
769 try:
774 try:
770 return self[attr]
775 return self[attr]
771 except KeyError:
776 except KeyError:
772 raise AttributeError('%s object has no attribute %s' % (
777 raise AttributeError('%s object has no attribute %s' % (
773 self.__class__, attr))
778 self.__class__, attr))
774
779
775
780
776 class AttributeDict(AttributeDictBase):
781 class AttributeDict(AttributeDictBase):
777 def __getattr__(self, attr):
782 def __getattr__(self, attr):
778 return self.get(attr, None)
783 return self.get(attr, None)
779
784
780
785
781
786
782 class OrderedDefaultDict(collections.OrderedDict, collections.defaultdict):
787 class OrderedDefaultDict(collections.OrderedDict, collections.defaultdict):
783 def __init__(self, default_factory=None, *args, **kwargs):
788 def __init__(self, default_factory=None, *args, **kwargs):
784 # in python3 you can omit the args to super
789 # in python3 you can omit the args to super
785 super(OrderedDefaultDict, self).__init__(*args, **kwargs)
790 super(OrderedDefaultDict, self).__init__(*args, **kwargs)
786 self.default_factory = default_factory
791 self.default_factory = default_factory
787
792
788
793
789 def fix_PATH(os_=None):
794 def fix_PATH(os_=None):
790 """
795 """
791 Get current active python path, and append it to PATH variable to fix
796 Get current active python path, and append it to PATH variable to fix
792 issues of subprocess calls and different python versions
797 issues of subprocess calls and different python versions
793 """
798 """
794 if os_ is None:
799 if os_ is None:
795 import os
800 import os
796 else:
801 else:
797 os = os_
802 os = os_
798
803
799 cur_path = os.path.split(sys.executable)[0]
804 cur_path = os.path.split(sys.executable)[0]
800 if not os.environ['PATH'].startswith(cur_path):
805 if not os.environ['PATH'].startswith(cur_path):
801 os.environ['PATH'] = '%s:%s' % (cur_path, os.environ['PATH'])
806 os.environ['PATH'] = '%s:%s' % (cur_path, os.environ['PATH'])
802
807
803
808
804 def obfuscate_url_pw(engine):
809 def obfuscate_url_pw(engine):
805 _url = engine or ''
810 _url = engine or ''
806 try:
811 try:
807 _url = sqlalchemy.engine.url.make_url(engine)
812 _url = sqlalchemy.engine.url.make_url(engine)
808 if _url.password:
813 if _url.password:
809 _url.password = 'XXXXX'
814 _url.password = 'XXXXX'
810 except Exception:
815 except Exception:
811 pass
816 pass
812 return unicode(_url)
817 return unicode(_url)
813
818
814
819
815 def get_server_url(environ):
820 def get_server_url(environ):
816 req = webob.Request(environ)
821 req = webob.Request(environ)
817 return req.host_url + req.script_name
822 return req.host_url + req.script_name
818
823
819
824
820 def unique_id(hexlen=32):
825 def unique_id(hexlen=32):
821 alphabet = "23456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghjklmnpqrstuvwxyz"
826 alphabet = "23456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghjklmnpqrstuvwxyz"
822 return suuid(truncate_to=hexlen, alphabet=alphabet)
827 return suuid(truncate_to=hexlen, alphabet=alphabet)
823
828
824
829
825 def suuid(url=None, truncate_to=22, alphabet=None):
830 def suuid(url=None, truncate_to=22, alphabet=None):
826 """
831 """
827 Generate and return a short URL safe UUID.
832 Generate and return a short URL safe UUID.
828
833
829 If the url parameter is provided, set the namespace to the provided
834 If the url parameter is provided, set the namespace to the provided
830 URL and generate a UUID.
835 URL and generate a UUID.
831
836
832 :param url to get the uuid for
837 :param url to get the uuid for
833 :truncate_to: truncate the basic 22 UUID to shorter version
838 :truncate_to: truncate the basic 22 UUID to shorter version
834
839
835 The IDs won't be universally unique any longer, but the probability of
840 The IDs won't be universally unique any longer, but the probability of
836 a collision will still be very low.
841 a collision will still be very low.
837 """
842 """
838 # Define our alphabet.
843 # Define our alphabet.
839 _ALPHABET = alphabet or "23456789ABCDEFGHJKLMNPQRSTUVWXYZ"
844 _ALPHABET = alphabet or "23456789ABCDEFGHJKLMNPQRSTUVWXYZ"
840
845
841 # If no URL is given, generate a random UUID.
846 # If no URL is given, generate a random UUID.
842 if url is None:
847 if url is None:
843 unique_id = uuid.uuid4().int
848 unique_id = uuid.uuid4().int
844 else:
849 else:
845 unique_id = uuid.uuid3(uuid.NAMESPACE_URL, url).int
850 unique_id = uuid.uuid3(uuid.NAMESPACE_URL, url).int
846
851
847 alphabet_length = len(_ALPHABET)
852 alphabet_length = len(_ALPHABET)
848 output = []
853 output = []
849 while unique_id > 0:
854 while unique_id > 0:
850 digit = unique_id % alphabet_length
855 digit = unique_id % alphabet_length
851 output.append(_ALPHABET[digit])
856 output.append(_ALPHABET[digit])
852 unique_id = int(unique_id / alphabet_length)
857 unique_id = int(unique_id / alphabet_length)
853 return "".join(output)[:truncate_to]
858 return "".join(output)[:truncate_to]
854
859
855
860
856 def get_current_rhodecode_user(request=None):
861 def get_current_rhodecode_user(request=None):
857 """
862 """
858 Gets rhodecode user from request
863 Gets rhodecode user from request
859 """
864 """
860 pyramid_request = request or pyramid.threadlocal.get_current_request()
865 pyramid_request = request or pyramid.threadlocal.get_current_request()
861
866
862 # web case
867 # web case
863 if pyramid_request and hasattr(pyramid_request, 'user'):
868 if pyramid_request and hasattr(pyramid_request, 'user'):
864 return pyramid_request.user
869 return pyramid_request.user
865
870
866 # api case
871 # api case
867 if pyramid_request and hasattr(pyramid_request, 'rpc_user'):
872 if pyramid_request and hasattr(pyramid_request, 'rpc_user'):
868 return pyramid_request.rpc_user
873 return pyramid_request.rpc_user
869
874
870 return None
875 return None
871
876
872
877
873 def action_logger_generic(action, namespace=''):
878 def action_logger_generic(action, namespace=''):
874 """
879 """
875 A generic logger for actions useful to the system overview, tries to find
880 A generic logger for actions useful to the system overview, tries to find
876 an acting user for the context of the call otherwise reports unknown user
881 an acting user for the context of the call otherwise reports unknown user
877
882
878 :param action: logging message eg 'comment 5 deleted'
883 :param action: logging message eg 'comment 5 deleted'
879 :param type: string
884 :param type: string
880
885
881 :param namespace: namespace of the logging message eg. 'repo.comments'
886 :param namespace: namespace of the logging message eg. 'repo.comments'
882 :param type: string
887 :param type: string
883
888
884 """
889 """
885
890
886 logger_name = 'rhodecode.actions'
891 logger_name = 'rhodecode.actions'
887
892
888 if namespace:
893 if namespace:
889 logger_name += '.' + namespace
894 logger_name += '.' + namespace
890
895
891 log = logging.getLogger(logger_name)
896 log = logging.getLogger(logger_name)
892
897
893 # get a user if we can
898 # get a user if we can
894 user = get_current_rhodecode_user()
899 user = get_current_rhodecode_user()
895
900
896 logfunc = log.info
901 logfunc = log.info
897
902
898 if not user:
903 if not user:
899 user = '<unknown user>'
904 user = '<unknown user>'
900 logfunc = log.warning
905 logfunc = log.warning
901
906
902 logfunc('Logging action by {}: {}'.format(user, action))
907 logfunc('Logging action by {}: {}'.format(user, action))
903
908
904
909
905 def escape_split(text, sep=',', maxsplit=-1):
910 def escape_split(text, sep=',', maxsplit=-1):
906 r"""
911 r"""
907 Allows for escaping of the separator: e.g. arg='foo\, bar'
912 Allows for escaping of the separator: e.g. arg='foo\, bar'
908
913
909 It should be noted that the way bash et. al. do command line parsing, those
914 It should be noted that the way bash et. al. do command line parsing, those
910 single quotes are required.
915 single quotes are required.
911 """
916 """
912 escaped_sep = r'\%s' % sep
917 escaped_sep = r'\%s' % sep
913
918
914 if escaped_sep not in text:
919 if escaped_sep not in text:
915 return text.split(sep, maxsplit)
920 return text.split(sep, maxsplit)
916
921
917 before, _mid, after = text.partition(escaped_sep)
922 before, _mid, after = text.partition(escaped_sep)
918 startlist = before.split(sep, maxsplit) # a regular split is fine here
923 startlist = before.split(sep, maxsplit) # a regular split is fine here
919 unfinished = startlist[-1]
924 unfinished = startlist[-1]
920 startlist = startlist[:-1]
925 startlist = startlist[:-1]
921
926
922 # recurse because there may be more escaped separators
927 # recurse because there may be more escaped separators
923 endlist = escape_split(after, sep, maxsplit)
928 endlist = escape_split(after, sep, maxsplit)
924
929
925 # finish building the escaped value. we use endlist[0] becaue the first
930 # finish building the escaped value. we use endlist[0] becaue the first
926 # part of the string sent in recursion is the rest of the escaped value.
931 # part of the string sent in recursion is the rest of the escaped value.
927 unfinished += sep + endlist[0]
932 unfinished += sep + endlist[0]
928
933
929 return startlist + [unfinished] + endlist[1:] # put together all the parts
934 return startlist + [unfinished] + endlist[1:] # put together all the parts
930
935
931
936
932 class OptionalAttr(object):
937 class OptionalAttr(object):
933 """
938 """
934 Special Optional Option that defines other attribute. Example::
939 Special Optional Option that defines other attribute. Example::
935
940
936 def test(apiuser, userid=Optional(OAttr('apiuser')):
941 def test(apiuser, userid=Optional(OAttr('apiuser')):
937 user = Optional.extract(userid)
942 user = Optional.extract(userid)
938 # calls
943 # calls
939
944
940 """
945 """
941
946
942 def __init__(self, attr_name):
947 def __init__(self, attr_name):
943 self.attr_name = attr_name
948 self.attr_name = attr_name
944
949
945 def __repr__(self):
950 def __repr__(self):
946 return '<OptionalAttr:%s>' % self.attr_name
951 return '<OptionalAttr:%s>' % self.attr_name
947
952
948 def __call__(self):
953 def __call__(self):
949 return self
954 return self
950
955
951
956
952 # alias
957 # alias
953 OAttr = OptionalAttr
958 OAttr = OptionalAttr
954
959
955
960
956 class Optional(object):
961 class Optional(object):
957 """
962 """
958 Defines an optional parameter::
963 Defines an optional parameter::
959
964
960 param = param.getval() if isinstance(param, Optional) else param
965 param = param.getval() if isinstance(param, Optional) else param
961 param = param() if isinstance(param, Optional) else param
966 param = param() if isinstance(param, Optional) else param
962
967
963 is equivalent of::
968 is equivalent of::
964
969
965 param = Optional.extract(param)
970 param = Optional.extract(param)
966
971
967 """
972 """
968
973
969 def __init__(self, type_):
974 def __init__(self, type_):
970 self.type_ = type_
975 self.type_ = type_
971
976
972 def __repr__(self):
977 def __repr__(self):
973 return '<Optional:%s>' % self.type_.__repr__()
978 return '<Optional:%s>' % self.type_.__repr__()
974
979
975 def __call__(self):
980 def __call__(self):
976 return self.getval()
981 return self.getval()
977
982
978 def getval(self):
983 def getval(self):
979 """
984 """
980 returns value from this Optional instance
985 returns value from this Optional instance
981 """
986 """
982 if isinstance(self.type_, OAttr):
987 if isinstance(self.type_, OAttr):
983 # use params name
988 # use params name
984 return self.type_.attr_name
989 return self.type_.attr_name
985 return self.type_
990 return self.type_
986
991
987 @classmethod
992 @classmethod
988 def extract(cls, val):
993 def extract(cls, val):
989 """
994 """
990 Extracts value from Optional() instance
995 Extracts value from Optional() instance
991
996
992 :param val:
997 :param val:
993 :return: original value if it's not Optional instance else
998 :return: original value if it's not Optional instance else
994 value of instance
999 value of instance
995 """
1000 """
996 if isinstance(val, cls):
1001 if isinstance(val, cls):
997 return val.getval()
1002 return val.getval()
998 return val
1003 return val
999
1004
1000
1005
1001 def glob2re(pat):
1006 def glob2re(pat):
1002 """
1007 """
1003 Translate a shell PATTERN to a regular expression.
1008 Translate a shell PATTERN to a regular expression.
1004
1009
1005 There is no way to quote meta-characters.
1010 There is no way to quote meta-characters.
1006 """
1011 """
1007
1012
1008 i, n = 0, len(pat)
1013 i, n = 0, len(pat)
1009 res = ''
1014 res = ''
1010 while i < n:
1015 while i < n:
1011 c = pat[i]
1016 c = pat[i]
1012 i = i+1
1017 i = i+1
1013 if c == '*':
1018 if c == '*':
1014 #res = res + '.*'
1019 #res = res + '.*'
1015 res = res + '[^/]*'
1020 res = res + '[^/]*'
1016 elif c == '?':
1021 elif c == '?':
1017 #res = res + '.'
1022 #res = res + '.'
1018 res = res + '[^/]'
1023 res = res + '[^/]'
1019 elif c == '[':
1024 elif c == '[':
1020 j = i
1025 j = i
1021 if j < n and pat[j] == '!':
1026 if j < n and pat[j] == '!':
1022 j = j+1
1027 j = j+1
1023 if j < n and pat[j] == ']':
1028 if j < n and pat[j] == ']':
1024 j = j+1
1029 j = j+1
1025 while j < n and pat[j] != ']':
1030 while j < n and pat[j] != ']':
1026 j = j+1
1031 j = j+1
1027 if j >= n:
1032 if j >= n:
1028 res = res + '\\['
1033 res = res + '\\['
1029 else:
1034 else:
1030 stuff = pat[i:j].replace('\\','\\\\')
1035 stuff = pat[i:j].replace('\\','\\\\')
1031 i = j+1
1036 i = j+1
1032 if stuff[0] == '!':
1037 if stuff[0] == '!':
1033 stuff = '^' + stuff[1:]
1038 stuff = '^' + stuff[1:]
1034 elif stuff[0] == '^':
1039 elif stuff[0] == '^':
1035 stuff = '\\' + stuff
1040 stuff = '\\' + stuff
1036 res = '%s[%s]' % (res, stuff)
1041 res = '%s[%s]' % (res, stuff)
1037 else:
1042 else:
1038 res = res + re.escape(c)
1043 res = res + re.escape(c)
1039 return res + '\Z(?ms)'
1044 return res + '\Z(?ms)'
1040
1045
1041
1046
1042 def parse_byte_string(size_str):
1047 def parse_byte_string(size_str):
1043 match = re.match(r'(\d+)(MB|KB)', size_str, re.IGNORECASE)
1048 match = re.match(r'(\d+)(MB|KB)', size_str, re.IGNORECASE)
1044 if not match:
1049 if not match:
1045 raise ValueError('Given size:%s is invalid, please make sure '
1050 raise ValueError('Given size:%s is invalid, please make sure '
1046 'to use format of <num>(MB|KB)' % size_str)
1051 'to use format of <num>(MB|KB)' % size_str)
1047
1052
1048 _parts = match.groups()
1053 _parts = match.groups()
1049 num, type_ = _parts
1054 num, type_ = _parts
1050 return long(num) * {'mb': 1024*1024, 'kb': 1024}[type_.lower()]
1055 return long(num) * {'mb': 1024*1024, 'kb': 1024}[type_.lower()]
1051
1056
1052
1057
1053 class CachedProperty(object):
1058 class CachedProperty(object):
1054 """
1059 """
1055 Lazy Attributes. With option to invalidate the cache by running a method
1060 Lazy Attributes. With option to invalidate the cache by running a method
1056
1061
1057 class Foo():
1062 class Foo():
1058
1063
1059 @CachedProperty
1064 @CachedProperty
1060 def heavy_func():
1065 def heavy_func():
1061 return 'super-calculation'
1066 return 'super-calculation'
1062
1067
1063 foo = Foo()
1068 foo = Foo()
1064 foo.heavy_func() # first computions
1069 foo.heavy_func() # first computions
1065 foo.heavy_func() # fetch from cache
1070 foo.heavy_func() # fetch from cache
1066 foo._invalidate_prop_cache('heavy_func')
1071 foo._invalidate_prop_cache('heavy_func')
1067 # at this point calling foo.heavy_func() will be re-computed
1072 # at this point calling foo.heavy_func() will be re-computed
1068 """
1073 """
1069
1074
1070 def __init__(self, func, func_name=None):
1075 def __init__(self, func, func_name=None):
1071
1076
1072 if func_name is None:
1077 if func_name is None:
1073 func_name = func.__name__
1078 func_name = func.__name__
1074 self.data = (func, func_name)
1079 self.data = (func, func_name)
1075 update_wrapper(self, func)
1080 update_wrapper(self, func)
1076
1081
1077 def __get__(self, inst, class_):
1082 def __get__(self, inst, class_):
1078 if inst is None:
1083 if inst is None:
1079 return self
1084 return self
1080
1085
1081 func, func_name = self.data
1086 func, func_name = self.data
1082 value = func(inst)
1087 value = func(inst)
1083 inst.__dict__[func_name] = value
1088 inst.__dict__[func_name] = value
1084 if '_invalidate_prop_cache' not in inst.__dict__:
1089 if '_invalidate_prop_cache' not in inst.__dict__:
1085 inst.__dict__['_invalidate_prop_cache'] = partial(
1090 inst.__dict__['_invalidate_prop_cache'] = partial(
1086 self._invalidate_prop_cache, inst)
1091 self._invalidate_prop_cache, inst)
1087 return value
1092 return value
1088
1093
1089 def _invalidate_prop_cache(self, inst, name):
1094 def _invalidate_prop_cache(self, inst, name):
1090 inst.__dict__.pop(name, None)
1095 inst.__dict__.pop(name, None)
General Comments 0
You need to be logged in to leave comments. Login now