Show More
@@ -1,407 +1,410 b'' | |||||
1 | # -*- coding: utf-8 -*- |
|
1 | # -*- coding: utf-8 -*- | |
2 | """ |
|
2 | """ | |
3 | rhodecode.lib.__init__ |
|
3 | rhodecode.lib.__init__ | |
4 | ~~~~~~~~~~~~~~~~~~~~~~~ |
|
4 | ~~~~~~~~~~~~~~~~~~~~~~~ | |
5 |
|
5 | |||
6 | Some simple helper functions |
|
6 | Some simple helper functions | |
7 |
|
7 | |||
8 | :created_on: Jan 5, 2011 |
|
8 | :created_on: Jan 5, 2011 | |
9 | :author: marcink |
|
9 | :author: marcink | |
10 | :copyright: (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com> |
|
10 | :copyright: (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com> | |
11 | :license: GPLv3, see COPYING for more details. |
|
11 | :license: GPLv3, see COPYING for more details. | |
12 | """ |
|
12 | """ | |
13 | # This program is free software: you can redistribute it and/or modify |
|
13 | # This program is free software: you can redistribute it and/or modify | |
14 | # it under the terms of the GNU General Public License as published by |
|
14 | # it under the terms of the GNU General Public License as published by | |
15 | # the Free Software Foundation, either version 3 of the License, or |
|
15 | # the Free Software Foundation, either version 3 of the License, or | |
16 | # (at your option) any later version. |
|
16 | # (at your option) any later version. | |
17 | # |
|
17 | # | |
18 | # This program is distributed in the hope that it will be useful, |
|
18 | # This program is distributed in the hope that it will be useful, | |
19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
21 | # GNU General Public License for more details. |
|
21 | # GNU General Public License for more details. | |
22 | # |
|
22 | # | |
23 | # You should have received a copy of the GNU General Public License |
|
23 | # You should have received a copy of the GNU General Public License | |
24 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
24 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
25 |
|
25 | |||
26 | import os |
|
26 | import os | |
27 |
|
27 | |||
28 | def __get_lem(): |
|
28 | def __get_lem(): | |
29 | from pygments import lexers |
|
29 | from pygments import lexers | |
30 | from string import lower |
|
30 | from string import lower | |
31 | from collections import defaultdict |
|
31 | from collections import defaultdict | |
32 |
|
32 | |||
33 | d = defaultdict(lambda: []) |
|
33 | d = defaultdict(lambda: []) | |
34 |
|
34 | |||
35 | def __clean(s): |
|
35 | def __clean(s): | |
36 | s = s.lstrip('*') |
|
36 | s = s.lstrip('*') | |
37 | s = s.lstrip('.') |
|
37 | s = s.lstrip('.') | |
38 |
|
38 | |||
39 | if s.find('[') != -1: |
|
39 | if s.find('[') != -1: | |
40 | exts = [] |
|
40 | exts = [] | |
41 | start, stop = s.find('['), s.find(']') |
|
41 | start, stop = s.find('['), s.find(']') | |
42 |
|
42 | |||
43 | for suffix in s[start + 1:stop]: |
|
43 | for suffix in s[start + 1:stop]: | |
44 | exts.append(s[:s.find('[')] + suffix) |
|
44 | exts.append(s[:s.find('[')] + suffix) | |
45 | return map(lower, exts) |
|
45 | return map(lower, exts) | |
46 | else: |
|
46 | else: | |
47 | return map(lower, [s]) |
|
47 | return map(lower, [s]) | |
48 |
|
48 | |||
49 | for lx, t in sorted(lexers.LEXERS.items()): |
|
49 | for lx, t in sorted(lexers.LEXERS.items()): | |
50 | m = map(__clean, t[-2]) |
|
50 | m = map(__clean, t[-2]) | |
51 | if m: |
|
51 | if m: | |
52 | m = reduce(lambda x, y: x + y, m) |
|
52 | m = reduce(lambda x, y: x + y, m) | |
53 | for ext in m: |
|
53 | for ext in m: | |
54 | desc = lx.replace('Lexer', '') |
|
54 | desc = lx.replace('Lexer', '') | |
55 | d[ext].append(desc) |
|
55 | d[ext].append(desc) | |
56 |
|
56 | |||
57 | return dict(d) |
|
57 | return dict(d) | |
58 |
|
58 | |||
59 | # language map is also used by whoosh indexer, which for those specified |
|
59 | # language map is also used by whoosh indexer, which for those specified | |
60 | # extensions will index it's content |
|
60 | # extensions will index it's content | |
61 | LANGUAGES_EXTENSIONS_MAP = __get_lem() |
|
61 | LANGUAGES_EXTENSIONS_MAP = __get_lem() | |
62 |
|
62 | |||
63 | # Additional mappings that are not present in the pygments lexers |
|
63 | # Additional mappings that are not present in the pygments lexers | |
64 | # NOTE: that this will overide any mappings in LANGUAGES_EXTENSIONS_MAP |
|
64 | # NOTE: that this will overide any mappings in LANGUAGES_EXTENSIONS_MAP | |
65 | ADDITIONAL_MAPPINGS = {'xaml': 'XAML'} |
|
65 | ADDITIONAL_MAPPINGS = {'xaml': 'XAML'} | |
66 |
|
66 | |||
67 | LANGUAGES_EXTENSIONS_MAP.update(ADDITIONAL_MAPPINGS) |
|
67 | LANGUAGES_EXTENSIONS_MAP.update(ADDITIONAL_MAPPINGS) | |
68 |
|
68 | |||
69 |
|
69 | |||
70 | def str2bool(_str): |
|
70 | def str2bool(_str): | |
71 | """ |
|
71 | """ | |
72 | returs True/False value from given string, it tries to translate the |
|
72 | returs True/False value from given string, it tries to translate the | |
73 | string into boolean |
|
73 | string into boolean | |
74 |
|
74 | |||
75 | :param _str: string value to translate into boolean |
|
75 | :param _str: string value to translate into boolean | |
76 | :rtype: boolean |
|
76 | :rtype: boolean | |
77 | :returns: boolean from given string |
|
77 | :returns: boolean from given string | |
78 | """ |
|
78 | """ | |
79 | if _str is None: |
|
79 | if _str is None: | |
80 | return False |
|
80 | return False | |
81 | if _str in (True, False): |
|
81 | if _str in (True, False): | |
82 | return _str |
|
82 | return _str | |
83 | _str = str(_str).strip().lower() |
|
83 | _str = str(_str).strip().lower() | |
84 | return _str in ('t', 'true', 'y', 'yes', 'on', '1') |
|
84 | return _str in ('t', 'true', 'y', 'yes', 'on', '1') | |
85 |
|
85 | |||
86 |
|
86 | |||
87 | def convert_line_endings(line, mode): |
|
87 | def convert_line_endings(line, mode): | |
88 | """ |
|
88 | """ | |
89 | Converts a given line "line end" accordingly to given mode |
|
89 | Converts a given line "line end" accordingly to given mode | |
90 |
|
90 | |||
91 | Available modes are:: |
|
91 | Available modes are:: | |
92 | 0 - Unix |
|
92 | 0 - Unix | |
93 | 1 - Mac |
|
93 | 1 - Mac | |
94 | 2 - DOS |
|
94 | 2 - DOS | |
95 |
|
95 | |||
96 | :param line: given line to convert |
|
96 | :param line: given line to convert | |
97 | :param mode: mode to convert to |
|
97 | :param mode: mode to convert to | |
98 | :rtype: str |
|
98 | :rtype: str | |
99 | :return: converted line according to mode |
|
99 | :return: converted line according to mode | |
100 | """ |
|
100 | """ | |
101 | from string import replace |
|
101 | from string import replace | |
102 |
|
102 | |||
103 | if mode == 0: |
|
103 | if mode == 0: | |
104 | line = replace(line, '\r\n', '\n') |
|
104 | line = replace(line, '\r\n', '\n') | |
105 | line = replace(line, '\r', '\n') |
|
105 | line = replace(line, '\r', '\n') | |
106 | elif mode == 1: |
|
106 | elif mode == 1: | |
107 | line = replace(line, '\r\n', '\r') |
|
107 | line = replace(line, '\r\n', '\r') | |
108 | line = replace(line, '\n', '\r') |
|
108 | line = replace(line, '\n', '\r') | |
109 | elif mode == 2: |
|
109 | elif mode == 2: | |
110 | import re |
|
110 | import re | |
111 | line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line) |
|
111 | line = re.sub("\r(?!\n)|(?<!\r)\n", "\r\n", line) | |
112 | return line |
|
112 | return line | |
113 |
|
113 | |||
114 |
|
114 | |||
115 | def detect_mode(line, default): |
|
115 | def detect_mode(line, default): | |
116 | """ |
|
116 | """ | |
117 | Detects line break for given line, if line break couldn't be found |
|
117 | Detects line break for given line, if line break couldn't be found | |
118 | given default value is returned |
|
118 | given default value is returned | |
119 |
|
119 | |||
120 | :param line: str line |
|
120 | :param line: str line | |
121 | :param default: default |
|
121 | :param default: default | |
122 | :rtype: int |
|
122 | :rtype: int | |
123 | :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS |
|
123 | :return: value of line end on of 0 - Unix, 1 - Mac, 2 - DOS | |
124 | """ |
|
124 | """ | |
125 | if line.endswith('\r\n'): |
|
125 | if line.endswith('\r\n'): | |
126 | return 2 |
|
126 | return 2 | |
127 | elif line.endswith('\n'): |
|
127 | elif line.endswith('\n'): | |
128 | return 0 |
|
128 | return 0 | |
129 | elif line.endswith('\r'): |
|
129 | elif line.endswith('\r'): | |
130 | return 1 |
|
130 | return 1 | |
131 | else: |
|
131 | else: | |
132 | return default |
|
132 | return default | |
133 |
|
133 | |||
134 |
|
134 | |||
135 | def generate_api_key(username, salt=None): |
|
135 | def generate_api_key(username, salt=None): | |
136 | """ |
|
136 | """ | |
137 | Generates unique API key for given username, if salt is not given |
|
137 | Generates unique API key for given username, if salt is not given | |
138 | it'll be generated from some random string |
|
138 | it'll be generated from some random string | |
139 |
|
139 | |||
140 | :param username: username as string |
|
140 | :param username: username as string | |
141 | :param salt: salt to hash generate KEY |
|
141 | :param salt: salt to hash generate KEY | |
142 | :rtype: str |
|
142 | :rtype: str | |
143 | :returns: sha1 hash from username+salt |
|
143 | :returns: sha1 hash from username+salt | |
144 | """ |
|
144 | """ | |
145 | from tempfile import _RandomNameSequence |
|
145 | from tempfile import _RandomNameSequence | |
146 | import hashlib |
|
146 | import hashlib | |
147 |
|
147 | |||
148 | if salt is None: |
|
148 | if salt is None: | |
149 | salt = _RandomNameSequence().next() |
|
149 | salt = _RandomNameSequence().next() | |
150 |
|
150 | |||
151 | return hashlib.sha1(username + salt).hexdigest() |
|
151 | return hashlib.sha1(username + salt).hexdigest() | |
152 |
|
152 | |||
153 |
|
153 | |||
154 | def safe_unicode(str_, from_encoding='utf8'): |
|
154 | def safe_unicode(str_, from_encoding='utf8'): | |
155 | """ |
|
155 | """ | |
156 | safe unicode function. Does few trick to turn str_ into unicode |
|
156 | safe unicode function. Does few trick to turn str_ into unicode | |
157 |
|
157 | |||
158 | In case of UnicodeDecode error we try to return it with encoding detected |
|
158 | In case of UnicodeDecode error we try to return it with encoding detected | |
159 | by chardet library if it fails fallback to unicode with errors replaced |
|
159 | by chardet library if it fails fallback to unicode with errors replaced | |
160 |
|
160 | |||
161 | :param str_: string to decode |
|
161 | :param str_: string to decode | |
162 | :rtype: unicode |
|
162 | :rtype: unicode | |
163 | :returns: unicode object |
|
163 | :returns: unicode object | |
164 | """ |
|
164 | """ | |
165 | if isinstance(str_, unicode): |
|
165 | if isinstance(str_, unicode): | |
166 | return str_ |
|
166 | return str_ | |
167 |
|
167 | |||
168 | try: |
|
168 | try: | |
169 | return unicode(str_) |
|
169 | return unicode(str_) | |
170 | except UnicodeDecodeError: |
|
170 | except UnicodeDecodeError: | |
171 | pass |
|
171 | pass | |
172 |
|
172 | |||
173 | try: |
|
173 | try: | |
174 | return unicode(str_, from_encoding) |
|
174 | return unicode(str_, from_encoding) | |
175 | except UnicodeDecodeError: |
|
175 | except UnicodeDecodeError: | |
176 | pass |
|
176 | pass | |
177 |
|
177 | |||
178 | try: |
|
178 | try: | |
179 | import chardet |
|
179 | import chardet | |
180 | encoding = chardet.detect(str_)['encoding'] |
|
180 | encoding = chardet.detect(str_)['encoding'] | |
181 | if encoding is None: |
|
181 | if encoding is None: | |
182 | raise Exception() |
|
182 | raise Exception() | |
183 | return str_.decode(encoding) |
|
183 | return str_.decode(encoding) | |
184 | except (ImportError, UnicodeDecodeError, Exception): |
|
184 | except (ImportError, UnicodeDecodeError, Exception): | |
185 | return unicode(str_, from_encoding, 'replace') |
|
185 | return unicode(str_, from_encoding, 'replace') | |
186 |
|
186 | |||
187 | def safe_str(unicode_, to_encoding='utf8'): |
|
187 | def safe_str(unicode_, to_encoding='utf8'): | |
188 | """ |
|
188 | """ | |
189 | safe str function. Does few trick to turn unicode_ into string |
|
189 | safe str function. Does few trick to turn unicode_ into string | |
190 |
|
190 | |||
191 | In case of UnicodeEncodeError we try to return it with encoding detected |
|
191 | In case of UnicodeEncodeError we try to return it with encoding detected | |
192 | by chardet library if it fails fallback to string with errors replaced |
|
192 | by chardet library if it fails fallback to string with errors replaced | |
193 |
|
193 | |||
194 | :param unicode_: unicode to encode |
|
194 | :param unicode_: unicode to encode | |
195 | :rtype: str |
|
195 | :rtype: str | |
196 | :returns: str object |
|
196 | :returns: str object | |
197 | """ |
|
197 | """ | |
|
198 | ||||
|
199 | if not isinstance(unicode_, basestring): | |||
|
200 | return str(unicode_) | |||
198 |
|
201 | |||
199 | if isinstance(unicode_, str): |
|
202 | if isinstance(unicode_, str): | |
200 | return unicode_ |
|
203 | return unicode_ | |
201 |
|
204 | |||
202 | try: |
|
205 | try: | |
203 | return unicode_.encode(to_encoding) |
|
206 | return unicode_.encode(to_encoding) | |
204 | except UnicodeEncodeError: |
|
207 | except UnicodeEncodeError: | |
205 | pass |
|
208 | pass | |
206 |
|
209 | |||
207 | try: |
|
210 | try: | |
208 | import chardet |
|
211 | import chardet | |
209 | encoding = chardet.detect(unicode_)['encoding'] |
|
212 | encoding = chardet.detect(unicode_)['encoding'] | |
210 | print encoding |
|
213 | print encoding | |
211 | if encoding is None: |
|
214 | if encoding is None: | |
212 | raise UnicodeEncodeError() |
|
215 | raise UnicodeEncodeError() | |
213 |
|
216 | |||
214 | return unicode_.encode(encoding) |
|
217 | return unicode_.encode(encoding) | |
215 | except (ImportError, UnicodeEncodeError): |
|
218 | except (ImportError, UnicodeEncodeError): | |
216 | return unicode_.encode(to_encoding, 'replace') |
|
219 | return unicode_.encode(to_encoding, 'replace') | |
217 |
|
220 | |||
218 | return safe_str |
|
221 | return safe_str | |
219 |
|
222 | |||
220 |
|
223 | |||
221 |
|
224 | |||
222 | def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs): |
|
225 | def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs): | |
223 | """ |
|
226 | """ | |
224 | Custom engine_from_config functions that makes sure we use NullPool for |
|
227 | Custom engine_from_config functions that makes sure we use NullPool for | |
225 | file based sqlite databases. This prevents errors on sqlite. This only |
|
228 | file based sqlite databases. This prevents errors on sqlite. This only | |
226 | applies to sqlalchemy versions < 0.7.0 |
|
229 | applies to sqlalchemy versions < 0.7.0 | |
227 |
|
230 | |||
228 | """ |
|
231 | """ | |
229 | import sqlalchemy |
|
232 | import sqlalchemy | |
230 | from sqlalchemy import engine_from_config as efc |
|
233 | from sqlalchemy import engine_from_config as efc | |
231 | import logging |
|
234 | import logging | |
232 |
|
235 | |||
233 | if int(sqlalchemy.__version__.split('.')[1]) < 7: |
|
236 | if int(sqlalchemy.__version__.split('.')[1]) < 7: | |
234 |
|
237 | |||
235 | # This solution should work for sqlalchemy < 0.7.0, and should use |
|
238 | # This solution should work for sqlalchemy < 0.7.0, and should use | |
236 | # proxy=TimerProxy() for execution time profiling |
|
239 | # proxy=TimerProxy() for execution time profiling | |
237 |
|
240 | |||
238 | from sqlalchemy.pool import NullPool |
|
241 | from sqlalchemy.pool import NullPool | |
239 | url = configuration[prefix + 'url'] |
|
242 | url = configuration[prefix + 'url'] | |
240 |
|
243 | |||
241 | if url.startswith('sqlite'): |
|
244 | if url.startswith('sqlite'): | |
242 | kwargs.update({'poolclass': NullPool}) |
|
245 | kwargs.update({'poolclass': NullPool}) | |
243 | return efc(configuration, prefix, **kwargs) |
|
246 | return efc(configuration, prefix, **kwargs) | |
244 | else: |
|
247 | else: | |
245 | import time |
|
248 | import time | |
246 | from sqlalchemy import event |
|
249 | from sqlalchemy import event | |
247 | from sqlalchemy.engine import Engine |
|
250 | from sqlalchemy.engine import Engine | |
248 |
|
251 | |||
249 | log = logging.getLogger('sqlalchemy.engine') |
|
252 | log = logging.getLogger('sqlalchemy.engine') | |
250 | BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38) |
|
253 | BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = xrange(30, 38) | |
251 | engine = efc(configuration, prefix, **kwargs) |
|
254 | engine = efc(configuration, prefix, **kwargs) | |
252 |
|
255 | |||
253 | def color_sql(sql): |
|
256 | def color_sql(sql): | |
254 | COLOR_SEQ = "\033[1;%dm" |
|
257 | COLOR_SEQ = "\033[1;%dm" | |
255 | COLOR_SQL = YELLOW |
|
258 | COLOR_SQL = YELLOW | |
256 | normal = '\x1b[0m' |
|
259 | normal = '\x1b[0m' | |
257 | return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal]) |
|
260 | return ''.join([COLOR_SEQ % COLOR_SQL, sql, normal]) | |
258 |
|
261 | |||
259 | if configuration['debug']: |
|
262 | if configuration['debug']: | |
260 | #attach events only for debug configuration |
|
263 | #attach events only for debug configuration | |
261 |
|
264 | |||
262 | def before_cursor_execute(conn, cursor, statement, |
|
265 | def before_cursor_execute(conn, cursor, statement, | |
263 | parameters, context, executemany): |
|
266 | parameters, context, executemany): | |
264 | context._query_start_time = time.time() |
|
267 | context._query_start_time = time.time() | |
265 | log.info(color_sql(">>>>> STARTING QUERY >>>>>")) |
|
268 | log.info(color_sql(">>>>> STARTING QUERY >>>>>")) | |
266 |
|
269 | |||
267 |
|
270 | |||
268 | def after_cursor_execute(conn, cursor, statement, |
|
271 | def after_cursor_execute(conn, cursor, statement, | |
269 | parameters, context, executemany): |
|
272 | parameters, context, executemany): | |
270 | total = time.time() - context._query_start_time |
|
273 | total = time.time() - context._query_start_time | |
271 | log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total)) |
|
274 | log.info(color_sql("<<<<< TOTAL TIME: %f <<<<<" % total)) | |
272 |
|
275 | |||
273 | event.listen(engine, "before_cursor_execute", |
|
276 | event.listen(engine, "before_cursor_execute", | |
274 | before_cursor_execute) |
|
277 | before_cursor_execute) | |
275 | event.listen(engine, "after_cursor_execute", |
|
278 | event.listen(engine, "after_cursor_execute", | |
276 | after_cursor_execute) |
|
279 | after_cursor_execute) | |
277 |
|
280 | |||
278 | return engine |
|
281 | return engine | |
279 |
|
282 | |||
280 |
|
283 | |||
281 | def age(curdate): |
|
284 | def age(curdate): | |
282 | """ |
|
285 | """ | |
283 | turns a datetime into an age string. |
|
286 | turns a datetime into an age string. | |
284 |
|
287 | |||
285 | :param curdate: datetime object |
|
288 | :param curdate: datetime object | |
286 | :rtype: unicode |
|
289 | :rtype: unicode | |
287 | :returns: unicode words describing age |
|
290 | :returns: unicode words describing age | |
288 | """ |
|
291 | """ | |
289 |
|
292 | |||
290 | from datetime import datetime |
|
293 | from datetime import datetime | |
291 | from webhelpers.date import time_ago_in_words |
|
294 | from webhelpers.date import time_ago_in_words | |
292 |
|
295 | |||
293 | _ = lambda s:s |
|
296 | _ = lambda s:s | |
294 |
|
297 | |||
295 | if not curdate: |
|
298 | if not curdate: | |
296 | return '' |
|
299 | return '' | |
297 |
|
300 | |||
298 | agescales = [(_(u"year"), 3600 * 24 * 365), |
|
301 | agescales = [(_(u"year"), 3600 * 24 * 365), | |
299 | (_(u"month"), 3600 * 24 * 30), |
|
302 | (_(u"month"), 3600 * 24 * 30), | |
300 | (_(u"day"), 3600 * 24), |
|
303 | (_(u"day"), 3600 * 24), | |
301 | (_(u"hour"), 3600), |
|
304 | (_(u"hour"), 3600), | |
302 | (_(u"minute"), 60), |
|
305 | (_(u"minute"), 60), | |
303 | (_(u"second"), 1), ] |
|
306 | (_(u"second"), 1), ] | |
304 |
|
307 | |||
305 | age = datetime.now() - curdate |
|
308 | age = datetime.now() - curdate | |
306 | age_seconds = (age.days * agescales[2][1]) + age.seconds |
|
309 | age_seconds = (age.days * agescales[2][1]) + age.seconds | |
307 | pos = 1 |
|
310 | pos = 1 | |
308 | for scale in agescales: |
|
311 | for scale in agescales: | |
309 | if scale[1] <= age_seconds: |
|
312 | if scale[1] <= age_seconds: | |
310 | if pos == 6:pos = 5 |
|
313 | if pos == 6:pos = 5 | |
311 | return '%s %s' % (time_ago_in_words(curdate, |
|
314 | return '%s %s' % (time_ago_in_words(curdate, | |
312 | agescales[pos][0]), _('ago')) |
|
315 | agescales[pos][0]), _('ago')) | |
313 | pos += 1 |
|
316 | pos += 1 | |
314 |
|
317 | |||
315 | return _(u'just now') |
|
318 | return _(u'just now') | |
316 |
|
319 | |||
317 |
|
320 | |||
318 | def uri_filter(uri): |
|
321 | def uri_filter(uri): | |
319 | """ |
|
322 | """ | |
320 | Removes user:password from given url string |
|
323 | Removes user:password from given url string | |
321 |
|
324 | |||
322 | :param uri: |
|
325 | :param uri: | |
323 | :rtype: unicode |
|
326 | :rtype: unicode | |
324 | :returns: filtered list of strings |
|
327 | :returns: filtered list of strings | |
325 | """ |
|
328 | """ | |
326 | if not uri: |
|
329 | if not uri: | |
327 | return '' |
|
330 | return '' | |
328 |
|
331 | |||
329 | proto = '' |
|
332 | proto = '' | |
330 |
|
333 | |||
331 | for pat in ('https://', 'http://'): |
|
334 | for pat in ('https://', 'http://'): | |
332 | if uri.startswith(pat): |
|
335 | if uri.startswith(pat): | |
333 | uri = uri[len(pat):] |
|
336 | uri = uri[len(pat):] | |
334 | proto = pat |
|
337 | proto = pat | |
335 | break |
|
338 | break | |
336 |
|
339 | |||
337 | # remove passwords and username |
|
340 | # remove passwords and username | |
338 | uri = uri[uri.find('@') + 1:] |
|
341 | uri = uri[uri.find('@') + 1:] | |
339 |
|
342 | |||
340 | # get the port |
|
343 | # get the port | |
341 | cred_pos = uri.find(':') |
|
344 | cred_pos = uri.find(':') | |
342 | if cred_pos == -1: |
|
345 | if cred_pos == -1: | |
343 | host, port = uri, None |
|
346 | host, port = uri, None | |
344 | else: |
|
347 | else: | |
345 | host, port = uri[:cred_pos], uri[cred_pos + 1:] |
|
348 | host, port = uri[:cred_pos], uri[cred_pos + 1:] | |
346 |
|
349 | |||
347 | return filter(None, [proto, host, port]) |
|
350 | return filter(None, [proto, host, port]) | |
348 |
|
351 | |||
349 |
|
352 | |||
350 | def credentials_filter(uri): |
|
353 | def credentials_filter(uri): | |
351 | """ |
|
354 | """ | |
352 | Returns a url with removed credentials |
|
355 | Returns a url with removed credentials | |
353 |
|
356 | |||
354 | :param uri: |
|
357 | :param uri: | |
355 | """ |
|
358 | """ | |
356 |
|
359 | |||
357 | uri = uri_filter(uri) |
|
360 | uri = uri_filter(uri) | |
358 | #check if we have port |
|
361 | #check if we have port | |
359 | if len(uri) > 2 and uri[2]: |
|
362 | if len(uri) > 2 and uri[2]: | |
360 | uri[2] = ':' + uri[2] |
|
363 | uri[2] = ':' + uri[2] | |
361 |
|
364 | |||
362 | return ''.join(uri) |
|
365 | return ''.join(uri) | |
363 |
|
366 | |||
364 | def get_changeset_safe(repo, rev): |
|
367 | def get_changeset_safe(repo, rev): | |
365 | """ |
|
368 | """ | |
366 | Safe version of get_changeset if this changeset doesn't exists for a |
|
369 | Safe version of get_changeset if this changeset doesn't exists for a | |
367 | repo it returns a Dummy one instead |
|
370 | repo it returns a Dummy one instead | |
368 |
|
371 | |||
369 | :param repo: |
|
372 | :param repo: | |
370 | :param rev: |
|
373 | :param rev: | |
371 | """ |
|
374 | """ | |
372 | from vcs.backends.base import BaseRepository |
|
375 | from vcs.backends.base import BaseRepository | |
373 | from vcs.exceptions import RepositoryError |
|
376 | from vcs.exceptions import RepositoryError | |
374 | if not isinstance(repo, BaseRepository): |
|
377 | if not isinstance(repo, BaseRepository): | |
375 | raise Exception('You must pass an Repository ' |
|
378 | raise Exception('You must pass an Repository ' | |
376 | 'object as first argument got %s', type(repo)) |
|
379 | 'object as first argument got %s', type(repo)) | |
377 |
|
380 | |||
378 | try: |
|
381 | try: | |
379 | cs = repo.get_changeset(rev) |
|
382 | cs = repo.get_changeset(rev) | |
380 | except RepositoryError: |
|
383 | except RepositoryError: | |
381 | from rhodecode.lib.utils import EmptyChangeset |
|
384 | from rhodecode.lib.utils import EmptyChangeset | |
382 | cs = EmptyChangeset(requested_revision=rev) |
|
385 | cs = EmptyChangeset(requested_revision=rev) | |
383 | return cs |
|
386 | return cs | |
384 |
|
387 | |||
385 |
|
388 | |||
386 | def get_current_revision(quiet=False): |
|
389 | def get_current_revision(quiet=False): | |
387 | """ |
|
390 | """ | |
388 | Returns tuple of (number, id) from repository containing this package |
|
391 | Returns tuple of (number, id) from repository containing this package | |
389 | or None if repository could not be found. |
|
392 | or None if repository could not be found. | |
390 |
|
393 | |||
391 | :param quiet: prints error for fetching revision if True |
|
394 | :param quiet: prints error for fetching revision if True | |
392 | """ |
|
395 | """ | |
393 |
|
396 | |||
394 | try: |
|
397 | try: | |
395 | from vcs import get_repo |
|
398 | from vcs import get_repo | |
396 | from vcs.utils.helpers import get_scm |
|
399 | from vcs.utils.helpers import get_scm | |
397 | repopath = os.path.join(os.path.dirname(__file__), '..', '..') |
|
400 | repopath = os.path.join(os.path.dirname(__file__), '..', '..') | |
398 | scm = get_scm(repopath)[0] |
|
401 | scm = get_scm(repopath)[0] | |
399 | repo = get_repo(path=repopath, alias=scm) |
|
402 | repo = get_repo(path=repopath, alias=scm) | |
400 | tip = repo.get_changeset() |
|
403 | tip = repo.get_changeset() | |
401 | return (tip.revision, tip.short_id) |
|
404 | return (tip.revision, tip.short_id) | |
402 | except Exception, err: |
|
405 | except Exception, err: | |
403 | if not quiet: |
|
406 | if not quiet: | |
404 | print ("Cannot retrieve rhodecode's revision. Original error " |
|
407 | print ("Cannot retrieve rhodecode's revision. Original error " | |
405 | "was: %s" % err) |
|
408 | "was: %s" % err) | |
406 | return None |
|
409 | return None | |
407 |
|
410 |
@@ -1,109 +1,109 b'' | |||||
1 | # -*- coding: utf-8 -*- |
|
1 | # -*- coding: utf-8 -*- | |
2 | """ |
|
2 | """ | |
3 | rhodecode.lib.celerylib.__init__ |
|
3 | rhodecode.lib.celerylib.__init__ | |
4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
|
4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | |
5 |
|
5 | |||
6 | celery libs for RhodeCode |
|
6 | celery libs for RhodeCode | |
7 |
|
7 | |||
8 | :created_on: Nov 27, 2010 |
|
8 | :created_on: Nov 27, 2010 | |
9 | :author: marcink |
|
9 | :author: marcink | |
10 | :copyright: (C) 2009-2011 Marcin Kuzminski <marcin@python-works.com> |
|
10 | :copyright: (C) 2009-2011 Marcin Kuzminski <marcin@python-works.com> | |
11 | :license: GPLv3, see COPYING for more details. |
|
11 | :license: GPLv3, see COPYING for more details. | |
12 | """ |
|
12 | """ | |
13 | # This program is free software: you can redistribute it and/or modify |
|
13 | # This program is free software: you can redistribute it and/or modify | |
14 | # it under the terms of the GNU General Public License as published by |
|
14 | # it under the terms of the GNU General Public License as published by | |
15 | # the Free Software Foundation, either version 3 of the License, or |
|
15 | # the Free Software Foundation, either version 3 of the License, or | |
16 | # (at your option) any later version. |
|
16 | # (at your option) any later version. | |
17 | # |
|
17 | # | |
18 | # This program is distributed in the hope that it will be useful, |
|
18 | # This program is distributed in the hope that it will be useful, | |
19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
21 | # GNU General Public License for more details. |
|
21 | # GNU General Public License for more details. | |
22 | # |
|
22 | # | |
23 | # You should have received a copy of the GNU General Public License |
|
23 | # You should have received a copy of the GNU General Public License | |
24 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
24 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
25 |
|
25 | |||
26 | import os |
|
26 | import os | |
27 | import sys |
|
27 | import sys | |
28 | import socket |
|
28 | import socket | |
29 | import traceback |
|
29 | import traceback | |
30 | import logging |
|
30 | import logging | |
31 | from os.path import dirname as dn, join as jn |
|
31 | from os.path import dirname as dn, join as jn | |
32 |
|
32 | |||
33 | from hashlib import md5 |
|
33 | from hashlib import md5 | |
34 | from decorator import decorator |
|
34 | from decorator import decorator | |
35 | from pylons import config |
|
35 | from pylons import config | |
36 |
|
36 | |||
37 | from vcs.utils.lazy import LazyProperty |
|
37 | from vcs.utils.lazy import LazyProperty | |
38 |
|
38 | |||
39 | from rhodecode.lib import str2bool |
|
39 | from rhodecode.lib import str2bool, safe_str | |
40 | from rhodecode.lib.pidlock import DaemonLock, LockHeld |
|
40 | from rhodecode.lib.pidlock import DaemonLock, LockHeld | |
41 |
|
41 | |||
42 | from celery.messaging import establish_connection |
|
42 | from celery.messaging import establish_connection | |
43 |
|
43 | |||
44 |
|
44 | |||
45 | log = logging.getLogger(__name__) |
|
45 | log = logging.getLogger(__name__) | |
46 |
|
46 | |||
47 | try: |
|
47 | try: | |
48 | CELERY_ON = str2bool(config['app_conf'].get('use_celery')) |
|
48 | CELERY_ON = str2bool(config['app_conf'].get('use_celery')) | |
49 | except KeyError: |
|
49 | except KeyError: | |
50 | CELERY_ON = False |
|
50 | CELERY_ON = False | |
51 |
|
51 | |||
52 |
|
52 | |||
53 | class ResultWrapper(object): |
|
53 | class ResultWrapper(object): | |
54 | def __init__(self, task): |
|
54 | def __init__(self, task): | |
55 | self.task = task |
|
55 | self.task = task | |
56 |
|
56 | |||
57 | @LazyProperty |
|
57 | @LazyProperty | |
58 | def result(self): |
|
58 | def result(self): | |
59 | return self.task |
|
59 | return self.task | |
60 |
|
60 | |||
61 |
|
61 | |||
62 | def run_task(task, *args, **kwargs): |
|
62 | def run_task(task, *args, **kwargs): | |
63 | if CELERY_ON: |
|
63 | if CELERY_ON: | |
64 | try: |
|
64 | try: | |
65 | t = task.apply_async(args=args, kwargs=kwargs) |
|
65 | t = task.apply_async(args=args, kwargs=kwargs) | |
66 | log.info('running task %s:%s', t.task_id, task) |
|
66 | log.info('running task %s:%s', t.task_id, task) | |
67 | return t |
|
67 | return t | |
68 |
|
68 | |||
69 | except socket.error, e: |
|
69 | except socket.error, e: | |
70 | if isinstance(e, IOError) and e.errno == 111: |
|
70 | if isinstance(e, IOError) and e.errno == 111: | |
71 | log.debug('Unable to connect to celeryd. Sync execution') |
|
71 | log.debug('Unable to connect to celeryd. Sync execution') | |
72 | else: |
|
72 | else: | |
73 | log.error(traceback.format_exc()) |
|
73 | log.error(traceback.format_exc()) | |
74 | except KeyError, e: |
|
74 | except KeyError, e: | |
75 | log.debug('Unable to connect to celeryd. Sync execution') |
|
75 | log.debug('Unable to connect to celeryd. Sync execution') | |
76 | except Exception, e: |
|
76 | except Exception, e: | |
77 | log.error(traceback.format_exc()) |
|
77 | log.error(traceback.format_exc()) | |
78 |
|
78 | |||
79 | log.debug('executing task %s in sync mode', task) |
|
79 | log.debug('executing task %s in sync mode', task) | |
80 | return ResultWrapper(task(*args, **kwargs)) |
|
80 | return ResultWrapper(task(*args, **kwargs)) | |
81 |
|
81 | |||
82 |
|
82 | |||
83 | def __get_lockkey(func, *fargs, **fkwargs): |
|
83 | def __get_lockkey(func, *fargs, **fkwargs): | |
84 | params = list(fargs) |
|
84 | params = list(fargs) | |
85 | params.extend(['%s-%s' % ar for ar in fkwargs.items()]) |
|
85 | params.extend(['%s-%s' % ar for ar in fkwargs.items()]) | |
86 |
|
86 | |||
87 | func_name = str(func.__name__) if hasattr(func, '__name__') else str(func) |
|
87 | func_name = str(func.__name__) if hasattr(func, '__name__') else str(func) | |
88 |
|
88 | |||
89 | lockkey = 'task_%s.lock' % \ |
|
89 | lockkey = 'task_%s.lock' % \ | |
90 | md5(func_name + '-' + '-'.join(map(str, params))).hexdigest() |
|
90 | md5(func_name + '-' + '-'.join(map(safe_str, params))).hexdigest() | |
91 | return lockkey |
|
91 | return lockkey | |
92 |
|
92 | |||
93 |
|
93 | |||
94 | def locked_task(func): |
|
94 | def locked_task(func): | |
95 | def __wrapper(func, *fargs, **fkwargs): |
|
95 | def __wrapper(func, *fargs, **fkwargs): | |
96 | lockkey = __get_lockkey(func, *fargs, **fkwargs) |
|
96 | lockkey = __get_lockkey(func, *fargs, **fkwargs) | |
97 | lockkey_path = config['here'] |
|
97 | lockkey_path = config['here'] | |
98 |
|
98 | |||
99 | log.info('running task with lockkey %s', lockkey) |
|
99 | log.info('running task with lockkey %s', lockkey) | |
100 | try: |
|
100 | try: | |
101 | l = DaemonLock(file_=jn(lockkey_path, lockkey)) |
|
101 | l = DaemonLock(file_=jn(lockkey_path, lockkey)) | |
102 | ret = func(*fargs, **fkwargs) |
|
102 | ret = func(*fargs, **fkwargs) | |
103 | l.release() |
|
103 | l.release() | |
104 | return ret |
|
104 | return ret | |
105 | except LockHeld: |
|
105 | except LockHeld: | |
106 | log.info('LockHeld') |
|
106 | log.info('LockHeld') | |
107 | return 'Task with key %s already running' % lockkey |
|
107 | return 'Task with key %s already running' % lockkey | |
108 |
|
108 | |||
109 | return decorator(__wrapper, func) |
|
109 | return decorator(__wrapper, func) |
General Comments 0
You need to be logged in to leave comments.
Login now