Show More
@@ -1,318 +1,320 b'' | |||
|
1 | 1 | .. _changelog: |
|
2 | 2 | |
|
3 | 3 | Changelog |
|
4 | 4 | ========= |
|
5 | 5 | |
|
6 | 6 | 1.2.0 (**2011-XX-XX**) |
|
7 | 7 | ====================== |
|
8 | 8 | |
|
9 | 9 | :status: in-progress |
|
10 | 10 | :branch: beta |
|
11 | 11 | |
|
12 | 12 | news |
|
13 | 13 | ---- |
|
14 | 14 | |
|
15 | 15 | - implemented #89 Can setup google analytics code from settings menu |
|
16 | 16 | - implemented #91 added nicer looking archive urls with more download options |
|
17 | 17 | like tags, branches |
|
18 | 18 | - implemented #44 into file browsing, and added follow branch option |
|
19 | 19 | - implemented #84 downloads can be enabled/disabled for each repository |
|
20 | 20 | - anonymous repository can be cloned without having to pass default:default |
|
21 | 21 | into clone url |
|
22 | 22 | - fixed #90 whoosh indexer can index chooses repositories passed in command |
|
23 | 23 | line |
|
24 | 24 | - extended journal with day aggregates and paging |
|
25 | 25 | - implemented #107 source code lines highlight ranges |
|
26 | 26 | - implemented #93 customizable changelog on combined revision ranges - |
|
27 | 27 | equivalent of githubs compare view |
|
28 | 28 | - implemented #108 extended and more powerful LDAP configuration |
|
29 | 29 | - implemented #56 users groups |
|
30 | 30 | - major code rewrites optimized codes for speed and memory usage |
|
31 | 31 | - raw and diff downloads are now in git format |
|
32 | 32 | - setup command checks for write access to given path |
|
33 | 33 | - fixed many issues with international characters and unicode. It uses utf8 |
|
34 | 34 | decode with replace to provide less errors even with non utf8 encoded strings |
|
35 | 35 | - #125 added API KEY access to feeds |
|
36 | 36 | - #109 Repository can be created from external Mercurial link (aka. remote |
|
37 | 37 | repository, and manually updated (via pull) from admin panel |
|
38 | 38 | - beta git support - push/pull server + basic view for git repos |
|
39 | - added followers page | |
|
39 | - added followers page and forks page | |
|
40 | 40 | |
|
41 | 41 | fixes |
|
42 | 42 | ----- |
|
43 | 43 | |
|
44 | 44 | - fixed file browser bug, when switching into given form revision the url was |
|
45 | 45 | not changing |
|
46 | 46 | - fixed propagation to error controller on simplehg and simplegit middlewares |
|
47 | 47 | - fixed error when trying to make a download on empty repository |
|
48 | 48 | - fixed problem with '[' chars in commit messages in journal |
|
49 | 49 | - fixed #99 Unicode errors, on file node paths with non utf-8 characters |
|
50 | 50 | - journal fork fixes |
|
51 | 51 | - removed issue with space inside renamed repository after deletion |
|
52 | 52 | - fixed strange issue on formencode imports |
|
53 | 53 | - fixed #126 Deleting repository on Windows, rename used incompatible chars. |
|
54 | 54 | - #150 fixes for errors on repositories mapped in db but corrupted in |
|
55 | 55 | filesystem |
|
56 | 56 | - fixed problem with ascendant characters in realm #181 |
|
57 | - fixed problem with sqlite file based database connection pool | |
|
58 | - whoosh indexer and code stats share the same dynamic extensions map | |
|
57 | 59 | |
|
58 | 60 | 1.1.8 (**2011-04-12**) |
|
59 | 61 | ====================== |
|
60 | 62 | |
|
61 | 63 | news |
|
62 | 64 | ---- |
|
63 | 65 | |
|
64 | 66 | - improved windows support |
|
65 | 67 | |
|
66 | 68 | fixes |
|
67 | 69 | ----- |
|
68 | 70 | |
|
69 | 71 | - fixed #140 freeze of python dateutil library, since new version is python2.x |
|
70 | 72 | incompatible |
|
71 | 73 | - setup-app will check for write permission in given path |
|
72 | 74 | - cleaned up license info issue #149 |
|
73 | 75 | - fixes for issues #137,#116 and problems with unicode and accented characters. |
|
74 | 76 | - fixes crashes on gravatar, when passed in email as unicode |
|
75 | 77 | - fixed tooltip flickering problems |
|
76 | 78 | - fixed came_from redirection on windows |
|
77 | 79 | - fixed logging modules, and sql formatters |
|
78 | 80 | - windows fixes for os.kill issue #133 |
|
79 | 81 | - fixes path splitting for windows issues #148 |
|
80 | 82 | - fixed issue #143 wrong import on migration to 1.1.X |
|
81 | 83 | - fixed problems with displaying binary files, thanks to Thomas Waldmann |
|
82 | 84 | - removed name from archive files since it's breaking ui for long repo names |
|
83 | 85 | - fixed issue with archive headers sent to browser, thanks to Thomas Waldmann |
|
84 | 86 | - fixed compatibility for 1024px displays, and larger dpi settings, thanks to |
|
85 | 87 | Thomas Waldmann |
|
86 | 88 | - fixed issue #166 summary pager was skipping 10 revisions on second page |
|
87 | 89 | |
|
88 | 90 | |
|
89 | 91 | 1.1.7 (**2011-03-23**) |
|
90 | 92 | ====================== |
|
91 | 93 | |
|
92 | 94 | news |
|
93 | 95 | ---- |
|
94 | 96 | |
|
95 | 97 | fixes |
|
96 | 98 | ----- |
|
97 | 99 | |
|
98 | 100 | - fixed (again) #136 installation support for FreeBSD |
|
99 | 101 | |
|
100 | 102 | |
|
101 | 103 | 1.1.6 (**2011-03-21**) |
|
102 | 104 | ====================== |
|
103 | 105 | |
|
104 | 106 | news |
|
105 | 107 | ---- |
|
106 | 108 | |
|
107 | 109 | fixes |
|
108 | 110 | ----- |
|
109 | 111 | |
|
110 | 112 | - fixed #136 installation support for FreeBSD |
|
111 | 113 | - RhodeCode will check for python version during installation |
|
112 | 114 | |
|
113 | 115 | 1.1.5 (**2011-03-17**) |
|
114 | 116 | ====================== |
|
115 | 117 | |
|
116 | 118 | news |
|
117 | 119 | ---- |
|
118 | 120 | |
|
119 | 121 | - basic windows support, by exchanging pybcrypt into sha256 for windows only |
|
120 | 122 | highly inspired by idea of mantis406 |
|
121 | 123 | |
|
122 | 124 | fixes |
|
123 | 125 | ----- |
|
124 | 126 | |
|
125 | 127 | - fixed sorting by author in main page |
|
126 | 128 | - fixed crashes with diffs on binary files |
|
127 | 129 | - fixed #131 problem with boolean values for LDAP |
|
128 | 130 | - fixed #122 mysql problems thanks to striker69 |
|
129 | 131 | - fixed problem with errors on calling raw/raw_files/annotate functions |
|
130 | 132 | with unknown revisions |
|
131 | 133 | - fixed returned rawfiles attachment names with international character |
|
132 | 134 | - cleaned out docs, big thanks to Jason Harris |
|
133 | 135 | |
|
134 | 136 | 1.1.4 (**2011-02-19**) |
|
135 | 137 | ====================== |
|
136 | 138 | |
|
137 | 139 | news |
|
138 | 140 | ---- |
|
139 | 141 | |
|
140 | 142 | fixes |
|
141 | 143 | ----- |
|
142 | 144 | |
|
143 | 145 | - fixed formencode import problem on settings page, that caused server crash |
|
144 | 146 | when that page was accessed as first after server start |
|
145 | 147 | - journal fixes |
|
146 | 148 | - fixed option to access repository just by entering http://server/<repo_name> |
|
147 | 149 | |
|
148 | 150 | 1.1.3 (**2011-02-16**) |
|
149 | 151 | ====================== |
|
150 | 152 | |
|
151 | 153 | news |
|
152 | 154 | ---- |
|
153 | 155 | |
|
154 | 156 | - implemented #102 allowing the '.' character in username |
|
155 | 157 | - added option to access repository just by entering http://server/<repo_name> |
|
156 | 158 | - celery task ignores result for better performance |
|
157 | 159 | |
|
158 | 160 | fixes |
|
159 | 161 | ----- |
|
160 | 162 | |
|
161 | 163 | - fixed ehlo command and non auth mail servers on smtp_lib. Thanks to |
|
162 | 164 | apollo13 and Johan Walles |
|
163 | 165 | - small fixes in journal |
|
164 | 166 | - fixed problems with getting setting for celery from .ini files |
|
165 | 167 | - registration, password reset and login boxes share the same title as main |
|
166 | 168 | application now |
|
167 | 169 | - fixed #113: to high permissions to fork repository |
|
168 | 170 | - fixed problem with '[' chars in commit messages in journal |
|
169 | 171 | - removed issue with space inside renamed repository after deletion |
|
170 | 172 | - db transaction fixes when filesystem repository creation failed |
|
171 | 173 | - fixed #106 relation issues on databases different than sqlite |
|
172 | 174 | - fixed static files paths links to use of url() method |
|
173 | 175 | |
|
174 | 176 | 1.1.2 (**2011-01-12**) |
|
175 | 177 | ====================== |
|
176 | 178 | |
|
177 | 179 | news |
|
178 | 180 | ---- |
|
179 | 181 | |
|
180 | 182 | |
|
181 | 183 | fixes |
|
182 | 184 | ----- |
|
183 | 185 | |
|
184 | 186 | - fixes #98 protection against float division of percentage stats |
|
185 | 187 | - fixed graph bug |
|
186 | 188 | - forced webhelpers version since it was making troubles during installation |
|
187 | 189 | |
|
188 | 190 | 1.1.1 (**2011-01-06**) |
|
189 | 191 | ====================== |
|
190 | 192 | |
|
191 | 193 | news |
|
192 | 194 | ---- |
|
193 | 195 | |
|
194 | 196 | - added force https option into ini files for easier https usage (no need to |
|
195 | 197 | set server headers with this options) |
|
196 | 198 | - small css updates |
|
197 | 199 | |
|
198 | 200 | fixes |
|
199 | 201 | ----- |
|
200 | 202 | |
|
201 | 203 | - fixed #96 redirect loop on files view on repositories without changesets |
|
202 | 204 | - fixed #97 unicode string passed into server header in special cases (mod_wsgi) |
|
203 | 205 | and server crashed with errors |
|
204 | 206 | - fixed large tooltips problems on main page |
|
205 | 207 | - fixed #92 whoosh indexer is more error proof |
|
206 | 208 | |
|
207 | 209 | 1.1.0 (**2010-12-18**) |
|
208 | 210 | ====================== |
|
209 | 211 | |
|
210 | 212 | news |
|
211 | 213 | ---- |
|
212 | 214 | |
|
213 | 215 | - rewrite of internals for vcs >=0.1.10 |
|
214 | 216 | - uses mercurial 1.7 with dotencode disabled for maintaining compatibility |
|
215 | 217 | with older clients |
|
216 | 218 | - anonymous access, authentication via ldap |
|
217 | 219 | - performance upgrade for cached repos list - each repository has it's own |
|
218 | 220 | cache that's invalidated when needed. |
|
219 | 221 | - performance upgrades on repositories with large amount of commits (20K+) |
|
220 | 222 | - main page quick filter for filtering repositories |
|
221 | 223 | - user dashboards with ability to follow chosen repositories actions |
|
222 | 224 | - sends email to admin on new user registration |
|
223 | 225 | - added cache/statistics reset options into repository settings |
|
224 | 226 | - more detailed action logger (based on hooks) with pushed changesets lists |
|
225 | 227 | and options to disable those hooks from admin panel |
|
226 | 228 | - introduced new enhanced changelog for merges that shows more accurate results |
|
227 | 229 | - new improved and faster code stats (based on pygments lexers mapping tables, |
|
228 | 230 | showing up to 10 trending sources for each repository. Additionally stats |
|
229 | 231 | can be disabled in repository settings. |
|
230 | 232 | - gui optimizations, fixed application width to 1024px |
|
231 | 233 | - added cut off (for large files/changesets) limit into config files |
|
232 | 234 | - whoosh, celeryd, upgrade moved to paster command |
|
233 | 235 | - other than sqlite database backends can be used |
|
234 | 236 | |
|
235 | 237 | fixes |
|
236 | 238 | ----- |
|
237 | 239 | |
|
238 | 240 | - fixes #61 forked repo was showing only after cache expired |
|
239 | 241 | - fixes #76 no confirmation on user deletes |
|
240 | 242 | - fixes #66 Name field misspelled |
|
241 | 243 | - fixes #72 block user removal when he owns repositories |
|
242 | 244 | - fixes #69 added password confirmation fields |
|
243 | 245 | - fixes #87 RhodeCode crashes occasionally on updating repository owner |
|
244 | 246 | - fixes #82 broken annotations on files with more than 1 blank line at the end |
|
245 | 247 | - a lot of fixes and tweaks for file browser |
|
246 | 248 | - fixed detached session issues |
|
247 | 249 | - fixed when user had no repos he would see all repos listed in my account |
|
248 | 250 | - fixed ui() instance bug when global hgrc settings was loaded for server |
|
249 | 251 | instance and all hgrc options were merged with our db ui() object |
|
250 | 252 | - numerous small bugfixes |
|
251 | 253 | |
|
252 | 254 | (special thanks for TkSoh for detailed feedback) |
|
253 | 255 | |
|
254 | 256 | |
|
255 | 257 | 1.0.2 (**2010-11-12**) |
|
256 | 258 | ====================== |
|
257 | 259 | |
|
258 | 260 | news |
|
259 | 261 | ---- |
|
260 | 262 | |
|
261 | 263 | - tested under python2.7 |
|
262 | 264 | - bumped sqlalchemy and celery versions |
|
263 | 265 | |
|
264 | 266 | fixes |
|
265 | 267 | ----- |
|
266 | 268 | |
|
267 | 269 | - fixed #59 missing graph.js |
|
268 | 270 | - fixed repo_size crash when repository had broken symlinks |
|
269 | 271 | - fixed python2.5 crashes. |
|
270 | 272 | |
|
271 | 273 | |
|
272 | 274 | 1.0.1 (**2010-11-10**) |
|
273 | 275 | ====================== |
|
274 | 276 | |
|
275 | 277 | news |
|
276 | 278 | ---- |
|
277 | 279 | |
|
278 | 280 | - small css updated |
|
279 | 281 | |
|
280 | 282 | fixes |
|
281 | 283 | ----- |
|
282 | 284 | |
|
283 | 285 | - fixed #53 python2.5 incompatible enumerate calls |
|
284 | 286 | - fixed #52 disable mercurial extension for web |
|
285 | 287 | - fixed #51 deleting repositories don't delete it's dependent objects |
|
286 | 288 | |
|
287 | 289 | |
|
288 | 290 | 1.0.0 (**2010-11-02**) |
|
289 | 291 | ====================== |
|
290 | 292 | |
|
291 | 293 | - security bugfix simplehg wasn't checking for permissions on commands |
|
292 | 294 | other than pull or push. |
|
293 | 295 | - fixed doubled messages after push or pull in admin journal |
|
294 | 296 | - templating and css corrections, fixed repo switcher on chrome, updated titles |
|
295 | 297 | - admin menu accessible from options menu on repository view |
|
296 | 298 | - permissions cached queries |
|
297 | 299 | |
|
298 | 300 | 1.0.0rc4 (**2010-10-12**) |
|
299 | 301 | ========================== |
|
300 | 302 | |
|
301 | 303 | - fixed python2.5 missing simplejson imports (thanks to Jens BΓ€ckman) |
|
302 | 304 | - removed cache_manager settings from sqlalchemy meta |
|
303 | 305 | - added sqlalchemy cache settings to ini files |
|
304 | 306 | - validated password length and added second try of failure on paster setup-app |
|
305 | 307 | - fixed setup database destroy prompt even when there was no db |
|
306 | 308 | |
|
307 | 309 | |
|
308 | 310 | 1.0.0rc3 (**2010-10-11**) |
|
309 | 311 | ========================= |
|
310 | 312 | |
|
311 | 313 | - fixed i18n during installation. |
|
312 | 314 | |
|
313 | 315 | 1.0.0rc2 (**2010-10-11**) |
|
314 | 316 | ========================= |
|
315 | 317 | |
|
316 | 318 | - Disabled dirsize in file browser, it's causing nasty bug when dir renames |
|
317 | 319 | occure. After vcs is fixed it'll be put back again. |
|
318 | 320 | - templating/css rewrites, optimized css. No newline at end of file |
@@ -1,100 +1,142 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | """ |
|
3 | 3 | rhodecode.lib.__init__ |
|
4 | 4 | ~~~~~~~~~~~~~~~~~~~~~~~ |
|
5 | 5 | |
|
6 | 6 | Some simple helper functions |
|
7 | 7 | |
|
8 | 8 | :created_on: Jan 5, 2011 |
|
9 | 9 | :author: marcink |
|
10 | 10 | :copyright: (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com> |
|
11 | 11 | :license: GPLv3, see COPYING for more details. |
|
12 | 12 | """ |
|
13 | 13 | # This program is free software: you can redistribute it and/or modify |
|
14 | 14 | # it under the terms of the GNU General Public License as published by |
|
15 | 15 | # the Free Software Foundation, either version 3 of the License, or |
|
16 | 16 | # (at your option) any later version. |
|
17 | 17 | # |
|
18 | 18 | # This program is distributed in the hope that it will be useful, |
|
19 | 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
20 | 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
21 | 21 | # GNU General Public License for more details. |
|
22 | 22 | # |
|
23 | 23 | # You should have received a copy of the GNU General Public License |
|
24 | 24 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
25 | 25 | |
|
26 | 26 | |
|
27 | ||
|
28 | def __get_lem(): | |
|
29 | from pygments import lexers | |
|
30 | from string import lower | |
|
31 | from collections import defaultdict | |
|
32 | ||
|
33 | d = defaultdict(lambda: []) | |
|
34 | ||
|
35 | def __clean(s): | |
|
36 | s = s.lstrip('*') | |
|
37 | s = s.lstrip('.') | |
|
38 | ||
|
39 | if s.find('[') != -1: | |
|
40 | exts = [] | |
|
41 | start, stop = s.find('['), s.find(']') | |
|
42 | ||
|
43 | for suffix in s[start + 1:stop]: | |
|
44 | exts.append(s[:s.find('[')] + suffix) | |
|
45 | return map(lower, exts) | |
|
46 | else: | |
|
47 | return map(lower, [s]) | |
|
48 | ||
|
49 | for lx, t in sorted(lexers.LEXERS.items()): | |
|
50 | m = map(__clean, t[-2]) | |
|
51 | if m: | |
|
52 | m = reduce(lambda x, y: x + y, m) | |
|
53 | for ext in m: | |
|
54 | desc = lx.replace('Lexer', '') | |
|
55 | d[ext].append(desc) | |
|
56 | ||
|
57 | return dict(d) | |
|
58 | ||
|
59 | # language map is also used by whoosh indexer, which for those specified | |
|
60 | # extensions will index it's content | |
|
61 | LANGUAGES_EXTENSIONS_MAP = __get_lem() | |
|
62 | ||
|
63 | #Additional mappings that are not present in the pygments lexers | |
|
64 | # NOTE: that this will overide any mappings in LANGUAGES_EXTENSIONS_MAP | |
|
65 | ADDITIONAL_MAPPINGS = {'xaml': 'XAML'} | |
|
66 | ||
|
67 | LANGUAGES_EXTENSIONS_MAP.update(ADDITIONAL_MAPPINGS) | |
|
68 | ||
|
27 | 69 | def str2bool(_str): |
|
28 | 70 | """ |
|
29 | 71 | returs True/False value from given string, it tries to translate the |
|
30 | 72 | string into boolean |
|
31 | 73 | |
|
32 | 74 | :param _str: string value to translate into boolean |
|
33 | 75 | :rtype: boolean |
|
34 | 76 | :returns: boolean from given string |
|
35 | 77 | """ |
|
36 | 78 | if _str is None: |
|
37 | 79 | return False |
|
38 | 80 | if _str in (True, False): |
|
39 | 81 | return _str |
|
40 | 82 | _str = str(_str).strip().lower() |
|
41 | 83 | return _str in ('t', 'true', 'y', 'yes', 'on', '1') |
|
42 | 84 | |
|
43 | 85 | |
|
44 | 86 | def generate_api_key(username, salt=None): |
|
45 | 87 | """ |
|
46 | 88 | Generates unique API key for given username,if salt is not given |
|
47 | 89 | it'll be generated from some random string |
|
48 | 90 | |
|
49 | 91 | :param username: username as string |
|
50 | 92 | :param salt: salt to hash generate KEY |
|
51 | 93 | :rtype: str |
|
52 | 94 | :returns: sha1 hash from username+salt |
|
53 | 95 | """ |
|
54 | 96 | from tempfile import _RandomNameSequence |
|
55 | 97 | import hashlib |
|
56 | 98 | |
|
57 | 99 | if salt is None: |
|
58 | 100 | salt = _RandomNameSequence().next() |
|
59 | 101 | |
|
60 | 102 | return hashlib.sha1(username + salt).hexdigest() |
|
61 | 103 | |
|
62 | 104 | |
|
63 | 105 | def safe_unicode(_str, from_encoding='utf8'): |
|
64 | 106 | """ |
|
65 | 107 | safe unicode function. In case of UnicodeDecode error we try to return |
|
66 | 108 | unicode with errors replace |
|
67 | 109 | |
|
68 | 110 | :param _str: string to decode |
|
69 | 111 | :rtype: unicode |
|
70 | 112 | :returns: unicode object |
|
71 | 113 | """ |
|
72 | 114 | |
|
73 | 115 | if isinstance(_str, unicode): |
|
74 | 116 | return _str |
|
75 | 117 | |
|
76 | 118 | try: |
|
77 | 119 | u_str = unicode(_str, from_encoding) |
|
78 | 120 | except UnicodeDecodeError: |
|
79 | 121 | u_str = unicode(_str, from_encoding, 'replace') |
|
80 | 122 | |
|
81 | 123 | return u_str |
|
82 | 124 | |
|
83 | 125 | |
|
84 | 126 | def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs): |
|
85 | 127 | """ |
|
86 | 128 | Custom engine_from_config functions that makes sure we use NullPool for |
|
87 | 129 | file based sqlite databases. This prevents errors on sqlite. |
|
88 | 130 | |
|
89 | 131 | """ |
|
90 | 132 | from sqlalchemy import engine_from_config as efc |
|
91 | 133 | from sqlalchemy.pool import NullPool |
|
92 | 134 | |
|
93 | 135 | url = configuration[prefix + 'url'] |
|
94 | 136 | |
|
95 | 137 | if url.startswith('sqlite'): |
|
96 | 138 | kwargs.update({'poolclass':NullPool}) |
|
97 | 139 | |
|
98 | 140 | return efc(configuration, prefix, **kwargs) |
|
99 | 141 | |
|
100 | 142 |
@@ -1,409 +1,374 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | """ |
|
3 | 3 | rhodecode.lib.celerylib.tasks |
|
4 | 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
|
5 | 5 | |
|
6 | 6 | RhodeCode task modules, containing all task that suppose to be run |
|
7 | 7 | by celery daemon |
|
8 | 8 | |
|
9 | 9 | :created_on: Oct 6, 2010 |
|
10 | 10 | :author: marcink |
|
11 | 11 | :copyright: (C) 2009-2011 Marcin Kuzminski <marcin@python-works.com> |
|
12 | 12 | :license: GPLv3, see COPYING for more details. |
|
13 | 13 | """ |
|
14 | 14 | # This program is free software: you can redistribute it and/or modify |
|
15 | 15 | # it under the terms of the GNU General Public License as published by |
|
16 | 16 | # the Free Software Foundation, either version 3 of the License, or |
|
17 | 17 | # (at your option) any later version. |
|
18 | 18 | # |
|
19 | 19 | # This program is distributed in the hope that it will be useful, |
|
20 | 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
21 | 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
22 | 22 | # GNU General Public License for more details. |
|
23 | 23 | # |
|
24 | 24 | # You should have received a copy of the GNU General Public License |
|
25 | 25 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
26 | 26 | from celery.decorators import task |
|
27 | 27 | |
|
28 | 28 | import os |
|
29 | 29 | import traceback |
|
30 | 30 | import logging |
|
31 | 31 | |
|
32 | 32 | from time import mktime |
|
33 | 33 | from operator import itemgetter |
|
34 | from pygments import lexers | |
|
35 | 34 | from string import lower |
|
36 | 35 | |
|
37 | 36 | from pylons import config |
|
38 | 37 | from pylons.i18n.translation import _ |
|
39 | 38 | |
|
39 | from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP | |
|
40 | 40 | from rhodecode.lib.celerylib import run_task, locked_task, str2bool, \ |
|
41 | 41 | __get_lockkey, LockHeld, DaemonLock |
|
42 | 42 | from rhodecode.lib.helpers import person |
|
43 | 43 | from rhodecode.lib.smtp_mailer import SmtpMailer |
|
44 | 44 | from rhodecode.lib.utils import OrderedDict, add_cache |
|
45 | 45 | from rhodecode.model import init_model |
|
46 | 46 | from rhodecode.model import meta |
|
47 | 47 | from rhodecode.model.db import RhodeCodeUi, Statistics, Repository |
|
48 | 48 | |
|
49 | 49 | from vcs.backends import get_repo |
|
50 | 50 | |
|
51 | 51 | from sqlalchemy import engine_from_config |
|
52 | 52 | |
|
53 | 53 | add_cache(config) |
|
54 | 54 | |
|
55 | 55 | try: |
|
56 | 56 | import json |
|
57 | 57 | except ImportError: |
|
58 | 58 | #python 2.5 compatibility |
|
59 | 59 | import simplejson as json |
|
60 | 60 | |
|
61 | 61 | __all__ = ['whoosh_index', 'get_commits_stats', |
|
62 | 62 | 'reset_user_password', 'send_email'] |
|
63 | 63 | |
|
64 | 64 | CELERY_ON = str2bool(config['app_conf'].get('use_celery')) |
|
65 | 65 | |
|
66 | LANGUAGES_EXTENSIONS_MAP = {} | |
|
67 | ||
|
68 | ||
|
69 | def __clean(s): | |
|
70 | ||
|
71 | s = s.lstrip('*') | |
|
72 | s = s.lstrip('.') | |
|
73 | ||
|
74 | if s.find('[') != -1: | |
|
75 | exts = [] | |
|
76 | start, stop = s.find('['), s.find(']') | |
|
77 | ||
|
78 | for suffix in s[start + 1:stop]: | |
|
79 | exts.append(s[:s.find('[')] + suffix) | |
|
80 | return map(lower, exts) | |
|
81 | else: | |
|
82 | return map(lower, [s]) | |
|
83 | ||
|
84 | for lx, t in sorted(lexers.LEXERS.items()): | |
|
85 | m = map(__clean, t[-2]) | |
|
86 | if m: | |
|
87 | m = reduce(lambda x, y: x + y, m) | |
|
88 | for ext in m: | |
|
89 | desc = lx.replace('Lexer', '') | |
|
90 | if ext in LANGUAGES_EXTENSIONS_MAP: | |
|
91 | if desc not in LANGUAGES_EXTENSIONS_MAP[ext]: | |
|
92 | LANGUAGES_EXTENSIONS_MAP[ext].append(desc) | |
|
93 | else: | |
|
94 | LANGUAGES_EXTENSIONS_MAP[ext] = [desc] | |
|
95 | ||
|
96 | #Additional mappings that are not present in the pygments lexers | |
|
97 | # NOTE: that this will overide any mappings in LANGUAGES_EXTENSIONS_MAP | |
|
98 | ADDITIONAL_MAPPINGS = {'xaml': 'XAML'} | |
|
99 | ||
|
100 | LANGUAGES_EXTENSIONS_MAP.update(ADDITIONAL_MAPPINGS) | |
|
101 | 66 | |
|
102 | 67 | |
|
103 | 68 | def get_session(): |
|
104 | 69 | if CELERY_ON: |
|
105 | 70 | engine = engine_from_config(config, 'sqlalchemy.db1.') |
|
106 | 71 | init_model(engine) |
|
107 | 72 | sa = meta.Session() |
|
108 | 73 | return sa |
|
109 | 74 | |
|
110 | 75 | |
|
111 | 76 | def get_repos_path(): |
|
112 | 77 | sa = get_session() |
|
113 | 78 | q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one() |
|
114 | 79 | return q.ui_value |
|
115 | 80 | |
|
116 | 81 | |
|
117 | 82 | @task(ignore_result=True) |
|
118 | 83 | @locked_task |
|
119 | 84 | def whoosh_index(repo_location, full_index): |
|
120 | 85 | #log = whoosh_index.get_logger() |
|
121 | 86 | from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon |
|
122 | 87 | index_location = config['index_dir'] |
|
123 | 88 | WhooshIndexingDaemon(index_location=index_location, |
|
124 | 89 | repo_location=repo_location, sa=get_session())\ |
|
125 | 90 | .run(full_index=full_index) |
|
126 | 91 | |
|
127 | 92 | |
|
128 | 93 | @task(ignore_result=True) |
|
129 | 94 | def get_commits_stats(repo_name, ts_min_y, ts_max_y): |
|
130 | 95 | try: |
|
131 | 96 | log = get_commits_stats.get_logger() |
|
132 | 97 | except: |
|
133 | 98 | log = logging.getLogger(__name__) |
|
134 | 99 | |
|
135 | 100 | lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y, |
|
136 | 101 | ts_max_y) |
|
137 | 102 | log.info('running task with lockkey %s', lockkey) |
|
138 | 103 | try: |
|
139 | 104 | lock = DaemonLock(lockkey) |
|
140 | 105 | |
|
141 | 106 | #for js data compatibilty cleans the key for person from ' |
|
142 | 107 | akc = lambda k: person(k).replace('"', "") |
|
143 | 108 | |
|
144 | 109 | co_day_auth_aggr = {} |
|
145 | 110 | commits_by_day_aggregate = {} |
|
146 | 111 | repos_path = get_repos_path() |
|
147 | 112 | p = os.path.join(repos_path, repo_name) |
|
148 | 113 | repo = get_repo(p) |
|
149 | 114 | repo_size = len(repo.revisions) |
|
150 | 115 | #return if repo have no revisions |
|
151 | 116 | if repo_size < 1: |
|
152 | 117 | lock.release() |
|
153 | 118 | return True |
|
154 | 119 | |
|
155 | 120 | skip_date_limit = True |
|
156 | 121 | parse_limit = int(config['app_conf'].get('commit_parse_limit')) |
|
157 | 122 | last_rev = 0 |
|
158 | 123 | last_cs = None |
|
159 | 124 | timegetter = itemgetter('time') |
|
160 | 125 | |
|
161 | 126 | sa = get_session() |
|
162 | 127 | |
|
163 | 128 | dbrepo = sa.query(Repository)\ |
|
164 | 129 | .filter(Repository.repo_name == repo_name).scalar() |
|
165 | 130 | cur_stats = sa.query(Statistics)\ |
|
166 | 131 | .filter(Statistics.repository == dbrepo).scalar() |
|
167 | 132 | |
|
168 | 133 | if cur_stats is not None: |
|
169 | 134 | last_rev = cur_stats.stat_on_revision |
|
170 | 135 | |
|
171 | 136 | if last_rev == repo.get_changeset().revision and repo_size > 1: |
|
172 | 137 | #pass silently without any work if we're not on first revision or |
|
173 | 138 | #current state of parsing revision(from db marker) is the |
|
174 | 139 | #last revision |
|
175 | 140 | lock.release() |
|
176 | 141 | return True |
|
177 | 142 | |
|
178 | 143 | if cur_stats: |
|
179 | 144 | commits_by_day_aggregate = OrderedDict(json.loads( |
|
180 | 145 | cur_stats.commit_activity_combined)) |
|
181 | 146 | co_day_auth_aggr = json.loads(cur_stats.commit_activity) |
|
182 | 147 | |
|
183 | 148 | log.debug('starting parsing %s', parse_limit) |
|
184 | 149 | lmktime = mktime |
|
185 | 150 | |
|
186 | 151 | last_rev = last_rev + 1 if last_rev > 0 else last_rev |
|
187 | 152 | |
|
188 | 153 | for cs in repo[last_rev:last_rev + parse_limit]: |
|
189 | 154 | last_cs = cs # remember last parsed changeset |
|
190 | 155 | k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1], |
|
191 | 156 | cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0]) |
|
192 | 157 | |
|
193 | 158 | if akc(cs.author) in co_day_auth_aggr: |
|
194 | 159 | try: |
|
195 | 160 | l = [timegetter(x) for x in |
|
196 | 161 | co_day_auth_aggr[akc(cs.author)]['data']] |
|
197 | 162 | time_pos = l.index(k) |
|
198 | 163 | except ValueError: |
|
199 | 164 | time_pos = False |
|
200 | 165 | |
|
201 | 166 | if time_pos >= 0 and time_pos is not False: |
|
202 | 167 | |
|
203 | 168 | datadict = \ |
|
204 | 169 | co_day_auth_aggr[akc(cs.author)]['data'][time_pos] |
|
205 | 170 | |
|
206 | 171 | datadict["commits"] += 1 |
|
207 | 172 | datadict["added"] += len(cs.added) |
|
208 | 173 | datadict["changed"] += len(cs.changed) |
|
209 | 174 | datadict["removed"] += len(cs.removed) |
|
210 | 175 | |
|
211 | 176 | else: |
|
212 | 177 | if k >= ts_min_y and k <= ts_max_y or skip_date_limit: |
|
213 | 178 | |
|
214 | 179 | datadict = {"time": k, |
|
215 | 180 | "commits": 1, |
|
216 | 181 | "added": len(cs.added), |
|
217 | 182 | "changed": len(cs.changed), |
|
218 | 183 | "removed": len(cs.removed), |
|
219 | 184 | } |
|
220 | 185 | co_day_auth_aggr[akc(cs.author)]['data']\ |
|
221 | 186 | .append(datadict) |
|
222 | 187 | |
|
223 | 188 | else: |
|
224 | 189 | if k >= ts_min_y and k <= ts_max_y or skip_date_limit: |
|
225 | 190 | co_day_auth_aggr[akc(cs.author)] = { |
|
226 | 191 | "label": akc(cs.author), |
|
227 | 192 | "data": [{"time":k, |
|
228 | 193 | "commits":1, |
|
229 | 194 | "added":len(cs.added), |
|
230 | 195 | "changed":len(cs.changed), |
|
231 | 196 | "removed":len(cs.removed), |
|
232 | 197 | }], |
|
233 | 198 | "schema": ["commits"], |
|
234 | 199 | } |
|
235 | 200 | |
|
236 | 201 | #gather all data by day |
|
237 | 202 | if k in commits_by_day_aggregate: |
|
238 | 203 | commits_by_day_aggregate[k] += 1 |
|
239 | 204 | else: |
|
240 | 205 | commits_by_day_aggregate[k] = 1 |
|
241 | 206 | |
|
242 | 207 | overview_data = sorted(commits_by_day_aggregate.items(), |
|
243 | 208 | key=itemgetter(0)) |
|
244 | 209 | |
|
245 | 210 | if not co_day_auth_aggr: |
|
246 | 211 | co_day_auth_aggr[akc(repo.contact)] = { |
|
247 | 212 | "label": akc(repo.contact), |
|
248 | 213 | "data": [0, 1], |
|
249 | 214 | "schema": ["commits"], |
|
250 | 215 | } |
|
251 | 216 | |
|
252 | 217 | stats = cur_stats if cur_stats else Statistics() |
|
253 | 218 | stats.commit_activity = json.dumps(co_day_auth_aggr) |
|
254 | 219 | stats.commit_activity_combined = json.dumps(overview_data) |
|
255 | 220 | |
|
256 | 221 | log.debug('last revison %s', last_rev) |
|
257 | 222 | leftovers = len(repo.revisions[last_rev:]) |
|
258 | 223 | log.debug('revisions to parse %s', leftovers) |
|
259 | 224 | |
|
260 | 225 | if last_rev == 0 or leftovers < parse_limit: |
|
261 | 226 | log.debug('getting code trending stats') |
|
262 | 227 | stats.languages = json.dumps(__get_codes_stats(repo_name)) |
|
263 | 228 | |
|
264 | 229 | try: |
|
265 | 230 | stats.repository = dbrepo |
|
266 | 231 | stats.stat_on_revision = last_cs.revision if last_cs else 0 |
|
267 | 232 | sa.add(stats) |
|
268 | 233 | sa.commit() |
|
269 | 234 | except: |
|
270 | 235 | log.error(traceback.format_exc()) |
|
271 | 236 | sa.rollback() |
|
272 | 237 | lock.release() |
|
273 | 238 | return False |
|
274 | 239 | |
|
275 | 240 | #final release |
|
276 | 241 | lock.release() |
|
277 | 242 | |
|
278 | 243 | #execute another task if celery is enabled |
|
279 | 244 | if len(repo.revisions) > 1 and CELERY_ON: |
|
280 | 245 | run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y) |
|
281 | 246 | return True |
|
282 | 247 | except LockHeld: |
|
283 | 248 | log.info('LockHeld') |
|
284 | 249 | return 'Task with key %s already running' % lockkey |
|
285 | 250 | |
|
286 | 251 | |
|
287 | 252 | @task(ignore_result=True) |
|
288 | 253 | def reset_user_password(user_email): |
|
289 | 254 | try: |
|
290 | 255 | log = reset_user_password.get_logger() |
|
291 | 256 | except: |
|
292 | 257 | log = logging.getLogger(__name__) |
|
293 | 258 | |
|
294 | 259 | from rhodecode.lib import auth |
|
295 | 260 | from rhodecode.model.db import User |
|
296 | 261 | |
|
297 | 262 | try: |
|
298 | 263 | try: |
|
299 | 264 | sa = get_session() |
|
300 | 265 | user = sa.query(User).filter(User.email == user_email).scalar() |
|
301 | 266 | new_passwd = auth.PasswordGenerator().gen_password(8, |
|
302 | 267 | auth.PasswordGenerator.ALPHABETS_BIG_SMALL) |
|
303 | 268 | if user: |
|
304 | 269 | user.password = auth.get_crypt_password(new_passwd) |
|
305 | 270 | user.api_key = auth.generate_api_key(user.username) |
|
306 | 271 | sa.add(user) |
|
307 | 272 | sa.commit() |
|
308 | 273 | log.info('change password for %s', user_email) |
|
309 | 274 | if new_passwd is None: |
|
310 | 275 | raise Exception('unable to generate new password') |
|
311 | 276 | |
|
312 | 277 | except: |
|
313 | 278 | log.error(traceback.format_exc()) |
|
314 | 279 | sa.rollback() |
|
315 | 280 | |
|
316 | 281 | run_task(send_email, user_email, |
|
317 | 282 | "Your new rhodecode password", |
|
318 | 283 | 'Your new rhodecode password:%s' % (new_passwd)) |
|
319 | 284 | log.info('send new password mail to %s', user_email) |
|
320 | 285 | |
|
321 | 286 | except: |
|
322 | 287 | log.error('Failed to update user password') |
|
323 | 288 | log.error(traceback.format_exc()) |
|
324 | 289 | |
|
325 | 290 | return True |
|
326 | 291 | |
|
327 | 292 | |
|
328 | 293 | @task(ignore_result=True) |
|
329 | 294 | def send_email(recipients, subject, body): |
|
330 | 295 | """ |
|
331 | 296 | Sends an email with defined parameters from the .ini files. |
|
332 | 297 | |
|
333 | 298 | :param recipients: list of recipients, it this is empty the defined email |
|
334 | 299 | address from field 'email_to' is used instead |
|
335 | 300 | :param subject: subject of the mail |
|
336 | 301 | :param body: body of the mail |
|
337 | 302 | """ |
|
338 | 303 | try: |
|
339 | 304 | log = send_email.get_logger() |
|
340 | 305 | except: |
|
341 | 306 | log = logging.getLogger(__name__) |
|
342 | 307 | |
|
343 | 308 | email_config = config |
|
344 | 309 | |
|
345 | 310 | if not recipients: |
|
346 | 311 | recipients = [email_config.get('email_to')] |
|
347 | 312 | |
|
348 | 313 | mail_from = email_config.get('app_email_from') |
|
349 | 314 | user = email_config.get('smtp_username') |
|
350 | 315 | passwd = email_config.get('smtp_password') |
|
351 | 316 | mail_server = email_config.get('smtp_server') |
|
352 | 317 | mail_port = email_config.get('smtp_port') |
|
353 | 318 | tls = str2bool(email_config.get('smtp_use_tls')) |
|
354 | 319 | ssl = str2bool(email_config.get('smtp_use_ssl')) |
|
355 | 320 | debug = str2bool(config.get('debug')) |
|
356 | 321 | |
|
357 | 322 | try: |
|
358 | 323 | m = SmtpMailer(mail_from, user, passwd, mail_server, |
|
359 | 324 | mail_port, ssl, tls, debug=debug) |
|
360 | 325 | m.send(recipients, subject, body) |
|
361 | 326 | except: |
|
362 | 327 | log.error('Mail sending failed') |
|
363 | 328 | log.error(traceback.format_exc()) |
|
364 | 329 | return False |
|
365 | 330 | return True |
|
366 | 331 | |
|
367 | 332 | |
|
368 | 333 | @task(ignore_result=True) |
|
369 | 334 | def create_repo_fork(form_data, cur_user): |
|
370 | 335 | from rhodecode.model.repo import RepoModel |
|
371 | 336 | from vcs import get_backend |
|
372 | 337 | |
|
373 | 338 | try: |
|
374 | 339 | log = create_repo_fork.get_logger() |
|
375 | 340 | except: |
|
376 | 341 | log = logging.getLogger(__name__) |
|
377 | 342 | |
|
378 | 343 | repo_model = RepoModel(get_session()) |
|
379 | 344 | repo_model.create(form_data, cur_user, just_db=True, fork=True) |
|
380 | 345 | repo_name = form_data['repo_name'] |
|
381 | 346 | repos_path = get_repos_path() |
|
382 | 347 | repo_path = os.path.join(repos_path, repo_name) |
|
383 | 348 | repo_fork_path = os.path.join(repos_path, form_data['fork_name']) |
|
384 | 349 | alias = form_data['repo_type'] |
|
385 | 350 | |
|
386 | 351 | log.info('creating repo fork %s as %s', repo_name, repo_path) |
|
387 | 352 | backend = get_backend(alias) |
|
388 | 353 | backend(str(repo_fork_path), create=True, src_url=str(repo_path)) |
|
389 | 354 | |
|
390 | 355 | |
|
391 | 356 | def __get_codes_stats(repo_name): |
|
392 | 357 | repos_path = get_repos_path() |
|
393 | 358 | p = os.path.join(repos_path, repo_name) |
|
394 | 359 | repo = get_repo(p) |
|
395 | 360 | tip = repo.get_changeset() |
|
396 | 361 | code_stats = {} |
|
397 | 362 | |
|
398 | 363 | def aggregate(cs): |
|
399 | 364 | for f in cs[2]: |
|
400 | 365 | ext = lower(f.extension) |
|
401 | 366 | if ext in LANGUAGES_EXTENSIONS_MAP.keys() and not f.is_binary: |
|
402 | 367 | if ext in code_stats: |
|
403 | 368 | code_stats[ext] += 1 |
|
404 | 369 | else: |
|
405 | 370 | code_stats[ext] = 1 |
|
406 | 371 | |
|
407 | 372 | map(aggregate, tip.walk('/')) |
|
408 | 373 | |
|
409 | 374 | return code_stats or {} |
@@ -1,230 +1,224 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | """ |
|
3 | 3 | rhodecode.lib.indexers.__init__ |
|
4 | 4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
|
5 | 5 | |
|
6 | 6 | Whoosh indexing module for RhodeCode |
|
7 | 7 | |
|
8 | 8 | :created_on: Aug 17, 2010 |
|
9 | 9 | :author: marcink |
|
10 | 10 | :copyright: (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com> |
|
11 | 11 | :license: GPLv3, see COPYING for more details. |
|
12 | 12 | """ |
|
13 | 13 | # This program is free software: you can redistribute it and/or modify |
|
14 | 14 | # it under the terms of the GNU General Public License as published by |
|
15 | 15 | # the Free Software Foundation, either version 3 of the License, or |
|
16 | 16 | # (at your option) any later version. |
|
17 | 17 | # |
|
18 | 18 | # This program is distributed in the hope that it will be useful, |
|
19 | 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
20 | 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
21 | 21 | # GNU General Public License for more details. |
|
22 | 22 | # |
|
23 | 23 | # You should have received a copy of the GNU General Public License |
|
24 | 24 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
25 | 25 | import os |
|
26 | 26 | import sys |
|
27 | 27 | import traceback |
|
28 | 28 | from os.path import dirname as dn, join as jn |
|
29 | 29 | |
|
30 | 30 | #to get the rhodecode import |
|
31 | 31 | sys.path.append(dn(dn(dn(os.path.realpath(__file__))))) |
|
32 | 32 | |
|
33 | 33 | from string import strip |
|
34 | ||
|
35 | from rhodecode.model import init_model | |
|
36 | from rhodecode.model.scm import ScmModel | |
|
37 | from rhodecode.config.environment import load_environment | |
|
38 | from rhodecode.lib.utils import BasePasterCommand, Command, add_cache | |
|
39 | ||
|
40 | 34 | from shutil import rmtree |
|
41 | from webhelpers.html.builder import escape | |
|
42 | from vcs.utils.lazy import LazyProperty | |
|
43 | ||
|
44 | from sqlalchemy import engine_from_config | |
|
45 | 35 | |
|
46 | 36 | from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter |
|
47 | 37 | from whoosh.fields import TEXT, ID, STORED, Schema, FieldType |
|
48 | 38 | from whoosh.index import create_in, open_dir |
|
49 | 39 | from whoosh.formats import Characters |
|
50 | 40 | from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter |
|
51 | 41 | |
|
42 | from webhelpers.html.builder import escape | |
|
43 | from sqlalchemy import engine_from_config | |
|
44 | from vcs.utils.lazy import LazyProperty | |
|
45 | ||
|
46 | from rhodecode.model import init_model | |
|
47 | from rhodecode.model.scm import ScmModel | |
|
48 | from rhodecode.config.environment import load_environment | |
|
49 | from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP | |
|
50 | from rhodecode.lib.utils import BasePasterCommand, Command, add_cache | |
|
52 | 51 | |
|
53 | 52 | #EXTENSIONS WE WANT TO INDEX CONTENT OFF |
|
54 | INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c', | |
|
55 | 'cfg', 'cfm', 'cpp', 'cs', 'css', 'diff', 'do', 'el', 'erl', | |
|
56 | 'h', 'htm', 'html', 'ini', 'java', 'js', 'jsp', 'jspx', 'lisp', | |
|
57 | 'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3', | |
|
58 | 'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', 'sql', | |
|
59 | 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', | |
|
60 | 'yaws'] | |
|
53 | INDEX_EXTENSIONS = LANGUAGES_EXTENSIONS_MAP.keys() | |
|
61 | 54 | |
|
62 | 55 | #CUSTOM ANALYZER wordsplit + lowercase filter |
|
63 | 56 | ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter() |
|
64 | 57 | |
|
65 | 58 | |
|
66 | 59 | #INDEX SCHEMA DEFINITION |
|
67 | 60 | SCHEMA = Schema(owner=TEXT(), |
|
68 | 61 | repository=TEXT(stored=True), |
|
69 | 62 | path=TEXT(stored=True), |
|
70 | 63 | content=FieldType(format=Characters(ANALYZER), |
|
71 | 64 | scorable=True, stored=True), |
|
72 | 65 | modtime=STORED(), extension=TEXT(stored=True)) |
|
73 | 66 | |
|
74 | 67 | |
|
75 | 68 | IDX_NAME = 'HG_INDEX' |
|
76 | 69 | FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n') |
|
77 | 70 | FRAGMENTER = SimpleFragmenter(200) |
|
78 | 71 | |
|
79 | 72 | |
|
80 | 73 | class MakeIndex(BasePasterCommand): |
|
81 | 74 | |
|
82 | 75 | max_args = 1 |
|
83 | 76 | min_args = 1 |
|
84 | 77 | |
|
85 | 78 | usage = "CONFIG_FILE" |
|
86 | 79 | summary = "Creates index for full text search given configuration file" |
|
87 | 80 | group_name = "RhodeCode" |
|
88 | 81 | takes_config_file = -1 |
|
89 | 82 | parser = Command.standard_parser(verbose=True) |
|
90 | 83 | |
|
91 | 84 | def command(self): |
|
92 | 85 | |
|
93 | 86 | from pylons import config |
|
94 | 87 | add_cache(config) |
|
95 | 88 | engine = engine_from_config(config, 'sqlalchemy.db1.') |
|
96 | 89 | init_model(engine) |
|
97 | 90 | |
|
98 | 91 | index_location = config['index_dir'] |
|
99 | 92 | repo_location = self.options.repo_location |
|
100 | 93 | repo_list = map(strip, self.options.repo_list.split(',')) \ |
|
101 | 94 | if self.options.repo_list else None |
|
102 | 95 | |
|
103 | 96 | #====================================================================== |
|
104 | 97 | # WHOOSH DAEMON |
|
105 | 98 | #====================================================================== |
|
106 | 99 | from rhodecode.lib.pidlock import LockHeld, DaemonLock |
|
107 | 100 | from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon |
|
108 | 101 | try: |
|
109 | 102 | l = DaemonLock() |
|
110 | 103 | WhooshIndexingDaemon(index_location=index_location, |
|
111 | 104 | repo_location=repo_location, |
|
112 | 105 | repo_list=repo_list)\ |
|
113 | 106 | .run(full_index=self.options.full_index) |
|
114 | 107 | l.release() |
|
115 | 108 | except LockHeld: |
|
116 | 109 | sys.exit(1) |
|
117 | 110 | |
|
118 | 111 | def update_parser(self): |
|
119 | 112 | self.parser.add_option('--repo-location', |
|
120 | 113 | action='store', |
|
121 | 114 | dest='repo_location', |
|
122 | 115 | help="Specifies repositories location to index REQUIRED", |
|
123 | 116 | ) |
|
124 | 117 | self.parser.add_option('--index-only', |
|
125 | 118 | action='store', |
|
126 | 119 | dest='repo_list', |
|
127 | 120 | help="Specifies a comma separated list of repositores " |
|
128 | 121 | "to build index on OPTIONAL", |
|
129 | 122 | ) |
|
130 | 123 | self.parser.add_option('-f', |
|
131 | 124 | action='store_true', |
|
132 | 125 | dest='full_index', |
|
133 | 126 | help="Specifies that index should be made full i.e" |
|
134 | 127 | " destroy old and build from scratch", |
|
135 | 128 | default=False) |
|
136 | 129 | |
|
137 | 130 | class ResultWrapper(object): |
|
138 | 131 | def __init__(self, search_type, searcher, matcher, highlight_items): |
|
139 | 132 | self.search_type = search_type |
|
140 | 133 | self.searcher = searcher |
|
141 | 134 | self.matcher = matcher |
|
142 | 135 | self.highlight_items = highlight_items |
|
143 | 136 | self.fragment_size = 200 / 2 |
|
144 | 137 | |
|
145 | 138 | @LazyProperty |
|
146 | 139 | def doc_ids(self): |
|
147 | 140 | docs_id = [] |
|
148 | 141 | while self.matcher.is_active(): |
|
149 | 142 | docnum = self.matcher.id() |
|
150 | 143 | chunks = [offsets for offsets in self.get_chunks()] |
|
151 | 144 | docs_id.append([docnum, chunks]) |
|
152 | 145 | self.matcher.next() |
|
153 | 146 | return docs_id |
|
154 | 147 | |
|
155 | 148 | def __str__(self): |
|
156 | 149 | return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids)) |
|
157 | 150 | |
|
158 | 151 | def __repr__(self): |
|
159 | 152 | return self.__str__() |
|
160 | 153 | |
|
161 | 154 | def __len__(self): |
|
162 | 155 | return len(self.doc_ids) |
|
163 | 156 | |
|
164 | 157 | def __iter__(self): |
|
165 | 158 | """ |
|
166 | 159 | Allows Iteration over results,and lazy generate content |
|
167 | 160 | |
|
168 | 161 | *Requires* implementation of ``__getitem__`` method. |
|
169 | 162 | """ |
|
170 | 163 | for docid in self.doc_ids: |
|
171 | 164 | yield self.get_full_content(docid) |
|
172 | 165 | |
|
173 | 166 | def __getitem__(self, key): |
|
174 | 167 | """ |
|
175 | 168 | Slicing of resultWrapper |
|
176 | 169 | """ |
|
177 | 170 | i, j = key.start, key.stop |
|
178 | 171 | |
|
179 | 172 | slice = [] |
|
180 | 173 | for docid in self.doc_ids[i:j]: |
|
181 | 174 | slice.append(self.get_full_content(docid)) |
|
182 | 175 | return slice |
|
183 | 176 | |
|
184 | 177 | |
|
185 | 178 | def get_full_content(self, docid): |
|
186 | 179 | res = self.searcher.stored_fields(docid[0]) |
|
187 | 180 | f_path = res['path'][res['path'].find(res['repository']) \ |
|
188 | 181 | + len(res['repository']):].lstrip('/') |
|
189 | 182 | |
|
190 | 183 | content_short = self.get_short_content(res, docid[1]) |
|
191 | 184 | res.update({'content_short':content_short, |
|
192 | 185 | 'content_short_hl':self.highlight(content_short), |
|
193 | 186 | 'f_path':f_path}) |
|
194 | 187 | |
|
195 | 188 | return res |
|
196 | 189 | |
|
197 | 190 | def get_short_content(self, res, chunks): |
|
198 | 191 | |
|
199 | 192 | return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks]) |
|
200 | 193 | |
|
201 | 194 | def get_chunks(self): |
|
202 | 195 | """ |
|
203 | 196 | Smart function that implements chunking the content |
|
204 | 197 | but not overlap chunks so it doesn't highlight the same |
|
205 | 198 | close occurrences twice. |
|
206 | @param matcher: | |
|
207 |
|
|
|
199 | ||
|
200 | :param matcher: | |
|
201 | :param size: | |
|
208 | 202 | """ |
|
209 | 203 | memory = [(0, 0)] |
|
210 | 204 | for span in self.matcher.spans(): |
|
211 | 205 | start = span.startchar or 0 |
|
212 | 206 | end = span.endchar or 0 |
|
213 | 207 | start_offseted = max(0, start - self.fragment_size) |
|
214 | 208 | end_offseted = end + self.fragment_size |
|
215 | 209 | |
|
216 | 210 | if start_offseted < memory[-1][1]: |
|
217 | 211 | start_offseted = memory[-1][1] |
|
218 | 212 | memory.append((start_offseted, end_offseted,)) |
|
219 | 213 | yield (start_offseted, end_offseted,) |
|
220 | 214 | |
|
221 | 215 | def highlight(self, content, top=5): |
|
222 | 216 | if self.search_type != 'content': |
|
223 | 217 | return '' |
|
224 | 218 | hl = highlight(escape(content), |
|
225 | 219 | self.highlight_items, |
|
226 | 220 | analyzer=ANALYZER, |
|
227 | 221 | fragmenter=FRAGMENTER, |
|
228 | 222 | formatter=FORMATTER, |
|
229 | 223 | top=top) |
|
230 | 224 | return hl |
General Comments 0
You need to be logged in to leave comments.
Login now