Show More
@@ -1,318 +1,320 b'' | |||||
1 | .. _changelog: |
|
1 | .. _changelog: | |
2 |
|
2 | |||
3 | Changelog |
|
3 | Changelog | |
4 | ========= |
|
4 | ========= | |
5 |
|
5 | |||
6 | 1.2.0 (**2011-XX-XX**) |
|
6 | 1.2.0 (**2011-XX-XX**) | |
7 | ====================== |
|
7 | ====================== | |
8 |
|
8 | |||
9 | :status: in-progress |
|
9 | :status: in-progress | |
10 | :branch: beta |
|
10 | :branch: beta | |
11 |
|
11 | |||
12 | news |
|
12 | news | |
13 | ---- |
|
13 | ---- | |
14 |
|
14 | |||
15 | - implemented #89 Can setup google analytics code from settings menu |
|
15 | - implemented #89 Can setup google analytics code from settings menu | |
16 | - implemented #91 added nicer looking archive urls with more download options |
|
16 | - implemented #91 added nicer looking archive urls with more download options | |
17 | like tags, branches |
|
17 | like tags, branches | |
18 | - implemented #44 into file browsing, and added follow branch option |
|
18 | - implemented #44 into file browsing, and added follow branch option | |
19 | - implemented #84 downloads can be enabled/disabled for each repository |
|
19 | - implemented #84 downloads can be enabled/disabled for each repository | |
20 | - anonymous repository can be cloned without having to pass default:default |
|
20 | - anonymous repository can be cloned without having to pass default:default | |
21 | into clone url |
|
21 | into clone url | |
22 | - fixed #90 whoosh indexer can index chooses repositories passed in command |
|
22 | - fixed #90 whoosh indexer can index chooses repositories passed in command | |
23 | line |
|
23 | line | |
24 | - extended journal with day aggregates and paging |
|
24 | - extended journal with day aggregates and paging | |
25 | - implemented #107 source code lines highlight ranges |
|
25 | - implemented #107 source code lines highlight ranges | |
26 | - implemented #93 customizable changelog on combined revision ranges - |
|
26 | - implemented #93 customizable changelog on combined revision ranges - | |
27 | equivalent of githubs compare view |
|
27 | equivalent of githubs compare view | |
28 | - implemented #108 extended and more powerful LDAP configuration |
|
28 | - implemented #108 extended and more powerful LDAP configuration | |
29 | - implemented #56 users groups |
|
29 | - implemented #56 users groups | |
30 | - major code rewrites optimized codes for speed and memory usage |
|
30 | - major code rewrites optimized codes for speed and memory usage | |
31 | - raw and diff downloads are now in git format |
|
31 | - raw and diff downloads are now in git format | |
32 | - setup command checks for write access to given path |
|
32 | - setup command checks for write access to given path | |
33 | - fixed many issues with international characters and unicode. It uses utf8 |
|
33 | - fixed many issues with international characters and unicode. It uses utf8 | |
34 | decode with replace to provide less errors even with non utf8 encoded strings |
|
34 | decode with replace to provide less errors even with non utf8 encoded strings | |
35 | - #125 added API KEY access to feeds |
|
35 | - #125 added API KEY access to feeds | |
36 | - #109 Repository can be created from external Mercurial link (aka. remote |
|
36 | - #109 Repository can be created from external Mercurial link (aka. remote | |
37 | repository, and manually updated (via pull) from admin panel |
|
37 | repository, and manually updated (via pull) from admin panel | |
38 | - beta git support - push/pull server + basic view for git repos |
|
38 | - beta git support - push/pull server + basic view for git repos | |
39 | - added followers page |
|
39 | - added followers page and forks page | |
40 |
|
40 | |||
41 | fixes |
|
41 | fixes | |
42 | ----- |
|
42 | ----- | |
43 |
|
43 | |||
44 | - fixed file browser bug, when switching into given form revision the url was |
|
44 | - fixed file browser bug, when switching into given form revision the url was | |
45 | not changing |
|
45 | not changing | |
46 | - fixed propagation to error controller on simplehg and simplegit middlewares |
|
46 | - fixed propagation to error controller on simplehg and simplegit middlewares | |
47 | - fixed error when trying to make a download on empty repository |
|
47 | - fixed error when trying to make a download on empty repository | |
48 | - fixed problem with '[' chars in commit messages in journal |
|
48 | - fixed problem with '[' chars in commit messages in journal | |
49 | - fixed #99 Unicode errors, on file node paths with non utf-8 characters |
|
49 | - fixed #99 Unicode errors, on file node paths with non utf-8 characters | |
50 | - journal fork fixes |
|
50 | - journal fork fixes | |
51 | - removed issue with space inside renamed repository after deletion |
|
51 | - removed issue with space inside renamed repository after deletion | |
52 | - fixed strange issue on formencode imports |
|
52 | - fixed strange issue on formencode imports | |
53 | - fixed #126 Deleting repository on Windows, rename used incompatible chars. |
|
53 | - fixed #126 Deleting repository on Windows, rename used incompatible chars. | |
54 | - #150 fixes for errors on repositories mapped in db but corrupted in |
|
54 | - #150 fixes for errors on repositories mapped in db but corrupted in | |
55 | filesystem |
|
55 | filesystem | |
56 | - fixed problem with ascendant characters in realm #181 |
|
56 | - fixed problem with ascendant characters in realm #181 | |
|
57 | - fixed problem with sqlite file based database connection pool | |||
|
58 | - whoosh indexer and code stats share the same dynamic extensions map | |||
57 |
|
59 | |||
58 | 1.1.8 (**2011-04-12**) |
|
60 | 1.1.8 (**2011-04-12**) | |
59 | ====================== |
|
61 | ====================== | |
60 |
|
62 | |||
61 | news |
|
63 | news | |
62 | ---- |
|
64 | ---- | |
63 |
|
65 | |||
64 | - improved windows support |
|
66 | - improved windows support | |
65 |
|
67 | |||
66 | fixes |
|
68 | fixes | |
67 | ----- |
|
69 | ----- | |
68 |
|
70 | |||
69 | - fixed #140 freeze of python dateutil library, since new version is python2.x |
|
71 | - fixed #140 freeze of python dateutil library, since new version is python2.x | |
70 | incompatible |
|
72 | incompatible | |
71 | - setup-app will check for write permission in given path |
|
73 | - setup-app will check for write permission in given path | |
72 | - cleaned up license info issue #149 |
|
74 | - cleaned up license info issue #149 | |
73 | - fixes for issues #137,#116 and problems with unicode and accented characters. |
|
75 | - fixes for issues #137,#116 and problems with unicode and accented characters. | |
74 | - fixes crashes on gravatar, when passed in email as unicode |
|
76 | - fixes crashes on gravatar, when passed in email as unicode | |
75 | - fixed tooltip flickering problems |
|
77 | - fixed tooltip flickering problems | |
76 | - fixed came_from redirection on windows |
|
78 | - fixed came_from redirection on windows | |
77 | - fixed logging modules, and sql formatters |
|
79 | - fixed logging modules, and sql formatters | |
78 | - windows fixes for os.kill issue #133 |
|
80 | - windows fixes for os.kill issue #133 | |
79 | - fixes path splitting for windows issues #148 |
|
81 | - fixes path splitting for windows issues #148 | |
80 | - fixed issue #143 wrong import on migration to 1.1.X |
|
82 | - fixed issue #143 wrong import on migration to 1.1.X | |
81 | - fixed problems with displaying binary files, thanks to Thomas Waldmann |
|
83 | - fixed problems with displaying binary files, thanks to Thomas Waldmann | |
82 | - removed name from archive files since it's breaking ui for long repo names |
|
84 | - removed name from archive files since it's breaking ui for long repo names | |
83 | - fixed issue with archive headers sent to browser, thanks to Thomas Waldmann |
|
85 | - fixed issue with archive headers sent to browser, thanks to Thomas Waldmann | |
84 | - fixed compatibility for 1024px displays, and larger dpi settings, thanks to |
|
86 | - fixed compatibility for 1024px displays, and larger dpi settings, thanks to | |
85 | Thomas Waldmann |
|
87 | Thomas Waldmann | |
86 | - fixed issue #166 summary pager was skipping 10 revisions on second page |
|
88 | - fixed issue #166 summary pager was skipping 10 revisions on second page | |
87 |
|
89 | |||
88 |
|
90 | |||
89 | 1.1.7 (**2011-03-23**) |
|
91 | 1.1.7 (**2011-03-23**) | |
90 | ====================== |
|
92 | ====================== | |
91 |
|
93 | |||
92 | news |
|
94 | news | |
93 | ---- |
|
95 | ---- | |
94 |
|
96 | |||
95 | fixes |
|
97 | fixes | |
96 | ----- |
|
98 | ----- | |
97 |
|
99 | |||
98 | - fixed (again) #136 installation support for FreeBSD |
|
100 | - fixed (again) #136 installation support for FreeBSD | |
99 |
|
101 | |||
100 |
|
102 | |||
101 | 1.1.6 (**2011-03-21**) |
|
103 | 1.1.6 (**2011-03-21**) | |
102 | ====================== |
|
104 | ====================== | |
103 |
|
105 | |||
104 | news |
|
106 | news | |
105 | ---- |
|
107 | ---- | |
106 |
|
108 | |||
107 | fixes |
|
109 | fixes | |
108 | ----- |
|
110 | ----- | |
109 |
|
111 | |||
110 | - fixed #136 installation support for FreeBSD |
|
112 | - fixed #136 installation support for FreeBSD | |
111 | - RhodeCode will check for python version during installation |
|
113 | - RhodeCode will check for python version during installation | |
112 |
|
114 | |||
113 | 1.1.5 (**2011-03-17**) |
|
115 | 1.1.5 (**2011-03-17**) | |
114 | ====================== |
|
116 | ====================== | |
115 |
|
117 | |||
116 | news |
|
118 | news | |
117 | ---- |
|
119 | ---- | |
118 |
|
120 | |||
119 | - basic windows support, by exchanging pybcrypt into sha256 for windows only |
|
121 | - basic windows support, by exchanging pybcrypt into sha256 for windows only | |
120 | highly inspired by idea of mantis406 |
|
122 | highly inspired by idea of mantis406 | |
121 |
|
123 | |||
122 | fixes |
|
124 | fixes | |
123 | ----- |
|
125 | ----- | |
124 |
|
126 | |||
125 | - fixed sorting by author in main page |
|
127 | - fixed sorting by author in main page | |
126 | - fixed crashes with diffs on binary files |
|
128 | - fixed crashes with diffs on binary files | |
127 | - fixed #131 problem with boolean values for LDAP |
|
129 | - fixed #131 problem with boolean values for LDAP | |
128 | - fixed #122 mysql problems thanks to striker69 |
|
130 | - fixed #122 mysql problems thanks to striker69 | |
129 | - fixed problem with errors on calling raw/raw_files/annotate functions |
|
131 | - fixed problem with errors on calling raw/raw_files/annotate functions | |
130 | with unknown revisions |
|
132 | with unknown revisions | |
131 | - fixed returned rawfiles attachment names with international character |
|
133 | - fixed returned rawfiles attachment names with international character | |
132 | - cleaned out docs, big thanks to Jason Harris |
|
134 | - cleaned out docs, big thanks to Jason Harris | |
133 |
|
135 | |||
134 | 1.1.4 (**2011-02-19**) |
|
136 | 1.1.4 (**2011-02-19**) | |
135 | ====================== |
|
137 | ====================== | |
136 |
|
138 | |||
137 | news |
|
139 | news | |
138 | ---- |
|
140 | ---- | |
139 |
|
141 | |||
140 | fixes |
|
142 | fixes | |
141 | ----- |
|
143 | ----- | |
142 |
|
144 | |||
143 | - fixed formencode import problem on settings page, that caused server crash |
|
145 | - fixed formencode import problem on settings page, that caused server crash | |
144 | when that page was accessed as first after server start |
|
146 | when that page was accessed as first after server start | |
145 | - journal fixes |
|
147 | - journal fixes | |
146 | - fixed option to access repository just by entering http://server/<repo_name> |
|
148 | - fixed option to access repository just by entering http://server/<repo_name> | |
147 |
|
149 | |||
148 | 1.1.3 (**2011-02-16**) |
|
150 | 1.1.3 (**2011-02-16**) | |
149 | ====================== |
|
151 | ====================== | |
150 |
|
152 | |||
151 | news |
|
153 | news | |
152 | ---- |
|
154 | ---- | |
153 |
|
155 | |||
154 | - implemented #102 allowing the '.' character in username |
|
156 | - implemented #102 allowing the '.' character in username | |
155 | - added option to access repository just by entering http://server/<repo_name> |
|
157 | - added option to access repository just by entering http://server/<repo_name> | |
156 | - celery task ignores result for better performance |
|
158 | - celery task ignores result for better performance | |
157 |
|
159 | |||
158 | fixes |
|
160 | fixes | |
159 | ----- |
|
161 | ----- | |
160 |
|
162 | |||
161 | - fixed ehlo command and non auth mail servers on smtp_lib. Thanks to |
|
163 | - fixed ehlo command and non auth mail servers on smtp_lib. Thanks to | |
162 | apollo13 and Johan Walles |
|
164 | apollo13 and Johan Walles | |
163 | - small fixes in journal |
|
165 | - small fixes in journal | |
164 | - fixed problems with getting setting for celery from .ini files |
|
166 | - fixed problems with getting setting for celery from .ini files | |
165 | - registration, password reset and login boxes share the same title as main |
|
167 | - registration, password reset and login boxes share the same title as main | |
166 | application now |
|
168 | application now | |
167 | - fixed #113: to high permissions to fork repository |
|
169 | - fixed #113: to high permissions to fork repository | |
168 | - fixed problem with '[' chars in commit messages in journal |
|
170 | - fixed problem with '[' chars in commit messages in journal | |
169 | - removed issue with space inside renamed repository after deletion |
|
171 | - removed issue with space inside renamed repository after deletion | |
170 | - db transaction fixes when filesystem repository creation failed |
|
172 | - db transaction fixes when filesystem repository creation failed | |
171 | - fixed #106 relation issues on databases different than sqlite |
|
173 | - fixed #106 relation issues on databases different than sqlite | |
172 | - fixed static files paths links to use of url() method |
|
174 | - fixed static files paths links to use of url() method | |
173 |
|
175 | |||
174 | 1.1.2 (**2011-01-12**) |
|
176 | 1.1.2 (**2011-01-12**) | |
175 | ====================== |
|
177 | ====================== | |
176 |
|
178 | |||
177 | news |
|
179 | news | |
178 | ---- |
|
180 | ---- | |
179 |
|
181 | |||
180 |
|
182 | |||
181 | fixes |
|
183 | fixes | |
182 | ----- |
|
184 | ----- | |
183 |
|
185 | |||
184 | - fixes #98 protection against float division of percentage stats |
|
186 | - fixes #98 protection against float division of percentage stats | |
185 | - fixed graph bug |
|
187 | - fixed graph bug | |
186 | - forced webhelpers version since it was making troubles during installation |
|
188 | - forced webhelpers version since it was making troubles during installation | |
187 |
|
189 | |||
188 | 1.1.1 (**2011-01-06**) |
|
190 | 1.1.1 (**2011-01-06**) | |
189 | ====================== |
|
191 | ====================== | |
190 |
|
192 | |||
191 | news |
|
193 | news | |
192 | ---- |
|
194 | ---- | |
193 |
|
195 | |||
194 | - added force https option into ini files for easier https usage (no need to |
|
196 | - added force https option into ini files for easier https usage (no need to | |
195 | set server headers with this options) |
|
197 | set server headers with this options) | |
196 | - small css updates |
|
198 | - small css updates | |
197 |
|
199 | |||
198 | fixes |
|
200 | fixes | |
199 | ----- |
|
201 | ----- | |
200 |
|
202 | |||
201 | - fixed #96 redirect loop on files view on repositories without changesets |
|
203 | - fixed #96 redirect loop on files view on repositories without changesets | |
202 | - fixed #97 unicode string passed into server header in special cases (mod_wsgi) |
|
204 | - fixed #97 unicode string passed into server header in special cases (mod_wsgi) | |
203 | and server crashed with errors |
|
205 | and server crashed with errors | |
204 | - fixed large tooltips problems on main page |
|
206 | - fixed large tooltips problems on main page | |
205 | - fixed #92 whoosh indexer is more error proof |
|
207 | - fixed #92 whoosh indexer is more error proof | |
206 |
|
208 | |||
207 | 1.1.0 (**2010-12-18**) |
|
209 | 1.1.0 (**2010-12-18**) | |
208 | ====================== |
|
210 | ====================== | |
209 |
|
211 | |||
210 | news |
|
212 | news | |
211 | ---- |
|
213 | ---- | |
212 |
|
214 | |||
213 | - rewrite of internals for vcs >=0.1.10 |
|
215 | - rewrite of internals for vcs >=0.1.10 | |
214 | - uses mercurial 1.7 with dotencode disabled for maintaining compatibility |
|
216 | - uses mercurial 1.7 with dotencode disabled for maintaining compatibility | |
215 | with older clients |
|
217 | with older clients | |
216 | - anonymous access, authentication via ldap |
|
218 | - anonymous access, authentication via ldap | |
217 | - performance upgrade for cached repos list - each repository has it's own |
|
219 | - performance upgrade for cached repos list - each repository has it's own | |
218 | cache that's invalidated when needed. |
|
220 | cache that's invalidated when needed. | |
219 | - performance upgrades on repositories with large amount of commits (20K+) |
|
221 | - performance upgrades on repositories with large amount of commits (20K+) | |
220 | - main page quick filter for filtering repositories |
|
222 | - main page quick filter for filtering repositories | |
221 | - user dashboards with ability to follow chosen repositories actions |
|
223 | - user dashboards with ability to follow chosen repositories actions | |
222 | - sends email to admin on new user registration |
|
224 | - sends email to admin on new user registration | |
223 | - added cache/statistics reset options into repository settings |
|
225 | - added cache/statistics reset options into repository settings | |
224 | - more detailed action logger (based on hooks) with pushed changesets lists |
|
226 | - more detailed action logger (based on hooks) with pushed changesets lists | |
225 | and options to disable those hooks from admin panel |
|
227 | and options to disable those hooks from admin panel | |
226 | - introduced new enhanced changelog for merges that shows more accurate results |
|
228 | - introduced new enhanced changelog for merges that shows more accurate results | |
227 | - new improved and faster code stats (based on pygments lexers mapping tables, |
|
229 | - new improved and faster code stats (based on pygments lexers mapping tables, | |
228 | showing up to 10 trending sources for each repository. Additionally stats |
|
230 | showing up to 10 trending sources for each repository. Additionally stats | |
229 | can be disabled in repository settings. |
|
231 | can be disabled in repository settings. | |
230 | - gui optimizations, fixed application width to 1024px |
|
232 | - gui optimizations, fixed application width to 1024px | |
231 | - added cut off (for large files/changesets) limit into config files |
|
233 | - added cut off (for large files/changesets) limit into config files | |
232 | - whoosh, celeryd, upgrade moved to paster command |
|
234 | - whoosh, celeryd, upgrade moved to paster command | |
233 | - other than sqlite database backends can be used |
|
235 | - other than sqlite database backends can be used | |
234 |
|
236 | |||
235 | fixes |
|
237 | fixes | |
236 | ----- |
|
238 | ----- | |
237 |
|
239 | |||
238 | - fixes #61 forked repo was showing only after cache expired |
|
240 | - fixes #61 forked repo was showing only after cache expired | |
239 | - fixes #76 no confirmation on user deletes |
|
241 | - fixes #76 no confirmation on user deletes | |
240 | - fixes #66 Name field misspelled |
|
242 | - fixes #66 Name field misspelled | |
241 | - fixes #72 block user removal when he owns repositories |
|
243 | - fixes #72 block user removal when he owns repositories | |
242 | - fixes #69 added password confirmation fields |
|
244 | - fixes #69 added password confirmation fields | |
243 | - fixes #87 RhodeCode crashes occasionally on updating repository owner |
|
245 | - fixes #87 RhodeCode crashes occasionally on updating repository owner | |
244 | - fixes #82 broken annotations on files with more than 1 blank line at the end |
|
246 | - fixes #82 broken annotations on files with more than 1 blank line at the end | |
245 | - a lot of fixes and tweaks for file browser |
|
247 | - a lot of fixes and tweaks for file browser | |
246 | - fixed detached session issues |
|
248 | - fixed detached session issues | |
247 | - fixed when user had no repos he would see all repos listed in my account |
|
249 | - fixed when user had no repos he would see all repos listed in my account | |
248 | - fixed ui() instance bug when global hgrc settings was loaded for server |
|
250 | - fixed ui() instance bug when global hgrc settings was loaded for server | |
249 | instance and all hgrc options were merged with our db ui() object |
|
251 | instance and all hgrc options were merged with our db ui() object | |
250 | - numerous small bugfixes |
|
252 | - numerous small bugfixes | |
251 |
|
253 | |||
252 | (special thanks for TkSoh for detailed feedback) |
|
254 | (special thanks for TkSoh for detailed feedback) | |
253 |
|
255 | |||
254 |
|
256 | |||
255 | 1.0.2 (**2010-11-12**) |
|
257 | 1.0.2 (**2010-11-12**) | |
256 | ====================== |
|
258 | ====================== | |
257 |
|
259 | |||
258 | news |
|
260 | news | |
259 | ---- |
|
261 | ---- | |
260 |
|
262 | |||
261 | - tested under python2.7 |
|
263 | - tested under python2.7 | |
262 | - bumped sqlalchemy and celery versions |
|
264 | - bumped sqlalchemy and celery versions | |
263 |
|
265 | |||
264 | fixes |
|
266 | fixes | |
265 | ----- |
|
267 | ----- | |
266 |
|
268 | |||
267 | - fixed #59 missing graph.js |
|
269 | - fixed #59 missing graph.js | |
268 | - fixed repo_size crash when repository had broken symlinks |
|
270 | - fixed repo_size crash when repository had broken symlinks | |
269 | - fixed python2.5 crashes. |
|
271 | - fixed python2.5 crashes. | |
270 |
|
272 | |||
271 |
|
273 | |||
272 | 1.0.1 (**2010-11-10**) |
|
274 | 1.0.1 (**2010-11-10**) | |
273 | ====================== |
|
275 | ====================== | |
274 |
|
276 | |||
275 | news |
|
277 | news | |
276 | ---- |
|
278 | ---- | |
277 |
|
279 | |||
278 | - small css updated |
|
280 | - small css updated | |
279 |
|
281 | |||
280 | fixes |
|
282 | fixes | |
281 | ----- |
|
283 | ----- | |
282 |
|
284 | |||
283 | - fixed #53 python2.5 incompatible enumerate calls |
|
285 | - fixed #53 python2.5 incompatible enumerate calls | |
284 | - fixed #52 disable mercurial extension for web |
|
286 | - fixed #52 disable mercurial extension for web | |
285 | - fixed #51 deleting repositories don't delete it's dependent objects |
|
287 | - fixed #51 deleting repositories don't delete it's dependent objects | |
286 |
|
288 | |||
287 |
|
289 | |||
288 | 1.0.0 (**2010-11-02**) |
|
290 | 1.0.0 (**2010-11-02**) | |
289 | ====================== |
|
291 | ====================== | |
290 |
|
292 | |||
291 | - security bugfix simplehg wasn't checking for permissions on commands |
|
293 | - security bugfix simplehg wasn't checking for permissions on commands | |
292 | other than pull or push. |
|
294 | other than pull or push. | |
293 | - fixed doubled messages after push or pull in admin journal |
|
295 | - fixed doubled messages after push or pull in admin journal | |
294 | - templating and css corrections, fixed repo switcher on chrome, updated titles |
|
296 | - templating and css corrections, fixed repo switcher on chrome, updated titles | |
295 | - admin menu accessible from options menu on repository view |
|
297 | - admin menu accessible from options menu on repository view | |
296 | - permissions cached queries |
|
298 | - permissions cached queries | |
297 |
|
299 | |||
298 | 1.0.0rc4 (**2010-10-12**) |
|
300 | 1.0.0rc4 (**2010-10-12**) | |
299 | ========================== |
|
301 | ========================== | |
300 |
|
302 | |||
301 | - fixed python2.5 missing simplejson imports (thanks to Jens BΓ€ckman) |
|
303 | - fixed python2.5 missing simplejson imports (thanks to Jens BΓ€ckman) | |
302 | - removed cache_manager settings from sqlalchemy meta |
|
304 | - removed cache_manager settings from sqlalchemy meta | |
303 | - added sqlalchemy cache settings to ini files |
|
305 | - added sqlalchemy cache settings to ini files | |
304 | - validated password length and added second try of failure on paster setup-app |
|
306 | - validated password length and added second try of failure on paster setup-app | |
305 | - fixed setup database destroy prompt even when there was no db |
|
307 | - fixed setup database destroy prompt even when there was no db | |
306 |
|
308 | |||
307 |
|
309 | |||
308 | 1.0.0rc3 (**2010-10-11**) |
|
310 | 1.0.0rc3 (**2010-10-11**) | |
309 | ========================= |
|
311 | ========================= | |
310 |
|
312 | |||
311 | - fixed i18n during installation. |
|
313 | - fixed i18n during installation. | |
312 |
|
314 | |||
313 | 1.0.0rc2 (**2010-10-11**) |
|
315 | 1.0.0rc2 (**2010-10-11**) | |
314 | ========================= |
|
316 | ========================= | |
315 |
|
317 | |||
316 | - Disabled dirsize in file browser, it's causing nasty bug when dir renames |
|
318 | - Disabled dirsize in file browser, it's causing nasty bug when dir renames | |
317 | occure. After vcs is fixed it'll be put back again. |
|
319 | occure. After vcs is fixed it'll be put back again. | |
318 | - templating/css rewrites, optimized css. No newline at end of file |
|
320 | - templating/css rewrites, optimized css. |
@@ -1,100 +1,142 b'' | |||||
1 | # -*- coding: utf-8 -*- |
|
1 | # -*- coding: utf-8 -*- | |
2 | """ |
|
2 | """ | |
3 | rhodecode.lib.__init__ |
|
3 | rhodecode.lib.__init__ | |
4 | ~~~~~~~~~~~~~~~~~~~~~~~ |
|
4 | ~~~~~~~~~~~~~~~~~~~~~~~ | |
5 |
|
5 | |||
6 | Some simple helper functions |
|
6 | Some simple helper functions | |
7 |
|
7 | |||
8 | :created_on: Jan 5, 2011 |
|
8 | :created_on: Jan 5, 2011 | |
9 | :author: marcink |
|
9 | :author: marcink | |
10 | :copyright: (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com> |
|
10 | :copyright: (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com> | |
11 | :license: GPLv3, see COPYING for more details. |
|
11 | :license: GPLv3, see COPYING for more details. | |
12 | """ |
|
12 | """ | |
13 | # This program is free software: you can redistribute it and/or modify |
|
13 | # This program is free software: you can redistribute it and/or modify | |
14 | # it under the terms of the GNU General Public License as published by |
|
14 | # it under the terms of the GNU General Public License as published by | |
15 | # the Free Software Foundation, either version 3 of the License, or |
|
15 | # the Free Software Foundation, either version 3 of the License, or | |
16 | # (at your option) any later version. |
|
16 | # (at your option) any later version. | |
17 | # |
|
17 | # | |
18 | # This program is distributed in the hope that it will be useful, |
|
18 | # This program is distributed in the hope that it will be useful, | |
19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
21 | # GNU General Public License for more details. |
|
21 | # GNU General Public License for more details. | |
22 | # |
|
22 | # | |
23 | # You should have received a copy of the GNU General Public License |
|
23 | # You should have received a copy of the GNU General Public License | |
24 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
24 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
25 |
|
25 | |||
26 |
|
26 | |||
|
27 | ||||
|
28 | def __get_lem(): | |||
|
29 | from pygments import lexers | |||
|
30 | from string import lower | |||
|
31 | from collections import defaultdict | |||
|
32 | ||||
|
33 | d = defaultdict(lambda: []) | |||
|
34 | ||||
|
35 | def __clean(s): | |||
|
36 | s = s.lstrip('*') | |||
|
37 | s = s.lstrip('.') | |||
|
38 | ||||
|
39 | if s.find('[') != -1: | |||
|
40 | exts = [] | |||
|
41 | start, stop = s.find('['), s.find(']') | |||
|
42 | ||||
|
43 | for suffix in s[start + 1:stop]: | |||
|
44 | exts.append(s[:s.find('[')] + suffix) | |||
|
45 | return map(lower, exts) | |||
|
46 | else: | |||
|
47 | return map(lower, [s]) | |||
|
48 | ||||
|
49 | for lx, t in sorted(lexers.LEXERS.items()): | |||
|
50 | m = map(__clean, t[-2]) | |||
|
51 | if m: | |||
|
52 | m = reduce(lambda x, y: x + y, m) | |||
|
53 | for ext in m: | |||
|
54 | desc = lx.replace('Lexer', '') | |||
|
55 | d[ext].append(desc) | |||
|
56 | ||||
|
57 | return dict(d) | |||
|
58 | ||||
|
59 | # language map is also used by whoosh indexer, which for those specified | |||
|
60 | # extensions will index it's content | |||
|
61 | LANGUAGES_EXTENSIONS_MAP = __get_lem() | |||
|
62 | ||||
|
63 | #Additional mappings that are not present in the pygments lexers | |||
|
64 | # NOTE: that this will overide any mappings in LANGUAGES_EXTENSIONS_MAP | |||
|
65 | ADDITIONAL_MAPPINGS = {'xaml': 'XAML'} | |||
|
66 | ||||
|
67 | LANGUAGES_EXTENSIONS_MAP.update(ADDITIONAL_MAPPINGS) | |||
|
68 | ||||
27 | def str2bool(_str): |
|
69 | def str2bool(_str): | |
28 | """ |
|
70 | """ | |
29 | returs True/False value from given string, it tries to translate the |
|
71 | returs True/False value from given string, it tries to translate the | |
30 | string into boolean |
|
72 | string into boolean | |
31 |
|
73 | |||
32 | :param _str: string value to translate into boolean |
|
74 | :param _str: string value to translate into boolean | |
33 | :rtype: boolean |
|
75 | :rtype: boolean | |
34 | :returns: boolean from given string |
|
76 | :returns: boolean from given string | |
35 | """ |
|
77 | """ | |
36 | if _str is None: |
|
78 | if _str is None: | |
37 | return False |
|
79 | return False | |
38 | if _str in (True, False): |
|
80 | if _str in (True, False): | |
39 | return _str |
|
81 | return _str | |
40 | _str = str(_str).strip().lower() |
|
82 | _str = str(_str).strip().lower() | |
41 | return _str in ('t', 'true', 'y', 'yes', 'on', '1') |
|
83 | return _str in ('t', 'true', 'y', 'yes', 'on', '1') | |
42 |
|
84 | |||
43 |
|
85 | |||
44 | def generate_api_key(username, salt=None): |
|
86 | def generate_api_key(username, salt=None): | |
45 | """ |
|
87 | """ | |
46 | Generates unique API key for given username,if salt is not given |
|
88 | Generates unique API key for given username,if salt is not given | |
47 | it'll be generated from some random string |
|
89 | it'll be generated from some random string | |
48 |
|
90 | |||
49 | :param username: username as string |
|
91 | :param username: username as string | |
50 | :param salt: salt to hash generate KEY |
|
92 | :param salt: salt to hash generate KEY | |
51 | :rtype: str |
|
93 | :rtype: str | |
52 | :returns: sha1 hash from username+salt |
|
94 | :returns: sha1 hash from username+salt | |
53 | """ |
|
95 | """ | |
54 | from tempfile import _RandomNameSequence |
|
96 | from tempfile import _RandomNameSequence | |
55 | import hashlib |
|
97 | import hashlib | |
56 |
|
98 | |||
57 | if salt is None: |
|
99 | if salt is None: | |
58 | salt = _RandomNameSequence().next() |
|
100 | salt = _RandomNameSequence().next() | |
59 |
|
101 | |||
60 | return hashlib.sha1(username + salt).hexdigest() |
|
102 | return hashlib.sha1(username + salt).hexdigest() | |
61 |
|
103 | |||
62 |
|
104 | |||
63 | def safe_unicode(_str, from_encoding='utf8'): |
|
105 | def safe_unicode(_str, from_encoding='utf8'): | |
64 | """ |
|
106 | """ | |
65 | safe unicode function. In case of UnicodeDecode error we try to return |
|
107 | safe unicode function. In case of UnicodeDecode error we try to return | |
66 | unicode with errors replace |
|
108 | unicode with errors replace | |
67 |
|
109 | |||
68 | :param _str: string to decode |
|
110 | :param _str: string to decode | |
69 | :rtype: unicode |
|
111 | :rtype: unicode | |
70 | :returns: unicode object |
|
112 | :returns: unicode object | |
71 | """ |
|
113 | """ | |
72 |
|
114 | |||
73 | if isinstance(_str, unicode): |
|
115 | if isinstance(_str, unicode): | |
74 | return _str |
|
116 | return _str | |
75 |
|
117 | |||
76 | try: |
|
118 | try: | |
77 | u_str = unicode(_str, from_encoding) |
|
119 | u_str = unicode(_str, from_encoding) | |
78 | except UnicodeDecodeError: |
|
120 | except UnicodeDecodeError: | |
79 | u_str = unicode(_str, from_encoding, 'replace') |
|
121 | u_str = unicode(_str, from_encoding, 'replace') | |
80 |
|
122 | |||
81 | return u_str |
|
123 | return u_str | |
82 |
|
124 | |||
83 |
|
125 | |||
84 | def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs): |
|
126 | def engine_from_config(configuration, prefix='sqlalchemy.', **kwargs): | |
85 | """ |
|
127 | """ | |
86 | Custom engine_from_config functions that makes sure we use NullPool for |
|
128 | Custom engine_from_config functions that makes sure we use NullPool for | |
87 | file based sqlite databases. This prevents errors on sqlite. |
|
129 | file based sqlite databases. This prevents errors on sqlite. | |
88 |
|
130 | |||
89 | """ |
|
131 | """ | |
90 | from sqlalchemy import engine_from_config as efc |
|
132 | from sqlalchemy import engine_from_config as efc | |
91 | from sqlalchemy.pool import NullPool |
|
133 | from sqlalchemy.pool import NullPool | |
92 |
|
134 | |||
93 | url = configuration[prefix + 'url'] |
|
135 | url = configuration[prefix + 'url'] | |
94 |
|
136 | |||
95 | if url.startswith('sqlite'): |
|
137 | if url.startswith('sqlite'): | |
96 | kwargs.update({'poolclass':NullPool}) |
|
138 | kwargs.update({'poolclass':NullPool}) | |
97 |
|
139 | |||
98 | return efc(configuration, prefix, **kwargs) |
|
140 | return efc(configuration, prefix, **kwargs) | |
99 |
|
141 | |||
100 |
|
142 |
@@ -1,409 +1,374 b'' | |||||
1 | # -*- coding: utf-8 -*- |
|
1 | # -*- coding: utf-8 -*- | |
2 | """ |
|
2 | """ | |
3 | rhodecode.lib.celerylib.tasks |
|
3 | rhodecode.lib.celerylib.tasks | |
4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
|
4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | |
5 |
|
5 | |||
6 | RhodeCode task modules, containing all task that suppose to be run |
|
6 | RhodeCode task modules, containing all task that suppose to be run | |
7 | by celery daemon |
|
7 | by celery daemon | |
8 |
|
8 | |||
9 | :created_on: Oct 6, 2010 |
|
9 | :created_on: Oct 6, 2010 | |
10 | :author: marcink |
|
10 | :author: marcink | |
11 | :copyright: (C) 2009-2011 Marcin Kuzminski <marcin@python-works.com> |
|
11 | :copyright: (C) 2009-2011 Marcin Kuzminski <marcin@python-works.com> | |
12 | :license: GPLv3, see COPYING for more details. |
|
12 | :license: GPLv3, see COPYING for more details. | |
13 | """ |
|
13 | """ | |
14 | # This program is free software: you can redistribute it and/or modify |
|
14 | # This program is free software: you can redistribute it and/or modify | |
15 | # it under the terms of the GNU General Public License as published by |
|
15 | # it under the terms of the GNU General Public License as published by | |
16 | # the Free Software Foundation, either version 3 of the License, or |
|
16 | # the Free Software Foundation, either version 3 of the License, or | |
17 | # (at your option) any later version. |
|
17 | # (at your option) any later version. | |
18 | # |
|
18 | # | |
19 | # This program is distributed in the hope that it will be useful, |
|
19 | # This program is distributed in the hope that it will be useful, | |
20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
22 | # GNU General Public License for more details. |
|
22 | # GNU General Public License for more details. | |
23 | # |
|
23 | # | |
24 | # You should have received a copy of the GNU General Public License |
|
24 | # You should have received a copy of the GNU General Public License | |
25 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
25 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
26 | from celery.decorators import task |
|
26 | from celery.decorators import task | |
27 |
|
27 | |||
28 | import os |
|
28 | import os | |
29 | import traceback |
|
29 | import traceback | |
30 | import logging |
|
30 | import logging | |
31 |
|
31 | |||
32 | from time import mktime |
|
32 | from time import mktime | |
33 | from operator import itemgetter |
|
33 | from operator import itemgetter | |
34 | from pygments import lexers |
|
|||
35 | from string import lower |
|
34 | from string import lower | |
36 |
|
35 | |||
37 | from pylons import config |
|
36 | from pylons import config | |
38 | from pylons.i18n.translation import _ |
|
37 | from pylons.i18n.translation import _ | |
39 |
|
38 | |||
|
39 | from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP | |||
40 | from rhodecode.lib.celerylib import run_task, locked_task, str2bool, \ |
|
40 | from rhodecode.lib.celerylib import run_task, locked_task, str2bool, \ | |
41 | __get_lockkey, LockHeld, DaemonLock |
|
41 | __get_lockkey, LockHeld, DaemonLock | |
42 | from rhodecode.lib.helpers import person |
|
42 | from rhodecode.lib.helpers import person | |
43 | from rhodecode.lib.smtp_mailer import SmtpMailer |
|
43 | from rhodecode.lib.smtp_mailer import SmtpMailer | |
44 | from rhodecode.lib.utils import OrderedDict, add_cache |
|
44 | from rhodecode.lib.utils import OrderedDict, add_cache | |
45 | from rhodecode.model import init_model |
|
45 | from rhodecode.model import init_model | |
46 | from rhodecode.model import meta |
|
46 | from rhodecode.model import meta | |
47 | from rhodecode.model.db import RhodeCodeUi, Statistics, Repository |
|
47 | from rhodecode.model.db import RhodeCodeUi, Statistics, Repository | |
48 |
|
48 | |||
49 | from vcs.backends import get_repo |
|
49 | from vcs.backends import get_repo | |
50 |
|
50 | |||
51 | from sqlalchemy import engine_from_config |
|
51 | from sqlalchemy import engine_from_config | |
52 |
|
52 | |||
53 | add_cache(config) |
|
53 | add_cache(config) | |
54 |
|
54 | |||
55 | try: |
|
55 | try: | |
56 | import json |
|
56 | import json | |
57 | except ImportError: |
|
57 | except ImportError: | |
58 | #python 2.5 compatibility |
|
58 | #python 2.5 compatibility | |
59 | import simplejson as json |
|
59 | import simplejson as json | |
60 |
|
60 | |||
61 | __all__ = ['whoosh_index', 'get_commits_stats', |
|
61 | __all__ = ['whoosh_index', 'get_commits_stats', | |
62 | 'reset_user_password', 'send_email'] |
|
62 | 'reset_user_password', 'send_email'] | |
63 |
|
63 | |||
64 | CELERY_ON = str2bool(config['app_conf'].get('use_celery')) |
|
64 | CELERY_ON = str2bool(config['app_conf'].get('use_celery')) | |
65 |
|
65 | |||
66 | LANGUAGES_EXTENSIONS_MAP = {} |
|
|||
67 |
|
||||
68 |
|
||||
69 | def __clean(s): |
|
|||
70 |
|
||||
71 | s = s.lstrip('*') |
|
|||
72 | s = s.lstrip('.') |
|
|||
73 |
|
||||
74 | if s.find('[') != -1: |
|
|||
75 | exts = [] |
|
|||
76 | start, stop = s.find('['), s.find(']') |
|
|||
77 |
|
||||
78 | for suffix in s[start + 1:stop]: |
|
|||
79 | exts.append(s[:s.find('[')] + suffix) |
|
|||
80 | return map(lower, exts) |
|
|||
81 | else: |
|
|||
82 | return map(lower, [s]) |
|
|||
83 |
|
||||
84 | for lx, t in sorted(lexers.LEXERS.items()): |
|
|||
85 | m = map(__clean, t[-2]) |
|
|||
86 | if m: |
|
|||
87 | m = reduce(lambda x, y: x + y, m) |
|
|||
88 | for ext in m: |
|
|||
89 | desc = lx.replace('Lexer', '') |
|
|||
90 | if ext in LANGUAGES_EXTENSIONS_MAP: |
|
|||
91 | if desc not in LANGUAGES_EXTENSIONS_MAP[ext]: |
|
|||
92 | LANGUAGES_EXTENSIONS_MAP[ext].append(desc) |
|
|||
93 | else: |
|
|||
94 | LANGUAGES_EXTENSIONS_MAP[ext] = [desc] |
|
|||
95 |
|
||||
96 | #Additional mappings that are not present in the pygments lexers |
|
|||
97 | # NOTE: that this will overide any mappings in LANGUAGES_EXTENSIONS_MAP |
|
|||
98 | ADDITIONAL_MAPPINGS = {'xaml': 'XAML'} |
|
|||
99 |
|
||||
100 | LANGUAGES_EXTENSIONS_MAP.update(ADDITIONAL_MAPPINGS) |
|
|||
101 |
|
66 | |||
102 |
|
67 | |||
103 | def get_session(): |
|
68 | def get_session(): | |
104 | if CELERY_ON: |
|
69 | if CELERY_ON: | |
105 | engine = engine_from_config(config, 'sqlalchemy.db1.') |
|
70 | engine = engine_from_config(config, 'sqlalchemy.db1.') | |
106 | init_model(engine) |
|
71 | init_model(engine) | |
107 | sa = meta.Session() |
|
72 | sa = meta.Session() | |
108 | return sa |
|
73 | return sa | |
109 |
|
74 | |||
110 |
|
75 | |||
111 | def get_repos_path(): |
|
76 | def get_repos_path(): | |
112 | sa = get_session() |
|
77 | sa = get_session() | |
113 | q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one() |
|
78 | q = sa.query(RhodeCodeUi).filter(RhodeCodeUi.ui_key == '/').one() | |
114 | return q.ui_value |
|
79 | return q.ui_value | |
115 |
|
80 | |||
116 |
|
81 | |||
117 | @task(ignore_result=True) |
|
82 | @task(ignore_result=True) | |
118 | @locked_task |
|
83 | @locked_task | |
119 | def whoosh_index(repo_location, full_index): |
|
84 | def whoosh_index(repo_location, full_index): | |
120 | #log = whoosh_index.get_logger() |
|
85 | #log = whoosh_index.get_logger() | |
121 | from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon |
|
86 | from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon | |
122 | index_location = config['index_dir'] |
|
87 | index_location = config['index_dir'] | |
123 | WhooshIndexingDaemon(index_location=index_location, |
|
88 | WhooshIndexingDaemon(index_location=index_location, | |
124 | repo_location=repo_location, sa=get_session())\ |
|
89 | repo_location=repo_location, sa=get_session())\ | |
125 | .run(full_index=full_index) |
|
90 | .run(full_index=full_index) | |
126 |
|
91 | |||
127 |
|
92 | |||
128 | @task(ignore_result=True) |
|
93 | @task(ignore_result=True) | |
129 | def get_commits_stats(repo_name, ts_min_y, ts_max_y): |
|
94 | def get_commits_stats(repo_name, ts_min_y, ts_max_y): | |
130 | try: |
|
95 | try: | |
131 | log = get_commits_stats.get_logger() |
|
96 | log = get_commits_stats.get_logger() | |
132 | except: |
|
97 | except: | |
133 | log = logging.getLogger(__name__) |
|
98 | log = logging.getLogger(__name__) | |
134 |
|
99 | |||
135 | lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y, |
|
100 | lockkey = __get_lockkey('get_commits_stats', repo_name, ts_min_y, | |
136 | ts_max_y) |
|
101 | ts_max_y) | |
137 | log.info('running task with lockkey %s', lockkey) |
|
102 | log.info('running task with lockkey %s', lockkey) | |
138 | try: |
|
103 | try: | |
139 | lock = DaemonLock(lockkey) |
|
104 | lock = DaemonLock(lockkey) | |
140 |
|
105 | |||
141 | #for js data compatibilty cleans the key for person from ' |
|
106 | #for js data compatibilty cleans the key for person from ' | |
142 | akc = lambda k: person(k).replace('"', "") |
|
107 | akc = lambda k: person(k).replace('"', "") | |
143 |
|
108 | |||
144 | co_day_auth_aggr = {} |
|
109 | co_day_auth_aggr = {} | |
145 | commits_by_day_aggregate = {} |
|
110 | commits_by_day_aggregate = {} | |
146 | repos_path = get_repos_path() |
|
111 | repos_path = get_repos_path() | |
147 | p = os.path.join(repos_path, repo_name) |
|
112 | p = os.path.join(repos_path, repo_name) | |
148 | repo = get_repo(p) |
|
113 | repo = get_repo(p) | |
149 | repo_size = len(repo.revisions) |
|
114 | repo_size = len(repo.revisions) | |
150 | #return if repo have no revisions |
|
115 | #return if repo have no revisions | |
151 | if repo_size < 1: |
|
116 | if repo_size < 1: | |
152 | lock.release() |
|
117 | lock.release() | |
153 | return True |
|
118 | return True | |
154 |
|
119 | |||
155 | skip_date_limit = True |
|
120 | skip_date_limit = True | |
156 | parse_limit = int(config['app_conf'].get('commit_parse_limit')) |
|
121 | parse_limit = int(config['app_conf'].get('commit_parse_limit')) | |
157 | last_rev = 0 |
|
122 | last_rev = 0 | |
158 | last_cs = None |
|
123 | last_cs = None | |
159 | timegetter = itemgetter('time') |
|
124 | timegetter = itemgetter('time') | |
160 |
|
125 | |||
161 | sa = get_session() |
|
126 | sa = get_session() | |
162 |
|
127 | |||
163 | dbrepo = sa.query(Repository)\ |
|
128 | dbrepo = sa.query(Repository)\ | |
164 | .filter(Repository.repo_name == repo_name).scalar() |
|
129 | .filter(Repository.repo_name == repo_name).scalar() | |
165 | cur_stats = sa.query(Statistics)\ |
|
130 | cur_stats = sa.query(Statistics)\ | |
166 | .filter(Statistics.repository == dbrepo).scalar() |
|
131 | .filter(Statistics.repository == dbrepo).scalar() | |
167 |
|
132 | |||
168 | if cur_stats is not None: |
|
133 | if cur_stats is not None: | |
169 | last_rev = cur_stats.stat_on_revision |
|
134 | last_rev = cur_stats.stat_on_revision | |
170 |
|
135 | |||
171 | if last_rev == repo.get_changeset().revision and repo_size > 1: |
|
136 | if last_rev == repo.get_changeset().revision and repo_size > 1: | |
172 | #pass silently without any work if we're not on first revision or |
|
137 | #pass silently without any work if we're not on first revision or | |
173 | #current state of parsing revision(from db marker) is the |
|
138 | #current state of parsing revision(from db marker) is the | |
174 | #last revision |
|
139 | #last revision | |
175 | lock.release() |
|
140 | lock.release() | |
176 | return True |
|
141 | return True | |
177 |
|
142 | |||
178 | if cur_stats: |
|
143 | if cur_stats: | |
179 | commits_by_day_aggregate = OrderedDict(json.loads( |
|
144 | commits_by_day_aggregate = OrderedDict(json.loads( | |
180 | cur_stats.commit_activity_combined)) |
|
145 | cur_stats.commit_activity_combined)) | |
181 | co_day_auth_aggr = json.loads(cur_stats.commit_activity) |
|
146 | co_day_auth_aggr = json.loads(cur_stats.commit_activity) | |
182 |
|
147 | |||
183 | log.debug('starting parsing %s', parse_limit) |
|
148 | log.debug('starting parsing %s', parse_limit) | |
184 | lmktime = mktime |
|
149 | lmktime = mktime | |
185 |
|
150 | |||
186 | last_rev = last_rev + 1 if last_rev > 0 else last_rev |
|
151 | last_rev = last_rev + 1 if last_rev > 0 else last_rev | |
187 |
|
152 | |||
188 | for cs in repo[last_rev:last_rev + parse_limit]: |
|
153 | for cs in repo[last_rev:last_rev + parse_limit]: | |
189 | last_cs = cs # remember last parsed changeset |
|
154 | last_cs = cs # remember last parsed changeset | |
190 | k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1], |
|
155 | k = lmktime([cs.date.timetuple()[0], cs.date.timetuple()[1], | |
191 | cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0]) |
|
156 | cs.date.timetuple()[2], 0, 0, 0, 0, 0, 0]) | |
192 |
|
157 | |||
193 | if akc(cs.author) in co_day_auth_aggr: |
|
158 | if akc(cs.author) in co_day_auth_aggr: | |
194 | try: |
|
159 | try: | |
195 | l = [timegetter(x) for x in |
|
160 | l = [timegetter(x) for x in | |
196 | co_day_auth_aggr[akc(cs.author)]['data']] |
|
161 | co_day_auth_aggr[akc(cs.author)]['data']] | |
197 | time_pos = l.index(k) |
|
162 | time_pos = l.index(k) | |
198 | except ValueError: |
|
163 | except ValueError: | |
199 | time_pos = False |
|
164 | time_pos = False | |
200 |
|
165 | |||
201 | if time_pos >= 0 and time_pos is not False: |
|
166 | if time_pos >= 0 and time_pos is not False: | |
202 |
|
167 | |||
203 | datadict = \ |
|
168 | datadict = \ | |
204 | co_day_auth_aggr[akc(cs.author)]['data'][time_pos] |
|
169 | co_day_auth_aggr[akc(cs.author)]['data'][time_pos] | |
205 |
|
170 | |||
206 | datadict["commits"] += 1 |
|
171 | datadict["commits"] += 1 | |
207 | datadict["added"] += len(cs.added) |
|
172 | datadict["added"] += len(cs.added) | |
208 | datadict["changed"] += len(cs.changed) |
|
173 | datadict["changed"] += len(cs.changed) | |
209 | datadict["removed"] += len(cs.removed) |
|
174 | datadict["removed"] += len(cs.removed) | |
210 |
|
175 | |||
211 | else: |
|
176 | else: | |
212 | if k >= ts_min_y and k <= ts_max_y or skip_date_limit: |
|
177 | if k >= ts_min_y and k <= ts_max_y or skip_date_limit: | |
213 |
|
178 | |||
214 | datadict = {"time": k, |
|
179 | datadict = {"time": k, | |
215 | "commits": 1, |
|
180 | "commits": 1, | |
216 | "added": len(cs.added), |
|
181 | "added": len(cs.added), | |
217 | "changed": len(cs.changed), |
|
182 | "changed": len(cs.changed), | |
218 | "removed": len(cs.removed), |
|
183 | "removed": len(cs.removed), | |
219 | } |
|
184 | } | |
220 | co_day_auth_aggr[akc(cs.author)]['data']\ |
|
185 | co_day_auth_aggr[akc(cs.author)]['data']\ | |
221 | .append(datadict) |
|
186 | .append(datadict) | |
222 |
|
187 | |||
223 | else: |
|
188 | else: | |
224 | if k >= ts_min_y and k <= ts_max_y or skip_date_limit: |
|
189 | if k >= ts_min_y and k <= ts_max_y or skip_date_limit: | |
225 | co_day_auth_aggr[akc(cs.author)] = { |
|
190 | co_day_auth_aggr[akc(cs.author)] = { | |
226 | "label": akc(cs.author), |
|
191 | "label": akc(cs.author), | |
227 | "data": [{"time":k, |
|
192 | "data": [{"time":k, | |
228 | "commits":1, |
|
193 | "commits":1, | |
229 | "added":len(cs.added), |
|
194 | "added":len(cs.added), | |
230 | "changed":len(cs.changed), |
|
195 | "changed":len(cs.changed), | |
231 | "removed":len(cs.removed), |
|
196 | "removed":len(cs.removed), | |
232 | }], |
|
197 | }], | |
233 | "schema": ["commits"], |
|
198 | "schema": ["commits"], | |
234 | } |
|
199 | } | |
235 |
|
200 | |||
236 | #gather all data by day |
|
201 | #gather all data by day | |
237 | if k in commits_by_day_aggregate: |
|
202 | if k in commits_by_day_aggregate: | |
238 | commits_by_day_aggregate[k] += 1 |
|
203 | commits_by_day_aggregate[k] += 1 | |
239 | else: |
|
204 | else: | |
240 | commits_by_day_aggregate[k] = 1 |
|
205 | commits_by_day_aggregate[k] = 1 | |
241 |
|
206 | |||
242 | overview_data = sorted(commits_by_day_aggregate.items(), |
|
207 | overview_data = sorted(commits_by_day_aggregate.items(), | |
243 | key=itemgetter(0)) |
|
208 | key=itemgetter(0)) | |
244 |
|
209 | |||
245 | if not co_day_auth_aggr: |
|
210 | if not co_day_auth_aggr: | |
246 | co_day_auth_aggr[akc(repo.contact)] = { |
|
211 | co_day_auth_aggr[akc(repo.contact)] = { | |
247 | "label": akc(repo.contact), |
|
212 | "label": akc(repo.contact), | |
248 | "data": [0, 1], |
|
213 | "data": [0, 1], | |
249 | "schema": ["commits"], |
|
214 | "schema": ["commits"], | |
250 | } |
|
215 | } | |
251 |
|
216 | |||
252 | stats = cur_stats if cur_stats else Statistics() |
|
217 | stats = cur_stats if cur_stats else Statistics() | |
253 | stats.commit_activity = json.dumps(co_day_auth_aggr) |
|
218 | stats.commit_activity = json.dumps(co_day_auth_aggr) | |
254 | stats.commit_activity_combined = json.dumps(overview_data) |
|
219 | stats.commit_activity_combined = json.dumps(overview_data) | |
255 |
|
220 | |||
256 | log.debug('last revison %s', last_rev) |
|
221 | log.debug('last revison %s', last_rev) | |
257 | leftovers = len(repo.revisions[last_rev:]) |
|
222 | leftovers = len(repo.revisions[last_rev:]) | |
258 | log.debug('revisions to parse %s', leftovers) |
|
223 | log.debug('revisions to parse %s', leftovers) | |
259 |
|
224 | |||
260 | if last_rev == 0 or leftovers < parse_limit: |
|
225 | if last_rev == 0 or leftovers < parse_limit: | |
261 | log.debug('getting code trending stats') |
|
226 | log.debug('getting code trending stats') | |
262 | stats.languages = json.dumps(__get_codes_stats(repo_name)) |
|
227 | stats.languages = json.dumps(__get_codes_stats(repo_name)) | |
263 |
|
228 | |||
264 | try: |
|
229 | try: | |
265 | stats.repository = dbrepo |
|
230 | stats.repository = dbrepo | |
266 | stats.stat_on_revision = last_cs.revision if last_cs else 0 |
|
231 | stats.stat_on_revision = last_cs.revision if last_cs else 0 | |
267 | sa.add(stats) |
|
232 | sa.add(stats) | |
268 | sa.commit() |
|
233 | sa.commit() | |
269 | except: |
|
234 | except: | |
270 | log.error(traceback.format_exc()) |
|
235 | log.error(traceback.format_exc()) | |
271 | sa.rollback() |
|
236 | sa.rollback() | |
272 | lock.release() |
|
237 | lock.release() | |
273 | return False |
|
238 | return False | |
274 |
|
239 | |||
275 | #final release |
|
240 | #final release | |
276 | lock.release() |
|
241 | lock.release() | |
277 |
|
242 | |||
278 | #execute another task if celery is enabled |
|
243 | #execute another task if celery is enabled | |
279 | if len(repo.revisions) > 1 and CELERY_ON: |
|
244 | if len(repo.revisions) > 1 and CELERY_ON: | |
280 | run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y) |
|
245 | run_task(get_commits_stats, repo_name, ts_min_y, ts_max_y) | |
281 | return True |
|
246 | return True | |
282 | except LockHeld: |
|
247 | except LockHeld: | |
283 | log.info('LockHeld') |
|
248 | log.info('LockHeld') | |
284 | return 'Task with key %s already running' % lockkey |
|
249 | return 'Task with key %s already running' % lockkey | |
285 |
|
250 | |||
286 |
|
251 | |||
287 | @task(ignore_result=True) |
|
252 | @task(ignore_result=True) | |
288 | def reset_user_password(user_email): |
|
253 | def reset_user_password(user_email): | |
289 | try: |
|
254 | try: | |
290 | log = reset_user_password.get_logger() |
|
255 | log = reset_user_password.get_logger() | |
291 | except: |
|
256 | except: | |
292 | log = logging.getLogger(__name__) |
|
257 | log = logging.getLogger(__name__) | |
293 |
|
258 | |||
294 | from rhodecode.lib import auth |
|
259 | from rhodecode.lib import auth | |
295 | from rhodecode.model.db import User |
|
260 | from rhodecode.model.db import User | |
296 |
|
261 | |||
297 | try: |
|
262 | try: | |
298 | try: |
|
263 | try: | |
299 | sa = get_session() |
|
264 | sa = get_session() | |
300 | user = sa.query(User).filter(User.email == user_email).scalar() |
|
265 | user = sa.query(User).filter(User.email == user_email).scalar() | |
301 | new_passwd = auth.PasswordGenerator().gen_password(8, |
|
266 | new_passwd = auth.PasswordGenerator().gen_password(8, | |
302 | auth.PasswordGenerator.ALPHABETS_BIG_SMALL) |
|
267 | auth.PasswordGenerator.ALPHABETS_BIG_SMALL) | |
303 | if user: |
|
268 | if user: | |
304 | user.password = auth.get_crypt_password(new_passwd) |
|
269 | user.password = auth.get_crypt_password(new_passwd) | |
305 | user.api_key = auth.generate_api_key(user.username) |
|
270 | user.api_key = auth.generate_api_key(user.username) | |
306 | sa.add(user) |
|
271 | sa.add(user) | |
307 | sa.commit() |
|
272 | sa.commit() | |
308 | log.info('change password for %s', user_email) |
|
273 | log.info('change password for %s', user_email) | |
309 | if new_passwd is None: |
|
274 | if new_passwd is None: | |
310 | raise Exception('unable to generate new password') |
|
275 | raise Exception('unable to generate new password') | |
311 |
|
276 | |||
312 | except: |
|
277 | except: | |
313 | log.error(traceback.format_exc()) |
|
278 | log.error(traceback.format_exc()) | |
314 | sa.rollback() |
|
279 | sa.rollback() | |
315 |
|
280 | |||
316 | run_task(send_email, user_email, |
|
281 | run_task(send_email, user_email, | |
317 | "Your new rhodecode password", |
|
282 | "Your new rhodecode password", | |
318 | 'Your new rhodecode password:%s' % (new_passwd)) |
|
283 | 'Your new rhodecode password:%s' % (new_passwd)) | |
319 | log.info('send new password mail to %s', user_email) |
|
284 | log.info('send new password mail to %s', user_email) | |
320 |
|
285 | |||
321 | except: |
|
286 | except: | |
322 | log.error('Failed to update user password') |
|
287 | log.error('Failed to update user password') | |
323 | log.error(traceback.format_exc()) |
|
288 | log.error(traceback.format_exc()) | |
324 |
|
289 | |||
325 | return True |
|
290 | return True | |
326 |
|
291 | |||
327 |
|
292 | |||
328 | @task(ignore_result=True) |
|
293 | @task(ignore_result=True) | |
329 | def send_email(recipients, subject, body): |
|
294 | def send_email(recipients, subject, body): | |
330 | """ |
|
295 | """ | |
331 | Sends an email with defined parameters from the .ini files. |
|
296 | Sends an email with defined parameters from the .ini files. | |
332 |
|
297 | |||
333 | :param recipients: list of recipients, it this is empty the defined email |
|
298 | :param recipients: list of recipients, it this is empty the defined email | |
334 | address from field 'email_to' is used instead |
|
299 | address from field 'email_to' is used instead | |
335 | :param subject: subject of the mail |
|
300 | :param subject: subject of the mail | |
336 | :param body: body of the mail |
|
301 | :param body: body of the mail | |
337 | """ |
|
302 | """ | |
338 | try: |
|
303 | try: | |
339 | log = send_email.get_logger() |
|
304 | log = send_email.get_logger() | |
340 | except: |
|
305 | except: | |
341 | log = logging.getLogger(__name__) |
|
306 | log = logging.getLogger(__name__) | |
342 |
|
307 | |||
343 | email_config = config |
|
308 | email_config = config | |
344 |
|
309 | |||
345 | if not recipients: |
|
310 | if not recipients: | |
346 | recipients = [email_config.get('email_to')] |
|
311 | recipients = [email_config.get('email_to')] | |
347 |
|
312 | |||
348 | mail_from = email_config.get('app_email_from') |
|
313 | mail_from = email_config.get('app_email_from') | |
349 | user = email_config.get('smtp_username') |
|
314 | user = email_config.get('smtp_username') | |
350 | passwd = email_config.get('smtp_password') |
|
315 | passwd = email_config.get('smtp_password') | |
351 | mail_server = email_config.get('smtp_server') |
|
316 | mail_server = email_config.get('smtp_server') | |
352 | mail_port = email_config.get('smtp_port') |
|
317 | mail_port = email_config.get('smtp_port') | |
353 | tls = str2bool(email_config.get('smtp_use_tls')) |
|
318 | tls = str2bool(email_config.get('smtp_use_tls')) | |
354 | ssl = str2bool(email_config.get('smtp_use_ssl')) |
|
319 | ssl = str2bool(email_config.get('smtp_use_ssl')) | |
355 | debug = str2bool(config.get('debug')) |
|
320 | debug = str2bool(config.get('debug')) | |
356 |
|
321 | |||
357 | try: |
|
322 | try: | |
358 | m = SmtpMailer(mail_from, user, passwd, mail_server, |
|
323 | m = SmtpMailer(mail_from, user, passwd, mail_server, | |
359 | mail_port, ssl, tls, debug=debug) |
|
324 | mail_port, ssl, tls, debug=debug) | |
360 | m.send(recipients, subject, body) |
|
325 | m.send(recipients, subject, body) | |
361 | except: |
|
326 | except: | |
362 | log.error('Mail sending failed') |
|
327 | log.error('Mail sending failed') | |
363 | log.error(traceback.format_exc()) |
|
328 | log.error(traceback.format_exc()) | |
364 | return False |
|
329 | return False | |
365 | return True |
|
330 | return True | |
366 |
|
331 | |||
367 |
|
332 | |||
368 | @task(ignore_result=True) |
|
333 | @task(ignore_result=True) | |
369 | def create_repo_fork(form_data, cur_user): |
|
334 | def create_repo_fork(form_data, cur_user): | |
370 | from rhodecode.model.repo import RepoModel |
|
335 | from rhodecode.model.repo import RepoModel | |
371 | from vcs import get_backend |
|
336 | from vcs import get_backend | |
372 |
|
337 | |||
373 | try: |
|
338 | try: | |
374 | log = create_repo_fork.get_logger() |
|
339 | log = create_repo_fork.get_logger() | |
375 | except: |
|
340 | except: | |
376 | log = logging.getLogger(__name__) |
|
341 | log = logging.getLogger(__name__) | |
377 |
|
342 | |||
378 | repo_model = RepoModel(get_session()) |
|
343 | repo_model = RepoModel(get_session()) | |
379 | repo_model.create(form_data, cur_user, just_db=True, fork=True) |
|
344 | repo_model.create(form_data, cur_user, just_db=True, fork=True) | |
380 | repo_name = form_data['repo_name'] |
|
345 | repo_name = form_data['repo_name'] | |
381 | repos_path = get_repos_path() |
|
346 | repos_path = get_repos_path() | |
382 | repo_path = os.path.join(repos_path, repo_name) |
|
347 | repo_path = os.path.join(repos_path, repo_name) | |
383 | repo_fork_path = os.path.join(repos_path, form_data['fork_name']) |
|
348 | repo_fork_path = os.path.join(repos_path, form_data['fork_name']) | |
384 | alias = form_data['repo_type'] |
|
349 | alias = form_data['repo_type'] | |
385 |
|
350 | |||
386 | log.info('creating repo fork %s as %s', repo_name, repo_path) |
|
351 | log.info('creating repo fork %s as %s', repo_name, repo_path) | |
387 | backend = get_backend(alias) |
|
352 | backend = get_backend(alias) | |
388 | backend(str(repo_fork_path), create=True, src_url=str(repo_path)) |
|
353 | backend(str(repo_fork_path), create=True, src_url=str(repo_path)) | |
389 |
|
354 | |||
390 |
|
355 | |||
391 | def __get_codes_stats(repo_name): |
|
356 | def __get_codes_stats(repo_name): | |
392 | repos_path = get_repos_path() |
|
357 | repos_path = get_repos_path() | |
393 | p = os.path.join(repos_path, repo_name) |
|
358 | p = os.path.join(repos_path, repo_name) | |
394 | repo = get_repo(p) |
|
359 | repo = get_repo(p) | |
395 | tip = repo.get_changeset() |
|
360 | tip = repo.get_changeset() | |
396 | code_stats = {} |
|
361 | code_stats = {} | |
397 |
|
362 | |||
398 | def aggregate(cs): |
|
363 | def aggregate(cs): | |
399 | for f in cs[2]: |
|
364 | for f in cs[2]: | |
400 | ext = lower(f.extension) |
|
365 | ext = lower(f.extension) | |
401 | if ext in LANGUAGES_EXTENSIONS_MAP.keys() and not f.is_binary: |
|
366 | if ext in LANGUAGES_EXTENSIONS_MAP.keys() and not f.is_binary: | |
402 | if ext in code_stats: |
|
367 | if ext in code_stats: | |
403 | code_stats[ext] += 1 |
|
368 | code_stats[ext] += 1 | |
404 | else: |
|
369 | else: | |
405 | code_stats[ext] = 1 |
|
370 | code_stats[ext] = 1 | |
406 |
|
371 | |||
407 | map(aggregate, tip.walk('/')) |
|
372 | map(aggregate, tip.walk('/')) | |
408 |
|
373 | |||
409 | return code_stats or {} |
|
374 | return code_stats or {} |
@@ -1,230 +1,224 b'' | |||||
1 | # -*- coding: utf-8 -*- |
|
1 | # -*- coding: utf-8 -*- | |
2 | """ |
|
2 | """ | |
3 | rhodecode.lib.indexers.__init__ |
|
3 | rhodecode.lib.indexers.__init__ | |
4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
|
4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | |
5 |
|
5 | |||
6 | Whoosh indexing module for RhodeCode |
|
6 | Whoosh indexing module for RhodeCode | |
7 |
|
7 | |||
8 | :created_on: Aug 17, 2010 |
|
8 | :created_on: Aug 17, 2010 | |
9 | :author: marcink |
|
9 | :author: marcink | |
10 | :copyright: (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com> |
|
10 | :copyright: (C) 2009-2010 Marcin Kuzminski <marcin@python-works.com> | |
11 | :license: GPLv3, see COPYING for more details. |
|
11 | :license: GPLv3, see COPYING for more details. | |
12 | """ |
|
12 | """ | |
13 | # This program is free software: you can redistribute it and/or modify |
|
13 | # This program is free software: you can redistribute it and/or modify | |
14 | # it under the terms of the GNU General Public License as published by |
|
14 | # it under the terms of the GNU General Public License as published by | |
15 | # the Free Software Foundation, either version 3 of the License, or |
|
15 | # the Free Software Foundation, either version 3 of the License, or | |
16 | # (at your option) any later version. |
|
16 | # (at your option) any later version. | |
17 | # |
|
17 | # | |
18 | # This program is distributed in the hope that it will be useful, |
|
18 | # This program is distributed in the hope that it will be useful, | |
19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
21 | # GNU General Public License for more details. |
|
21 | # GNU General Public License for more details. | |
22 | # |
|
22 | # | |
23 | # You should have received a copy of the GNU General Public License |
|
23 | # You should have received a copy of the GNU General Public License | |
24 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
24 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
25 | import os |
|
25 | import os | |
26 | import sys |
|
26 | import sys | |
27 | import traceback |
|
27 | import traceback | |
28 | from os.path import dirname as dn, join as jn |
|
28 | from os.path import dirname as dn, join as jn | |
29 |
|
29 | |||
30 | #to get the rhodecode import |
|
30 | #to get the rhodecode import | |
31 | sys.path.append(dn(dn(dn(os.path.realpath(__file__))))) |
|
31 | sys.path.append(dn(dn(dn(os.path.realpath(__file__))))) | |
32 |
|
32 | |||
33 | from string import strip |
|
33 | from string import strip | |
34 |
|
||||
35 | from rhodecode.model import init_model |
|
|||
36 | from rhodecode.model.scm import ScmModel |
|
|||
37 | from rhodecode.config.environment import load_environment |
|
|||
38 | from rhodecode.lib.utils import BasePasterCommand, Command, add_cache |
|
|||
39 |
|
||||
40 | from shutil import rmtree |
|
34 | from shutil import rmtree | |
41 | from webhelpers.html.builder import escape |
|
|||
42 | from vcs.utils.lazy import LazyProperty |
|
|||
43 |
|
||||
44 | from sqlalchemy import engine_from_config |
|
|||
45 |
|
35 | |||
46 | from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter |
|
36 | from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter | |
47 | from whoosh.fields import TEXT, ID, STORED, Schema, FieldType |
|
37 | from whoosh.fields import TEXT, ID, STORED, Schema, FieldType | |
48 | from whoosh.index import create_in, open_dir |
|
38 | from whoosh.index import create_in, open_dir | |
49 | from whoosh.formats import Characters |
|
39 | from whoosh.formats import Characters | |
50 | from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter |
|
40 | from whoosh.highlight import highlight, SimpleFragmenter, HtmlFormatter | |
51 |
|
41 | |||
|
42 | from webhelpers.html.builder import escape | |||
|
43 | from sqlalchemy import engine_from_config | |||
|
44 | from vcs.utils.lazy import LazyProperty | |||
|
45 | ||||
|
46 | from rhodecode.model import init_model | |||
|
47 | from rhodecode.model.scm import ScmModel | |||
|
48 | from rhodecode.config.environment import load_environment | |||
|
49 | from rhodecode.lib import LANGUAGES_EXTENSIONS_MAP | |||
|
50 | from rhodecode.lib.utils import BasePasterCommand, Command, add_cache | |||
52 |
|
51 | |||
53 | #EXTENSIONS WE WANT TO INDEX CONTENT OFF |
|
52 | #EXTENSIONS WE WANT TO INDEX CONTENT OFF | |
54 | INDEX_EXTENSIONS = ['action', 'adp', 'ashx', 'asmx', 'aspx', 'asx', 'axd', 'c', |
|
53 | INDEX_EXTENSIONS = LANGUAGES_EXTENSIONS_MAP.keys() | |
55 | 'cfg', 'cfm', 'cpp', 'cs', 'css', 'diff', 'do', 'el', 'erl', |
|
|||
56 | 'h', 'htm', 'html', 'ini', 'java', 'js', 'jsp', 'jspx', 'lisp', |
|
|||
57 | 'lua', 'm', 'mako', 'ml', 'pas', 'patch', 'php', 'php3', |
|
|||
58 | 'php4', 'phtml', 'pm', 'py', 'rb', 'rst', 's', 'sh', 'sql', |
|
|||
59 | 'tpl', 'txt', 'vim', 'wss', 'xhtml', 'xml', 'xsl', 'xslt', |
|
|||
60 | 'yaws'] |
|
|||
61 |
|
54 | |||
62 | #CUSTOM ANALYZER wordsplit + lowercase filter |
|
55 | #CUSTOM ANALYZER wordsplit + lowercase filter | |
63 | ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter() |
|
56 | ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter() | |
64 |
|
57 | |||
65 |
|
58 | |||
66 | #INDEX SCHEMA DEFINITION |
|
59 | #INDEX SCHEMA DEFINITION | |
67 | SCHEMA = Schema(owner=TEXT(), |
|
60 | SCHEMA = Schema(owner=TEXT(), | |
68 | repository=TEXT(stored=True), |
|
61 | repository=TEXT(stored=True), | |
69 | path=TEXT(stored=True), |
|
62 | path=TEXT(stored=True), | |
70 | content=FieldType(format=Characters(ANALYZER), |
|
63 | content=FieldType(format=Characters(ANALYZER), | |
71 | scorable=True, stored=True), |
|
64 | scorable=True, stored=True), | |
72 | modtime=STORED(), extension=TEXT(stored=True)) |
|
65 | modtime=STORED(), extension=TEXT(stored=True)) | |
73 |
|
66 | |||
74 |
|
67 | |||
75 | IDX_NAME = 'HG_INDEX' |
|
68 | IDX_NAME = 'HG_INDEX' | |
76 | FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n') |
|
69 | FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n') | |
77 | FRAGMENTER = SimpleFragmenter(200) |
|
70 | FRAGMENTER = SimpleFragmenter(200) | |
78 |
|
71 | |||
79 |
|
72 | |||
80 | class MakeIndex(BasePasterCommand): |
|
73 | class MakeIndex(BasePasterCommand): | |
81 |
|
74 | |||
82 | max_args = 1 |
|
75 | max_args = 1 | |
83 | min_args = 1 |
|
76 | min_args = 1 | |
84 |
|
77 | |||
85 | usage = "CONFIG_FILE" |
|
78 | usage = "CONFIG_FILE" | |
86 | summary = "Creates index for full text search given configuration file" |
|
79 | summary = "Creates index for full text search given configuration file" | |
87 | group_name = "RhodeCode" |
|
80 | group_name = "RhodeCode" | |
88 | takes_config_file = -1 |
|
81 | takes_config_file = -1 | |
89 | parser = Command.standard_parser(verbose=True) |
|
82 | parser = Command.standard_parser(verbose=True) | |
90 |
|
83 | |||
91 | def command(self): |
|
84 | def command(self): | |
92 |
|
85 | |||
93 | from pylons import config |
|
86 | from pylons import config | |
94 | add_cache(config) |
|
87 | add_cache(config) | |
95 | engine = engine_from_config(config, 'sqlalchemy.db1.') |
|
88 | engine = engine_from_config(config, 'sqlalchemy.db1.') | |
96 | init_model(engine) |
|
89 | init_model(engine) | |
97 |
|
90 | |||
98 | index_location = config['index_dir'] |
|
91 | index_location = config['index_dir'] | |
99 | repo_location = self.options.repo_location |
|
92 | repo_location = self.options.repo_location | |
100 | repo_list = map(strip, self.options.repo_list.split(',')) \ |
|
93 | repo_list = map(strip, self.options.repo_list.split(',')) \ | |
101 | if self.options.repo_list else None |
|
94 | if self.options.repo_list else None | |
102 |
|
95 | |||
103 | #====================================================================== |
|
96 | #====================================================================== | |
104 | # WHOOSH DAEMON |
|
97 | # WHOOSH DAEMON | |
105 | #====================================================================== |
|
98 | #====================================================================== | |
106 | from rhodecode.lib.pidlock import LockHeld, DaemonLock |
|
99 | from rhodecode.lib.pidlock import LockHeld, DaemonLock | |
107 | from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon |
|
100 | from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon | |
108 | try: |
|
101 | try: | |
109 | l = DaemonLock() |
|
102 | l = DaemonLock() | |
110 | WhooshIndexingDaemon(index_location=index_location, |
|
103 | WhooshIndexingDaemon(index_location=index_location, | |
111 | repo_location=repo_location, |
|
104 | repo_location=repo_location, | |
112 | repo_list=repo_list)\ |
|
105 | repo_list=repo_list)\ | |
113 | .run(full_index=self.options.full_index) |
|
106 | .run(full_index=self.options.full_index) | |
114 | l.release() |
|
107 | l.release() | |
115 | except LockHeld: |
|
108 | except LockHeld: | |
116 | sys.exit(1) |
|
109 | sys.exit(1) | |
117 |
|
110 | |||
118 | def update_parser(self): |
|
111 | def update_parser(self): | |
119 | self.parser.add_option('--repo-location', |
|
112 | self.parser.add_option('--repo-location', | |
120 | action='store', |
|
113 | action='store', | |
121 | dest='repo_location', |
|
114 | dest='repo_location', | |
122 | help="Specifies repositories location to index REQUIRED", |
|
115 | help="Specifies repositories location to index REQUIRED", | |
123 | ) |
|
116 | ) | |
124 | self.parser.add_option('--index-only', |
|
117 | self.parser.add_option('--index-only', | |
125 | action='store', |
|
118 | action='store', | |
126 | dest='repo_list', |
|
119 | dest='repo_list', | |
127 | help="Specifies a comma separated list of repositores " |
|
120 | help="Specifies a comma separated list of repositores " | |
128 | "to build index on OPTIONAL", |
|
121 | "to build index on OPTIONAL", | |
129 | ) |
|
122 | ) | |
130 | self.parser.add_option('-f', |
|
123 | self.parser.add_option('-f', | |
131 | action='store_true', |
|
124 | action='store_true', | |
132 | dest='full_index', |
|
125 | dest='full_index', | |
133 | help="Specifies that index should be made full i.e" |
|
126 | help="Specifies that index should be made full i.e" | |
134 | " destroy old and build from scratch", |
|
127 | " destroy old and build from scratch", | |
135 | default=False) |
|
128 | default=False) | |
136 |
|
129 | |||
137 | class ResultWrapper(object): |
|
130 | class ResultWrapper(object): | |
138 | def __init__(self, search_type, searcher, matcher, highlight_items): |
|
131 | def __init__(self, search_type, searcher, matcher, highlight_items): | |
139 | self.search_type = search_type |
|
132 | self.search_type = search_type | |
140 | self.searcher = searcher |
|
133 | self.searcher = searcher | |
141 | self.matcher = matcher |
|
134 | self.matcher = matcher | |
142 | self.highlight_items = highlight_items |
|
135 | self.highlight_items = highlight_items | |
143 | self.fragment_size = 200 / 2 |
|
136 | self.fragment_size = 200 / 2 | |
144 |
|
137 | |||
145 | @LazyProperty |
|
138 | @LazyProperty | |
146 | def doc_ids(self): |
|
139 | def doc_ids(self): | |
147 | docs_id = [] |
|
140 | docs_id = [] | |
148 | while self.matcher.is_active(): |
|
141 | while self.matcher.is_active(): | |
149 | docnum = self.matcher.id() |
|
142 | docnum = self.matcher.id() | |
150 | chunks = [offsets for offsets in self.get_chunks()] |
|
143 | chunks = [offsets for offsets in self.get_chunks()] | |
151 | docs_id.append([docnum, chunks]) |
|
144 | docs_id.append([docnum, chunks]) | |
152 | self.matcher.next() |
|
145 | self.matcher.next() | |
153 | return docs_id |
|
146 | return docs_id | |
154 |
|
147 | |||
155 | def __str__(self): |
|
148 | def __str__(self): | |
156 | return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids)) |
|
149 | return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids)) | |
157 |
|
150 | |||
158 | def __repr__(self): |
|
151 | def __repr__(self): | |
159 | return self.__str__() |
|
152 | return self.__str__() | |
160 |
|
153 | |||
161 | def __len__(self): |
|
154 | def __len__(self): | |
162 | return len(self.doc_ids) |
|
155 | return len(self.doc_ids) | |
163 |
|
156 | |||
164 | def __iter__(self): |
|
157 | def __iter__(self): | |
165 | """ |
|
158 | """ | |
166 | Allows Iteration over results,and lazy generate content |
|
159 | Allows Iteration over results,and lazy generate content | |
167 |
|
160 | |||
168 | *Requires* implementation of ``__getitem__`` method. |
|
161 | *Requires* implementation of ``__getitem__`` method. | |
169 | """ |
|
162 | """ | |
170 | for docid in self.doc_ids: |
|
163 | for docid in self.doc_ids: | |
171 | yield self.get_full_content(docid) |
|
164 | yield self.get_full_content(docid) | |
172 |
|
165 | |||
173 | def __getitem__(self, key): |
|
166 | def __getitem__(self, key): | |
174 | """ |
|
167 | """ | |
175 | Slicing of resultWrapper |
|
168 | Slicing of resultWrapper | |
176 | """ |
|
169 | """ | |
177 | i, j = key.start, key.stop |
|
170 | i, j = key.start, key.stop | |
178 |
|
171 | |||
179 | slice = [] |
|
172 | slice = [] | |
180 | for docid in self.doc_ids[i:j]: |
|
173 | for docid in self.doc_ids[i:j]: | |
181 | slice.append(self.get_full_content(docid)) |
|
174 | slice.append(self.get_full_content(docid)) | |
182 | return slice |
|
175 | return slice | |
183 |
|
176 | |||
184 |
|
177 | |||
185 | def get_full_content(self, docid): |
|
178 | def get_full_content(self, docid): | |
186 | res = self.searcher.stored_fields(docid[0]) |
|
179 | res = self.searcher.stored_fields(docid[0]) | |
187 | f_path = res['path'][res['path'].find(res['repository']) \ |
|
180 | f_path = res['path'][res['path'].find(res['repository']) \ | |
188 | + len(res['repository']):].lstrip('/') |
|
181 | + len(res['repository']):].lstrip('/') | |
189 |
|
182 | |||
190 | content_short = self.get_short_content(res, docid[1]) |
|
183 | content_short = self.get_short_content(res, docid[1]) | |
191 | res.update({'content_short':content_short, |
|
184 | res.update({'content_short':content_short, | |
192 | 'content_short_hl':self.highlight(content_short), |
|
185 | 'content_short_hl':self.highlight(content_short), | |
193 | 'f_path':f_path}) |
|
186 | 'f_path':f_path}) | |
194 |
|
187 | |||
195 | return res |
|
188 | return res | |
196 |
|
189 | |||
197 | def get_short_content(self, res, chunks): |
|
190 | def get_short_content(self, res, chunks): | |
198 |
|
191 | |||
199 | return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks]) |
|
192 | return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks]) | |
200 |
|
193 | |||
201 | def get_chunks(self): |
|
194 | def get_chunks(self): | |
202 | """ |
|
195 | """ | |
203 | Smart function that implements chunking the content |
|
196 | Smart function that implements chunking the content | |
204 | but not overlap chunks so it doesn't highlight the same |
|
197 | but not overlap chunks so it doesn't highlight the same | |
205 | close occurrences twice. |
|
198 | close occurrences twice. | |
206 | @param matcher: |
|
199 | ||
207 |
|
|
200 | :param matcher: | |
|
201 | :param size: | |||
208 | """ |
|
202 | """ | |
209 | memory = [(0, 0)] |
|
203 | memory = [(0, 0)] | |
210 | for span in self.matcher.spans(): |
|
204 | for span in self.matcher.spans(): | |
211 | start = span.startchar or 0 |
|
205 | start = span.startchar or 0 | |
212 | end = span.endchar or 0 |
|
206 | end = span.endchar or 0 | |
213 | start_offseted = max(0, start - self.fragment_size) |
|
207 | start_offseted = max(0, start - self.fragment_size) | |
214 | end_offseted = end + self.fragment_size |
|
208 | end_offseted = end + self.fragment_size | |
215 |
|
209 | |||
216 | if start_offseted < memory[-1][1]: |
|
210 | if start_offseted < memory[-1][1]: | |
217 | start_offseted = memory[-1][1] |
|
211 | start_offseted = memory[-1][1] | |
218 | memory.append((start_offseted, end_offseted,)) |
|
212 | memory.append((start_offseted, end_offseted,)) | |
219 | yield (start_offseted, end_offseted,) |
|
213 | yield (start_offseted, end_offseted,) | |
220 |
|
214 | |||
221 | def highlight(self, content, top=5): |
|
215 | def highlight(self, content, top=5): | |
222 | if self.search_type != 'content': |
|
216 | if self.search_type != 'content': | |
223 | return '' |
|
217 | return '' | |
224 | hl = highlight(escape(content), |
|
218 | hl = highlight(escape(content), | |
225 | self.highlight_items, |
|
219 | self.highlight_items, | |
226 | analyzer=ANALYZER, |
|
220 | analyzer=ANALYZER, | |
227 | fragmenter=FRAGMENTER, |
|
221 | fragmenter=FRAGMENTER, | |
228 | formatter=FORMATTER, |
|
222 | formatter=FORMATTER, | |
229 | top=top) |
|
223 | top=top) | |
230 | return hl |
|
224 | return hl |
General Comments 0
You need to be logged in to leave comments.
Login now