Show More
@@ -1,66 +1,66 b'' | |||||
1 | # -*- coding: utf-8 -*- |
|
1 | # -*- coding: utf-8 -*- | |
2 | """ |
|
2 | """ | |
3 | rhodecode.lib.dbmigrate.__init__ |
|
3 | rhodecode.lib.dbmigrate.__init__ | |
4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
|
4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | |
5 |
|
5 | |||
6 | Database migration modules |
|
6 | Database migration modules | |
7 |
|
7 | |||
8 | :created_on: Dec 11, 2010 |
|
8 | :created_on: Dec 11, 2010 | |
9 | :author: marcink |
|
9 | :author: marcink | |
10 | :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com> |
|
10 | :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com> | |
11 | :license: GPLv3, see COPYING for more details. |
|
11 | :license: GPLv3, see COPYING for more details. | |
12 | """ |
|
12 | """ | |
13 | # This program is free software: you can redistribute it and/or modify |
|
13 | # This program is free software: you can redistribute it and/or modify | |
14 | # it under the terms of the GNU General Public License as published by |
|
14 | # it under the terms of the GNU General Public License as published by | |
15 | # the Free Software Foundation, either version 3 of the License, or |
|
15 | # the Free Software Foundation, either version 3 of the License, or | |
16 | # (at your option) any later version. |
|
16 | # (at your option) any later version. | |
17 | # |
|
17 | # | |
18 | # This program is distributed in the hope that it will be useful, |
|
18 | # This program is distributed in the hope that it will be useful, | |
19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
21 | # GNU General Public License for more details. |
|
21 | # GNU General Public License for more details. | |
22 | # |
|
22 | # | |
23 | # You should have received a copy of the GNU General Public License |
|
23 | # You should have received a copy of the GNU General Public License | |
24 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
24 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
25 |
|
25 | |||
26 | import logging |
|
26 | import logging | |
27 | from sqlalchemy import engine_from_config |
|
27 | from sqlalchemy import engine_from_config | |
28 |
|
28 | |||
29 |
|
29 | |||
30 | from rhodecode.lib.utils import BasePasterCommand, Command, add_cache |
|
30 | from rhodecode.lib.utils import BasePasterCommand, Command, add_cache | |
31 | from rhodecode.lib.db_manage import DbManage |
|
31 | from rhodecode.lib.db_manage import DbManage | |
32 |
|
32 | |||
33 | log = logging.getLogger(__name__) |
|
33 | log = logging.getLogger(__name__) | |
34 |
|
34 | |||
35 |
|
35 | |||
36 | class UpgradeDb(BasePasterCommand): |
|
36 | class UpgradeDb(BasePasterCommand): | |
37 | """Command used for paster to upgrade our database to newer version |
|
37 | """Command used for paster to upgrade our database to newer version | |
38 | """ |
|
38 | """ | |
39 |
|
39 | |||
40 | max_args = 1 |
|
40 | max_args = 1 | |
41 | min_args = 1 |
|
41 | min_args = 1 | |
42 |
|
42 | |||
43 | usage = "CONFIG_FILE" |
|
43 | usage = "CONFIG_FILE" | |
44 |
summary = "Upgrades current db to newer version |
|
44 | summary = "Upgrades current db to newer version" | |
45 | group_name = "RhodeCode" |
|
45 | group_name = "RhodeCode" | |
46 |
|
46 | |||
47 | parser = Command.standard_parser(verbose=True) |
|
47 | parser = Command.standard_parser(verbose=True) | |
48 |
|
48 | |||
49 | def command(self): |
|
49 | def command(self): | |
50 | from pylons import config |
|
50 | from pylons import config | |
51 |
|
51 | |||
52 | add_cache(config) |
|
52 | add_cache(config) | |
53 |
|
53 | |||
54 | db_uri = config['sqlalchemy.db1.url'] |
|
54 | db_uri = config['sqlalchemy.db1.url'] | |
55 |
|
55 | |||
56 | dbmanage = DbManage(log_sql=True, dbconf=db_uri, |
|
56 | dbmanage = DbManage(log_sql=True, dbconf=db_uri, | |
57 | root=config['here'], tests=False) |
|
57 | root=config['here'], tests=False) | |
58 |
|
58 | |||
59 | dbmanage.upgrade() |
|
59 | dbmanage.upgrade() | |
60 |
|
60 | |||
61 | def update_parser(self): |
|
61 | def update_parser(self): | |
62 | self.parser.add_option('--sql', |
|
62 | self.parser.add_option('--sql', | |
63 | action='store_true', |
|
63 | action='store_true', | |
64 | dest='just_sql', |
|
64 | dest='just_sql', | |
65 | help="Prints upgrade sql for further investigation", |
|
65 | help="Prints upgrade sql for further investigation", | |
66 | default=False) |
|
66 | default=False) |
@@ -1,280 +1,280 b'' | |||||
1 | # -*- coding: utf-8 -*- |
|
1 | # -*- coding: utf-8 -*- | |
2 | """ |
|
2 | """ | |
3 | rhodecode.lib.indexers.__init__ |
|
3 | rhodecode.lib.indexers.__init__ | |
4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
|
4 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | |
5 |
|
5 | |||
6 | Whoosh indexing module for RhodeCode |
|
6 | Whoosh indexing module for RhodeCode | |
7 |
|
7 | |||
8 | :created_on: Aug 17, 2010 |
|
8 | :created_on: Aug 17, 2010 | |
9 | :author: marcink |
|
9 | :author: marcink | |
10 | :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com> |
|
10 | :copyright: (C) 2010-2012 Marcin Kuzminski <marcin@python-works.com> | |
11 | :license: GPLv3, see COPYING for more details. |
|
11 | :license: GPLv3, see COPYING for more details. | |
12 | """ |
|
12 | """ | |
13 | # This program is free software: you can redistribute it and/or modify |
|
13 | # This program is free software: you can redistribute it and/or modify | |
14 | # it under the terms of the GNU General Public License as published by |
|
14 | # it under the terms of the GNU General Public License as published by | |
15 | # the Free Software Foundation, either version 3 of the License, or |
|
15 | # the Free Software Foundation, either version 3 of the License, or | |
16 | # (at your option) any later version. |
|
16 | # (at your option) any later version. | |
17 | # |
|
17 | # | |
18 | # This program is distributed in the hope that it will be useful, |
|
18 | # This program is distributed in the hope that it will be useful, | |
19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
21 | # GNU General Public License for more details. |
|
21 | # GNU General Public License for more details. | |
22 | # |
|
22 | # | |
23 | # You should have received a copy of the GNU General Public License |
|
23 | # You should have received a copy of the GNU General Public License | |
24 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
24 | # along with this program. If not, see <http://www.gnu.org/licenses/>. | |
25 | import os |
|
25 | import os | |
26 | import sys |
|
26 | import sys | |
27 | import traceback |
|
27 | import traceback | |
28 | import logging |
|
28 | import logging | |
29 | from os.path import dirname as dn, join as jn |
|
29 | from os.path import dirname as dn, join as jn | |
30 |
|
30 | |||
31 | #to get the rhodecode import |
|
31 | #to get the rhodecode import | |
32 | sys.path.append(dn(dn(dn(os.path.realpath(__file__))))) |
|
32 | sys.path.append(dn(dn(dn(os.path.realpath(__file__))))) | |
33 |
|
33 | |||
34 | from string import strip |
|
34 | from string import strip | |
35 | from shutil import rmtree |
|
35 | from shutil import rmtree | |
36 |
|
36 | |||
37 | from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter |
|
37 | from whoosh.analysis import RegexTokenizer, LowercaseFilter, StopFilter | |
38 | from whoosh.fields import TEXT, ID, STORED, NUMERIC, BOOLEAN, Schema, FieldType, DATETIME |
|
38 | from whoosh.fields import TEXT, ID, STORED, NUMERIC, BOOLEAN, Schema, FieldType, DATETIME | |
39 | from whoosh.index import create_in, open_dir |
|
39 | from whoosh.index import create_in, open_dir | |
40 | from whoosh.formats import Characters |
|
40 | from whoosh.formats import Characters | |
41 | from whoosh.highlight import highlight, HtmlFormatter, ContextFragmenter |
|
41 | from whoosh.highlight import highlight, HtmlFormatter, ContextFragmenter | |
42 |
|
42 | |||
43 | from webhelpers.html.builder import escape, literal |
|
43 | from webhelpers.html.builder import escape, literal | |
44 | from sqlalchemy import engine_from_config |
|
44 | from sqlalchemy import engine_from_config | |
45 |
|
45 | |||
46 | from rhodecode.model import init_model |
|
46 | from rhodecode.model import init_model | |
47 | from rhodecode.model.scm import ScmModel |
|
47 | from rhodecode.model.scm import ScmModel | |
48 | from rhodecode.model.repo import RepoModel |
|
48 | from rhodecode.model.repo import RepoModel | |
49 | from rhodecode.config.environment import load_environment |
|
49 | from rhodecode.config.environment import load_environment | |
50 | from rhodecode.lib.utils2 import LazyProperty |
|
50 | from rhodecode.lib.utils2 import LazyProperty | |
51 | from rhodecode.lib.utils import BasePasterCommand, Command, add_cache,\ |
|
51 | from rhodecode.lib.utils import BasePasterCommand, Command, add_cache,\ | |
52 | load_rcextensions |
|
52 | load_rcextensions | |
53 |
|
53 | |||
54 | log = logging.getLogger(__name__) |
|
54 | log = logging.getLogger(__name__) | |
55 |
|
55 | |||
56 | # CUSTOM ANALYZER wordsplit + lowercase filter |
|
56 | # CUSTOM ANALYZER wordsplit + lowercase filter | |
57 | ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter() |
|
57 | ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter() | |
58 |
|
58 | |||
59 | #INDEX SCHEMA DEFINITION |
|
59 | #INDEX SCHEMA DEFINITION | |
60 | SCHEMA = Schema( |
|
60 | SCHEMA = Schema( | |
61 | fileid=ID(unique=True), |
|
61 | fileid=ID(unique=True), | |
62 | owner=TEXT(), |
|
62 | owner=TEXT(), | |
63 | repository=TEXT(stored=True), |
|
63 | repository=TEXT(stored=True), | |
64 | path=TEXT(stored=True), |
|
64 | path=TEXT(stored=True), | |
65 | content=FieldType(format=Characters(), analyzer=ANALYZER, |
|
65 | content=FieldType(format=Characters(), analyzer=ANALYZER, | |
66 | scorable=True, stored=True), |
|
66 | scorable=True, stored=True), | |
67 | modtime=STORED(), |
|
67 | modtime=STORED(), | |
68 | extension=TEXT(stored=True) |
|
68 | extension=TEXT(stored=True) | |
69 | ) |
|
69 | ) | |
70 |
|
70 | |||
71 | IDX_NAME = 'HG_INDEX' |
|
71 | IDX_NAME = 'HG_INDEX' | |
72 | FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n') |
|
72 | FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n') | |
73 | FRAGMENTER = ContextFragmenter(200) |
|
73 | FRAGMENTER = ContextFragmenter(200) | |
74 |
|
74 | |||
75 | CHGSETS_SCHEMA = Schema( |
|
75 | CHGSETS_SCHEMA = Schema( | |
76 | raw_id=ID(unique=True, stored=True), |
|
76 | raw_id=ID(unique=True, stored=True), | |
77 | date=NUMERIC(stored=True), |
|
77 | date=NUMERIC(stored=True), | |
78 | last=BOOLEAN(), |
|
78 | last=BOOLEAN(), | |
79 | owner=TEXT(), |
|
79 | owner=TEXT(), | |
80 | repository=ID(unique=True, stored=True), |
|
80 | repository=ID(unique=True, stored=True), | |
81 | author=TEXT(stored=True), |
|
81 | author=TEXT(stored=True), | |
82 | message=FieldType(format=Characters(), analyzer=ANALYZER, |
|
82 | message=FieldType(format=Characters(), analyzer=ANALYZER, | |
83 | scorable=True, stored=True), |
|
83 | scorable=True, stored=True), | |
84 | parents=TEXT(), |
|
84 | parents=TEXT(), | |
85 | added=TEXT(), |
|
85 | added=TEXT(), | |
86 | removed=TEXT(), |
|
86 | removed=TEXT(), | |
87 | changed=TEXT(), |
|
87 | changed=TEXT(), | |
88 | ) |
|
88 | ) | |
89 |
|
89 | |||
90 | CHGSET_IDX_NAME = 'CHGSET_INDEX' |
|
90 | CHGSET_IDX_NAME = 'CHGSET_INDEX' | |
91 |
|
91 | |||
92 | # used only to generate queries in journal |
|
92 | # used only to generate queries in journal | |
93 | JOURNAL_SCHEMA = Schema( |
|
93 | JOURNAL_SCHEMA = Schema( | |
94 | username=TEXT(), |
|
94 | username=TEXT(), | |
95 | date=DATETIME(), |
|
95 | date=DATETIME(), | |
96 | action=TEXT(), |
|
96 | action=TEXT(), | |
97 | repository=TEXT(), |
|
97 | repository=TEXT(), | |
98 | ip=TEXT(), |
|
98 | ip=TEXT(), | |
99 | ) |
|
99 | ) | |
100 |
|
100 | |||
101 |
|
101 | |||
102 | class MakeIndex(BasePasterCommand): |
|
102 | class MakeIndex(BasePasterCommand): | |
103 |
|
103 | |||
104 | max_args = 1 |
|
104 | max_args = 1 | |
105 | min_args = 1 |
|
105 | min_args = 1 | |
106 |
|
106 | |||
107 | usage = "CONFIG_FILE" |
|
107 | usage = "CONFIG_FILE" | |
108 |
summary = "Creates |
|
108 | summary = "Creates or update full text search index" | |
109 | group_name = "RhodeCode" |
|
109 | group_name = "RhodeCode" | |
110 | takes_config_file = -1 |
|
110 | takes_config_file = -1 | |
111 | parser = Command.standard_parser(verbose=True) |
|
111 | parser = Command.standard_parser(verbose=True) | |
112 |
|
112 | |||
113 | def command(self): |
|
113 | def command(self): | |
114 | logging.config.fileConfig(self.path_to_ini_file) |
|
114 | logging.config.fileConfig(self.path_to_ini_file) | |
115 | from pylons import config |
|
115 | from pylons import config | |
116 | add_cache(config) |
|
116 | add_cache(config) | |
117 | engine = engine_from_config(config, 'sqlalchemy.db1.') |
|
117 | engine = engine_from_config(config, 'sqlalchemy.db1.') | |
118 | init_model(engine) |
|
118 | init_model(engine) | |
119 | index_location = config['index_dir'] |
|
119 | index_location = config['index_dir'] | |
120 | repo_location = self.options.repo_location \ |
|
120 | repo_location = self.options.repo_location \ | |
121 | if self.options.repo_location else RepoModel().repos_path |
|
121 | if self.options.repo_location else RepoModel().repos_path | |
122 | repo_list = map(strip, self.options.repo_list.split(',')) \ |
|
122 | repo_list = map(strip, self.options.repo_list.split(',')) \ | |
123 | if self.options.repo_list else None |
|
123 | if self.options.repo_list else None | |
124 | repo_update_list = map(strip, self.options.repo_update_list.split(',')) \ |
|
124 | repo_update_list = map(strip, self.options.repo_update_list.split(',')) \ | |
125 | if self.options.repo_update_list else None |
|
125 | if self.options.repo_update_list else None | |
126 | load_rcextensions(config['here']) |
|
126 | load_rcextensions(config['here']) | |
127 | #====================================================================== |
|
127 | #====================================================================== | |
128 | # WHOOSH DAEMON |
|
128 | # WHOOSH DAEMON | |
129 | #====================================================================== |
|
129 | #====================================================================== | |
130 | from rhodecode.lib.pidlock import LockHeld, DaemonLock |
|
130 | from rhodecode.lib.pidlock import LockHeld, DaemonLock | |
131 | from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon |
|
131 | from rhodecode.lib.indexers.daemon import WhooshIndexingDaemon | |
132 | try: |
|
132 | try: | |
133 | l = DaemonLock(file_=jn(dn(dn(index_location)), 'make_index.lock')) |
|
133 | l = DaemonLock(file_=jn(dn(dn(index_location)), 'make_index.lock')) | |
134 | WhooshIndexingDaemon(index_location=index_location, |
|
134 | WhooshIndexingDaemon(index_location=index_location, | |
135 | repo_location=repo_location, |
|
135 | repo_location=repo_location, | |
136 | repo_list=repo_list, |
|
136 | repo_list=repo_list, | |
137 | repo_update_list=repo_update_list)\ |
|
137 | repo_update_list=repo_update_list)\ | |
138 | .run(full_index=self.options.full_index) |
|
138 | .run(full_index=self.options.full_index) | |
139 | l.release() |
|
139 | l.release() | |
140 | except LockHeld: |
|
140 | except LockHeld: | |
141 | sys.exit(1) |
|
141 | sys.exit(1) | |
142 |
|
142 | |||
143 | def update_parser(self): |
|
143 | def update_parser(self): | |
144 | self.parser.add_option('--repo-location', |
|
144 | self.parser.add_option('--repo-location', | |
145 | action='store', |
|
145 | action='store', | |
146 | dest='repo_location', |
|
146 | dest='repo_location', | |
147 | help="Specifies repositories location to index OPTIONAL", |
|
147 | help="Specifies repositories location to index OPTIONAL", | |
148 | ) |
|
148 | ) | |
149 | self.parser.add_option('--index-only', |
|
149 | self.parser.add_option('--index-only', | |
150 | action='store', |
|
150 | action='store', | |
151 | dest='repo_list', |
|
151 | dest='repo_list', | |
152 | help="Specifies a comma separated list of repositores " |
|
152 | help="Specifies a comma separated list of repositores " | |
153 | "to build index on. If not given all repositories " |
|
153 | "to build index on. If not given all repositories " | |
154 | "are scanned for indexing. OPTIONAL", |
|
154 | "are scanned for indexing. OPTIONAL", | |
155 | ) |
|
155 | ) | |
156 | self.parser.add_option('--update-only', |
|
156 | self.parser.add_option('--update-only', | |
157 | action='store', |
|
157 | action='store', | |
158 | dest='repo_update_list', |
|
158 | dest='repo_update_list', | |
159 | help="Specifies a comma separated list of repositores " |
|
159 | help="Specifies a comma separated list of repositores " | |
160 | "to re-build index on. OPTIONAL", |
|
160 | "to re-build index on. OPTIONAL", | |
161 | ) |
|
161 | ) | |
162 | self.parser.add_option('-f', |
|
162 | self.parser.add_option('-f', | |
163 | action='store_true', |
|
163 | action='store_true', | |
164 | dest='full_index', |
|
164 | dest='full_index', | |
165 | help="Specifies that index should be made full i.e" |
|
165 | help="Specifies that index should be made full i.e" | |
166 | " destroy old and build from scratch", |
|
166 | " destroy old and build from scratch", | |
167 | default=False) |
|
167 | default=False) | |
168 |
|
168 | |||
169 |
|
169 | |||
170 | class WhooshResultWrapper(object): |
|
170 | class WhooshResultWrapper(object): | |
171 | def __init__(self, search_type, searcher, matcher, highlight_items, |
|
171 | def __init__(self, search_type, searcher, matcher, highlight_items, | |
172 | repo_location): |
|
172 | repo_location): | |
173 | self.search_type = search_type |
|
173 | self.search_type = search_type | |
174 | self.searcher = searcher |
|
174 | self.searcher = searcher | |
175 | self.matcher = matcher |
|
175 | self.matcher = matcher | |
176 | self.highlight_items = highlight_items |
|
176 | self.highlight_items = highlight_items | |
177 | self.fragment_size = 200 |
|
177 | self.fragment_size = 200 | |
178 | self.repo_location = repo_location |
|
178 | self.repo_location = repo_location | |
179 |
|
179 | |||
180 | @LazyProperty |
|
180 | @LazyProperty | |
181 | def doc_ids(self): |
|
181 | def doc_ids(self): | |
182 | docs_id = [] |
|
182 | docs_id = [] | |
183 | while self.matcher.is_active(): |
|
183 | while self.matcher.is_active(): | |
184 | docnum = self.matcher.id() |
|
184 | docnum = self.matcher.id() | |
185 | chunks = [offsets for offsets in self.get_chunks()] |
|
185 | chunks = [offsets for offsets in self.get_chunks()] | |
186 | docs_id.append([docnum, chunks]) |
|
186 | docs_id.append([docnum, chunks]) | |
187 | self.matcher.next() |
|
187 | self.matcher.next() | |
188 | return docs_id |
|
188 | return docs_id | |
189 |
|
189 | |||
190 | def __str__(self): |
|
190 | def __str__(self): | |
191 | return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids)) |
|
191 | return '<%s at %s>' % (self.__class__.__name__, len(self.doc_ids)) | |
192 |
|
192 | |||
193 | def __repr__(self): |
|
193 | def __repr__(self): | |
194 | return self.__str__() |
|
194 | return self.__str__() | |
195 |
|
195 | |||
196 | def __len__(self): |
|
196 | def __len__(self): | |
197 | return len(self.doc_ids) |
|
197 | return len(self.doc_ids) | |
198 |
|
198 | |||
199 | def __iter__(self): |
|
199 | def __iter__(self): | |
200 | """ |
|
200 | """ | |
201 | Allows Iteration over results,and lazy generate content |
|
201 | Allows Iteration over results,and lazy generate content | |
202 |
|
202 | |||
203 | *Requires* implementation of ``__getitem__`` method. |
|
203 | *Requires* implementation of ``__getitem__`` method. | |
204 | """ |
|
204 | """ | |
205 | for docid in self.doc_ids: |
|
205 | for docid in self.doc_ids: | |
206 | yield self.get_full_content(docid) |
|
206 | yield self.get_full_content(docid) | |
207 |
|
207 | |||
208 | def __getitem__(self, key): |
|
208 | def __getitem__(self, key): | |
209 | """ |
|
209 | """ | |
210 | Slicing of resultWrapper |
|
210 | Slicing of resultWrapper | |
211 | """ |
|
211 | """ | |
212 | i, j = key.start, key.stop |
|
212 | i, j = key.start, key.stop | |
213 |
|
213 | |||
214 | slices = [] |
|
214 | slices = [] | |
215 | for docid in self.doc_ids[i:j]: |
|
215 | for docid in self.doc_ids[i:j]: | |
216 | slices.append(self.get_full_content(docid)) |
|
216 | slices.append(self.get_full_content(docid)) | |
217 | return slices |
|
217 | return slices | |
218 |
|
218 | |||
219 | def get_full_content(self, docid): |
|
219 | def get_full_content(self, docid): | |
220 | res = self.searcher.stored_fields(docid[0]) |
|
220 | res = self.searcher.stored_fields(docid[0]) | |
221 | log.debug('result: %s' % res) |
|
221 | log.debug('result: %s' % res) | |
222 | if self.search_type == 'content': |
|
222 | if self.search_type == 'content': | |
223 | full_repo_path = jn(self.repo_location, res['repository']) |
|
223 | full_repo_path = jn(self.repo_location, res['repository']) | |
224 | f_path = res['path'].split(full_repo_path)[-1] |
|
224 | f_path = res['path'].split(full_repo_path)[-1] | |
225 | f_path = f_path.lstrip(os.sep) |
|
225 | f_path = f_path.lstrip(os.sep) | |
226 | content_short = self.get_short_content(res, docid[1]) |
|
226 | content_short = self.get_short_content(res, docid[1]) | |
227 | res.update({'content_short': content_short, |
|
227 | res.update({'content_short': content_short, | |
228 | 'content_short_hl': self.highlight(content_short), |
|
228 | 'content_short_hl': self.highlight(content_short), | |
229 | 'f_path': f_path |
|
229 | 'f_path': f_path | |
230 | }) |
|
230 | }) | |
231 | elif self.search_type == 'path': |
|
231 | elif self.search_type == 'path': | |
232 | full_repo_path = jn(self.repo_location, res['repository']) |
|
232 | full_repo_path = jn(self.repo_location, res['repository']) | |
233 | f_path = res['path'].split(full_repo_path)[-1] |
|
233 | f_path = res['path'].split(full_repo_path)[-1] | |
234 | f_path = f_path.lstrip(os.sep) |
|
234 | f_path = f_path.lstrip(os.sep) | |
235 | res.update({'f_path': f_path}) |
|
235 | res.update({'f_path': f_path}) | |
236 | elif self.search_type == 'message': |
|
236 | elif self.search_type == 'message': | |
237 | res.update({'message_hl': self.highlight(res['message'])}) |
|
237 | res.update({'message_hl': self.highlight(res['message'])}) | |
238 |
|
238 | |||
239 | log.debug('result: %s' % res) |
|
239 | log.debug('result: %s' % res) | |
240 |
|
240 | |||
241 | return res |
|
241 | return res | |
242 |
|
242 | |||
243 | def get_short_content(self, res, chunks): |
|
243 | def get_short_content(self, res, chunks): | |
244 |
|
244 | |||
245 | return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks]) |
|
245 | return ''.join([res['content'][chunk[0]:chunk[1]] for chunk in chunks]) | |
246 |
|
246 | |||
247 | def get_chunks(self): |
|
247 | def get_chunks(self): | |
248 | """ |
|
248 | """ | |
249 | Smart function that implements chunking the content |
|
249 | Smart function that implements chunking the content | |
250 | but not overlap chunks so it doesn't highlight the same |
|
250 | but not overlap chunks so it doesn't highlight the same | |
251 | close occurrences twice. |
|
251 | close occurrences twice. | |
252 |
|
252 | |||
253 | :param matcher: |
|
253 | :param matcher: | |
254 | :param size: |
|
254 | :param size: | |
255 | """ |
|
255 | """ | |
256 | memory = [(0, 0)] |
|
256 | memory = [(0, 0)] | |
257 | if self.matcher.supports('positions'): |
|
257 | if self.matcher.supports('positions'): | |
258 | for span in self.matcher.spans(): |
|
258 | for span in self.matcher.spans(): | |
259 | start = span.startchar or 0 |
|
259 | start = span.startchar or 0 | |
260 | end = span.endchar or 0 |
|
260 | end = span.endchar or 0 | |
261 | start_offseted = max(0, start - self.fragment_size) |
|
261 | start_offseted = max(0, start - self.fragment_size) | |
262 | end_offseted = end + self.fragment_size |
|
262 | end_offseted = end + self.fragment_size | |
263 |
|
263 | |||
264 | if start_offseted < memory[-1][1]: |
|
264 | if start_offseted < memory[-1][1]: | |
265 | start_offseted = memory[-1][1] |
|
265 | start_offseted = memory[-1][1] | |
266 | memory.append((start_offseted, end_offseted,)) |
|
266 | memory.append((start_offseted, end_offseted,)) | |
267 | yield (start_offseted, end_offseted,) |
|
267 | yield (start_offseted, end_offseted,) | |
268 |
|
268 | |||
269 | def highlight(self, content, top=5): |
|
269 | def highlight(self, content, top=5): | |
270 | if self.search_type not in ['content', 'message']: |
|
270 | if self.search_type not in ['content', 'message']: | |
271 | return '' |
|
271 | return '' | |
272 | hl = highlight( |
|
272 | hl = highlight( | |
273 | text=content, |
|
273 | text=content, | |
274 | terms=self.highlight_items, |
|
274 | terms=self.highlight_items, | |
275 | analyzer=ANALYZER, |
|
275 | analyzer=ANALYZER, | |
276 | fragmenter=FRAGMENTER, |
|
276 | fragmenter=FRAGMENTER, | |
277 | formatter=FORMATTER, |
|
277 | formatter=FORMATTER, | |
278 | top=top |
|
278 | top=top | |
279 | ) |
|
279 | ) | |
280 | return hl |
|
280 | return hl |
General Comments 0
You need to be logged in to leave comments.
Login now