Show More
@@ -20,6 +20,8 b' news' | |||||
20 | - new git repos are created as bare now by default |
|
20 | - new git repos are created as bare now by default | |
21 | - #464 added links to groups in permission box |
|
21 | - #464 added links to groups in permission box | |
22 | - #465 mentions autocomplete inside comments boxes |
|
22 | - #465 mentions autocomplete inside comments boxes | |
|
23 | - #469 added --update-only option to whoosh to re-index only given list | |||
|
24 | of repos in index | |||
23 |
|
25 | |||
24 | fixes |
|
26 | fixes | |
25 | +++++ |
|
27 | +++++ |
@@ -93,6 +93,8 b' class MakeIndex(BasePasterCommand):' | |||||
93 | if self.options.repo_location else RepoModel().repos_path |
|
93 | if self.options.repo_location else RepoModel().repos_path | |
94 | repo_list = map(strip, self.options.repo_list.split(',')) \ |
|
94 | repo_list = map(strip, self.options.repo_list.split(',')) \ | |
95 | if self.options.repo_list else None |
|
95 | if self.options.repo_list else None | |
|
96 | repo_update_list = map(strip, self.options.repo_update_list.split(',')) \ | |||
|
97 | if self.options.repo_update_list else None | |||
96 | load_rcextensions(config['here']) |
|
98 | load_rcextensions(config['here']) | |
97 | #====================================================================== |
|
99 | #====================================================================== | |
98 | # WHOOSH DAEMON |
|
100 | # WHOOSH DAEMON | |
@@ -103,7 +105,8 b' class MakeIndex(BasePasterCommand):' | |||||
103 | l = DaemonLock(file_=jn(dn(dn(index_location)), 'make_index.lock')) |
|
105 | l = DaemonLock(file_=jn(dn(dn(index_location)), 'make_index.lock')) | |
104 | WhooshIndexingDaemon(index_location=index_location, |
|
106 | WhooshIndexingDaemon(index_location=index_location, | |
105 | repo_location=repo_location, |
|
107 | repo_location=repo_location, | |
106 |
repo_list=repo_list, |
|
108 | repo_list=repo_list, | |
|
109 | repo_update_list=repo_update_list)\ | |||
107 | .run(full_index=self.options.full_index) |
|
110 | .run(full_index=self.options.full_index) | |
108 | l.release() |
|
111 | l.release() | |
109 | except LockHeld: |
|
112 | except LockHeld: | |
@@ -119,7 +122,14 b' class MakeIndex(BasePasterCommand):' | |||||
119 | action='store', |
|
122 | action='store', | |
120 | dest='repo_list', |
|
123 | dest='repo_list', | |
121 | help="Specifies a comma separated list of repositores " |
|
124 | help="Specifies a comma separated list of repositores " | |
122 |
"to build index on |
|
125 | "to build index on. If not given all repositories " | |
|
126 | "are scanned for indexing. OPTIONAL", | |||
|
127 | ) | |||
|
128 | self.parser.add_option('--update-only', | |||
|
129 | action='store', | |||
|
130 | dest='repo_update_list', | |||
|
131 | help="Specifies a comma separated list of repositores " | |||
|
132 | "to re-build index on. OPTIONAL", | |||
123 | ) |
|
133 | ) | |
124 | self.parser.add_option('-f', |
|
134 | self.parser.add_option('-f', | |
125 | action='store_true', |
|
135 | action='store_true', |
@@ -53,11 +53,12 b" log = logging.getLogger('whoosh_indexer'" | |||||
53 |
|
53 | |||
54 | class WhooshIndexingDaemon(object): |
|
54 | class WhooshIndexingDaemon(object): | |
55 | """ |
|
55 | """ | |
56 | Daemon for atomic jobs |
|
56 | Daemon for atomic indexing jobs | |
57 | """ |
|
57 | """ | |
58 |
|
58 | |||
59 | def __init__(self, indexname=IDX_NAME, index_location=None, |
|
59 | def __init__(self, indexname=IDX_NAME, index_location=None, | |
60 |
repo_location=None, sa=None, repo_list=None |
|
60 | repo_location=None, sa=None, repo_list=None, | |
|
61 | repo_update_list=None): | |||
61 | self.indexname = indexname |
|
62 | self.indexname = indexname | |
62 |
|
63 | |||
63 | self.index_location = index_location |
|
64 | self.index_location = index_location | |
@@ -70,13 +71,23 b' class WhooshIndexingDaemon(object):' | |||||
70 |
|
71 | |||
71 | self.repo_paths = ScmModel(sa).repo_scan(self.repo_location) |
|
72 | self.repo_paths = ScmModel(sa).repo_scan(self.repo_location) | |
72 |
|
73 | |||
|
74 | #filter repo list | |||
73 | if repo_list: |
|
75 | if repo_list: | |
74 | filtered_repo_paths = {} |
|
76 | self.filtered_repo_paths = {} | |
75 | for repo_name, repo in self.repo_paths.items(): |
|
77 | for repo_name, repo in self.repo_paths.items(): | |
76 | if repo_name in repo_list: |
|
78 | if repo_name in repo_list: | |
77 | filtered_repo_paths[repo_name] = repo |
|
79 | self.filtered_repo_paths[repo_name] = repo | |
|
80 | ||||
|
81 | self.repo_paths = self.filtered_repo_paths | |||
78 |
|
82 | |||
79 | self.repo_paths = filtered_repo_paths |
|
83 | #filter update repo list | |
|
84 | self.filtered_repo_update_paths = {} | |||
|
85 | if repo_update_list: | |||
|
86 | self.filtered_repo_update_paths = {} | |||
|
87 | for repo_name, repo in self.repo_paths.items(): | |||
|
88 | if repo_name in repo_update_list: | |||
|
89 | self.filtered_repo_update_paths[repo_name] = repo | |||
|
90 | self.repo_paths = self.filtered_repo_update_paths | |||
80 |
|
91 | |||
81 | self.initial = False |
|
92 | self.initial = False | |
82 | if not os.path.isdir(self.index_location): |
|
93 | if not os.path.isdir(self.index_location): | |
@@ -172,8 +183,8 b' class WhooshIndexingDaemon(object):' | |||||
172 | log.debug('>>> FINISHED BUILDING INDEX <<<') |
|
183 | log.debug('>>> FINISHED BUILDING INDEX <<<') | |
173 |
|
184 | |||
174 | def update_index(self): |
|
185 | def update_index(self): | |
175 | log.debug(('STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s ' |
|
186 | log.debug((u'STARTING INCREMENTAL INDEXING UPDATE FOR EXTENSIONS %s ' | |
176 | 'AND REPOS %s') % (INDEX_EXTENSIONS, self.repo_paths)) |
|
187 | 'AND REPOS %s') % (INDEX_EXTENSIONS, self.repo_paths.keys())) | |
177 |
|
188 | |||
178 | idx = open_dir(self.index_location, indexname=self.indexname) |
|
189 | idx = open_dir(self.index_location, indexname=self.indexname) | |
179 | # The set of all paths in the index |
|
190 | # The set of all paths in the index | |
@@ -187,18 +198,16 b' class WhooshIndexingDaemon(object):' | |||||
187 | # Loop over the stored fields in the index |
|
198 | # Loop over the stored fields in the index | |
188 | for fields in reader.all_stored_fields(): |
|
199 | for fields in reader.all_stored_fields(): | |
189 | indexed_path = fields['path'] |
|
200 | indexed_path = fields['path'] | |
|
201 | indexed_repo_path = fields['repository'] | |||
190 | indexed_paths.add(indexed_path) |
|
202 | indexed_paths.add(indexed_path) | |
191 |
|
203 | |||
192 | repo = self.repo_paths[fields['repository']] |
|
204 | if not indexed_repo_path in self.filtered_repo_update_paths: | |
|
205 | continue | |||
|
206 | ||||
|
207 | repo = self.repo_paths[indexed_repo_path] | |||
193 |
|
208 | |||
194 | try: |
|
209 | try: | |
195 | node = self.get_node(repo, indexed_path) |
|
210 | node = self.get_node(repo, indexed_path) | |
196 | except (ChangesetError, NodeDoesNotExistError): |
|
|||
197 | # This file was deleted since it was indexed |
|
|||
198 | log.debug('removing from index %s' % indexed_path) |
|
|||
199 | writer.delete_by_term('path', indexed_path) |
|
|||
200 |
|
||||
201 | else: |
|
|||
202 | # Check if this file was changed since it was indexed |
|
211 | # Check if this file was changed since it was indexed | |
203 | indexed_time = fields['modtime'] |
|
212 | indexed_time = fields['modtime'] | |
204 | mtime = self.get_node_mtime(node) |
|
213 | mtime = self.get_node_mtime(node) | |
@@ -208,6 +217,10 b' class WhooshIndexingDaemon(object):' | |||||
208 | log.debug('adding to reindex list %s' % indexed_path) |
|
217 | log.debug('adding to reindex list %s' % indexed_path) | |
209 | writer.delete_by_term('path', indexed_path) |
|
218 | writer.delete_by_term('path', indexed_path) | |
210 | to_index.add(indexed_path) |
|
219 | to_index.add(indexed_path) | |
|
220 | except (ChangesetError, NodeDoesNotExistError): | |||
|
221 | # This file was deleted since it was indexed | |||
|
222 | log.debug('removing from index %s' % indexed_path) | |||
|
223 | writer.delete_by_term('path', indexed_path) | |||
211 |
|
224 | |||
212 | # Loop over the files in the filesystem |
|
225 | # Loop over the files in the filesystem | |
213 | # Assume we have a function that gathers the filenames of the |
|
226 | # Assume we have a function that gathers the filenames of the |
General Comments 0
You need to be logged in to leave comments.
Login now