Show More
@@ -1,109 +1,118 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | |
|
3 | 3 | # Copyright (C) 2017-2017 RhodeCode GmbH |
|
4 | 4 | # |
|
5 | 5 | # This program is free software: you can redistribute it and/or modify |
|
6 | 6 | # it under the terms of the GNU Affero General Public License, version 3 |
|
7 | 7 | # (only), as published by the Free Software Foundation. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU Affero General Public License |
|
15 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | 16 | # |
|
17 | 17 | # This program is dual-licensed. If you wish to learn more about the |
|
18 | 18 | # RhodeCode Enterprise Edition, including its added features, Support services, |
|
19 | 19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ |
|
20 | 20 | import logging |
|
21 | 21 | |
|
22 | 22 | log = logging.getLogger(__name__) |
|
23 | 23 | |
|
24 | 24 | |
|
25 | 25 | class MaintenanceTask(object): |
|
26 | 26 | human_name = 'undefined' |
|
27 | 27 | |
|
28 | 28 | def __init__(self, db_repo): |
|
29 | 29 | self.db_repo = db_repo |
|
30 | 30 | |
|
31 | 31 | def run(self): |
|
32 | 32 | """Execute task and return task human value""" |
|
33 | 33 | raise NotImplementedError() |
|
34 | 34 | |
|
35 | 35 | |
|
36 | 36 | class GitGC(MaintenanceTask): |
|
37 | 37 | human_name = 'GIT Garbage collect' |
|
38 | 38 | |
|
39 | 39 | def _count_objects(self, repo): |
|
40 | 40 | stdout, stderr = repo.run_git_command( |
|
41 | 41 | ['count-objects', '-v'], fail_on_stderr=False) |
|
42 | 42 | |
|
43 | 43 | errors = ' ' |
|
44 | 44 | objects = ' '.join(stdout.splitlines()) |
|
45 | 45 | |
|
46 | 46 | if stderr: |
|
47 | 47 | errors = '\nSTD ERR:' + '\n'.join(stderr.splitlines()) |
|
48 | 48 | return objects + errors |
|
49 | 49 | |
|
50 | 50 | def run(self): |
|
51 | 51 | output = [] |
|
52 | 52 | instance = self.db_repo.scm_instance() |
|
53 | 53 | |
|
54 | 54 | objects = self._count_objects(instance) |
|
55 | 55 | output.append(objects) |
|
56 | 56 | log.debug('GIT objects:%s', objects) |
|
57 | 57 | |
|
58 | 58 | stdout, stderr = instance.run_git_command( |
|
59 | 59 | ['gc', '--aggressive'], fail_on_stderr=False) |
|
60 | 60 | |
|
61 | 61 | out = 'executed git gc --aggressive' |
|
62 | 62 | if stderr: |
|
63 | 63 | out = ''.join(stderr.splitlines()) |
|
64 | 64 | |
|
65 | 65 | elif stdout: |
|
66 | 66 | out = ''.join(stdout.splitlines()) |
|
67 | 67 | |
|
68 | 68 | output.append(out) |
|
69 | 69 | |
|
70 | 70 | objects = self._count_objects(instance) |
|
71 | 71 | log.debug('GIT objects:%s', objects) |
|
72 | 72 | output.append(objects) |
|
73 | 73 | |
|
74 | 74 | return '\n'.join(output) |
|
75 | 75 | |
|
76 | 76 | |
|
77 | 77 | class HGVerify(MaintenanceTask): |
|
78 | 78 | human_name = 'HG Verify repo' |
|
79 | 79 | |
|
80 | 80 | def run(self): |
|
81 | 81 | instance = self.db_repo.scm_instance() |
|
82 | 82 | res = instance.verify() |
|
83 | 83 | return res |
|
84 | 84 | |
|
85 | 85 | |
|
86 | class SVNVerify(MaintenanceTask): | |
|
87 | human_name = 'SVN Verify repo' | |
|
88 | ||
|
89 | def run(self): | |
|
90 | instance = self.db_repo.scm_instance() | |
|
91 | res = instance.verify() | |
|
92 | return res | |
|
93 | ||
|
94 | ||
|
86 | 95 | class RepoMaintenance(object): |
|
87 | 96 | """ |
|
88 | 97 | Performs maintenance of repository based on it's type |
|
89 | 98 | """ |
|
90 | 99 | tasks = { |
|
91 | 100 | 'hg': [HGVerify], |
|
92 | 101 | 'git': [GitGC], |
|
93 | 'svn': [], | |
|
102 | 'svn': [SVNVerify], | |
|
94 | 103 | } |
|
95 | 104 | |
|
96 | 105 | def get_tasks_for_repo(self, db_repo): |
|
97 | 106 | """ |
|
98 | 107 | fetches human names of tasks pending for execution for given type of repo |
|
99 | 108 | """ |
|
100 | 109 | tasks = [] |
|
101 | 110 | for task in self.tasks[db_repo.repo_type]: |
|
102 | 111 | tasks.append(task.human_name) |
|
103 | 112 | return tasks |
|
104 | 113 | |
|
105 | 114 | def execute(self, db_repo): |
|
106 | 115 | executed_tasks = [] |
|
107 | 116 | for task in self.tasks[db_repo.repo_type]: |
|
108 | 117 | executed_tasks.append(task(db_repo).run()) |
|
109 | 118 | return executed_tasks |
@@ -1,333 +1,339 b'' | |||
|
1 | 1 | # -*- coding: utf-8 -*- |
|
2 | 2 | |
|
3 | 3 | # Copyright (C) 2014-2017 RhodeCode GmbH |
|
4 | 4 | # |
|
5 | 5 | # This program is free software: you can redistribute it and/or modify |
|
6 | 6 | # it under the terms of the GNU Affero General Public License, version 3 |
|
7 | 7 | # (only), as published by the Free Software Foundation. |
|
8 | 8 | # |
|
9 | 9 | # This program is distributed in the hope that it will be useful, |
|
10 | 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
11 | 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
12 | 12 | # GNU General Public License for more details. |
|
13 | 13 | # |
|
14 | 14 | # You should have received a copy of the GNU Affero General Public License |
|
15 | 15 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
16 | 16 | # |
|
17 | 17 | # This program is dual-licensed. If you wish to learn more about the |
|
18 | 18 | # RhodeCode Enterprise Edition, including its added features, Support services, |
|
19 | 19 | # and proprietary license terms, please see https://rhodecode.com/licenses/ |
|
20 | 20 | |
|
21 | 21 | """ |
|
22 | 22 | SVN repository module |
|
23 | 23 | """ |
|
24 | 24 | |
|
25 | 25 | import logging |
|
26 | 26 | import os |
|
27 | 27 | import urllib |
|
28 | 28 | |
|
29 | 29 | from zope.cachedescriptors.property import Lazy as LazyProperty |
|
30 | 30 | |
|
31 | 31 | from rhodecode.lib.compat import OrderedDict |
|
32 | 32 | from rhodecode.lib.datelib import date_astimestamp |
|
33 | 33 | from rhodecode.lib.utils import safe_str, safe_unicode |
|
34 | 34 | from rhodecode.lib.vcs import connection, path as vcspath |
|
35 | 35 | from rhodecode.lib.vcs.backends import base |
|
36 | 36 | from rhodecode.lib.vcs.backends.svn.commit import ( |
|
37 | 37 | SubversionCommit, _date_from_svn_properties) |
|
38 | 38 | from rhodecode.lib.vcs.backends.svn.diff import SubversionDiff |
|
39 | 39 | from rhodecode.lib.vcs.backends.svn.inmemory import SubversionInMemoryCommit |
|
40 | 40 | from rhodecode.lib.vcs.conf import settings |
|
41 | 41 | from rhodecode.lib.vcs.exceptions import ( |
|
42 | 42 | CommitDoesNotExistError, EmptyRepositoryError, RepositoryError, |
|
43 | 43 | VCSError, NodeDoesNotExistError) |
|
44 | 44 | |
|
45 | 45 | |
|
46 | 46 | log = logging.getLogger(__name__) |
|
47 | 47 | |
|
48 | 48 | |
|
49 | 49 | class SubversionRepository(base.BaseRepository): |
|
50 | 50 | """ |
|
51 | 51 | Subversion backend implementation |
|
52 | 52 | |
|
53 | 53 | .. important:: |
|
54 | 54 | |
|
55 | 55 | It is very important to distinguish the commit index and the commit id |
|
56 | 56 | which is assigned by Subversion. The first one is always handled as an |
|
57 | 57 | `int` by this implementation. The commit id assigned by Subversion on |
|
58 | 58 | the other side will always be a `str`. |
|
59 | 59 | |
|
60 | 60 | There is a specific trap since the first commit will have the index |
|
61 | 61 | ``0`` but the svn id will be ``"1"``. |
|
62 | 62 | |
|
63 | 63 | """ |
|
64 | 64 | |
|
65 | 65 | # Note: Subversion does not really have a default branch name. |
|
66 | 66 | DEFAULT_BRANCH_NAME = None |
|
67 | 67 | |
|
68 | 68 | contact = base.BaseRepository.DEFAULT_CONTACT |
|
69 | 69 | description = base.BaseRepository.DEFAULT_DESCRIPTION |
|
70 | 70 | |
|
71 | 71 | def __init__(self, repo_path, config=None, create=False, src_url=None, |
|
72 | 72 | **kwargs): |
|
73 | 73 | self.path = safe_str(os.path.abspath(repo_path)) |
|
74 | 74 | self.config = config if config else base.Config() |
|
75 | 75 | self._remote = connection.Svn( |
|
76 | 76 | self.path, self.config) |
|
77 | 77 | |
|
78 | 78 | self._init_repo(create, src_url) |
|
79 | 79 | |
|
80 | 80 | self.bookmarks = {} |
|
81 | 81 | |
|
82 | 82 | def _init_repo(self, create, src_url): |
|
83 | 83 | if create and os.path.exists(self.path): |
|
84 | 84 | raise RepositoryError( |
|
85 | 85 | "Cannot create repository at %s, location already exist" |
|
86 | 86 | % self.path) |
|
87 | 87 | |
|
88 | 88 | if create: |
|
89 | 89 | self._remote.create_repository(settings.SVN_COMPATIBLE_VERSION) |
|
90 | 90 | if src_url: |
|
91 | 91 | src_url = _sanitize_url(src_url) |
|
92 | 92 | self._remote.import_remote_repository(src_url) |
|
93 | 93 | else: |
|
94 | 94 | self._check_path() |
|
95 | 95 | |
|
96 | 96 | @LazyProperty |
|
97 | 97 | def commit_ids(self): |
|
98 | 98 | head = self._remote.lookup(None) |
|
99 | 99 | return [str(r) for r in xrange(1, head + 1)] |
|
100 | 100 | |
|
101 | 101 | @LazyProperty |
|
102 | 102 | def branches(self): |
|
103 | 103 | return self._tags_or_branches('vcs_svn_branch') |
|
104 | 104 | |
|
105 | 105 | @LazyProperty |
|
106 | 106 | def branches_closed(self): |
|
107 | 107 | return {} |
|
108 | 108 | |
|
109 | 109 | @LazyProperty |
|
110 | 110 | def branches_all(self): |
|
111 | 111 | # TODO: johbo: Implement proper branch support |
|
112 | 112 | all_branches = {} |
|
113 | 113 | all_branches.update(self.branches) |
|
114 | 114 | all_branches.update(self.branches_closed) |
|
115 | 115 | return all_branches |
|
116 | 116 | |
|
117 | 117 | @LazyProperty |
|
118 | 118 | def tags(self): |
|
119 | 119 | return self._tags_or_branches('vcs_svn_tag') |
|
120 | 120 | |
|
121 | 121 | def _tags_or_branches(self, config_section): |
|
122 | 122 | found_items = {} |
|
123 | 123 | |
|
124 | 124 | if self.is_empty(): |
|
125 | 125 | return {} |
|
126 | 126 | |
|
127 | 127 | for pattern in self._patterns_from_section(config_section): |
|
128 | 128 | pattern = vcspath.sanitize(pattern) |
|
129 | 129 | tip = self.get_commit() |
|
130 | 130 | try: |
|
131 | 131 | if pattern.endswith('*'): |
|
132 | 132 | basedir = tip.get_node(vcspath.dirname(pattern)) |
|
133 | 133 | directories = basedir.dirs |
|
134 | 134 | else: |
|
135 | 135 | directories = (tip.get_node(pattern), ) |
|
136 | 136 | except NodeDoesNotExistError: |
|
137 | 137 | continue |
|
138 | 138 | found_items.update( |
|
139 | 139 | (safe_unicode(n.path), |
|
140 | 140 | self.commit_ids[-1]) |
|
141 | 141 | for n in directories) |
|
142 | 142 | |
|
143 | 143 | def get_name(item): |
|
144 | 144 | return item[0] |
|
145 | 145 | |
|
146 | 146 | return OrderedDict(sorted(found_items.items(), key=get_name)) |
|
147 | 147 | |
|
148 | 148 | def _patterns_from_section(self, section): |
|
149 | 149 | return (pattern for key, pattern in self.config.items(section)) |
|
150 | 150 | |
|
151 | 151 | def get_common_ancestor(self, commit_id1, commit_id2, repo2): |
|
152 | 152 | if self != repo2: |
|
153 | 153 | raise ValueError( |
|
154 | 154 | "Subversion does not support getting common ancestor of" |
|
155 | 155 | " different repositories.") |
|
156 | 156 | |
|
157 | 157 | if int(commit_id1) < int(commit_id2): |
|
158 | 158 | return commit_id1 |
|
159 | 159 | return commit_id2 |
|
160 | 160 | |
|
161 | def verify(self): | |
|
162 | verify = self._remote.verify() | |
|
163 | ||
|
164 | self._remote.invalidate_vcs_cache() | |
|
165 | return verify | |
|
166 | ||
|
161 | 167 | def compare(self, commit_id1, commit_id2, repo2, merge, pre_load=None): |
|
162 | 168 | # TODO: johbo: Implement better comparison, this is a very naive |
|
163 | 169 | # version which does not allow to compare branches, tags or folders |
|
164 | 170 | # at all. |
|
165 | 171 | if repo2 != self: |
|
166 | 172 | raise ValueError( |
|
167 | 173 | "Subversion does not support comparison of of different " |
|
168 | 174 | "repositories.") |
|
169 | 175 | |
|
170 | 176 | if commit_id1 == commit_id2: |
|
171 | 177 | return [] |
|
172 | 178 | |
|
173 | 179 | commit_idx1 = self._get_commit_idx(commit_id1) |
|
174 | 180 | commit_idx2 = self._get_commit_idx(commit_id2) |
|
175 | 181 | |
|
176 | 182 | commits = [ |
|
177 | 183 | self.get_commit(commit_idx=idx) |
|
178 | 184 | for idx in range(commit_idx1 + 1, commit_idx2 + 1)] |
|
179 | 185 | |
|
180 | 186 | return commits |
|
181 | 187 | |
|
182 | 188 | def _get_commit_idx(self, commit_id): |
|
183 | 189 | try: |
|
184 | 190 | svn_rev = int(commit_id) |
|
185 | 191 | except: |
|
186 | 192 | # TODO: johbo: this might be only one case, HEAD, check this |
|
187 | 193 | svn_rev = self._remote.lookup(commit_id) |
|
188 | 194 | commit_idx = svn_rev - 1 |
|
189 | 195 | if commit_idx >= len(self.commit_ids): |
|
190 | 196 | raise CommitDoesNotExistError( |
|
191 | 197 | "Commit at index %s does not exist." % (commit_idx, )) |
|
192 | 198 | return commit_idx |
|
193 | 199 | |
|
194 | 200 | @staticmethod |
|
195 | 201 | def check_url(url, config): |
|
196 | 202 | """ |
|
197 | 203 | Check if `url` is a valid source to import a Subversion repository. |
|
198 | 204 | """ |
|
199 | 205 | # convert to URL if it's a local directory |
|
200 | 206 | if os.path.isdir(url): |
|
201 | 207 | url = 'file://' + urllib.pathname2url(url) |
|
202 | 208 | return connection.Svn.check_url(url, config.serialize()) |
|
203 | 209 | |
|
204 | 210 | @staticmethod |
|
205 | 211 | def is_valid_repository(path): |
|
206 | 212 | try: |
|
207 | 213 | SubversionRepository(path) |
|
208 | 214 | return True |
|
209 | 215 | except VCSError: |
|
210 | 216 | pass |
|
211 | 217 | return False |
|
212 | 218 | |
|
213 | 219 | def _check_path(self): |
|
214 | 220 | if not os.path.exists(self.path): |
|
215 | 221 | raise VCSError('Path "%s" does not exist!' % (self.path, )) |
|
216 | 222 | if not self._remote.is_path_valid_repository(self.path): |
|
217 | 223 | raise VCSError( |
|
218 | 224 | 'Path "%s" does not contain a Subversion repository' % |
|
219 | 225 | (self.path, )) |
|
220 | 226 | |
|
221 | 227 | @LazyProperty |
|
222 | 228 | def last_change(self): |
|
223 | 229 | """ |
|
224 | 230 | Returns last change made on this repository as |
|
225 | 231 | `datetime.datetime` object. |
|
226 | 232 | """ |
|
227 | 233 | # Subversion always has a first commit which has id "0" and contains |
|
228 | 234 | # what we are looking for. |
|
229 | 235 | last_id = len(self.commit_ids) |
|
230 | 236 | properties = self._remote.revision_properties(last_id) |
|
231 | 237 | return _date_from_svn_properties(properties) |
|
232 | 238 | |
|
233 | 239 | @LazyProperty |
|
234 | 240 | def in_memory_commit(self): |
|
235 | 241 | return SubversionInMemoryCommit(self) |
|
236 | 242 | |
|
237 | 243 | def get_hook_location(self): |
|
238 | 244 | """ |
|
239 | 245 | returns absolute path to location where hooks are stored |
|
240 | 246 | """ |
|
241 | 247 | return os.path.join(self.path, 'hooks') |
|
242 | 248 | |
|
243 | 249 | def get_commit(self, commit_id=None, commit_idx=None, pre_load=None): |
|
244 | 250 | if self.is_empty(): |
|
245 | 251 | raise EmptyRepositoryError("There are no commits yet") |
|
246 | 252 | if commit_id is not None: |
|
247 | 253 | self._validate_commit_id(commit_id) |
|
248 | 254 | elif commit_idx is not None: |
|
249 | 255 | self._validate_commit_idx(commit_idx) |
|
250 | 256 | try: |
|
251 | 257 | commit_id = self.commit_ids[commit_idx] |
|
252 | 258 | except IndexError: |
|
253 | 259 | raise CommitDoesNotExistError |
|
254 | 260 | |
|
255 | 261 | commit_id = self._sanitize_commit_id(commit_id) |
|
256 | 262 | commit = SubversionCommit(repository=self, commit_id=commit_id) |
|
257 | 263 | return commit |
|
258 | 264 | |
|
259 | 265 | def get_commits( |
|
260 | 266 | self, start_id=None, end_id=None, start_date=None, end_date=None, |
|
261 | 267 | branch_name=None, pre_load=None): |
|
262 | 268 | if self.is_empty(): |
|
263 | 269 | raise EmptyRepositoryError("There are no commit_ids yet") |
|
264 | 270 | self._validate_branch_name(branch_name) |
|
265 | 271 | |
|
266 | 272 | if start_id is not None: |
|
267 | 273 | self._validate_commit_id(start_id) |
|
268 | 274 | if end_id is not None: |
|
269 | 275 | self._validate_commit_id(end_id) |
|
270 | 276 | |
|
271 | 277 | start_raw_id = self._sanitize_commit_id(start_id) |
|
272 | 278 | start_pos = self.commit_ids.index(start_raw_id) if start_id else None |
|
273 | 279 | end_raw_id = self._sanitize_commit_id(end_id) |
|
274 | 280 | end_pos = max(0, self.commit_ids.index(end_raw_id)) if end_id else None |
|
275 | 281 | |
|
276 | 282 | if None not in [start_id, end_id] and start_pos > end_pos: |
|
277 | 283 | raise RepositoryError( |
|
278 | 284 | "Start commit '%s' cannot be after end commit '%s'" % |
|
279 | 285 | (start_id, end_id)) |
|
280 | 286 | if end_pos is not None: |
|
281 | 287 | end_pos += 1 |
|
282 | 288 | |
|
283 | 289 | # Date based filtering |
|
284 | 290 | if start_date or end_date: |
|
285 | 291 | start_raw_id, end_raw_id = self._remote.lookup_interval( |
|
286 | 292 | date_astimestamp(start_date) if start_date else None, |
|
287 | 293 | date_astimestamp(end_date) if end_date else None) |
|
288 | 294 | start_pos = start_raw_id - 1 |
|
289 | 295 | end_pos = end_raw_id |
|
290 | 296 | |
|
291 | 297 | commit_ids = self.commit_ids |
|
292 | 298 | |
|
293 | 299 | # TODO: johbo: Reconsider impact of DEFAULT_BRANCH_NAME here |
|
294 | 300 | if branch_name not in [None, self.DEFAULT_BRANCH_NAME]: |
|
295 | 301 | svn_rev = long(self.commit_ids[-1]) |
|
296 | 302 | commit_ids = self._remote.node_history( |
|
297 | 303 | path=branch_name, revision=svn_rev, limit=None) |
|
298 | 304 | commit_ids = [str(i) for i in reversed(commit_ids)] |
|
299 | 305 | |
|
300 | 306 | if start_pos or end_pos: |
|
301 | 307 | commit_ids = commit_ids[start_pos:end_pos] |
|
302 | 308 | return base.CollectionGenerator(self, commit_ids, pre_load=pre_load) |
|
303 | 309 | |
|
304 | 310 | def _sanitize_commit_id(self, commit_id): |
|
305 | 311 | if commit_id and commit_id.isdigit(): |
|
306 | 312 | if int(commit_id) <= len(self.commit_ids): |
|
307 | 313 | return commit_id |
|
308 | 314 | else: |
|
309 | 315 | raise CommitDoesNotExistError( |
|
310 | 316 | "Commit %s does not exist." % (commit_id, )) |
|
311 | 317 | if commit_id not in [ |
|
312 | 318 | None, 'HEAD', 'tip', self.DEFAULT_BRANCH_NAME]: |
|
313 | 319 | raise CommitDoesNotExistError( |
|
314 | 320 | "Commit id %s not understood." % (commit_id, )) |
|
315 | 321 | svn_rev = self._remote.lookup('HEAD') |
|
316 | 322 | return str(svn_rev) |
|
317 | 323 | |
|
318 | 324 | def get_diff( |
|
319 | 325 | self, commit1, commit2, path=None, ignore_whitespace=False, |
|
320 | 326 | context=3, path1=None): |
|
321 | 327 | self._validate_diff_commits(commit1, commit2) |
|
322 | 328 | svn_rev1 = long(commit1.raw_id) |
|
323 | 329 | svn_rev2 = long(commit2.raw_id) |
|
324 | 330 | diff = self._remote.diff( |
|
325 | 331 | svn_rev1, svn_rev2, path1=path1, path2=path, |
|
326 | 332 | ignore_whitespace=ignore_whitespace, context=context) |
|
327 | 333 | return SubversionDiff(diff) |
|
328 | 334 | |
|
329 | 335 | |
|
330 | 336 | def _sanitize_url(url): |
|
331 | 337 | if '://' not in url: |
|
332 | 338 | url = 'file://' + urllib.pathname2url(url) |
|
333 | 339 | return url |
General Comments 0
You need to be logged in to leave comments.
Login now