##// END OF EJS Templates
svn: fixed case for float timestamp
super-admin -
r1086:43b5c59f python3
parent child Browse files
Show More
@@ -1,888 +1,888 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18
18
19 import os
19 import os
20 import subprocess
20 import subprocess
21 from urllib.error import URLError
21 from urllib.error import URLError
22 import urllib.parse
22 import urllib.parse
23 import logging
23 import logging
24 import posixpath as vcspath
24 import posixpath as vcspath
25 import io
25 import io
26 import urllib.request
26 import urllib.request
27 import urllib.parse
27 import urllib.parse
28 import urllib.error
28 import urllib.error
29 import traceback
29 import traceback
30
30
31 import svn.client
31 import svn.client
32 import svn.core
32 import svn.core
33 import svn.delta
33 import svn.delta
34 import svn.diff
34 import svn.diff
35 import svn.fs
35 import svn.fs
36 import svn.repos
36 import svn.repos
37
37
38 from vcsserver import svn_diff, exceptions, subprocessio, settings
38 from vcsserver import svn_diff, exceptions, subprocessio, settings
39 from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, archive_repo
39 from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, archive_repo
40 from vcsserver.exceptions import NoContentException
40 from vcsserver.exceptions import NoContentException
41 from vcsserver.str_utils import safe_str, safe_bytes
41 from vcsserver.str_utils import safe_str, safe_bytes
42 from vcsserver.vcs_base import RemoteBase
42 from vcsserver.vcs_base import RemoteBase
43 from vcsserver.lib.svnremoterepo import svnremoterepo
43 from vcsserver.lib.svnremoterepo import svnremoterepo
44 log = logging.getLogger(__name__)
44 log = logging.getLogger(__name__)
45
45
46
46
47 svn_compatible_versions_map = {
47 svn_compatible_versions_map = {
48 'pre-1.4-compatible': '1.3',
48 'pre-1.4-compatible': '1.3',
49 'pre-1.5-compatible': '1.4',
49 'pre-1.5-compatible': '1.4',
50 'pre-1.6-compatible': '1.5',
50 'pre-1.6-compatible': '1.5',
51 'pre-1.8-compatible': '1.7',
51 'pre-1.8-compatible': '1.7',
52 'pre-1.9-compatible': '1.8',
52 'pre-1.9-compatible': '1.8',
53 }
53 }
54
54
55 current_compatible_version = '1.14'
55 current_compatible_version = '1.14'
56
56
57
57
58 def reraise_safe_exceptions(func):
58 def reraise_safe_exceptions(func):
59 """Decorator for converting svn exceptions to something neutral."""
59 """Decorator for converting svn exceptions to something neutral."""
60 def wrapper(*args, **kwargs):
60 def wrapper(*args, **kwargs):
61 try:
61 try:
62 return func(*args, **kwargs)
62 return func(*args, **kwargs)
63 except Exception as e:
63 except Exception as e:
64 if not hasattr(e, '_vcs_kind'):
64 if not hasattr(e, '_vcs_kind'):
65 log.exception("Unhandled exception in svn remote call")
65 log.exception("Unhandled exception in svn remote call")
66 raise_from_original(exceptions.UnhandledException(e), e)
66 raise_from_original(exceptions.UnhandledException(e), e)
67 raise
67 raise
68 return wrapper
68 return wrapper
69
69
70
70
71 class SubversionFactory(RepoFactory):
71 class SubversionFactory(RepoFactory):
72 repo_type = 'svn'
72 repo_type = 'svn'
73
73
74 def _create_repo(self, wire, create, compatible_version):
74 def _create_repo(self, wire, create, compatible_version):
75 path = svn.core.svn_path_canonicalize(wire['path'])
75 path = svn.core.svn_path_canonicalize(wire['path'])
76 if create:
76 if create:
77 fs_config = {'compatible-version': current_compatible_version}
77 fs_config = {'compatible-version': current_compatible_version}
78 if compatible_version:
78 if compatible_version:
79
79
80 compatible_version_string = \
80 compatible_version_string = \
81 svn_compatible_versions_map.get(compatible_version) \
81 svn_compatible_versions_map.get(compatible_version) \
82 or compatible_version
82 or compatible_version
83 fs_config['compatible-version'] = compatible_version_string
83 fs_config['compatible-version'] = compatible_version_string
84
84
85 log.debug('Create SVN repo with config "%s"', fs_config)
85 log.debug('Create SVN repo with config "%s"', fs_config)
86 repo = svn.repos.create(path, "", "", None, fs_config)
86 repo = svn.repos.create(path, "", "", None, fs_config)
87 else:
87 else:
88 repo = svn.repos.open(path)
88 repo = svn.repos.open(path)
89
89
90 log.debug('Got SVN object: %s', repo)
90 log.debug('Got SVN object: %s', repo)
91 return repo
91 return repo
92
92
93 def repo(self, wire, create=False, compatible_version=None):
93 def repo(self, wire, create=False, compatible_version=None):
94 """
94 """
95 Get a repository instance for the given path.
95 Get a repository instance for the given path.
96 """
96 """
97 return self._create_repo(wire, create, compatible_version)
97 return self._create_repo(wire, create, compatible_version)
98
98
99
99
100 NODE_TYPE_MAPPING = {
100 NODE_TYPE_MAPPING = {
101 svn.core.svn_node_file: 'file',
101 svn.core.svn_node_file: 'file',
102 svn.core.svn_node_dir: 'dir',
102 svn.core.svn_node_dir: 'dir',
103 }
103 }
104
104
105
105
106 class SvnRemote(RemoteBase):
106 class SvnRemote(RemoteBase):
107
107
108 def __init__(self, factory, hg_factory=None):
108 def __init__(self, factory, hg_factory=None):
109 self._factory = factory
109 self._factory = factory
110
110
111 @reraise_safe_exceptions
111 @reraise_safe_exceptions
112 def discover_svn_version(self):
112 def discover_svn_version(self):
113 try:
113 try:
114 import svn.core
114 import svn.core
115 svn_ver = svn.core.SVN_VERSION
115 svn_ver = svn.core.SVN_VERSION
116 except ImportError:
116 except ImportError:
117 svn_ver = None
117 svn_ver = None
118 return safe_str(svn_ver)
118 return safe_str(svn_ver)
119
119
120 @reraise_safe_exceptions
120 @reraise_safe_exceptions
121 def is_empty(self, wire):
121 def is_empty(self, wire):
122
122
123 try:
123 try:
124 return self.lookup(wire, -1) == 0
124 return self.lookup(wire, -1) == 0
125 except Exception:
125 except Exception:
126 log.exception("failed to read object_store")
126 log.exception("failed to read object_store")
127 return False
127 return False
128
128
129 def check_url(self, url):
129 def check_url(self, url):
130
130
131 # uuid function get's only valid UUID from proper repo, else
131 # uuid function get's only valid UUID from proper repo, else
132 # throws exception
132 # throws exception
133 username, password, src_url = self.get_url_and_credentials(url)
133 username, password, src_url = self.get_url_and_credentials(url)
134 try:
134 try:
135 svnremoterepo(username, password, src_url).svn().uuid
135 svnremoterepo(username, password, src_url).svn().uuid
136 except Exception:
136 except Exception:
137 tb = traceback.format_exc()
137 tb = traceback.format_exc()
138 log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
138 log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
139 raise URLError(
139 raise URLError(
140 '"%s" is not a valid Subversion source url.' % (url, ))
140 '"%s" is not a valid Subversion source url.' % (url, ))
141 return True
141 return True
142
142
143 def is_path_valid_repository(self, wire, path):
143 def is_path_valid_repository(self, wire, path):
144
144
145 # NOTE(marcink): short circuit the check for SVN repo
145 # NOTE(marcink): short circuit the check for SVN repo
146 # the repos.open might be expensive to check, but we have one cheap
146 # the repos.open might be expensive to check, but we have one cheap
147 # pre condition that we can use, to check for 'format' file
147 # pre condition that we can use, to check for 'format' file
148
148
149 if not os.path.isfile(os.path.join(path, 'format')):
149 if not os.path.isfile(os.path.join(path, 'format')):
150 return False
150 return False
151
151
152 try:
152 try:
153 svn.repos.open(path)
153 svn.repos.open(path)
154 except svn.core.SubversionException:
154 except svn.core.SubversionException:
155 tb = traceback.format_exc()
155 tb = traceback.format_exc()
156 log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
156 log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
157 return False
157 return False
158 return True
158 return True
159
159
160 @reraise_safe_exceptions
160 @reraise_safe_exceptions
161 def verify(self, wire,):
161 def verify(self, wire,):
162 repo_path = wire['path']
162 repo_path = wire['path']
163 if not self.is_path_valid_repository(wire, repo_path):
163 if not self.is_path_valid_repository(wire, repo_path):
164 raise Exception(
164 raise Exception(
165 "Path %s is not a valid Subversion repository." % repo_path)
165 "Path %s is not a valid Subversion repository." % repo_path)
166
166
167 cmd = ['svnadmin', 'info', repo_path]
167 cmd = ['svnadmin', 'info', repo_path]
168 stdout, stderr = subprocessio.run_command(cmd)
168 stdout, stderr = subprocessio.run_command(cmd)
169 return stdout
169 return stdout
170
170
171 def lookup(self, wire, revision):
171 def lookup(self, wire, revision):
172 if revision not in [-1, None, 'HEAD']:
172 if revision not in [-1, None, 'HEAD']:
173 raise NotImplementedError
173 raise NotImplementedError
174 repo = self._factory.repo(wire)
174 repo = self._factory.repo(wire)
175 fs_ptr = svn.repos.fs(repo)
175 fs_ptr = svn.repos.fs(repo)
176 head = svn.fs.youngest_rev(fs_ptr)
176 head = svn.fs.youngest_rev(fs_ptr)
177 return head
177 return head
178
178
179 def lookup_interval(self, wire, start_ts, end_ts):
179 def lookup_interval(self, wire, start_ts, end_ts):
180 repo = self._factory.repo(wire)
180 repo = self._factory.repo(wire)
181 fsobj = svn.repos.fs(repo)
181 fsobj = svn.repos.fs(repo)
182 start_rev = None
182 start_rev = None
183 end_rev = None
183 end_rev = None
184 if start_ts:
184 if start_ts:
185 start_ts_svn = apr_time_t(start_ts)
185 start_ts_svn = apr_time_t(start_ts)
186 start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
186 start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
187 else:
187 else:
188 start_rev = 1
188 start_rev = 1
189 if end_ts:
189 if end_ts:
190 end_ts_svn = apr_time_t(end_ts)
190 end_ts_svn = apr_time_t(end_ts)
191 end_rev = svn.repos.dated_revision(repo, end_ts_svn)
191 end_rev = svn.repos.dated_revision(repo, end_ts_svn)
192 else:
192 else:
193 end_rev = svn.fs.youngest_rev(fsobj)
193 end_rev = svn.fs.youngest_rev(fsobj)
194 return start_rev, end_rev
194 return start_rev, end_rev
195
195
196 def revision_properties(self, wire, revision):
196 def revision_properties(self, wire, revision):
197
197
198 cache_on, context_uid, repo_id = self._cache_on(wire)
198 cache_on, context_uid, repo_id = self._cache_on(wire)
199 region = self._region(wire)
199 region = self._region(wire)
200 @region.conditional_cache_on_arguments(condition=cache_on)
200 @region.conditional_cache_on_arguments(condition=cache_on)
201 def _revision_properties(_repo_id, _revision):
201 def _revision_properties(_repo_id, _revision):
202 repo = self._factory.repo(wire)
202 repo = self._factory.repo(wire)
203 fs_ptr = svn.repos.fs(repo)
203 fs_ptr = svn.repos.fs(repo)
204 return svn.fs.revision_proplist(fs_ptr, revision)
204 return svn.fs.revision_proplist(fs_ptr, revision)
205 return _revision_properties(repo_id, revision)
205 return _revision_properties(repo_id, revision)
206
206
207 def revision_changes(self, wire, revision):
207 def revision_changes(self, wire, revision):
208
208
209 repo = self._factory.repo(wire)
209 repo = self._factory.repo(wire)
210 fsobj = svn.repos.fs(repo)
210 fsobj = svn.repos.fs(repo)
211 rev_root = svn.fs.revision_root(fsobj, revision)
211 rev_root = svn.fs.revision_root(fsobj, revision)
212
212
213 editor = svn.repos.ChangeCollector(fsobj, rev_root)
213 editor = svn.repos.ChangeCollector(fsobj, rev_root)
214 editor_ptr, editor_baton = svn.delta.make_editor(editor)
214 editor_ptr, editor_baton = svn.delta.make_editor(editor)
215 base_dir = ""
215 base_dir = ""
216 send_deltas = False
216 send_deltas = False
217 svn.repos.replay2(
217 svn.repos.replay2(
218 rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
218 rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
219 editor_ptr, editor_baton, None)
219 editor_ptr, editor_baton, None)
220
220
221 added = []
221 added = []
222 changed = []
222 changed = []
223 removed = []
223 removed = []
224
224
225 # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
225 # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
226 for path, change in editor.changes.items():
226 for path, change in editor.changes.items():
227 # TODO: Decide what to do with directory nodes. Subversion can add
227 # TODO: Decide what to do with directory nodes. Subversion can add
228 # empty directories.
228 # empty directories.
229
229
230 if change.item_kind == svn.core.svn_node_dir:
230 if change.item_kind == svn.core.svn_node_dir:
231 continue
231 continue
232 if change.action in [svn.repos.CHANGE_ACTION_ADD]:
232 if change.action in [svn.repos.CHANGE_ACTION_ADD]:
233 added.append(path)
233 added.append(path)
234 elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
234 elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
235 svn.repos.CHANGE_ACTION_REPLACE]:
235 svn.repos.CHANGE_ACTION_REPLACE]:
236 changed.append(path)
236 changed.append(path)
237 elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
237 elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
238 removed.append(path)
238 removed.append(path)
239 else:
239 else:
240 raise NotImplementedError(
240 raise NotImplementedError(
241 "Action %s not supported on path %s" % (
241 "Action %s not supported on path %s" % (
242 change.action, path))
242 change.action, path))
243
243
244 changes = {
244 changes = {
245 'added': added,
245 'added': added,
246 'changed': changed,
246 'changed': changed,
247 'removed': removed,
247 'removed': removed,
248 }
248 }
249 return changes
249 return changes
250
250
251 @reraise_safe_exceptions
251 @reraise_safe_exceptions
252 def node_history(self, wire, path, revision, limit):
252 def node_history(self, wire, path, revision, limit):
253 cache_on, context_uid, repo_id = self._cache_on(wire)
253 cache_on, context_uid, repo_id = self._cache_on(wire)
254 region = self._region(wire)
254 region = self._region(wire)
255 @region.conditional_cache_on_arguments(condition=cache_on)
255 @region.conditional_cache_on_arguments(condition=cache_on)
256 def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
256 def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
257 cross_copies = False
257 cross_copies = False
258 repo = self._factory.repo(wire)
258 repo = self._factory.repo(wire)
259 fsobj = svn.repos.fs(repo)
259 fsobj = svn.repos.fs(repo)
260 rev_root = svn.fs.revision_root(fsobj, revision)
260 rev_root = svn.fs.revision_root(fsobj, revision)
261
261
262 history_revisions = []
262 history_revisions = []
263 history = svn.fs.node_history(rev_root, path)
263 history = svn.fs.node_history(rev_root, path)
264 history = svn.fs.history_prev(history, cross_copies)
264 history = svn.fs.history_prev(history, cross_copies)
265 while history:
265 while history:
266 __, node_revision = svn.fs.history_location(history)
266 __, node_revision = svn.fs.history_location(history)
267 history_revisions.append(node_revision)
267 history_revisions.append(node_revision)
268 if limit and len(history_revisions) >= limit:
268 if limit and len(history_revisions) >= limit:
269 break
269 break
270 history = svn.fs.history_prev(history, cross_copies)
270 history = svn.fs.history_prev(history, cross_copies)
271 return history_revisions
271 return history_revisions
272 return _assert_correct_path(context_uid, repo_id, path, revision, limit)
272 return _assert_correct_path(context_uid, repo_id, path, revision, limit)
273
273
274 def node_properties(self, wire, path, revision):
274 def node_properties(self, wire, path, revision):
275 cache_on, context_uid, repo_id = self._cache_on(wire)
275 cache_on, context_uid, repo_id = self._cache_on(wire)
276 region = self._region(wire)
276 region = self._region(wire)
277 @region.conditional_cache_on_arguments(condition=cache_on)
277 @region.conditional_cache_on_arguments(condition=cache_on)
278 def _node_properties(_repo_id, _path, _revision):
278 def _node_properties(_repo_id, _path, _revision):
279 repo = self._factory.repo(wire)
279 repo = self._factory.repo(wire)
280 fsobj = svn.repos.fs(repo)
280 fsobj = svn.repos.fs(repo)
281 rev_root = svn.fs.revision_root(fsobj, revision)
281 rev_root = svn.fs.revision_root(fsobj, revision)
282 return svn.fs.node_proplist(rev_root, path)
282 return svn.fs.node_proplist(rev_root, path)
283 return _node_properties(repo_id, path, revision)
283 return _node_properties(repo_id, path, revision)
284
284
285 def file_annotate(self, wire, path, revision):
285 def file_annotate(self, wire, path, revision):
286 abs_path = 'file://' + urllib.request.pathname2url(
286 abs_path = 'file://' + urllib.request.pathname2url(
287 vcspath.join(wire['path'], path))
287 vcspath.join(wire['path'], path))
288 file_uri = svn.core.svn_path_canonicalize(abs_path)
288 file_uri = svn.core.svn_path_canonicalize(abs_path)
289
289
290 start_rev = svn_opt_revision_value_t(0)
290 start_rev = svn_opt_revision_value_t(0)
291 peg_rev = svn_opt_revision_value_t(revision)
291 peg_rev = svn_opt_revision_value_t(revision)
292 end_rev = peg_rev
292 end_rev = peg_rev
293
293
294 annotations = []
294 annotations = []
295
295
296 def receiver(line_no, revision, author, date, line, pool):
296 def receiver(line_no, revision, author, date, line, pool):
297 annotations.append((line_no, revision, line))
297 annotations.append((line_no, revision, line))
298
298
299 # TODO: Cannot use blame5, missing typemap function in the swig code
299 # TODO: Cannot use blame5, missing typemap function in the swig code
300 try:
300 try:
301 svn.client.blame2(
301 svn.client.blame2(
302 file_uri, peg_rev, start_rev, end_rev,
302 file_uri, peg_rev, start_rev, end_rev,
303 receiver, svn.client.create_context())
303 receiver, svn.client.create_context())
304 except svn.core.SubversionException as exc:
304 except svn.core.SubversionException as exc:
305 log.exception("Error during blame operation.")
305 log.exception("Error during blame operation.")
306 raise Exception(
306 raise Exception(
307 "Blame not supported or file does not exist at path %s. "
307 "Blame not supported or file does not exist at path %s. "
308 "Error %s." % (path, exc))
308 "Error %s." % (path, exc))
309
309
310 return annotations
310 return annotations
311
311
312 def get_node_type(self, wire, path, revision=None):
312 def get_node_type(self, wire, path, revision=None):
313
313
314 cache_on, context_uid, repo_id = self._cache_on(wire)
314 cache_on, context_uid, repo_id = self._cache_on(wire)
315 region = self._region(wire)
315 region = self._region(wire)
316 @region.conditional_cache_on_arguments(condition=cache_on)
316 @region.conditional_cache_on_arguments(condition=cache_on)
317 def _get_node_type(_repo_id, _path, _revision):
317 def _get_node_type(_repo_id, _path, _revision):
318 repo = self._factory.repo(wire)
318 repo = self._factory.repo(wire)
319 fs_ptr = svn.repos.fs(repo)
319 fs_ptr = svn.repos.fs(repo)
320 if _revision is None:
320 if _revision is None:
321 _revision = svn.fs.youngest_rev(fs_ptr)
321 _revision = svn.fs.youngest_rev(fs_ptr)
322 root = svn.fs.revision_root(fs_ptr, _revision)
322 root = svn.fs.revision_root(fs_ptr, _revision)
323 node = svn.fs.check_path(root, path)
323 node = svn.fs.check_path(root, path)
324 return NODE_TYPE_MAPPING.get(node, None)
324 return NODE_TYPE_MAPPING.get(node, None)
325 return _get_node_type(repo_id, path, revision)
325 return _get_node_type(repo_id, path, revision)
326
326
327 def get_nodes(self, wire, path, revision=None):
327 def get_nodes(self, wire, path, revision=None):
328
328
329 cache_on, context_uid, repo_id = self._cache_on(wire)
329 cache_on, context_uid, repo_id = self._cache_on(wire)
330 region = self._region(wire)
330 region = self._region(wire)
331
331
332 @region.conditional_cache_on_arguments(condition=cache_on)
332 @region.conditional_cache_on_arguments(condition=cache_on)
333 def _get_nodes(_repo_id, _path, _revision):
333 def _get_nodes(_repo_id, _path, _revision):
334 repo = self._factory.repo(wire)
334 repo = self._factory.repo(wire)
335 fsobj = svn.repos.fs(repo)
335 fsobj = svn.repos.fs(repo)
336 if _revision is None:
336 if _revision is None:
337 _revision = svn.fs.youngest_rev(fsobj)
337 _revision = svn.fs.youngest_rev(fsobj)
338 root = svn.fs.revision_root(fsobj, _revision)
338 root = svn.fs.revision_root(fsobj, _revision)
339 entries = svn.fs.dir_entries(root, path)
339 entries = svn.fs.dir_entries(root, path)
340 result = []
340 result = []
341 for entry_path, entry_info in entries.items():
341 for entry_path, entry_info in entries.items():
342 result.append(
342 result.append(
343 (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
343 (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
344 return result
344 return result
345 return _get_nodes(repo_id, path, revision)
345 return _get_nodes(repo_id, path, revision)
346
346
347 def get_file_content(self, wire, path, rev=None):
347 def get_file_content(self, wire, path, rev=None):
348 repo = self._factory.repo(wire)
348 repo = self._factory.repo(wire)
349 fsobj = svn.repos.fs(repo)
349 fsobj = svn.repos.fs(repo)
350 if rev is None:
350 if rev is None:
351 rev = svn.fs.youngest_revision(fsobj)
351 rev = svn.fs.youngest_revision(fsobj)
352 root = svn.fs.revision_root(fsobj, rev)
352 root = svn.fs.revision_root(fsobj, rev)
353 content = svn.core.Stream(svn.fs.file_contents(root, path))
353 content = svn.core.Stream(svn.fs.file_contents(root, path))
354 return content.read()
354 return content.read()
355
355
356 def get_file_size(self, wire, path, revision=None):
356 def get_file_size(self, wire, path, revision=None):
357
357
358 cache_on, context_uid, repo_id = self._cache_on(wire)
358 cache_on, context_uid, repo_id = self._cache_on(wire)
359 region = self._region(wire)
359 region = self._region(wire)
360
360
361 @region.conditional_cache_on_arguments(condition=cache_on)
361 @region.conditional_cache_on_arguments(condition=cache_on)
362 def _get_file_size(_repo_id, _path, _revision):
362 def _get_file_size(_repo_id, _path, _revision):
363 repo = self._factory.repo(wire)
363 repo = self._factory.repo(wire)
364 fsobj = svn.repos.fs(repo)
364 fsobj = svn.repos.fs(repo)
365 if _revision is None:
365 if _revision is None:
366 _revision = svn.fs.youngest_revision(fsobj)
366 _revision = svn.fs.youngest_revision(fsobj)
367 root = svn.fs.revision_root(fsobj, _revision)
367 root = svn.fs.revision_root(fsobj, _revision)
368 size = svn.fs.file_length(root, path)
368 size = svn.fs.file_length(root, path)
369 return size
369 return size
370 return _get_file_size(repo_id, path, revision)
370 return _get_file_size(repo_id, path, revision)
371
371
372 def create_repository(self, wire, compatible_version=None):
372 def create_repository(self, wire, compatible_version=None):
373 log.info('Creating Subversion repository in path "%s"', wire['path'])
373 log.info('Creating Subversion repository in path "%s"', wire['path'])
374 self._factory.repo(wire, create=True,
374 self._factory.repo(wire, create=True,
375 compatible_version=compatible_version)
375 compatible_version=compatible_version)
376
376
377 def get_url_and_credentials(self, src_url):
377 def get_url_and_credentials(self, src_url):
378 obj = urllib.parse.urlparse(src_url)
378 obj = urllib.parse.urlparse(src_url)
379 username = obj.username or None
379 username = obj.username or None
380 password = obj.password or None
380 password = obj.password or None
381 return username, password, src_url
381 return username, password, src_url
382
382
383 def import_remote_repository(self, wire, src_url):
383 def import_remote_repository(self, wire, src_url):
384 repo_path = wire['path']
384 repo_path = wire['path']
385 if not self.is_path_valid_repository(wire, repo_path):
385 if not self.is_path_valid_repository(wire, repo_path):
386 raise Exception(
386 raise Exception(
387 "Path %s is not a valid Subversion repository." % repo_path)
387 "Path %s is not a valid Subversion repository." % repo_path)
388
388
389 username, password, src_url = self.get_url_and_credentials(src_url)
389 username, password, src_url = self.get_url_and_credentials(src_url)
390 rdump_cmd = ['svnrdump', 'dump', '--non-interactive',
390 rdump_cmd = ['svnrdump', 'dump', '--non-interactive',
391 '--trust-server-cert-failures=unknown-ca']
391 '--trust-server-cert-failures=unknown-ca']
392 if username and password:
392 if username and password:
393 rdump_cmd += ['--username', username, '--password', password]
393 rdump_cmd += ['--username', username, '--password', password]
394 rdump_cmd += [src_url]
394 rdump_cmd += [src_url]
395
395
396 rdump = subprocess.Popen(
396 rdump = subprocess.Popen(
397 rdump_cmd,
397 rdump_cmd,
398 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
398 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
399 load = subprocess.Popen(
399 load = subprocess.Popen(
400 ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
400 ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
401
401
402 # TODO: johbo: This can be a very long operation, might be better
402 # TODO: johbo: This can be a very long operation, might be better
403 # to track some kind of status and provide an api to check if the
403 # to track some kind of status and provide an api to check if the
404 # import is done.
404 # import is done.
405 rdump.wait()
405 rdump.wait()
406 load.wait()
406 load.wait()
407
407
408 log.debug('Return process ended with code: %s', rdump.returncode)
408 log.debug('Return process ended with code: %s', rdump.returncode)
409 if rdump.returncode != 0:
409 if rdump.returncode != 0:
410 errors = rdump.stderr.read()
410 errors = rdump.stderr.read()
411 log.error('svnrdump dump failed: statuscode %s: message: %s', rdump.returncode, errors)
411 log.error('svnrdump dump failed: statuscode %s: message: %s', rdump.returncode, errors)
412
412
413 reason = 'UNKNOWN'
413 reason = 'UNKNOWN'
414 if b'svnrdump: E230001:' in errors:
414 if b'svnrdump: E230001:' in errors:
415 reason = 'INVALID_CERTIFICATE'
415 reason = 'INVALID_CERTIFICATE'
416
416
417 if reason == 'UNKNOWN':
417 if reason == 'UNKNOWN':
418 reason = 'UNKNOWN:{}'.format(safe_str(errors))
418 reason = 'UNKNOWN:{}'.format(safe_str(errors))
419
419
420 raise Exception(
420 raise Exception(
421 'Failed to dump the remote repository from %s. Reason:%s' % (
421 'Failed to dump the remote repository from %s. Reason:%s' % (
422 src_url, reason))
422 src_url, reason))
423 if load.returncode != 0:
423 if load.returncode != 0:
424 raise Exception(
424 raise Exception(
425 'Failed to load the dump of remote repository from %s.' %
425 'Failed to load the dump of remote repository from %s.' %
426 (src_url, ))
426 (src_url, ))
427
427
428 def commit(self, wire, message, author, timestamp, updated, removed):
428 def commit(self, wire, message, author, timestamp, updated, removed):
429
429
430 updated = [{k: safe_bytes(v) for k, v in x.items() if isinstance(v, str)} for x in updated]
430 updated = [{k: safe_bytes(v) for k, v in x.items() if isinstance(v, str)} for x in updated]
431
431
432 message = safe_bytes(message)
432 message = safe_bytes(message)
433 author = safe_bytes(author)
433 author = safe_bytes(author)
434
434
435 repo = self._factory.repo(wire)
435 repo = self._factory.repo(wire)
436 fsobj = svn.repos.fs(repo)
436 fsobj = svn.repos.fs(repo)
437
437
438 rev = svn.fs.youngest_rev(fsobj)
438 rev = svn.fs.youngest_rev(fsobj)
439 txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
439 txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
440 txn_root = svn.fs.txn_root(txn)
440 txn_root = svn.fs.txn_root(txn)
441
441
442 for node in updated:
442 for node in updated:
443 TxnNodeProcessor(node, txn_root).update()
443 TxnNodeProcessor(node, txn_root).update()
444 for node in removed:
444 for node in removed:
445 TxnNodeProcessor(node, txn_root).remove()
445 TxnNodeProcessor(node, txn_root).remove()
446
446
447 commit_id = svn.repos.fs_commit_txn(repo, txn)
447 commit_id = svn.repos.fs_commit_txn(repo, txn)
448
448
449 if timestamp:
449 if timestamp:
450 apr_time = apr_time_t(timestamp)
450 apr_time = int(apr_time_t(timestamp))
451 ts_formatted = svn.core.svn_time_to_cstring(apr_time)
451 ts_formatted = svn.core.svn_time_to_cstring(apr_time)
452 svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
452 svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
453
453
454 log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
454 log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
455 return commit_id
455 return commit_id
456
456
457 def diff(self, wire, rev1, rev2, path1=None, path2=None,
457 def diff(self, wire, rev1, rev2, path1=None, path2=None,
458 ignore_whitespace=False, context=3):
458 ignore_whitespace=False, context=3):
459
459
460 wire.update(cache=False)
460 wire.update(cache=False)
461 repo = self._factory.repo(wire)
461 repo = self._factory.repo(wire)
462 diff_creator = SvnDiffer(
462 diff_creator = SvnDiffer(
463 repo, rev1, path1, rev2, path2, ignore_whitespace, context)
463 repo, rev1, path1, rev2, path2, ignore_whitespace, context)
464 try:
464 try:
465 return diff_creator.generate_diff()
465 return diff_creator.generate_diff()
466 except svn.core.SubversionException as e:
466 except svn.core.SubversionException as e:
467 log.exception(
467 log.exception(
468 "Error during diff operation operation. "
468 "Error during diff operation operation. "
469 "Path might not exist %s, %s" % (path1, path2))
469 "Path might not exist %s, %s" % (path1, path2))
470 return ""
470 return ""
471
471
472 @reraise_safe_exceptions
472 @reraise_safe_exceptions
473 def is_large_file(self, wire, path):
473 def is_large_file(self, wire, path):
474 return False
474 return False
475
475
476 @reraise_safe_exceptions
476 @reraise_safe_exceptions
477 def is_binary(self, wire, rev, path):
477 def is_binary(self, wire, rev, path):
478 cache_on, context_uid, repo_id = self._cache_on(wire)
478 cache_on, context_uid, repo_id = self._cache_on(wire)
479 region = self._region(wire)
479 region = self._region(wire)
480
480
481 @region.conditional_cache_on_arguments(condition=cache_on)
481 @region.conditional_cache_on_arguments(condition=cache_on)
482 def _is_binary(_repo_id, _rev, _path):
482 def _is_binary(_repo_id, _rev, _path):
483 raw_bytes = self.get_file_content(wire, path, rev)
483 raw_bytes = self.get_file_content(wire, path, rev)
484 return raw_bytes and b'\0' in raw_bytes
484 return raw_bytes and b'\0' in raw_bytes
485
485
486 return _is_binary(repo_id, rev, path)
486 return _is_binary(repo_id, rev, path)
487
487
488 @reraise_safe_exceptions
488 @reraise_safe_exceptions
489 def md5_hash(self, wire, rev, path):
489 def md5_hash(self, wire, rev, path):
490 cache_on, context_uid, repo_id = self._cache_on(wire)
490 cache_on, context_uid, repo_id = self._cache_on(wire)
491 region = self._region(wire)
491 region = self._region(wire)
492
492
493 @region.conditional_cache_on_arguments(condition=cache_on)
493 @region.conditional_cache_on_arguments(condition=cache_on)
494 def _md5_hash(_repo_id, _rev, _path):
494 def _md5_hash(_repo_id, _rev, _path):
495 return ''
495 return ''
496
496
497 return _md5_hash(repo_id, rev, path)
497 return _md5_hash(repo_id, rev, path)
498
498
499 @reraise_safe_exceptions
499 @reraise_safe_exceptions
500 def run_svn_command(self, wire, cmd, **opts):
500 def run_svn_command(self, wire, cmd, **opts):
501 path = wire.get('path', None)
501 path = wire.get('path', None)
502
502
503 if path and os.path.isdir(path):
503 if path and os.path.isdir(path):
504 opts['cwd'] = path
504 opts['cwd'] = path
505
505
506 safe_call = opts.pop('_safe', False)
506 safe_call = opts.pop('_safe', False)
507
507
508 svnenv = os.environ.copy()
508 svnenv = os.environ.copy()
509 svnenv.update(opts.pop('extra_env', {}))
509 svnenv.update(opts.pop('extra_env', {}))
510
510
511 _opts = {'env': svnenv, 'shell': False}
511 _opts = {'env': svnenv, 'shell': False}
512
512
513 try:
513 try:
514 _opts.update(opts)
514 _opts.update(opts)
515 proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
515 proc = subprocessio.SubprocessIOChunker(cmd, **_opts)
516
516
517 return b''.join(proc), b''.join(proc.stderr)
517 return b''.join(proc), b''.join(proc.stderr)
518 except OSError as err:
518 except OSError as err:
519 if safe_call:
519 if safe_call:
520 return '', safe_str(err).strip()
520 return '', safe_str(err).strip()
521 else:
521 else:
522 cmd = ' '.join(map(safe_str, cmd)) # human friendly CMD
522 cmd = ' '.join(map(safe_str, cmd)) # human friendly CMD
523 tb_err = ("Couldn't run svn command (%s).\n"
523 tb_err = ("Couldn't run svn command (%s).\n"
524 "Original error was:%s\n"
524 "Original error was:%s\n"
525 "Call options:%s\n"
525 "Call options:%s\n"
526 % (cmd, err, _opts))
526 % (cmd, err, _opts))
527 log.exception(tb_err)
527 log.exception(tb_err)
528 raise exceptions.VcsException()(tb_err)
528 raise exceptions.VcsException()(tb_err)
529
529
530 @reraise_safe_exceptions
530 @reraise_safe_exceptions
531 def install_hooks(self, wire, force=False):
531 def install_hooks(self, wire, force=False):
532 from vcsserver.hook_utils import install_svn_hooks
532 from vcsserver.hook_utils import install_svn_hooks
533 repo_path = wire['path']
533 repo_path = wire['path']
534 binary_dir = settings.BINARY_DIR
534 binary_dir = settings.BINARY_DIR
535 executable = None
535 executable = None
536 if binary_dir:
536 if binary_dir:
537 executable = os.path.join(binary_dir, 'python3')
537 executable = os.path.join(binary_dir, 'python3')
538 return install_svn_hooks(repo_path, force_create=force)
538 return install_svn_hooks(repo_path, force_create=force)
539
539
540 @reraise_safe_exceptions
540 @reraise_safe_exceptions
541 def get_hooks_info(self, wire):
541 def get_hooks_info(self, wire):
542 from vcsserver.hook_utils import (
542 from vcsserver.hook_utils import (
543 get_svn_pre_hook_version, get_svn_post_hook_version)
543 get_svn_pre_hook_version, get_svn_post_hook_version)
544 repo_path = wire['path']
544 repo_path = wire['path']
545 return {
545 return {
546 'pre_version': get_svn_pre_hook_version(repo_path),
546 'pre_version': get_svn_pre_hook_version(repo_path),
547 'post_version': get_svn_post_hook_version(repo_path),
547 'post_version': get_svn_post_hook_version(repo_path),
548 }
548 }
549
549
550 @reraise_safe_exceptions
550 @reraise_safe_exceptions
551 def set_head_ref(self, wire, head_name):
551 def set_head_ref(self, wire, head_name):
552 pass
552 pass
553
553
554 @reraise_safe_exceptions
554 @reraise_safe_exceptions
555 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
555 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
556 archive_dir_name, commit_id):
556 archive_dir_name, commit_id):
557
557
558 def walk_tree(root, root_dir, _commit_id):
558 def walk_tree(root, root_dir, _commit_id):
559 """
559 """
560 Special recursive svn repo walker
560 Special recursive svn repo walker
561 """
561 """
562 root_dir = safe_bytes(root_dir)
562 root_dir = safe_bytes(root_dir)
563
563
564 filemode_default = 0o100644
564 filemode_default = 0o100644
565 filemode_executable = 0o100755
565 filemode_executable = 0o100755
566
566
567 file_iter = svn.fs.dir_entries(root, root_dir)
567 file_iter = svn.fs.dir_entries(root, root_dir)
568 for f_name in file_iter:
568 for f_name in file_iter:
569 f_type = NODE_TYPE_MAPPING.get(file_iter[f_name].kind, None)
569 f_type = NODE_TYPE_MAPPING.get(file_iter[f_name].kind, None)
570
570
571 if f_type == 'dir':
571 if f_type == 'dir':
572 # return only DIR, and then all entries in that dir
572 # return only DIR, and then all entries in that dir
573 yield os.path.join(root_dir, f_name), {'mode': filemode_default}, f_type
573 yield os.path.join(root_dir, f_name), {'mode': filemode_default}, f_type
574 new_root = os.path.join(root_dir, f_name)
574 new_root = os.path.join(root_dir, f_name)
575 for _f_name, _f_data, _f_type in walk_tree(root, new_root, _commit_id):
575 for _f_name, _f_data, _f_type in walk_tree(root, new_root, _commit_id):
576 yield _f_name, _f_data, _f_type
576 yield _f_name, _f_data, _f_type
577 else:
577 else:
578
578
579 f_path = os.path.join(root_dir, f_name).rstrip(b'/')
579 f_path = os.path.join(root_dir, f_name).rstrip(b'/')
580 prop_list = svn.fs.node_proplist(root, f_path)
580 prop_list = svn.fs.node_proplist(root, f_path)
581
581
582 f_mode = filemode_default
582 f_mode = filemode_default
583 if prop_list.get('svn:executable'):
583 if prop_list.get('svn:executable'):
584 f_mode = filemode_executable
584 f_mode = filemode_executable
585
585
586 f_is_link = False
586 f_is_link = False
587 if prop_list.get('svn:special'):
587 if prop_list.get('svn:special'):
588 f_is_link = True
588 f_is_link = True
589
589
590 data = {
590 data = {
591 'is_link': f_is_link,
591 'is_link': f_is_link,
592 'mode': f_mode,
592 'mode': f_mode,
593 'content_stream': svn.core.Stream(svn.fs.file_contents(root, f_path)).read
593 'content_stream': svn.core.Stream(svn.fs.file_contents(root, f_path)).read
594 }
594 }
595
595
596 yield f_path, data, f_type
596 yield f_path, data, f_type
597
597
598 def file_walker(_commit_id, path):
598 def file_walker(_commit_id, path):
599 repo = self._factory.repo(wire)
599 repo = self._factory.repo(wire)
600 root = svn.fs.revision_root(svn.repos.fs(repo), int(commit_id))
600 root = svn.fs.revision_root(svn.repos.fs(repo), int(commit_id))
601
601
602 def no_content():
602 def no_content():
603 raise NoContentException()
603 raise NoContentException()
604
604
605 for f_name, f_data, f_type in walk_tree(root, path, _commit_id):
605 for f_name, f_data, f_type in walk_tree(root, path, _commit_id):
606 file_path = f_name
606 file_path = f_name
607
607
608 if f_type == 'dir':
608 if f_type == 'dir':
609 mode = f_data['mode']
609 mode = f_data['mode']
610 yield ArchiveNode(file_path, mode, False, no_content)
610 yield ArchiveNode(file_path, mode, False, no_content)
611 else:
611 else:
612 mode = f_data['mode']
612 mode = f_data['mode']
613 is_link = f_data['is_link']
613 is_link = f_data['is_link']
614 data_stream = f_data['content_stream']
614 data_stream = f_data['content_stream']
615 yield ArchiveNode(file_path, mode, is_link, data_stream)
615 yield ArchiveNode(file_path, mode, is_link, data_stream)
616
616
617 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
617 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
618 archive_dir_name, commit_id)
618 archive_dir_name, commit_id)
619
619
620
620
621 class SvnDiffer(object):
621 class SvnDiffer(object):
622 """
622 """
623 Utility to create diffs based on difflib and the Subversion api
623 Utility to create diffs based on difflib and the Subversion api
624 """
624 """
625
625
626 binary_content = False
626 binary_content = False
627
627
628 def __init__(
628 def __init__(
629 self, repo, src_rev, src_path, tgt_rev, tgt_path,
629 self, repo, src_rev, src_path, tgt_rev, tgt_path,
630 ignore_whitespace, context):
630 ignore_whitespace, context):
631 self.repo = repo
631 self.repo = repo
632 self.ignore_whitespace = ignore_whitespace
632 self.ignore_whitespace = ignore_whitespace
633 self.context = context
633 self.context = context
634
634
635 fsobj = svn.repos.fs(repo)
635 fsobj = svn.repos.fs(repo)
636
636
637 self.tgt_rev = tgt_rev
637 self.tgt_rev = tgt_rev
638 self.tgt_path = tgt_path or ''
638 self.tgt_path = tgt_path or ''
639 self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
639 self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
640 self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
640 self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
641
641
642 self.src_rev = src_rev
642 self.src_rev = src_rev
643 self.src_path = src_path or self.tgt_path
643 self.src_path = src_path or self.tgt_path
644 self.src_root = svn.fs.revision_root(fsobj, src_rev)
644 self.src_root = svn.fs.revision_root(fsobj, src_rev)
645 self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
645 self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
646
646
647 self._validate()
647 self._validate()
648
648
649 def _validate(self):
649 def _validate(self):
650 if (self.tgt_kind != svn.core.svn_node_none and
650 if (self.tgt_kind != svn.core.svn_node_none and
651 self.src_kind != svn.core.svn_node_none and
651 self.src_kind != svn.core.svn_node_none and
652 self.src_kind != self.tgt_kind):
652 self.src_kind != self.tgt_kind):
653 # TODO: johbo: proper error handling
653 # TODO: johbo: proper error handling
654 raise Exception(
654 raise Exception(
655 "Source and target are not compatible for diff generation. "
655 "Source and target are not compatible for diff generation. "
656 "Source type: %s, target type: %s" %
656 "Source type: %s, target type: %s" %
657 (self.src_kind, self.tgt_kind))
657 (self.src_kind, self.tgt_kind))
658
658
659 def generate_diff(self):
659 def generate_diff(self):
660 buf = io.StringIO()
660 buf = io.StringIO()
661 if self.tgt_kind == svn.core.svn_node_dir:
661 if self.tgt_kind == svn.core.svn_node_dir:
662 self._generate_dir_diff(buf)
662 self._generate_dir_diff(buf)
663 else:
663 else:
664 self._generate_file_diff(buf)
664 self._generate_file_diff(buf)
665 return buf.getvalue()
665 return buf.getvalue()
666
666
667 def _generate_dir_diff(self, buf):
667 def _generate_dir_diff(self, buf):
668 editor = DiffChangeEditor()
668 editor = DiffChangeEditor()
669 editor_ptr, editor_baton = svn.delta.make_editor(editor)
669 editor_ptr, editor_baton = svn.delta.make_editor(editor)
670 svn.repos.dir_delta2(
670 svn.repos.dir_delta2(
671 self.src_root,
671 self.src_root,
672 self.src_path,
672 self.src_path,
673 '', # src_entry
673 '', # src_entry
674 self.tgt_root,
674 self.tgt_root,
675 self.tgt_path,
675 self.tgt_path,
676 editor_ptr, editor_baton,
676 editor_ptr, editor_baton,
677 authorization_callback_allow_all,
677 authorization_callback_allow_all,
678 False, # text_deltas
678 False, # text_deltas
679 svn.core.svn_depth_infinity, # depth
679 svn.core.svn_depth_infinity, # depth
680 False, # entry_props
680 False, # entry_props
681 False, # ignore_ancestry
681 False, # ignore_ancestry
682 )
682 )
683
683
684 for path, __, change in sorted(editor.changes):
684 for path, __, change in sorted(editor.changes):
685 self._generate_node_diff(
685 self._generate_node_diff(
686 buf, change, path, self.tgt_path, path, self.src_path)
686 buf, change, path, self.tgt_path, path, self.src_path)
687
687
688 def _generate_file_diff(self, buf):
688 def _generate_file_diff(self, buf):
689 change = None
689 change = None
690 if self.src_kind == svn.core.svn_node_none:
690 if self.src_kind == svn.core.svn_node_none:
691 change = "add"
691 change = "add"
692 elif self.tgt_kind == svn.core.svn_node_none:
692 elif self.tgt_kind == svn.core.svn_node_none:
693 change = "delete"
693 change = "delete"
694 tgt_base, tgt_path = vcspath.split(self.tgt_path)
694 tgt_base, tgt_path = vcspath.split(self.tgt_path)
695 src_base, src_path = vcspath.split(self.src_path)
695 src_base, src_path = vcspath.split(self.src_path)
696 self._generate_node_diff(
696 self._generate_node_diff(
697 buf, change, tgt_path, tgt_base, src_path, src_base)
697 buf, change, tgt_path, tgt_base, src_path, src_base)
698
698
699 def _generate_node_diff(
699 def _generate_node_diff(
700 self, buf, change, tgt_path, tgt_base, src_path, src_base):
700 self, buf, change, tgt_path, tgt_base, src_path, src_base):
701
701
702
702
703 tgt_path = safe_str(tgt_path)
703 tgt_path = safe_str(tgt_path)
704 src_path = safe_str(src_path)
704 src_path = safe_str(src_path)
705
705
706
706
707 if self.src_rev == self.tgt_rev and tgt_base == src_base:
707 if self.src_rev == self.tgt_rev and tgt_base == src_base:
708 # makes consistent behaviour with git/hg to return empty diff if
708 # makes consistent behaviour with git/hg to return empty diff if
709 # we compare same revisions
709 # we compare same revisions
710 return
710 return
711
711
712 tgt_full_path = vcspath.join(tgt_base, tgt_path)
712 tgt_full_path = vcspath.join(tgt_base, tgt_path)
713 src_full_path = vcspath.join(src_base, src_path)
713 src_full_path = vcspath.join(src_base, src_path)
714
714
715 self.binary_content = False
715 self.binary_content = False
716 mime_type = self._get_mime_type(tgt_full_path)
716 mime_type = self._get_mime_type(tgt_full_path)
717
717
718 if mime_type and not mime_type.startswith('text'):
718 if mime_type and not mime_type.startswith('text'):
719 self.binary_content = True
719 self.binary_content = True
720 buf.write("=" * 67 + '\n')
720 buf.write("=" * 67 + '\n')
721 buf.write("Cannot display: file marked as a binary type.\n")
721 buf.write("Cannot display: file marked as a binary type.\n")
722 buf.write("svn:mime-type = %s\n" % mime_type)
722 buf.write("svn:mime-type = %s\n" % mime_type)
723 buf.write("Index: %s\n" % (tgt_path, ))
723 buf.write("Index: %s\n" % (tgt_path, ))
724 buf.write("=" * 67 + '\n')
724 buf.write("=" * 67 + '\n')
725 buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
725 buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
726 'tgt_path': tgt_path})
726 'tgt_path': tgt_path})
727
727
728 if change == 'add':
728 if change == 'add':
729 # TODO: johbo: SVN is missing a zero here compared to git
729 # TODO: johbo: SVN is missing a zero here compared to git
730 buf.write("new file mode 10644\n")
730 buf.write("new file mode 10644\n")
731
731
732 #TODO(marcink): intro to binary detection of svn patches
732 #TODO(marcink): intro to binary detection of svn patches
733 # if self.binary_content:
733 # if self.binary_content:
734 # buf.write('GIT binary patch\n')
734 # buf.write('GIT binary patch\n')
735
735
736 buf.write("--- /dev/null\t(revision 0)\n")
736 buf.write("--- /dev/null\t(revision 0)\n")
737 src_lines = []
737 src_lines = []
738 else:
738 else:
739 if change == 'delete':
739 if change == 'delete':
740 buf.write("deleted file mode 10644\n")
740 buf.write("deleted file mode 10644\n")
741
741
742 #TODO(marcink): intro to binary detection of svn patches
742 #TODO(marcink): intro to binary detection of svn patches
743 # if self.binary_content:
743 # if self.binary_content:
744 # buf.write('GIT binary patch\n')
744 # buf.write('GIT binary patch\n')
745
745
746 buf.write("--- a/%s\t(revision %s)\n" % (
746 buf.write("--- a/%s\t(revision %s)\n" % (
747 src_path, self.src_rev))
747 src_path, self.src_rev))
748 src_lines = self._svn_readlines(self.src_root, src_full_path)
748 src_lines = self._svn_readlines(self.src_root, src_full_path)
749
749
750 if change == 'delete':
750 if change == 'delete':
751 buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
751 buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
752 tgt_lines = []
752 tgt_lines = []
753 else:
753 else:
754 buf.write("+++ b/%s\t(revision %s)\n" % (
754 buf.write("+++ b/%s\t(revision %s)\n" % (
755 tgt_path, self.tgt_rev))
755 tgt_path, self.tgt_rev))
756 tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
756 tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
757
757
758 if not self.binary_content:
758 if not self.binary_content:
759 udiff = svn_diff.unified_diff(
759 udiff = svn_diff.unified_diff(
760 src_lines, tgt_lines, context=self.context,
760 src_lines, tgt_lines, context=self.context,
761 ignore_blank_lines=self.ignore_whitespace,
761 ignore_blank_lines=self.ignore_whitespace,
762 ignore_case=False,
762 ignore_case=False,
763 ignore_space_changes=self.ignore_whitespace)
763 ignore_space_changes=self.ignore_whitespace)
764
764
765 buf.writelines(udiff)
765 buf.writelines(udiff)
766
766
767 def _get_mime_type(self, path):
767 def _get_mime_type(self, path):
768 try:
768 try:
769 mime_type = svn.fs.node_prop(
769 mime_type = svn.fs.node_prop(
770 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
770 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
771 except svn.core.SubversionException:
771 except svn.core.SubversionException:
772 mime_type = svn.fs.node_prop(
772 mime_type = svn.fs.node_prop(
773 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
773 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
774 return mime_type
774 return mime_type
775
775
776 def _svn_readlines(self, fs_root, node_path):
776 def _svn_readlines(self, fs_root, node_path):
777 if self.binary_content:
777 if self.binary_content:
778 return []
778 return []
779 node_kind = svn.fs.check_path(fs_root, node_path)
779 node_kind = svn.fs.check_path(fs_root, node_path)
780 if node_kind not in (
780 if node_kind not in (
781 svn.core.svn_node_file, svn.core.svn_node_symlink):
781 svn.core.svn_node_file, svn.core.svn_node_symlink):
782 return []
782 return []
783 content = svn.core.Stream(
783 content = svn.core.Stream(
784 svn.fs.file_contents(fs_root, node_path)).read()
784 svn.fs.file_contents(fs_root, node_path)).read()
785
785
786 return content.splitlines(True)
786 return content.splitlines(True)
787
787
788
788
789 class DiffChangeEditor(svn.delta.Editor):
789 class DiffChangeEditor(svn.delta.Editor):
790 """
790 """
791 Records changes between two given revisions
791 Records changes between two given revisions
792 """
792 """
793
793
794 def __init__(self):
794 def __init__(self):
795 self.changes = []
795 self.changes = []
796
796
797 def delete_entry(self, path, revision, parent_baton, pool=None):
797 def delete_entry(self, path, revision, parent_baton, pool=None):
798 self.changes.append((path, None, 'delete'))
798 self.changes.append((path, None, 'delete'))
799
799
800 def add_file(
800 def add_file(
801 self, path, parent_baton, copyfrom_path, copyfrom_revision,
801 self, path, parent_baton, copyfrom_path, copyfrom_revision,
802 file_pool=None):
802 file_pool=None):
803 self.changes.append((path, 'file', 'add'))
803 self.changes.append((path, 'file', 'add'))
804
804
805 def open_file(self, path, parent_baton, base_revision, file_pool=None):
805 def open_file(self, path, parent_baton, base_revision, file_pool=None):
806 self.changes.append((path, 'file', 'change'))
806 self.changes.append((path, 'file', 'change'))
807
807
808
808
809 def authorization_callback_allow_all(root, path, pool):
809 def authorization_callback_allow_all(root, path, pool):
810 return True
810 return True
811
811
812
812
813 class TxnNodeProcessor(object):
813 class TxnNodeProcessor(object):
814 """
814 """
815 Utility to process the change of one node within a transaction root.
815 Utility to process the change of one node within a transaction root.
816
816
817 It encapsulates the knowledge of how to add, update or remove
817 It encapsulates the knowledge of how to add, update or remove
818 a node for a given transaction root. The purpose is to support the method
818 a node for a given transaction root. The purpose is to support the method
819 `SvnRemote.commit`.
819 `SvnRemote.commit`.
820 """
820 """
821
821
822 def __init__(self, node, txn_root):
822 def __init__(self, node, txn_root):
823 assert isinstance(node['path'], bytes)
823 assert isinstance(node['path'], bytes)
824
824
825 self.node = node
825 self.node = node
826 self.txn_root = txn_root
826 self.txn_root = txn_root
827
827
828 def update(self):
828 def update(self):
829 self._ensure_parent_dirs()
829 self._ensure_parent_dirs()
830 self._add_file_if_node_does_not_exist()
830 self._add_file_if_node_does_not_exist()
831 self._update_file_content()
831 self._update_file_content()
832 self._update_file_properties()
832 self._update_file_properties()
833
833
834 def remove(self):
834 def remove(self):
835 svn.fs.delete(self.txn_root, self.node['path'])
835 svn.fs.delete(self.txn_root, self.node['path'])
836 # TODO: Clean up directory if empty
836 # TODO: Clean up directory if empty
837
837
838 def _ensure_parent_dirs(self):
838 def _ensure_parent_dirs(self):
839 curdir = vcspath.dirname(self.node['path'])
839 curdir = vcspath.dirname(self.node['path'])
840 dirs_to_create = []
840 dirs_to_create = []
841 while not self._svn_path_exists(curdir):
841 while not self._svn_path_exists(curdir):
842 dirs_to_create.append(curdir)
842 dirs_to_create.append(curdir)
843 curdir = vcspath.dirname(curdir)
843 curdir = vcspath.dirname(curdir)
844
844
845 for curdir in reversed(dirs_to_create):
845 for curdir in reversed(dirs_to_create):
846 log.debug('Creating missing directory "%s"', curdir)
846 log.debug('Creating missing directory "%s"', curdir)
847 svn.fs.make_dir(self.txn_root, curdir)
847 svn.fs.make_dir(self.txn_root, curdir)
848
848
849 def _svn_path_exists(self, path):
849 def _svn_path_exists(self, path):
850 path_status = svn.fs.check_path(self.txn_root, path)
850 path_status = svn.fs.check_path(self.txn_root, path)
851 return path_status != svn.core.svn_node_none
851 return path_status != svn.core.svn_node_none
852
852
853 def _add_file_if_node_does_not_exist(self):
853 def _add_file_if_node_does_not_exist(self):
854 kind = svn.fs.check_path(self.txn_root, self.node['path'])
854 kind = svn.fs.check_path(self.txn_root, self.node['path'])
855 if kind == svn.core.svn_node_none:
855 if kind == svn.core.svn_node_none:
856 svn.fs.make_file(self.txn_root, self.node['path'])
856 svn.fs.make_file(self.txn_root, self.node['path'])
857
857
858 def _update_file_content(self):
858 def _update_file_content(self):
859 assert isinstance(self.node['content'], bytes)
859 assert isinstance(self.node['content'], bytes)
860
860
861 handler, baton = svn.fs.apply_textdelta(
861 handler, baton = svn.fs.apply_textdelta(
862 self.txn_root, self.node['path'], None, None)
862 self.txn_root, self.node['path'], None, None)
863 svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
863 svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
864
864
865 def _update_file_properties(self):
865 def _update_file_properties(self):
866 properties = self.node.get('properties', {})
866 properties = self.node.get('properties', {})
867 for key, value in properties.items():
867 for key, value in properties.items():
868 svn.fs.change_node_prop(
868 svn.fs.change_node_prop(
869 self.txn_root, self.node['path'], key, value)
869 self.txn_root, self.node['path'], key, value)
870
870
871
871
872 def apr_time_t(timestamp):
872 def apr_time_t(timestamp):
873 """
873 """
874 Convert a Python timestamp into APR timestamp type apr_time_t
874 Convert a Python timestamp into APR timestamp type apr_time_t
875 """
875 """
876 return timestamp * 1E6
876 return timestamp * 1E6
877
877
878
878
879 def svn_opt_revision_value_t(num):
879 def svn_opt_revision_value_t(num):
880 """
880 """
881 Put `num` into a `svn_opt_revision_value_t` structure.
881 Put `num` into a `svn_opt_revision_value_t` structure.
882 """
882 """
883 value = svn.core.svn_opt_revision_value_t()
883 value = svn.core.svn_opt_revision_value_t()
884 value.number = num
884 value.number = num
885 revision = svn.core.svn_opt_revision_t()
885 revision = svn.core.svn_opt_revision_t()
886 revision.kind = svn.core.svn_opt_revision_number
886 revision.kind = svn.core.svn_opt_revision_number
887 revision.value = value
887 revision.value = value
888 return revision
888 return revision
General Comments 0
You need to be logged in to leave comments. Login now