##// END OF EJS Templates
svn: fix subprocess problems on some of the calls for file checking.
milka -
r926:8b3be25b default
parent child Browse files
Show More
@@ -1,856 +1,855 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 from __future__ import absolute_import
18 from __future__ import absolute_import
19
19
20 import os
20 import os
21 import subprocess
21 import subprocess
22 import time
22 import time
23 from urllib2 import URLError
23 from urllib2 import URLError
24 import urlparse
24 import urlparse
25 import logging
25 import logging
26 import posixpath as vcspath
26 import posixpath as vcspath
27 import StringIO
27 import StringIO
28 import urllib
28 import urllib
29 import traceback
29 import traceback
30
30
31 import svn.client
31 import svn.client
32 import svn.core
32 import svn.core
33 import svn.delta
33 import svn.delta
34 import svn.diff
34 import svn.diff
35 import svn.fs
35 import svn.fs
36 import svn.repos
36 import svn.repos
37
37
38 from vcsserver import svn_diff, exceptions, subprocessio, settings
38 from vcsserver import svn_diff, exceptions, subprocessio, settings
39 from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, archive_repo
39 from vcsserver.base import RepoFactory, raise_from_original, ArchiveNode, archive_repo
40 from vcsserver.exceptions import NoContentException
40 from vcsserver.exceptions import NoContentException
41 from vcsserver.utils import safe_str
41 from vcsserver.vcs_base import RemoteBase
42 from vcsserver.vcs_base import RemoteBase
42
43
43 log = logging.getLogger(__name__)
44 log = logging.getLogger(__name__)
44
45
45
46
46 svn_compatible_versions_map = {
47 svn_compatible_versions_map = {
47 'pre-1.4-compatible': '1.3',
48 'pre-1.4-compatible': '1.3',
48 'pre-1.5-compatible': '1.4',
49 'pre-1.5-compatible': '1.4',
49 'pre-1.6-compatible': '1.5',
50 'pre-1.6-compatible': '1.5',
50 'pre-1.8-compatible': '1.7',
51 'pre-1.8-compatible': '1.7',
51 'pre-1.9-compatible': '1.8',
52 'pre-1.9-compatible': '1.8',
52 }
53 }
53
54
54 current_compatible_version = '1.12'
55 current_compatible_version = '1.12'
55
56
56
57
57 def reraise_safe_exceptions(func):
58 def reraise_safe_exceptions(func):
58 """Decorator for converting svn exceptions to something neutral."""
59 """Decorator for converting svn exceptions to something neutral."""
59 def wrapper(*args, **kwargs):
60 def wrapper(*args, **kwargs):
60 try:
61 try:
61 return func(*args, **kwargs)
62 return func(*args, **kwargs)
62 except Exception as e:
63 except Exception as e:
63 if not hasattr(e, '_vcs_kind'):
64 if not hasattr(e, '_vcs_kind'):
64 log.exception("Unhandled exception in svn remote call")
65 log.exception("Unhandled exception in svn remote call")
65 raise_from_original(exceptions.UnhandledException(e))
66 raise_from_original(exceptions.UnhandledException(e))
66 raise
67 raise
67 return wrapper
68 return wrapper
68
69
69
70
70 class SubversionFactory(RepoFactory):
71 class SubversionFactory(RepoFactory):
71 repo_type = 'svn'
72 repo_type = 'svn'
72
73
73 def _create_repo(self, wire, create, compatible_version):
74 def _create_repo(self, wire, create, compatible_version):
74 path = svn.core.svn_path_canonicalize(wire['path'])
75 path = svn.core.svn_path_canonicalize(wire['path'])
75 if create:
76 if create:
76 fs_config = {'compatible-version': current_compatible_version}
77 fs_config = {'compatible-version': current_compatible_version}
77 if compatible_version:
78 if compatible_version:
78
79
79 compatible_version_string = \
80 compatible_version_string = \
80 svn_compatible_versions_map.get(compatible_version) \
81 svn_compatible_versions_map.get(compatible_version) \
81 or compatible_version
82 or compatible_version
82 fs_config['compatible-version'] = compatible_version_string
83 fs_config['compatible-version'] = compatible_version_string
83
84
84 log.debug('Create SVN repo with config "%s"', fs_config)
85 log.debug('Create SVN repo with config "%s"', fs_config)
85 repo = svn.repos.create(path, "", "", None, fs_config)
86 repo = svn.repos.create(path, "", "", None, fs_config)
86 else:
87 else:
87 repo = svn.repos.open(path)
88 repo = svn.repos.open(path)
88
89
89 log.debug('Got SVN object: %s', repo)
90 log.debug('Got SVN object: %s', repo)
90 return repo
91 return repo
91
92
92 def repo(self, wire, create=False, compatible_version=None):
93 def repo(self, wire, create=False, compatible_version=None):
93 """
94 """
94 Get a repository instance for the given path.
95 Get a repository instance for the given path.
95 """
96 """
96 return self._create_repo(wire, create, compatible_version)
97 return self._create_repo(wire, create, compatible_version)
97
98
98
99
99 NODE_TYPE_MAPPING = {
100 NODE_TYPE_MAPPING = {
100 svn.core.svn_node_file: 'file',
101 svn.core.svn_node_file: 'file',
101 svn.core.svn_node_dir: 'dir',
102 svn.core.svn_node_dir: 'dir',
102 }
103 }
103
104
104
105
105 class SvnRemote(RemoteBase):
106 class SvnRemote(RemoteBase):
106
107
107 def __init__(self, factory, hg_factory=None):
108 def __init__(self, factory, hg_factory=None):
108 self._factory = factory
109 self._factory = factory
109 # TODO: Remove once we do not use internal Mercurial objects anymore
110 # TODO: Remove once we do not use internal Mercurial objects anymore
110 # for subversion
111 # for subversion
111 self._hg_factory = hg_factory
112 self._hg_factory = hg_factory
112
113
113 @reraise_safe_exceptions
114 @reraise_safe_exceptions
114 def discover_svn_version(self):
115 def discover_svn_version(self):
115 try:
116 try:
116 import svn.core
117 import svn.core
117 svn_ver = svn.core.SVN_VERSION
118 svn_ver = svn.core.SVN_VERSION
118 except ImportError:
119 except ImportError:
119 svn_ver = None
120 svn_ver = None
120 return svn_ver
121 return svn_ver
121
122
122 @reraise_safe_exceptions
123 @reraise_safe_exceptions
123 def is_empty(self, wire):
124 def is_empty(self, wire):
124
125
125 try:
126 try:
126 return self.lookup(wire, -1) == 0
127 return self.lookup(wire, -1) == 0
127 except Exception:
128 except Exception:
128 log.exception("failed to read object_store")
129 log.exception("failed to read object_store")
129 return False
130 return False
130
131
131 def check_url(self, url, config_items):
132 def check_url(self, url, config_items):
132 # this can throw exception if not installed, but we detect this
133 # this can throw exception if not installed, but we detect this
133 from hgsubversion import svnrepo
134 from hgsubversion import svnrepo
134
135
135 baseui = self._hg_factory._create_config(config_items)
136 baseui = self._hg_factory._create_config(config_items)
136 # uuid function get's only valid UUID from proper repo, else
137 # uuid function get's only valid UUID from proper repo, else
137 # throws exception
138 # throws exception
138 try:
139 try:
139 svnrepo.svnremoterepo(baseui, url).svn.uuid
140 svnrepo.svnremoterepo(baseui, url).svn.uuid
140 except Exception:
141 except Exception:
141 tb = traceback.format_exc()
142 tb = traceback.format_exc()
142 log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
143 log.debug("Invalid Subversion url: `%s`, tb: %s", url, tb)
143 raise URLError(
144 raise URLError(
144 '"%s" is not a valid Subversion source url.' % (url, ))
145 '"%s" is not a valid Subversion source url.' % (url, ))
145 return True
146 return True
146
147
147 def is_path_valid_repository(self, wire, path):
148 def is_path_valid_repository(self, wire, path):
148
149
149 # NOTE(marcink): short circuit the check for SVN repo
150 # NOTE(marcink): short circuit the check for SVN repo
150 # the repos.open might be expensive to check, but we have one cheap
151 # the repos.open might be expensive to check, but we have one cheap
151 # pre condition that we can use, to check for 'format' file
152 # pre condition that we can use, to check for 'format' file
152
153
153 if not os.path.isfile(os.path.join(path, 'format')):
154 if not os.path.isfile(os.path.join(path, 'format')):
154 return False
155 return False
155
156
156 try:
157 try:
157 svn.repos.open(path)
158 svn.repos.open(path)
158 except svn.core.SubversionException:
159 except svn.core.SubversionException:
159 tb = traceback.format_exc()
160 tb = traceback.format_exc()
160 log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
161 log.debug("Invalid Subversion path `%s`, tb: %s", path, tb)
161 return False
162 return False
162 return True
163 return True
163
164
164 @reraise_safe_exceptions
165 @reraise_safe_exceptions
165 def verify(self, wire,):
166 def verify(self, wire,):
166 repo_path = wire['path']
167 repo_path = wire['path']
167 if not self.is_path_valid_repository(wire, repo_path):
168 if not self.is_path_valid_repository(wire, repo_path):
168 raise Exception(
169 raise Exception(
169 "Path %s is not a valid Subversion repository." % repo_path)
170 "Path %s is not a valid Subversion repository." % repo_path)
170
171
171 cmd = ['svnadmin', 'info', repo_path]
172 cmd = ['svnadmin', 'info', repo_path]
172 stdout, stderr = subprocessio.run_command(cmd)
173 stdout, stderr = subprocessio.run_command(cmd)
173 return stdout
174 return stdout
174
175
175 def lookup(self, wire, revision):
176 def lookup(self, wire, revision):
176 if revision not in [-1, None, 'HEAD']:
177 if revision not in [-1, None, 'HEAD']:
177 raise NotImplementedError
178 raise NotImplementedError
178 repo = self._factory.repo(wire)
179 repo = self._factory.repo(wire)
179 fs_ptr = svn.repos.fs(repo)
180 fs_ptr = svn.repos.fs(repo)
180 head = svn.fs.youngest_rev(fs_ptr)
181 head = svn.fs.youngest_rev(fs_ptr)
181 return head
182 return head
182
183
183 def lookup_interval(self, wire, start_ts, end_ts):
184 def lookup_interval(self, wire, start_ts, end_ts):
184 repo = self._factory.repo(wire)
185 repo = self._factory.repo(wire)
185 fsobj = svn.repos.fs(repo)
186 fsobj = svn.repos.fs(repo)
186 start_rev = None
187 start_rev = None
187 end_rev = None
188 end_rev = None
188 if start_ts:
189 if start_ts:
189 start_ts_svn = apr_time_t(start_ts)
190 start_ts_svn = apr_time_t(start_ts)
190 start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
191 start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
191 else:
192 else:
192 start_rev = 1
193 start_rev = 1
193 if end_ts:
194 if end_ts:
194 end_ts_svn = apr_time_t(end_ts)
195 end_ts_svn = apr_time_t(end_ts)
195 end_rev = svn.repos.dated_revision(repo, end_ts_svn)
196 end_rev = svn.repos.dated_revision(repo, end_ts_svn)
196 else:
197 else:
197 end_rev = svn.fs.youngest_rev(fsobj)
198 end_rev = svn.fs.youngest_rev(fsobj)
198 return start_rev, end_rev
199 return start_rev, end_rev
199
200
200 def revision_properties(self, wire, revision):
201 def revision_properties(self, wire, revision):
201
202
202 cache_on, context_uid, repo_id = self._cache_on(wire)
203 cache_on, context_uid, repo_id = self._cache_on(wire)
203 @self.region.conditional_cache_on_arguments(condition=cache_on)
204 @self.region.conditional_cache_on_arguments(condition=cache_on)
204 def _revision_properties(_repo_id, _revision):
205 def _revision_properties(_repo_id, _revision):
205 repo = self._factory.repo(wire)
206 repo = self._factory.repo(wire)
206 fs_ptr = svn.repos.fs(repo)
207 fs_ptr = svn.repos.fs(repo)
207 return svn.fs.revision_proplist(fs_ptr, revision)
208 return svn.fs.revision_proplist(fs_ptr, revision)
208 return _revision_properties(repo_id, revision)
209 return _revision_properties(repo_id, revision)
209
210
210 def revision_changes(self, wire, revision):
211 def revision_changes(self, wire, revision):
211
212
212 repo = self._factory.repo(wire)
213 repo = self._factory.repo(wire)
213 fsobj = svn.repos.fs(repo)
214 fsobj = svn.repos.fs(repo)
214 rev_root = svn.fs.revision_root(fsobj, revision)
215 rev_root = svn.fs.revision_root(fsobj, revision)
215
216
216 editor = svn.repos.ChangeCollector(fsobj, rev_root)
217 editor = svn.repos.ChangeCollector(fsobj, rev_root)
217 editor_ptr, editor_baton = svn.delta.make_editor(editor)
218 editor_ptr, editor_baton = svn.delta.make_editor(editor)
218 base_dir = ""
219 base_dir = ""
219 send_deltas = False
220 send_deltas = False
220 svn.repos.replay2(
221 svn.repos.replay2(
221 rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
222 rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
222 editor_ptr, editor_baton, None)
223 editor_ptr, editor_baton, None)
223
224
224 added = []
225 added = []
225 changed = []
226 changed = []
226 removed = []
227 removed = []
227
228
228 # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
229 # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
229 for path, change in editor.changes.iteritems():
230 for path, change in editor.changes.iteritems():
230 # TODO: Decide what to do with directory nodes. Subversion can add
231 # TODO: Decide what to do with directory nodes. Subversion can add
231 # empty directories.
232 # empty directories.
232
233
233 if change.item_kind == svn.core.svn_node_dir:
234 if change.item_kind == svn.core.svn_node_dir:
234 continue
235 continue
235 if change.action in [svn.repos.CHANGE_ACTION_ADD]:
236 if change.action in [svn.repos.CHANGE_ACTION_ADD]:
236 added.append(path)
237 added.append(path)
237 elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
238 elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
238 svn.repos.CHANGE_ACTION_REPLACE]:
239 svn.repos.CHANGE_ACTION_REPLACE]:
239 changed.append(path)
240 changed.append(path)
240 elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
241 elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
241 removed.append(path)
242 removed.append(path)
242 else:
243 else:
243 raise NotImplementedError(
244 raise NotImplementedError(
244 "Action %s not supported on path %s" % (
245 "Action %s not supported on path %s" % (
245 change.action, path))
246 change.action, path))
246
247
247 changes = {
248 changes = {
248 'added': added,
249 'added': added,
249 'changed': changed,
250 'changed': changed,
250 'removed': removed,
251 'removed': removed,
251 }
252 }
252 return changes
253 return changes
253
254
254 @reraise_safe_exceptions
255 @reraise_safe_exceptions
255 def node_history(self, wire, path, revision, limit):
256 def node_history(self, wire, path, revision, limit):
256 cache_on, context_uid, repo_id = self._cache_on(wire)
257 cache_on, context_uid, repo_id = self._cache_on(wire)
257 @self.region.conditional_cache_on_arguments(condition=cache_on)
258 @self.region.conditional_cache_on_arguments(condition=cache_on)
258 def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
259 def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
259 cross_copies = False
260 cross_copies = False
260 repo = self._factory.repo(wire)
261 repo = self._factory.repo(wire)
261 fsobj = svn.repos.fs(repo)
262 fsobj = svn.repos.fs(repo)
262 rev_root = svn.fs.revision_root(fsobj, revision)
263 rev_root = svn.fs.revision_root(fsobj, revision)
263
264
264 history_revisions = []
265 history_revisions = []
265 history = svn.fs.node_history(rev_root, path)
266 history = svn.fs.node_history(rev_root, path)
266 history = svn.fs.history_prev(history, cross_copies)
267 history = svn.fs.history_prev(history, cross_copies)
267 while history:
268 while history:
268 __, node_revision = svn.fs.history_location(history)
269 __, node_revision = svn.fs.history_location(history)
269 history_revisions.append(node_revision)
270 history_revisions.append(node_revision)
270 if limit and len(history_revisions) >= limit:
271 if limit and len(history_revisions) >= limit:
271 break
272 break
272 history = svn.fs.history_prev(history, cross_copies)
273 history = svn.fs.history_prev(history, cross_copies)
273 return history_revisions
274 return history_revisions
274 return _assert_correct_path(context_uid, repo_id, path, revision, limit)
275 return _assert_correct_path(context_uid, repo_id, path, revision, limit)
275
276
276 def node_properties(self, wire, path, revision):
277 def node_properties(self, wire, path, revision):
277 cache_on, context_uid, repo_id = self._cache_on(wire)
278 cache_on, context_uid, repo_id = self._cache_on(wire)
278 @self.region.conditional_cache_on_arguments(condition=cache_on)
279 @self.region.conditional_cache_on_arguments(condition=cache_on)
279 def _node_properties(_repo_id, _path, _revision):
280 def _node_properties(_repo_id, _path, _revision):
280 repo = self._factory.repo(wire)
281 repo = self._factory.repo(wire)
281 fsobj = svn.repos.fs(repo)
282 fsobj = svn.repos.fs(repo)
282 rev_root = svn.fs.revision_root(fsobj, revision)
283 rev_root = svn.fs.revision_root(fsobj, revision)
283 return svn.fs.node_proplist(rev_root, path)
284 return svn.fs.node_proplist(rev_root, path)
284 return _node_properties(repo_id, path, revision)
285 return _node_properties(repo_id, path, revision)
285
286
286 def file_annotate(self, wire, path, revision):
287 def file_annotate(self, wire, path, revision):
287 abs_path = 'file://' + urllib.pathname2url(
288 abs_path = 'file://' + urllib.pathname2url(
288 vcspath.join(wire['path'], path))
289 vcspath.join(wire['path'], path))
289 file_uri = svn.core.svn_path_canonicalize(abs_path)
290 file_uri = svn.core.svn_path_canonicalize(abs_path)
290
291
291 start_rev = svn_opt_revision_value_t(0)
292 start_rev = svn_opt_revision_value_t(0)
292 peg_rev = svn_opt_revision_value_t(revision)
293 peg_rev = svn_opt_revision_value_t(revision)
293 end_rev = peg_rev
294 end_rev = peg_rev
294
295
295 annotations = []
296 annotations = []
296
297
297 def receiver(line_no, revision, author, date, line, pool):
298 def receiver(line_no, revision, author, date, line, pool):
298 annotations.append((line_no, revision, line))
299 annotations.append((line_no, revision, line))
299
300
300 # TODO: Cannot use blame5, missing typemap function in the swig code
301 # TODO: Cannot use blame5, missing typemap function in the swig code
301 try:
302 try:
302 svn.client.blame2(
303 svn.client.blame2(
303 file_uri, peg_rev, start_rev, end_rev,
304 file_uri, peg_rev, start_rev, end_rev,
304 receiver, svn.client.create_context())
305 receiver, svn.client.create_context())
305 except svn.core.SubversionException as exc:
306 except svn.core.SubversionException as exc:
306 log.exception("Error during blame operation.")
307 log.exception("Error during blame operation.")
307 raise Exception(
308 raise Exception(
308 "Blame not supported or file does not exist at path %s. "
309 "Blame not supported or file does not exist at path %s. "
309 "Error %s." % (path, exc))
310 "Error %s." % (path, exc))
310
311
311 return annotations
312 return annotations
312
313
313 def get_node_type(self, wire, path, revision=None):
314 def get_node_type(self, wire, path, revision=None):
314
315
315 cache_on, context_uid, repo_id = self._cache_on(wire)
316 cache_on, context_uid, repo_id = self._cache_on(wire)
316 @self.region.conditional_cache_on_arguments(condition=cache_on)
317 @self.region.conditional_cache_on_arguments(condition=cache_on)
317 def _get_node_type(_repo_id, _path, _revision):
318 def _get_node_type(_repo_id, _path, _revision):
318 repo = self._factory.repo(wire)
319 repo = self._factory.repo(wire)
319 fs_ptr = svn.repos.fs(repo)
320 fs_ptr = svn.repos.fs(repo)
320 if _revision is None:
321 if _revision is None:
321 _revision = svn.fs.youngest_rev(fs_ptr)
322 _revision = svn.fs.youngest_rev(fs_ptr)
322 root = svn.fs.revision_root(fs_ptr, _revision)
323 root = svn.fs.revision_root(fs_ptr, _revision)
323 node = svn.fs.check_path(root, path)
324 node = svn.fs.check_path(root, path)
324 return NODE_TYPE_MAPPING.get(node, None)
325 return NODE_TYPE_MAPPING.get(node, None)
325 return _get_node_type(repo_id, path, revision)
326 return _get_node_type(repo_id, path, revision)
326
327
327 def get_nodes(self, wire, path, revision=None):
328 def get_nodes(self, wire, path, revision=None):
328
329
329 cache_on, context_uid, repo_id = self._cache_on(wire)
330 cache_on, context_uid, repo_id = self._cache_on(wire)
330 @self.region.conditional_cache_on_arguments(condition=cache_on)
331 @self.region.conditional_cache_on_arguments(condition=cache_on)
331 def _get_nodes(_repo_id, _path, _revision):
332 def _get_nodes(_repo_id, _path, _revision):
332 repo = self._factory.repo(wire)
333 repo = self._factory.repo(wire)
333 fsobj = svn.repos.fs(repo)
334 fsobj = svn.repos.fs(repo)
334 if _revision is None:
335 if _revision is None:
335 _revision = svn.fs.youngest_rev(fsobj)
336 _revision = svn.fs.youngest_rev(fsobj)
336 root = svn.fs.revision_root(fsobj, _revision)
337 root = svn.fs.revision_root(fsobj, _revision)
337 entries = svn.fs.dir_entries(root, path)
338 entries = svn.fs.dir_entries(root, path)
338 result = []
339 result = []
339 for entry_path, entry_info in entries.iteritems():
340 for entry_path, entry_info in entries.iteritems():
340 result.append(
341 result.append(
341 (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
342 (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
342 return result
343 return result
343 return _get_nodes(repo_id, path, revision)
344 return _get_nodes(repo_id, path, revision)
344
345
345 def get_file_content(self, wire, path, rev=None):
346 def get_file_content(self, wire, path, rev=None):
346 repo = self._factory.repo(wire)
347 repo = self._factory.repo(wire)
347 fsobj = svn.repos.fs(repo)
348 fsobj = svn.repos.fs(repo)
348 if rev is None:
349 if rev is None:
349 rev = svn.fs.youngest_revision(fsobj)
350 rev = svn.fs.youngest_revision(fsobj)
350 root = svn.fs.revision_root(fsobj, rev)
351 root = svn.fs.revision_root(fsobj, rev)
351 content = svn.core.Stream(svn.fs.file_contents(root, path))
352 content = svn.core.Stream(svn.fs.file_contents(root, path))
352 return content.read()
353 return content.read()
353
354
354 def get_file_size(self, wire, path, revision=None):
355 def get_file_size(self, wire, path, revision=None):
355
356
356 cache_on, context_uid, repo_id = self._cache_on(wire)
357 cache_on, context_uid, repo_id = self._cache_on(wire)
357 @self.region.conditional_cache_on_arguments(condition=cache_on)
358 @self.region.conditional_cache_on_arguments(condition=cache_on)
358 def _get_file_size(_repo_id, _path, _revision):
359 def _get_file_size(_repo_id, _path, _revision):
359 repo = self._factory.repo(wire)
360 repo = self._factory.repo(wire)
360 fsobj = svn.repos.fs(repo)
361 fsobj = svn.repos.fs(repo)
361 if _revision is None:
362 if _revision is None:
362 _revision = svn.fs.youngest_revision(fsobj)
363 _revision = svn.fs.youngest_revision(fsobj)
363 root = svn.fs.revision_root(fsobj, _revision)
364 root = svn.fs.revision_root(fsobj, _revision)
364 size = svn.fs.file_length(root, path)
365 size = svn.fs.file_length(root, path)
365 return size
366 return size
366 return _get_file_size(repo_id, path, revision)
367 return _get_file_size(repo_id, path, revision)
367
368
368 def create_repository(self, wire, compatible_version=None):
369 def create_repository(self, wire, compatible_version=None):
369 log.info('Creating Subversion repository in path "%s"', wire['path'])
370 log.info('Creating Subversion repository in path "%s"', wire['path'])
370 self._factory.repo(wire, create=True,
371 self._factory.repo(wire, create=True,
371 compatible_version=compatible_version)
372 compatible_version=compatible_version)
372
373
373 def get_url_and_credentials(self, src_url):
374 def get_url_and_credentials(self, src_url):
374 obj = urlparse.urlparse(src_url)
375 obj = urlparse.urlparse(src_url)
375 username = obj.username or None
376 username = obj.username or None
376 password = obj.password or None
377 password = obj.password or None
377 return username, password, src_url
378 return username, password, src_url
378
379
379 def import_remote_repository(self, wire, src_url):
380 def import_remote_repository(self, wire, src_url):
380 repo_path = wire['path']
381 repo_path = wire['path']
381 if not self.is_path_valid_repository(wire, repo_path):
382 if not self.is_path_valid_repository(wire, repo_path):
382 raise Exception(
383 raise Exception(
383 "Path %s is not a valid Subversion repository." % repo_path)
384 "Path %s is not a valid Subversion repository." % repo_path)
384
385
385 username, password, src_url = self.get_url_and_credentials(src_url)
386 username, password, src_url = self.get_url_and_credentials(src_url)
386 rdump_cmd = ['svnrdump', 'dump', '--non-interactive',
387 rdump_cmd = ['svnrdump', 'dump', '--non-interactive',
387 '--trust-server-cert-failures=unknown-ca']
388 '--trust-server-cert-failures=unknown-ca']
388 if username and password:
389 if username and password:
389 rdump_cmd += ['--username', username, '--password', password]
390 rdump_cmd += ['--username', username, '--password', password]
390 rdump_cmd += [src_url]
391 rdump_cmd += [src_url]
391
392
392 rdump = subprocess.Popen(
393 rdump = subprocess.Popen(
393 rdump_cmd,
394 rdump_cmd,
394 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
395 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
395 load = subprocess.Popen(
396 load = subprocess.Popen(
396 ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
397 ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
397
398
398 # TODO: johbo: This can be a very long operation, might be better
399 # TODO: johbo: This can be a very long operation, might be better
399 # to track some kind of status and provide an api to check if the
400 # to track some kind of status and provide an api to check if the
400 # import is done.
401 # import is done.
401 rdump.wait()
402 rdump.wait()
402 load.wait()
403 load.wait()
403
404
404 log.debug('Return process ended with code: %s', rdump.returncode)
405 log.debug('Return process ended with code: %s', rdump.returncode)
405 if rdump.returncode != 0:
406 if rdump.returncode != 0:
406 errors = rdump.stderr.read()
407 errors = rdump.stderr.read()
407 log.error('svnrdump dump failed: statuscode %s: message: %s',
408 log.error('svnrdump dump failed: statuscode %s: message: %s',
408 rdump.returncode, errors)
409 rdump.returncode, errors)
409 reason = 'UNKNOWN'
410 reason = 'UNKNOWN'
410 if 'svnrdump: E230001:' in errors:
411 if 'svnrdump: E230001:' in errors:
411 reason = 'INVALID_CERTIFICATE'
412 reason = 'INVALID_CERTIFICATE'
412
413
413 if reason == 'UNKNOWN':
414 if reason == 'UNKNOWN':
414 reason = 'UNKNOWN:{}'.format(errors)
415 reason = 'UNKNOWN:{}'.format(errors)
415 raise Exception(
416 raise Exception(
416 'Failed to dump the remote repository from %s. Reason:%s' % (
417 'Failed to dump the remote repository from %s. Reason:%s' % (
417 src_url, reason))
418 src_url, reason))
418 if load.returncode != 0:
419 if load.returncode != 0:
419 raise Exception(
420 raise Exception(
420 'Failed to load the dump of remote repository from %s.' %
421 'Failed to load the dump of remote repository from %s.' %
421 (src_url, ))
422 (src_url, ))
422
423
423 def commit(self, wire, message, author, timestamp, updated, removed):
424 def commit(self, wire, message, author, timestamp, updated, removed):
424 assert isinstance(message, str)
425 assert isinstance(message, str)
425 assert isinstance(author, str)
426 assert isinstance(author, str)
426
427
427 repo = self._factory.repo(wire)
428 repo = self._factory.repo(wire)
428 fsobj = svn.repos.fs(repo)
429 fsobj = svn.repos.fs(repo)
429
430
430 rev = svn.fs.youngest_rev(fsobj)
431 rev = svn.fs.youngest_rev(fsobj)
431 txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
432 txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
432 txn_root = svn.fs.txn_root(txn)
433 txn_root = svn.fs.txn_root(txn)
433
434
434 for node in updated:
435 for node in updated:
435 TxnNodeProcessor(node, txn_root).update()
436 TxnNodeProcessor(node, txn_root).update()
436 for node in removed:
437 for node in removed:
437 TxnNodeProcessor(node, txn_root).remove()
438 TxnNodeProcessor(node, txn_root).remove()
438
439
439 commit_id = svn.repos.fs_commit_txn(repo, txn)
440 commit_id = svn.repos.fs_commit_txn(repo, txn)
440
441
441 if timestamp:
442 if timestamp:
442 apr_time = apr_time_t(timestamp)
443 apr_time = apr_time_t(timestamp)
443 ts_formatted = svn.core.svn_time_to_cstring(apr_time)
444 ts_formatted = svn.core.svn_time_to_cstring(apr_time)
444 svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
445 svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
445
446
446 log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
447 log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
447 return commit_id
448 return commit_id
448
449
449 def diff(self, wire, rev1, rev2, path1=None, path2=None,
450 def diff(self, wire, rev1, rev2, path1=None, path2=None,
450 ignore_whitespace=False, context=3):
451 ignore_whitespace=False, context=3):
451
452
452 wire.update(cache=False)
453 wire.update(cache=False)
453 repo = self._factory.repo(wire)
454 repo = self._factory.repo(wire)
454 diff_creator = SvnDiffer(
455 diff_creator = SvnDiffer(
455 repo, rev1, path1, rev2, path2, ignore_whitespace, context)
456 repo, rev1, path1, rev2, path2, ignore_whitespace, context)
456 try:
457 try:
457 return diff_creator.generate_diff()
458 return diff_creator.generate_diff()
458 except svn.core.SubversionException as e:
459 except svn.core.SubversionException as e:
459 log.exception(
460 log.exception(
460 "Error during diff operation operation. "
461 "Error during diff operation operation. "
461 "Path might not exist %s, %s" % (path1, path2))
462 "Path might not exist %s, %s" % (path1, path2))
462 return ""
463 return ""
463
464
464 @reraise_safe_exceptions
465 @reraise_safe_exceptions
465 def is_large_file(self, wire, path):
466 def is_large_file(self, wire, path):
466 return False
467 return False
467
468
468 @reraise_safe_exceptions
469 @reraise_safe_exceptions
469 def is_binary(self, wire, rev, path):
470 def is_binary(self, wire, rev, path):
470 cache_on, context_uid, repo_id = self._cache_on(wire)
471 cache_on, context_uid, repo_id = self._cache_on(wire)
471
472
472 @self.region.conditional_cache_on_arguments(condition=cache_on)
473 @self.region.conditional_cache_on_arguments(condition=cache_on)
473 def _is_binary(_repo_id, _rev, _path):
474 def _is_binary(_repo_id, _rev, _path):
474 raw_bytes = self.get_file_content(wire, path, rev)
475 raw_bytes = self.get_file_content(wire, path, rev)
475 return raw_bytes and '\0' in raw_bytes
476 return raw_bytes and '\0' in raw_bytes
476
477
477 return _is_binary(repo_id, rev, path)
478 return _is_binary(repo_id, rev, path)
478
479
479 @reraise_safe_exceptions
480 @reraise_safe_exceptions
480 def run_svn_command(self, wire, cmd, **opts):
481 def run_svn_command(self, wire, cmd, **opts):
481 path = wire.get('path', None)
482 path = wire.get('path', None)
482
483
483 if path and os.path.isdir(path):
484 if path and os.path.isdir(path):
484 opts['cwd'] = path
485 opts['cwd'] = path
485
486
486 safe_call = False
487 safe_call = opts.pop('_safe', False)
487 if '_safe' in opts:
488 safe_call = True
489
488
490 svnenv = os.environ.copy()
489 svnenv = os.environ.copy()
491 svnenv.update(opts.pop('extra_env', {}))
490 svnenv.update(opts.pop('extra_env', {}))
492
491
493 _opts = {'env': svnenv, 'shell': False}
492 _opts = {'env': svnenv, 'shell': False}
494
493
495 try:
494 try:
496 _opts.update(opts)
495 _opts.update(opts)
497 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
496 p = subprocessio.SubprocessIOChunker(cmd, **_opts)
498
497
499 return ''.join(p), ''.join(p.error)
498 return ''.join(p), ''.join(p.error)
500 except (EnvironmentError, OSError) as err:
499 except (EnvironmentError, OSError) as err:
501 if safe_call:
500 if safe_call:
502 return '', err
501 return '', safe_str(err).strip()
503 else:
502 else:
504 cmd = ' '.join(cmd) # human friendly CMD
503 cmd = ' '.join(cmd) # human friendly CMD
505 tb_err = ("Couldn't run svn command (%s).\n"
504 tb_err = ("Couldn't run svn command (%s).\n"
506 "Original error was:%s\n"
505 "Original error was:%s\n"
507 "Call options:%s\n"
506 "Call options:%s\n"
508 % (cmd, err, _opts))
507 % (cmd, err, _opts))
509 log.exception(tb_err)
508 log.exception(tb_err)
510 raise exceptions.VcsException()(tb_err)
509 raise exceptions.VcsException()(tb_err)
511
510
512 @reraise_safe_exceptions
511 @reraise_safe_exceptions
513 def install_hooks(self, wire, force=False):
512 def install_hooks(self, wire, force=False):
514 from vcsserver.hook_utils import install_svn_hooks
513 from vcsserver.hook_utils import install_svn_hooks
515 repo_path = wire['path']
514 repo_path = wire['path']
516 binary_dir = settings.BINARY_DIR
515 binary_dir = settings.BINARY_DIR
517 executable = None
516 executable = None
518 if binary_dir:
517 if binary_dir:
519 executable = os.path.join(binary_dir, 'python')
518 executable = os.path.join(binary_dir, 'python')
520 return install_svn_hooks(
519 return install_svn_hooks(
521 repo_path, executable=executable, force_create=force)
520 repo_path, executable=executable, force_create=force)
522
521
523 @reraise_safe_exceptions
522 @reraise_safe_exceptions
524 def get_hooks_info(self, wire):
523 def get_hooks_info(self, wire):
525 from vcsserver.hook_utils import (
524 from vcsserver.hook_utils import (
526 get_svn_pre_hook_version, get_svn_post_hook_version)
525 get_svn_pre_hook_version, get_svn_post_hook_version)
527 repo_path = wire['path']
526 repo_path = wire['path']
528 return {
527 return {
529 'pre_version': get_svn_pre_hook_version(repo_path),
528 'pre_version': get_svn_pre_hook_version(repo_path),
530 'post_version': get_svn_post_hook_version(repo_path),
529 'post_version': get_svn_post_hook_version(repo_path),
531 }
530 }
532
531
533 @reraise_safe_exceptions
532 @reraise_safe_exceptions
534 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
533 def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path,
535 archive_dir_name, commit_id):
534 archive_dir_name, commit_id):
536
535
537 def walk_tree(root, root_dir, _commit_id):
536 def walk_tree(root, root_dir, _commit_id):
538 """
537 """
539 Special recursive svn repo walker
538 Special recursive svn repo walker
540 """
539 """
541
540
542 filemode_default = 0o100644
541 filemode_default = 0o100644
543 filemode_executable = 0o100755
542 filemode_executable = 0o100755
544
543
545 file_iter = svn.fs.dir_entries(root, root_dir)
544 file_iter = svn.fs.dir_entries(root, root_dir)
546 for f_name in file_iter:
545 for f_name in file_iter:
547 f_type = NODE_TYPE_MAPPING.get(file_iter[f_name].kind, None)
546 f_type = NODE_TYPE_MAPPING.get(file_iter[f_name].kind, None)
548
547
549 if f_type == 'dir':
548 if f_type == 'dir':
550 # return only DIR, and then all entries in that dir
549 # return only DIR, and then all entries in that dir
551 yield os.path.join(root_dir, f_name), {'mode': filemode_default}, f_type
550 yield os.path.join(root_dir, f_name), {'mode': filemode_default}, f_type
552 new_root = os.path.join(root_dir, f_name)
551 new_root = os.path.join(root_dir, f_name)
553 for _f_name, _f_data, _f_type in walk_tree(root, new_root, _commit_id):
552 for _f_name, _f_data, _f_type in walk_tree(root, new_root, _commit_id):
554 yield _f_name, _f_data, _f_type
553 yield _f_name, _f_data, _f_type
555 else:
554 else:
556 f_path = os.path.join(root_dir, f_name).rstrip('/')
555 f_path = os.path.join(root_dir, f_name).rstrip('/')
557 prop_list = svn.fs.node_proplist(root, f_path)
556 prop_list = svn.fs.node_proplist(root, f_path)
558
557
559 f_mode = filemode_default
558 f_mode = filemode_default
560 if prop_list.get('svn:executable'):
559 if prop_list.get('svn:executable'):
561 f_mode = filemode_executable
560 f_mode = filemode_executable
562
561
563 f_is_link = False
562 f_is_link = False
564 if prop_list.get('svn:special'):
563 if prop_list.get('svn:special'):
565 f_is_link = True
564 f_is_link = True
566
565
567 data = {
566 data = {
568 'is_link': f_is_link,
567 'is_link': f_is_link,
569 'mode': f_mode,
568 'mode': f_mode,
570 'content_stream': svn.core.Stream(svn.fs.file_contents(root, f_path)).read
569 'content_stream': svn.core.Stream(svn.fs.file_contents(root, f_path)).read
571 }
570 }
572
571
573 yield f_path, data, f_type
572 yield f_path, data, f_type
574
573
575 def file_walker(_commit_id, path):
574 def file_walker(_commit_id, path):
576 repo = self._factory.repo(wire)
575 repo = self._factory.repo(wire)
577 root = svn.fs.revision_root(svn.repos.fs(repo), int(commit_id))
576 root = svn.fs.revision_root(svn.repos.fs(repo), int(commit_id))
578
577
579 def no_content():
578 def no_content():
580 raise NoContentException()
579 raise NoContentException()
581
580
582 for f_name, f_data, f_type in walk_tree(root, path, _commit_id):
581 for f_name, f_data, f_type in walk_tree(root, path, _commit_id):
583 file_path = f_name
582 file_path = f_name
584
583
585 if f_type == 'dir':
584 if f_type == 'dir':
586 mode = f_data['mode']
585 mode = f_data['mode']
587 yield ArchiveNode(file_path, mode, False, no_content)
586 yield ArchiveNode(file_path, mode, False, no_content)
588 else:
587 else:
589 mode = f_data['mode']
588 mode = f_data['mode']
590 is_link = f_data['is_link']
589 is_link = f_data['is_link']
591 data_stream = f_data['content_stream']
590 data_stream = f_data['content_stream']
592 yield ArchiveNode(file_path, mode, is_link, data_stream)
591 yield ArchiveNode(file_path, mode, is_link, data_stream)
593
592
594 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
593 return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path,
595 archive_dir_name, commit_id)
594 archive_dir_name, commit_id)
596
595
597
596
598 class SvnDiffer(object):
597 class SvnDiffer(object):
599 """
598 """
600 Utility to create diffs based on difflib and the Subversion api
599 Utility to create diffs based on difflib and the Subversion api
601 """
600 """
602
601
603 binary_content = False
602 binary_content = False
604
603
605 def __init__(
604 def __init__(
606 self, repo, src_rev, src_path, tgt_rev, tgt_path,
605 self, repo, src_rev, src_path, tgt_rev, tgt_path,
607 ignore_whitespace, context):
606 ignore_whitespace, context):
608 self.repo = repo
607 self.repo = repo
609 self.ignore_whitespace = ignore_whitespace
608 self.ignore_whitespace = ignore_whitespace
610 self.context = context
609 self.context = context
611
610
612 fsobj = svn.repos.fs(repo)
611 fsobj = svn.repos.fs(repo)
613
612
614 self.tgt_rev = tgt_rev
613 self.tgt_rev = tgt_rev
615 self.tgt_path = tgt_path or ''
614 self.tgt_path = tgt_path or ''
616 self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
615 self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
617 self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
616 self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
618
617
619 self.src_rev = src_rev
618 self.src_rev = src_rev
620 self.src_path = src_path or self.tgt_path
619 self.src_path = src_path or self.tgt_path
621 self.src_root = svn.fs.revision_root(fsobj, src_rev)
620 self.src_root = svn.fs.revision_root(fsobj, src_rev)
622 self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
621 self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
623
622
624 self._validate()
623 self._validate()
625
624
626 def _validate(self):
625 def _validate(self):
627 if (self.tgt_kind != svn.core.svn_node_none and
626 if (self.tgt_kind != svn.core.svn_node_none and
628 self.src_kind != svn.core.svn_node_none and
627 self.src_kind != svn.core.svn_node_none and
629 self.src_kind != self.tgt_kind):
628 self.src_kind != self.tgt_kind):
630 # TODO: johbo: proper error handling
629 # TODO: johbo: proper error handling
631 raise Exception(
630 raise Exception(
632 "Source and target are not compatible for diff generation. "
631 "Source and target are not compatible for diff generation. "
633 "Source type: %s, target type: %s" %
632 "Source type: %s, target type: %s" %
634 (self.src_kind, self.tgt_kind))
633 (self.src_kind, self.tgt_kind))
635
634
636 def generate_diff(self):
635 def generate_diff(self):
637 buf = StringIO.StringIO()
636 buf = StringIO.StringIO()
638 if self.tgt_kind == svn.core.svn_node_dir:
637 if self.tgt_kind == svn.core.svn_node_dir:
639 self._generate_dir_diff(buf)
638 self._generate_dir_diff(buf)
640 else:
639 else:
641 self._generate_file_diff(buf)
640 self._generate_file_diff(buf)
642 return buf.getvalue()
641 return buf.getvalue()
643
642
644 def _generate_dir_diff(self, buf):
643 def _generate_dir_diff(self, buf):
645 editor = DiffChangeEditor()
644 editor = DiffChangeEditor()
646 editor_ptr, editor_baton = svn.delta.make_editor(editor)
645 editor_ptr, editor_baton = svn.delta.make_editor(editor)
647 svn.repos.dir_delta2(
646 svn.repos.dir_delta2(
648 self.src_root,
647 self.src_root,
649 self.src_path,
648 self.src_path,
650 '', # src_entry
649 '', # src_entry
651 self.tgt_root,
650 self.tgt_root,
652 self.tgt_path,
651 self.tgt_path,
653 editor_ptr, editor_baton,
652 editor_ptr, editor_baton,
654 authorization_callback_allow_all,
653 authorization_callback_allow_all,
655 False, # text_deltas
654 False, # text_deltas
656 svn.core.svn_depth_infinity, # depth
655 svn.core.svn_depth_infinity, # depth
657 False, # entry_props
656 False, # entry_props
658 False, # ignore_ancestry
657 False, # ignore_ancestry
659 )
658 )
660
659
661 for path, __, change in sorted(editor.changes):
660 for path, __, change in sorted(editor.changes):
662 self._generate_node_diff(
661 self._generate_node_diff(
663 buf, change, path, self.tgt_path, path, self.src_path)
662 buf, change, path, self.tgt_path, path, self.src_path)
664
663
665 def _generate_file_diff(self, buf):
664 def _generate_file_diff(self, buf):
666 change = None
665 change = None
667 if self.src_kind == svn.core.svn_node_none:
666 if self.src_kind == svn.core.svn_node_none:
668 change = "add"
667 change = "add"
669 elif self.tgt_kind == svn.core.svn_node_none:
668 elif self.tgt_kind == svn.core.svn_node_none:
670 change = "delete"
669 change = "delete"
671 tgt_base, tgt_path = vcspath.split(self.tgt_path)
670 tgt_base, tgt_path = vcspath.split(self.tgt_path)
672 src_base, src_path = vcspath.split(self.src_path)
671 src_base, src_path = vcspath.split(self.src_path)
673 self._generate_node_diff(
672 self._generate_node_diff(
674 buf, change, tgt_path, tgt_base, src_path, src_base)
673 buf, change, tgt_path, tgt_base, src_path, src_base)
675
674
676 def _generate_node_diff(
675 def _generate_node_diff(
677 self, buf, change, tgt_path, tgt_base, src_path, src_base):
676 self, buf, change, tgt_path, tgt_base, src_path, src_base):
678
677
679 if self.src_rev == self.tgt_rev and tgt_base == src_base:
678 if self.src_rev == self.tgt_rev and tgt_base == src_base:
680 # makes consistent behaviour with git/hg to return empty diff if
679 # makes consistent behaviour with git/hg to return empty diff if
681 # we compare same revisions
680 # we compare same revisions
682 return
681 return
683
682
684 tgt_full_path = vcspath.join(tgt_base, tgt_path)
683 tgt_full_path = vcspath.join(tgt_base, tgt_path)
685 src_full_path = vcspath.join(src_base, src_path)
684 src_full_path = vcspath.join(src_base, src_path)
686
685
687 self.binary_content = False
686 self.binary_content = False
688 mime_type = self._get_mime_type(tgt_full_path)
687 mime_type = self._get_mime_type(tgt_full_path)
689
688
690 if mime_type and not mime_type.startswith('text'):
689 if mime_type and not mime_type.startswith('text'):
691 self.binary_content = True
690 self.binary_content = True
692 buf.write("=" * 67 + '\n')
691 buf.write("=" * 67 + '\n')
693 buf.write("Cannot display: file marked as a binary type.\n")
692 buf.write("Cannot display: file marked as a binary type.\n")
694 buf.write("svn:mime-type = %s\n" % mime_type)
693 buf.write("svn:mime-type = %s\n" % mime_type)
695 buf.write("Index: %s\n" % (tgt_path, ))
694 buf.write("Index: %s\n" % (tgt_path, ))
696 buf.write("=" * 67 + '\n')
695 buf.write("=" * 67 + '\n')
697 buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
696 buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
698 'tgt_path': tgt_path})
697 'tgt_path': tgt_path})
699
698
700 if change == 'add':
699 if change == 'add':
701 # TODO: johbo: SVN is missing a zero here compared to git
700 # TODO: johbo: SVN is missing a zero here compared to git
702 buf.write("new file mode 10644\n")
701 buf.write("new file mode 10644\n")
703
702
704 #TODO(marcink): intro to binary detection of svn patches
703 #TODO(marcink): intro to binary detection of svn patches
705 # if self.binary_content:
704 # if self.binary_content:
706 # buf.write('GIT binary patch\n')
705 # buf.write('GIT binary patch\n')
707
706
708 buf.write("--- /dev/null\t(revision 0)\n")
707 buf.write("--- /dev/null\t(revision 0)\n")
709 src_lines = []
708 src_lines = []
710 else:
709 else:
711 if change == 'delete':
710 if change == 'delete':
712 buf.write("deleted file mode 10644\n")
711 buf.write("deleted file mode 10644\n")
713
712
714 #TODO(marcink): intro to binary detection of svn patches
713 #TODO(marcink): intro to binary detection of svn patches
715 # if self.binary_content:
714 # if self.binary_content:
716 # buf.write('GIT binary patch\n')
715 # buf.write('GIT binary patch\n')
717
716
718 buf.write("--- a/%s\t(revision %s)\n" % (
717 buf.write("--- a/%s\t(revision %s)\n" % (
719 src_path, self.src_rev))
718 src_path, self.src_rev))
720 src_lines = self._svn_readlines(self.src_root, src_full_path)
719 src_lines = self._svn_readlines(self.src_root, src_full_path)
721
720
722 if change == 'delete':
721 if change == 'delete':
723 buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
722 buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
724 tgt_lines = []
723 tgt_lines = []
725 else:
724 else:
726 buf.write("+++ b/%s\t(revision %s)\n" % (
725 buf.write("+++ b/%s\t(revision %s)\n" % (
727 tgt_path, self.tgt_rev))
726 tgt_path, self.tgt_rev))
728 tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
727 tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
729
728
730 if not self.binary_content:
729 if not self.binary_content:
731 udiff = svn_diff.unified_diff(
730 udiff = svn_diff.unified_diff(
732 src_lines, tgt_lines, context=self.context,
731 src_lines, tgt_lines, context=self.context,
733 ignore_blank_lines=self.ignore_whitespace,
732 ignore_blank_lines=self.ignore_whitespace,
734 ignore_case=False,
733 ignore_case=False,
735 ignore_space_changes=self.ignore_whitespace)
734 ignore_space_changes=self.ignore_whitespace)
736 buf.writelines(udiff)
735 buf.writelines(udiff)
737
736
738 def _get_mime_type(self, path):
737 def _get_mime_type(self, path):
739 try:
738 try:
740 mime_type = svn.fs.node_prop(
739 mime_type = svn.fs.node_prop(
741 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
740 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
742 except svn.core.SubversionException:
741 except svn.core.SubversionException:
743 mime_type = svn.fs.node_prop(
742 mime_type = svn.fs.node_prop(
744 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
743 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
745 return mime_type
744 return mime_type
746
745
747 def _svn_readlines(self, fs_root, node_path):
746 def _svn_readlines(self, fs_root, node_path):
748 if self.binary_content:
747 if self.binary_content:
749 return []
748 return []
750 node_kind = svn.fs.check_path(fs_root, node_path)
749 node_kind = svn.fs.check_path(fs_root, node_path)
751 if node_kind not in (
750 if node_kind not in (
752 svn.core.svn_node_file, svn.core.svn_node_symlink):
751 svn.core.svn_node_file, svn.core.svn_node_symlink):
753 return []
752 return []
754 content = svn.core.Stream(svn.fs.file_contents(fs_root, node_path)).read()
753 content = svn.core.Stream(svn.fs.file_contents(fs_root, node_path)).read()
755 return content.splitlines(True)
754 return content.splitlines(True)
756
755
757
756
758 class DiffChangeEditor(svn.delta.Editor):
757 class DiffChangeEditor(svn.delta.Editor):
759 """
758 """
760 Records changes between two given revisions
759 Records changes between two given revisions
761 """
760 """
762
761
763 def __init__(self):
762 def __init__(self):
764 self.changes = []
763 self.changes = []
765
764
766 def delete_entry(self, path, revision, parent_baton, pool=None):
765 def delete_entry(self, path, revision, parent_baton, pool=None):
767 self.changes.append((path, None, 'delete'))
766 self.changes.append((path, None, 'delete'))
768
767
769 def add_file(
768 def add_file(
770 self, path, parent_baton, copyfrom_path, copyfrom_revision,
769 self, path, parent_baton, copyfrom_path, copyfrom_revision,
771 file_pool=None):
770 file_pool=None):
772 self.changes.append((path, 'file', 'add'))
771 self.changes.append((path, 'file', 'add'))
773
772
774 def open_file(self, path, parent_baton, base_revision, file_pool=None):
773 def open_file(self, path, parent_baton, base_revision, file_pool=None):
775 self.changes.append((path, 'file', 'change'))
774 self.changes.append((path, 'file', 'change'))
776
775
777
776
778 def authorization_callback_allow_all(root, path, pool):
777 def authorization_callback_allow_all(root, path, pool):
779 return True
778 return True
780
779
781
780
782 class TxnNodeProcessor(object):
781 class TxnNodeProcessor(object):
783 """
782 """
784 Utility to process the change of one node within a transaction root.
783 Utility to process the change of one node within a transaction root.
785
784
786 It encapsulates the knowledge of how to add, update or remove
785 It encapsulates the knowledge of how to add, update or remove
787 a node for a given transaction root. The purpose is to support the method
786 a node for a given transaction root. The purpose is to support the method
788 `SvnRemote.commit`.
787 `SvnRemote.commit`.
789 """
788 """
790
789
791 def __init__(self, node, txn_root):
790 def __init__(self, node, txn_root):
792 assert isinstance(node['path'], str)
791 assert isinstance(node['path'], str)
793
792
794 self.node = node
793 self.node = node
795 self.txn_root = txn_root
794 self.txn_root = txn_root
796
795
797 def update(self):
796 def update(self):
798 self._ensure_parent_dirs()
797 self._ensure_parent_dirs()
799 self._add_file_if_node_does_not_exist()
798 self._add_file_if_node_does_not_exist()
800 self._update_file_content()
799 self._update_file_content()
801 self._update_file_properties()
800 self._update_file_properties()
802
801
803 def remove(self):
802 def remove(self):
804 svn.fs.delete(self.txn_root, self.node['path'])
803 svn.fs.delete(self.txn_root, self.node['path'])
805 # TODO: Clean up directory if empty
804 # TODO: Clean up directory if empty
806
805
807 def _ensure_parent_dirs(self):
806 def _ensure_parent_dirs(self):
808 curdir = vcspath.dirname(self.node['path'])
807 curdir = vcspath.dirname(self.node['path'])
809 dirs_to_create = []
808 dirs_to_create = []
810 while not self._svn_path_exists(curdir):
809 while not self._svn_path_exists(curdir):
811 dirs_to_create.append(curdir)
810 dirs_to_create.append(curdir)
812 curdir = vcspath.dirname(curdir)
811 curdir = vcspath.dirname(curdir)
813
812
814 for curdir in reversed(dirs_to_create):
813 for curdir in reversed(dirs_to_create):
815 log.debug('Creating missing directory "%s"', curdir)
814 log.debug('Creating missing directory "%s"', curdir)
816 svn.fs.make_dir(self.txn_root, curdir)
815 svn.fs.make_dir(self.txn_root, curdir)
817
816
818 def _svn_path_exists(self, path):
817 def _svn_path_exists(self, path):
819 path_status = svn.fs.check_path(self.txn_root, path)
818 path_status = svn.fs.check_path(self.txn_root, path)
820 return path_status != svn.core.svn_node_none
819 return path_status != svn.core.svn_node_none
821
820
822 def _add_file_if_node_does_not_exist(self):
821 def _add_file_if_node_does_not_exist(self):
823 kind = svn.fs.check_path(self.txn_root, self.node['path'])
822 kind = svn.fs.check_path(self.txn_root, self.node['path'])
824 if kind == svn.core.svn_node_none:
823 if kind == svn.core.svn_node_none:
825 svn.fs.make_file(self.txn_root, self.node['path'])
824 svn.fs.make_file(self.txn_root, self.node['path'])
826
825
827 def _update_file_content(self):
826 def _update_file_content(self):
828 assert isinstance(self.node['content'], str)
827 assert isinstance(self.node['content'], str)
829 handler, baton = svn.fs.apply_textdelta(
828 handler, baton = svn.fs.apply_textdelta(
830 self.txn_root, self.node['path'], None, None)
829 self.txn_root, self.node['path'], None, None)
831 svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
830 svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
832
831
833 def _update_file_properties(self):
832 def _update_file_properties(self):
834 properties = self.node.get('properties', {})
833 properties = self.node.get('properties', {})
835 for key, value in properties.iteritems():
834 for key, value in properties.iteritems():
836 svn.fs.change_node_prop(
835 svn.fs.change_node_prop(
837 self.txn_root, self.node['path'], key, value)
836 self.txn_root, self.node['path'], key, value)
838
837
839
838
840 def apr_time_t(timestamp):
839 def apr_time_t(timestamp):
841 """
840 """
842 Convert a Python timestamp into APR timestamp type apr_time_t
841 Convert a Python timestamp into APR timestamp type apr_time_t
843 """
842 """
844 return timestamp * 1E6
843 return timestamp * 1E6
845
844
846
845
847 def svn_opt_revision_value_t(num):
846 def svn_opt_revision_value_t(num):
848 """
847 """
849 Put `num` into a `svn_opt_revision_value_t` structure.
848 Put `num` into a `svn_opt_revision_value_t` structure.
850 """
849 """
851 value = svn.core.svn_opt_revision_value_t()
850 value = svn.core.svn_opt_revision_value_t()
852 value.number = num
851 value.number = num
853 revision = svn.core.svn_opt_revision_t()
852 revision = svn.core.svn_opt_revision_t()
854 revision.kind = svn.core.svn_opt_revision_number
853 revision.kind = svn.core.svn_opt_revision_number
855 revision.value = value
854 revision.value = value
856 return revision
855 return revision
General Comments 0
You need to be logged in to leave comments. Login now