##// END OF EJS Templates
fixes #591 git backend was causing encoding errors when handling binary files...
marcink -
r2894:2654edfb beta
parent child Browse files
Show More
@@ -1,196 +1,202 b''
1 1 import time
2 2 import datetime
3 3 import posixpath
4 4 from dulwich import objects
5 5 from dulwich.repo import Repo
6 6 from rhodecode.lib.vcs.backends.base import BaseInMemoryChangeset
7 7 from rhodecode.lib.vcs.exceptions import RepositoryError
8 8 from rhodecode.lib.vcs.utils import safe_str
9 9
10 10
11 11 class GitInMemoryChangeset(BaseInMemoryChangeset):
12 12
13 13 def commit(self, message, author, parents=None, branch=None, date=None,
14 14 **kwargs):
15 15 """
16 16 Performs in-memory commit (doesn't check workdir in any way) and
17 17 returns newly created ``Changeset``. Updates repository's
18 18 ``revisions``.
19 19
20 20 :param message: message of the commit
21 21 :param author: full username, i.e. "Joe Doe <joe.doe@example.com>"
22 22 :param parents: single parent or sequence of parents from which commit
23 23 would be derieved
24 24 :param date: ``datetime.datetime`` instance. Defaults to
25 25 ``datetime.datetime.now()``.
26 26 :param branch: branch name, as string. If none given, default backend's
27 27 branch would be used.
28 28
29 29 :raises ``CommitError``: if any error occurs while committing
30 30 """
31 31 self.check_integrity(parents)
32 32
33 33 from .repository import GitRepository
34 34 if branch is None:
35 35 branch = GitRepository.DEFAULT_BRANCH_NAME
36 36
37 37 repo = self.repository._repo
38 38 object_store = repo.object_store
39 39
40 40 ENCODING = "UTF-8"
41 41 DIRMOD = 040000
42 42
43 43 # Create tree and populates it with blobs
44 44 commit_tree = self.parents[0] and repo[self.parents[0]._commit.tree] or\
45 45 objects.Tree()
46 46 for node in self.added + self.changed:
47 47 # Compute subdirs if needed
48 48 dirpath, nodename = posixpath.split(node.path)
49 49 dirnames = dirpath and dirpath.split('/') or []
50 50 parent = commit_tree
51 51 ancestors = [('', parent)]
52 52
53 53 # Tries to dig for the deepest existing tree
54 54 while dirnames:
55 55 curdir = dirnames.pop(0)
56 56 try:
57 57 dir_id = parent[curdir][1]
58 58 except KeyError:
59 59 # put curdir back into dirnames and stops
60 60 dirnames.insert(0, curdir)
61 61 break
62 62 else:
63 63 # If found, updates parent
64 64 parent = self.repository._repo[dir_id]
65 65 ancestors.append((curdir, parent))
66 # Now parent is deepest exising tree and we need to create subtrees
66 # Now parent is deepest existing tree and we need to create subtrees
67 67 # for dirnames (in reverse order) [this only applies for nodes from added]
68 68 new_trees = []
69 blob = objects.Blob.from_string(node.content.encode(ENCODING))
69
70 if not node.is_binary:
71 content = node.content.encode(ENCODING)
72 else:
73 content = node.content
74 blob = objects.Blob.from_string(content)
75
70 76 node_path = node.name.encode(ENCODING)
71 77 if dirnames:
72 78 # If there are trees which should be created we need to build
73 79 # them now (in reverse order)
74 80 reversed_dirnames = list(reversed(dirnames))
75 81 curtree = objects.Tree()
76 82 curtree[node_path] = node.mode, blob.id
77 83 new_trees.append(curtree)
78 84 for dirname in reversed_dirnames[:-1]:
79 85 newtree = objects.Tree()
80 86 #newtree.add(DIRMOD, dirname, curtree.id)
81 87 newtree[dirname] = DIRMOD, curtree.id
82 88 new_trees.append(newtree)
83 89 curtree = newtree
84 90 parent[reversed_dirnames[-1]] = DIRMOD, curtree.id
85 91 else:
86 92 parent.add(name=node_path, mode=node.mode, hexsha=blob.id)
87 93
88 94 new_trees.append(parent)
89 95 # Update ancestors
90 96 for parent, tree, path in reversed([(a[1], b[1], b[0]) for a, b in
91 97 zip(ancestors, ancestors[1:])]):
92 98 parent[path] = DIRMOD, tree.id
93 99 object_store.add_object(tree)
94 100
95 101 object_store.add_object(blob)
96 102 for tree in new_trees:
97 103 object_store.add_object(tree)
98 104 for node in self.removed:
99 105 paths = node.path.split('/')
100 106 tree = commit_tree
101 107 trees = [tree]
102 108 # Traverse deep into the forest...
103 109 for path in paths:
104 110 try:
105 111 obj = self.repository._repo[tree[path][1]]
106 112 if isinstance(obj, objects.Tree):
107 113 trees.append(obj)
108 114 tree = obj
109 115 except KeyError:
110 116 break
111 117 # Cut down the blob and all rotten trees on the way back...
112 118 for path, tree in reversed(zip(paths, trees)):
113 119 del tree[path]
114 120 if tree:
115 121 # This tree still has elements - don't remove it or any
116 122 # of it's parents
117 123 break
118 124
119 125 object_store.add_object(commit_tree)
120 126
121 127 # Create commit
122 128 commit = objects.Commit()
123 129 commit.tree = commit_tree.id
124 130 commit.parents = [p._commit.id for p in self.parents if p]
125 131 commit.author = commit.committer = safe_str(author)
126 132 commit.encoding = ENCODING
127 133 commit.message = safe_str(message)
128 134
129 135 # Compute date
130 136 if date is None:
131 137 date = time.time()
132 138 elif isinstance(date, datetime.datetime):
133 139 date = time.mktime(date.timetuple())
134 140
135 141 author_time = kwargs.pop('author_time', date)
136 142 commit.commit_time = int(date)
137 143 commit.author_time = int(author_time)
138 144 tz = time.timezone
139 145 author_tz = kwargs.pop('author_timezone', tz)
140 146 commit.commit_timezone = tz
141 147 commit.author_timezone = author_tz
142 148
143 149 object_store.add_object(commit)
144 150
145 151 ref = 'refs/heads/%s' % branch
146 152 repo.refs[ref] = commit.id
147 153 repo.refs.set_symbolic_ref('HEAD', ref)
148 154
149 155 # Update vcs repository object & recreate dulwich repo
150 156 self.repository.revisions.append(commit.id)
151 157 self.repository._repo = Repo(self.repository.path)
152 158 # invalidate parsed refs after commit
153 159 self.repository._parsed_refs = self.repository._get_parsed_refs()
154 160 tip = self.repository.get_changeset()
155 161 self.reset()
156 162 return tip
157 163
158 164 def _get_missing_trees(self, path, root_tree):
159 165 """
160 166 Creates missing ``Tree`` objects for the given path.
161 167
162 168 :param path: path given as a string. It may be a path to a file node
163 169 (i.e. ``foo/bar/baz.txt``) or directory path - in that case it must
164 170 end with slash (i.e. ``foo/bar/``).
165 171 :param root_tree: ``dulwich.objects.Tree`` object from which we start
166 172 traversing (should be commit's root tree)
167 173 """
168 174 dirpath = posixpath.split(path)[0]
169 175 dirs = dirpath.split('/')
170 176 if not dirs or dirs == ['']:
171 177 return []
172 178
173 179 def get_tree_for_dir(tree, dirname):
174 180 for name, mode, id in tree.iteritems():
175 181 if name == dirname:
176 182 obj = self.repository._repo[id]
177 183 if isinstance(obj, objects.Tree):
178 184 return obj
179 185 else:
180 186 raise RepositoryError("Cannot create directory %s "
181 187 "at tree %s as path is occupied and is not a "
182 188 "Tree" % (dirname, tree))
183 189 return None
184 190
185 191 trees = []
186 192 parent = root_tree
187 193 for dirname in dirs:
188 194 tree = get_tree_for_dir(parent, dirname)
189 195 if tree is None:
190 196 tree = objects.Tree()
191 197 dirmode = 040000
192 198 parent.add(dirmode, dirname, tree.id)
193 199 parent = tree
194 200 # Always append tree
195 201 trees.append(tree)
196 202 return trees
@@ -1,340 +1,341 b''
1 1 """
2 2 Tests so called "in memory changesets" commit API of vcs.
3 3 """
4 4 from __future__ import with_statement
5 5
6 6 from rhodecode.lib import vcs
7 7 import time
8 8 import datetime
9 9 from conf import SCM_TESTS, get_new_dir
10 10 from rhodecode.lib.vcs.exceptions import EmptyRepositoryError
11 11 from rhodecode.lib.vcs.exceptions import NodeAlreadyAddedError
12 12 from rhodecode.lib.vcs.exceptions import NodeAlreadyExistsError
13 13 from rhodecode.lib.vcs.exceptions import NodeAlreadyRemovedError
14 14 from rhodecode.lib.vcs.exceptions import NodeAlreadyChangedError
15 15 from rhodecode.lib.vcs.exceptions import NodeDoesNotExistError
16 16 from rhodecode.lib.vcs.exceptions import NodeNotChangedError
17 17 from rhodecode.lib.vcs.nodes import DirNode
18 18 from rhodecode.lib.vcs.nodes import FileNode
19 19 from rhodecode.lib.vcs.utils.compat import unittest
20 20
21 21
22 22 class InMemoryChangesetTestMixin(object):
23 23 """
24 24 This is a backend independent test case class which should be created
25 25 with ``type`` method.
26 26
27 27 It is required to set following attributes at subclass:
28 28
29 29 - ``backend_alias``: alias of used backend (see ``vcs.BACKENDS``)
30 30 - ``repo_path``: path to the repository which would be created for set of
31 31 tests
32 32 """
33 33
34 34 def get_backend(self):
35 35 return vcs.get_backend(self.backend_alias)
36 36
37 37 def setUp(self):
38 38 Backend = self.get_backend()
39 39 self.repo_path = get_new_dir(str(time.time()))
40 40 self.repo = Backend(self.repo_path, create=True)
41 41 self.imc = self.repo.in_memory_changeset
42 42 self.nodes = [
43 43 FileNode('foobar', content='Foo & bar'),
44 44 FileNode('foobar2', content='Foo & bar, doubled!'),
45 45 FileNode('foo bar with spaces', content=''),
46 46 FileNode('foo/bar/baz', content='Inside'),
47 FileNode('foo/bar/file.bin', content='\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00;\x00\x03\x00\xfe\xff\t\x00\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x1a\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x00\x18\x00\x00\x00\x01\x00\x00\x00\xfe\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff'),
47 48 ]
48 49
49 50 def test_add(self):
50 51 rev_count = len(self.repo.revisions)
51 52 to_add = [FileNode(node.path, content=node.content)
52 53 for node in self.nodes]
53 54 for node in to_add:
54 55 self.imc.add(node)
55 56 message = u'Added: %s' % ', '.join((node.path for node in self.nodes))
56 57 author = unicode(self.__class__)
57 58 changeset = self.imc.commit(message=message, author=author)
58 59
59 60 newtip = self.repo.get_changeset()
60 61 self.assertEqual(changeset, newtip)
61 62 self.assertEqual(rev_count + 1, len(self.repo.revisions))
62 63 self.assertEqual(newtip.message, message)
63 64 self.assertEqual(newtip.author, author)
64 65 self.assertTrue(not any((self.imc.added, self.imc.changed,
65 66 self.imc.removed)))
66 67 for node in to_add:
67 68 self.assertEqual(newtip.get_node(node.path).content, node.content)
68 69
69 70 def test_add_in_bulk(self):
70 71 rev_count = len(self.repo.revisions)
71 72 to_add = [FileNode(node.path, content=node.content)
72 73 for node in self.nodes]
73 74 self.imc.add(*to_add)
74 75 message = u'Added: %s' % ', '.join((node.path for node in self.nodes))
75 76 author = unicode(self.__class__)
76 77 changeset = self.imc.commit(message=message, author=author)
77 78
78 79 newtip = self.repo.get_changeset()
79 80 self.assertEqual(changeset, newtip)
80 81 self.assertEqual(rev_count + 1, len(self.repo.revisions))
81 82 self.assertEqual(newtip.message, message)
82 83 self.assertEqual(newtip.author, author)
83 84 self.assertTrue(not any((self.imc.added, self.imc.changed,
84 85 self.imc.removed)))
85 86 for node in to_add:
86 87 self.assertEqual(newtip.get_node(node.path).content, node.content)
87 88
88 89 def test_add_actually_adds_all_nodes_at_second_commit_too(self):
89 90 self.imc.add(FileNode('foo/bar/image.png', content='\0'))
90 91 self.imc.add(FileNode('foo/README.txt', content='readme!'))
91 92 changeset = self.imc.commit(u'Initial', u'joe.doe@example.com')
92 93 self.assertTrue(isinstance(changeset.get_node('foo'), DirNode))
93 94 self.assertTrue(isinstance(changeset.get_node('foo/bar'), DirNode))
94 95 self.assertEqual(changeset.get_node('foo/bar/image.png').content, '\0')
95 96 self.assertEqual(changeset.get_node('foo/README.txt').content, 'readme!')
96 97
97 98 # commit some more files again
98 99 to_add = [
99 100 FileNode('foo/bar/foobaz/bar', content='foo'),
100 101 FileNode('foo/bar/another/bar', content='foo'),
101 102 FileNode('foo/baz.txt', content='foo'),
102 103 FileNode('foobar/foobaz/file', content='foo'),
103 104 FileNode('foobar/barbaz', content='foo'),
104 105 ]
105 106 self.imc.add(*to_add)
106 107 changeset = self.imc.commit(u'Another', u'joe.doe@example.com')
107 108 self.assertEqual(changeset.get_node('foo/bar/foobaz/bar').content, 'foo')
108 109 self.assertEqual(changeset.get_node('foo/bar/another/bar').content, 'foo')
109 110 self.assertEqual(changeset.get_node('foo/baz.txt').content, 'foo')
110 111 self.assertEqual(changeset.get_node('foobar/foobaz/file').content, 'foo')
111 112 self.assertEqual(changeset.get_node('foobar/barbaz').content, 'foo')
112 113
113 114 def test_add_raise_already_added(self):
114 115 node = FileNode('foobar', content='baz')
115 116 self.imc.add(node)
116 117 self.assertRaises(NodeAlreadyAddedError, self.imc.add, node)
117 118
118 119 def test_check_integrity_raise_already_exist(self):
119 120 node = FileNode('foobar', content='baz')
120 121 self.imc.add(node)
121 122 self.imc.commit(message=u'Added foobar', author=unicode(self))
122 123 self.imc.add(node)
123 124 self.assertRaises(NodeAlreadyExistsError, self.imc.commit,
124 125 message='new message',
125 126 author=str(self))
126 127
127 128 def test_change(self):
128 129 self.imc.add(FileNode('foo/bar/baz', content='foo'))
129 130 self.imc.add(FileNode('foo/fbar', content='foobar'))
130 131 tip = self.imc.commit(u'Initial', u'joe.doe@example.com')
131 132
132 133 # Change node's content
133 134 node = FileNode('foo/bar/baz', content='My **changed** content')
134 135 self.imc.change(node)
135 136 self.imc.commit(u'Changed %s' % node.path, u'joe.doe@example.com')
136 137
137 138 newtip = self.repo.get_changeset()
138 139 self.assertNotEqual(tip, newtip)
139 140 self.assertNotEqual(tip.id, newtip.id)
140 141 self.assertEqual(newtip.get_node('foo/bar/baz').content,
141 142 'My **changed** content')
142 143
143 144 def test_change_raise_empty_repository(self):
144 145 node = FileNode('foobar')
145 146 self.assertRaises(EmptyRepositoryError, self.imc.change, node)
146 147
147 148 def test_check_integrity_change_raise_node_does_not_exist(self):
148 149 node = FileNode('foobar', content='baz')
149 150 self.imc.add(node)
150 151 self.imc.commit(message=u'Added foobar', author=unicode(self))
151 152 node = FileNode('not-foobar', content='')
152 153 self.imc.change(node)
153 154 self.assertRaises(NodeDoesNotExistError, self.imc.commit,
154 155 message='Changed not existing node',
155 156 author=str(self))
156 157
157 158 def test_change_raise_node_already_changed(self):
158 159 node = FileNode('foobar', content='baz')
159 160 self.imc.add(node)
160 161 self.imc.commit(message=u'Added foobar', author=unicode(self))
161 162 node = FileNode('foobar', content='more baz')
162 163 self.imc.change(node)
163 164 self.assertRaises(NodeAlreadyChangedError, self.imc.change, node)
164 165
165 166 def test_check_integrity_change_raise_node_not_changed(self):
166 167 self.test_add() # Performs first commit
167 168
168 169 node = FileNode(self.nodes[0].path, content=self.nodes[0].content)
169 170 self.imc.change(node)
170 171 self.assertRaises(NodeNotChangedError, self.imc.commit,
171 172 message=u'Trying to mark node as changed without touching it',
172 173 author=unicode(self))
173 174
174 175 def test_change_raise_node_already_removed(self):
175 176 node = FileNode('foobar', content='baz')
176 177 self.imc.add(node)
177 178 self.imc.commit(message=u'Added foobar', author=unicode(self))
178 179 self.imc.remove(FileNode('foobar'))
179 180 self.assertRaises(NodeAlreadyRemovedError, self.imc.change, node)
180 181
181 182 def test_remove(self):
182 183 self.test_add() # Performs first commit
183 184
184 185 tip = self.repo.get_changeset()
185 186 node = self.nodes[0]
186 187 self.assertEqual(node.content, tip.get_node(node.path).content)
187 188 self.imc.remove(node)
188 189 self.imc.commit(message=u'Removed %s' % node.path, author=unicode(self))
189 190
190 191 newtip = self.repo.get_changeset()
191 192 self.assertNotEqual(tip, newtip)
192 193 self.assertNotEqual(tip.id, newtip.id)
193 194 self.assertRaises(NodeDoesNotExistError, newtip.get_node, node.path)
194 195
195 196 def test_remove_last_file_from_directory(self):
196 197 node = FileNode('omg/qwe/foo/bar', content='foobar')
197 198 self.imc.add(node)
198 199 self.imc.commit(u'added', u'joe doe')
199 200
200 201 self.imc.remove(node)
201 202 tip = self.imc.commit(u'removed', u'joe doe')
202 203 self.assertRaises(NodeDoesNotExistError, tip.get_node, 'omg/qwe/foo/bar')
203 204
204 205 def test_remove_raise_node_does_not_exist(self):
205 206 self.imc.remove(self.nodes[0])
206 207 self.assertRaises(NodeDoesNotExistError, self.imc.commit,
207 208 message='Trying to remove node at empty repository',
208 209 author=str(self))
209 210
210 211 def test_check_integrity_remove_raise_node_does_not_exist(self):
211 212 self.test_add() # Performs first commit
212 213
213 214 node = FileNode('no-such-file')
214 215 self.imc.remove(node)
215 216 self.assertRaises(NodeDoesNotExistError, self.imc.commit,
216 217 message=u'Trying to remove not existing node',
217 218 author=unicode(self))
218 219
219 220 def test_remove_raise_node_already_removed(self):
220 221 self.test_add() # Performs first commit
221 222
222 223 node = FileNode(self.nodes[0].path)
223 224 self.imc.remove(node)
224 225 self.assertRaises(NodeAlreadyRemovedError, self.imc.remove, node)
225 226
226 227 def test_remove_raise_node_already_changed(self):
227 228 self.test_add() # Performs first commit
228 229
229 230 node = FileNode(self.nodes[0].path, content='Bending time')
230 231 self.imc.change(node)
231 232 self.assertRaises(NodeAlreadyChangedError, self.imc.remove, node)
232 233
233 234 def test_reset(self):
234 235 self.imc.add(FileNode('foo', content='bar'))
235 236 #self.imc.change(FileNode('baz', content='new'))
236 237 #self.imc.remove(FileNode('qwe'))
237 238 self.imc.reset()
238 239 self.assertTrue(not any((self.imc.added, self.imc.changed,
239 240 self.imc.removed)))
240 241
241 242 def test_multiple_commits(self):
242 243 N = 3 # number of commits to perform
243 244 last = None
244 245 for x in xrange(N):
245 246 fname = 'file%s' % str(x).rjust(5, '0')
246 247 content = 'foobar\n' * x
247 248 node = FileNode(fname, content=content)
248 249 self.imc.add(node)
249 250 commit = self.imc.commit(u"Commit no. %s" % (x + 1), author=u'vcs')
250 251 self.assertTrue(last != commit)
251 252 last = commit
252 253
253 254 # Check commit number for same repo
254 255 self.assertEqual(len(self.repo.revisions), N)
255 256
256 257 # Check commit number for recreated repo
257 258 backend = self.get_backend()
258 259 repo = backend(self.repo_path)
259 260 self.assertEqual(len(repo.revisions), N)
260 261
261 262 def test_date_attr(self):
262 263 node = FileNode('foobar.txt', content='Foobared!')
263 264 self.imc.add(node)
264 265 date = datetime.datetime(1985, 1, 30, 1, 45)
265 266 commit = self.imc.commit(u"Committed at time when I was born ;-)",
266 267 author=u'lb', date=date)
267 268
268 269 self.assertEqual(commit.date, date)
269 270
270 271
271 272 class BackendBaseTestCase(unittest.TestCase):
272 273 """
273 274 Base test class for tests which requires repository.
274 275 """
275 276 backend_alias = 'hg'
276 277 commits = [
277 278 {
278 279 'message': 'Initial commit',
279 280 'author': 'Joe Doe <joe.doe@example.com>',
280 281 'date': datetime.datetime(2010, 1, 1, 20),
281 282 'added': [
282 283 FileNode('foobar', content='Foobar'),
283 284 FileNode('foobar2', content='Foobar II'),
284 285 FileNode('foo/bar/baz', content='baz here!'),
285 286 ],
286 287 },
287 288 ]
288 289
289 290 def get_backend(self):
290 291 return vcs.get_backend(self.backend_alias)
291 292
292 293 def get_commits(self):
293 294 """
294 295 Returns list of commits which builds repository for each tests.
295 296 """
296 297 if hasattr(self, 'commits'):
297 298 return self.commits
298 299
299 300 def get_new_repo_path(self):
300 301 """
301 302 Returns newly created repository's directory.
302 303 """
303 304 backend = self.get_backend()
304 305 key = '%s-%s' % (backend.alias, str(time.time()))
305 306 repo_path = get_new_dir(key)
306 307 return repo_path
307 308
308 309 def setUp(self):
309 310 Backend = self.get_backend()
310 311 self.backend_class = Backend
311 312 self.repo_path = self.get_new_repo_path()
312 313 self.repo = Backend(self.repo_path, create=True)
313 314 self.imc = self.repo.in_memory_changeset
314 315
315 316 for commit in self.get_commits():
316 317 for node in commit.get('added', []):
317 318 self.imc.add(FileNode(node.path, content=node.content))
318 319 for node in commit.get('changed', []):
319 320 self.imc.change(FileNode(node.path, content=node.content))
320 321 for node in commit.get('removed', []):
321 322 self.imc.remove(FileNode(node.path))
322 323 self.imc.commit(message=unicode(commit['message']),
323 324 author=unicode(commit['author']),
324 325 date=commit['date'])
325 326
326 327 self.tip = self.repo.get_changeset()
327 328
328 329
329 330 # For each backend create test case class
330 331 for alias in SCM_TESTS:
331 332 attrs = {
332 333 'backend_alias': alias,
333 334 }
334 335 cls_name = ''.join(('%s in memory changeset test' % alias).title().split())
335 336 bases = (InMemoryChangesetTestMixin, unittest.TestCase)
336 337 globals()[cls_name] = type(cls_name, bases, attrs)
337 338
338 339
339 340 if __name__ == '__main__':
340 341 unittest.main()
General Comments 0
You need to be logged in to leave comments. Login now