Show More
@@ -7,6 +7,7 b'' | |||||
7 |
|
7 | |||
8 | from __future__ import absolute_import |
|
8 | from __future__ import absolute_import | |
9 |
|
9 | |||
|
10 | import collections | |||
10 | import weakref |
|
11 | import weakref | |
11 |
|
12 | |||
12 | from .i18n import _ |
|
13 | from .i18n import _ | |
@@ -58,7 +59,12 b' def pull(pullop):' | |||||
58 | remote.url(), pullop.gettransaction, |
|
59 | remote.url(), pullop.gettransaction, | |
59 | explicit=pullop.explicitbookmarks) |
|
60 | explicit=pullop.explicitbookmarks) | |
60 |
|
61 | |||
61 | _fetchmanifests(repo, tr, remote, csetres['manifestnodes']) |
|
62 | manres = _fetchmanifests(repo, tr, remote, csetres['manifestnodes']) | |
|
63 | ||||
|
64 | # Find all file nodes referenced by added manifests and fetch those | |||
|
65 | # revisions. | |||
|
66 | fnodes = _derivefilesfrommanifests(repo, manres['added']) | |||
|
67 | _fetchfiles(repo, tr, remote, fnodes, manres['linkrevs']) | |||
62 |
|
68 | |||
63 | def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True): |
|
69 | def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True): | |
64 | """Determine which changesets need to be pulled.""" |
|
70 | """Determine which changesets need to be pulled.""" | |
@@ -291,4 +297,98 b' def _fetchmanifests(repo, tr, remote, ma' | |||||
291 |
|
297 | |||
292 | return { |
|
298 | return { | |
293 | 'added': added, |
|
299 | 'added': added, | |
|
300 | 'linkrevs': linkrevs, | |||
294 | } |
|
301 | } | |
|
302 | ||||
|
303 | def _derivefilesfrommanifests(repo, manifestnodes): | |||
|
304 | """Determine what file nodes are relevant given a set of manifest nodes. | |||
|
305 | ||||
|
306 | Returns a dict mapping file paths to dicts of file node to first manifest | |||
|
307 | node. | |||
|
308 | """ | |||
|
309 | ml = repo.manifestlog | |||
|
310 | fnodes = collections.defaultdict(dict) | |||
|
311 | ||||
|
312 | for manifestnode in manifestnodes: | |||
|
313 | m = ml.get(b'', manifestnode) | |||
|
314 | ||||
|
315 | # TODO this will pull in unwanted nodes because it takes the storage | |||
|
316 | # delta into consideration. What we really want is something that takes | |||
|
317 | # the delta between the manifest's parents. And ideally we would | |||
|
318 | # ignore file nodes that are known locally. For now, ignore both | |||
|
319 | # these limitations. This will result in incremental fetches requesting | |||
|
320 | # data we already have. So this is far from ideal. | |||
|
321 | md = m.readfast() | |||
|
322 | ||||
|
323 | for path, fnode in md.items(): | |||
|
324 | fnodes[path].setdefault(fnode, manifestnode) | |||
|
325 | ||||
|
326 | return fnodes | |||
|
327 | ||||
|
328 | def _fetchfiles(repo, tr, remote, fnodes, linkrevs): | |||
|
329 | def iterrevisions(objs, progress): | |||
|
330 | for filerevision in objs: | |||
|
331 | node = filerevision[b'node'] | |||
|
332 | ||||
|
333 | if b'deltasize' in filerevision: | |||
|
334 | basenode = filerevision[b'deltabasenode'] | |||
|
335 | delta = next(objs) | |||
|
336 | elif b'revisionsize' in filerevision: | |||
|
337 | basenode = nullid | |||
|
338 | revision = next(objs) | |||
|
339 | delta = mdiff.trivialdiffheader(len(revision)) + revision | |||
|
340 | else: | |||
|
341 | continue | |||
|
342 | ||||
|
343 | yield ( | |||
|
344 | node, | |||
|
345 | filerevision[b'parents'][0], | |||
|
346 | filerevision[b'parents'][1], | |||
|
347 | node, | |||
|
348 | basenode, | |||
|
349 | delta, | |||
|
350 | # Flags not yet supported. | |||
|
351 | 0, | |||
|
352 | ) | |||
|
353 | ||||
|
354 | progress.increment() | |||
|
355 | ||||
|
356 | progress = repo.ui.makeprogress( | |||
|
357 | _('files'), unit=_('chunks'), | |||
|
358 | total=sum(len(v) for v in fnodes.itervalues())) | |||
|
359 | ||||
|
360 | # TODO make batch size configurable | |||
|
361 | batchsize = 10000 | |||
|
362 | fnodeslist = [x for x in sorted(fnodes.items())] | |||
|
363 | ||||
|
364 | for i in pycompat.xrange(0, len(fnodeslist), batchsize): | |||
|
365 | batch = [x for x in fnodeslist[i:i + batchsize]] | |||
|
366 | if not batch: | |||
|
367 | continue | |||
|
368 | ||||
|
369 | with remote.commandexecutor() as e: | |||
|
370 | fs = [] | |||
|
371 | locallinkrevs = {} | |||
|
372 | ||||
|
373 | for path, nodes in batch: | |||
|
374 | fs.append((path, e.callcommand(b'filedata', { | |||
|
375 | b'path': path, | |||
|
376 | b'nodes': sorted(nodes), | |||
|
377 | b'fields': {b'parents', b'revision'} | |||
|
378 | }))) | |||
|
379 | ||||
|
380 | locallinkrevs[path] = { | |||
|
381 | node: linkrevs[manifestnode] | |||
|
382 | for node, manifestnode in nodes.iteritems()} | |||
|
383 | ||||
|
384 | for path, f in fs: | |||
|
385 | objs = f.result() | |||
|
386 | ||||
|
387 | # Chomp off header objects. | |||
|
388 | next(objs) | |||
|
389 | ||||
|
390 | store = repo.file(path) | |||
|
391 | store.addgroup( | |||
|
392 | iterrevisions(objs, progress), | |||
|
393 | locallinkrevs[path].__getitem__, | |||
|
394 | weakref.proxy(tr)) |
@@ -94,6 +94,37 b' Test basic clone' | |||||
94 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) |
|
94 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |
95 | received frame(size=922; request=1; stream=2; streamflags=; type=command-response; flags=continuation) |
|
95 | received frame(size=922; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |
96 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) |
|
96 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |
|
97 | sending 2 commands | |||
|
98 | sending command filedata: { | |||
|
99 | 'fields': set([ | |||
|
100 | 'parents', | |||
|
101 | 'revision' | |||
|
102 | ]), | |||
|
103 | 'nodes': [ | |||
|
104 | '+N\xb0s\x19\xbf\xa0w\xa4\n/\x04\x916Y\xae\xf0\xdaB\xda', | |||
|
105 | '\x9a8\x12)\x97\xb3\xac\x97\xbe*\x9a\xa2\xe5V\x83\x83A\xfd\xf2\xcc', | |||
|
106 | '\xc2\xa2\x05\xc8\xb2\xad\xe2J\xf2`b\xe5<\xd5\xbc8\x01\xd6`\xda' | |||
|
107 | ], | |||
|
108 | 'path': 'a' | |||
|
109 | } | |||
|
110 | sending command filedata: { | |||
|
111 | 'fields': set([ | |||
|
112 | 'parents', | |||
|
113 | 'revision' | |||
|
114 | ]), | |||
|
115 | 'nodes': [ | |||
|
116 | '\x81\x9e%\x8d1\xa5\xe1`f)\xf3e\xbb\x90*\x1b!\xeeB\x16', | |||
|
117 | '\xb1zk\xd3g=\x9a\xb8\xce\xd5\x81\xa2\t\xf6/=\xa5\xccEx', | |||
|
118 | '\xc5\xb1\xf9\xd3n\x1c\xc18\xbf\xb6\xef\xb3\xde\xb7]\x8c\xcad\x94\xc3' | |||
|
119 | ], | |||
|
120 | 'path': 'b' | |||
|
121 | } | |||
|
122 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |||
|
123 | received frame(size=389; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
124 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |||
|
125 | received frame(size=11; request=3; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
126 | received frame(size=389; request=3; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
127 | received frame(size=0; request=3; stream=2; streamflags=; type=command-response; flags=eos) | |||
97 | updating the branch cache |
|
128 | updating the branch cache | |
98 | new changesets 3390ef850073:caa2a465451d (3 drafts) |
|
129 | new changesets 3390ef850073:caa2a465451d (3 drafts) | |
99 |
|
130 | |||
@@ -189,6 +220,34 b' Cloning only a specific revision works' | |||||
189 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) |
|
220 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |
190 | received frame(size=376; request=1; stream=2; streamflags=; type=command-response; flags=continuation) |
|
221 | received frame(size=376; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |
191 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) |
|
222 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |
|
223 | sending 2 commands | |||
|
224 | sending command filedata: { | |||
|
225 | 'fields': set([ | |||
|
226 | 'parents', | |||
|
227 | 'revision' | |||
|
228 | ]), | |||
|
229 | 'nodes': [ | |||
|
230 | '+N\xb0s\x19\xbf\xa0w\xa4\n/\x04\x916Y\xae\xf0\xdaB\xda', | |||
|
231 | '\x9a8\x12)\x97\xb3\xac\x97\xbe*\x9a\xa2\xe5V\x83\x83A\xfd\xf2\xcc' | |||
|
232 | ], | |||
|
233 | 'path': 'a' | |||
|
234 | } | |||
|
235 | sending command filedata: { | |||
|
236 | 'fields': set([ | |||
|
237 | 'parents', | |||
|
238 | 'revision' | |||
|
239 | ]), | |||
|
240 | 'nodes': [ | |||
|
241 | '\x81\x9e%\x8d1\xa5\xe1`f)\xf3e\xbb\x90*\x1b!\xeeB\x16' | |||
|
242 | ], | |||
|
243 | 'path': 'b' | |||
|
244 | } | |||
|
245 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |||
|
246 | received frame(size=249; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
247 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |||
|
248 | received frame(size=11; request=3; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
249 | received frame(size=109; request=3; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
250 | received frame(size=0; request=3; stream=2; streamflags=; type=command-response; flags=eos) | |||
192 | updating the branch cache |
|
251 | updating the branch cache | |
193 | new changesets 3390ef850073:4432d83626e8 |
|
252 | new changesets 3390ef850073:4432d83626e8 | |
194 |
|
253 | |||
@@ -268,6 +327,36 b' Incremental pull works' | |||||
268 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) |
|
327 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |
269 | received frame(size=559; request=1; stream=2; streamflags=; type=command-response; flags=continuation) |
|
328 | received frame(size=559; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |
270 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) |
|
329 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |
|
330 | sending 2 commands | |||
|
331 | sending command filedata: { | |||
|
332 | 'fields': set([ | |||
|
333 | 'parents', | |||
|
334 | 'revision' | |||
|
335 | ]), | |||
|
336 | 'nodes': [ | |||
|
337 | '+N\xb0s\x19\xbf\xa0w\xa4\n/\x04\x916Y\xae\xf0\xdaB\xda', | |||
|
338 | '\xc2\xa2\x05\xc8\xb2\xad\xe2J\xf2`b\xe5<\xd5\xbc8\x01\xd6`\xda' | |||
|
339 | ], | |||
|
340 | 'path': 'a' | |||
|
341 | } | |||
|
342 | sending command filedata: { | |||
|
343 | 'fields': set([ | |||
|
344 | 'parents', | |||
|
345 | 'revision' | |||
|
346 | ]), | |||
|
347 | 'nodes': [ | |||
|
348 | '\x81\x9e%\x8d1\xa5\xe1`f)\xf3e\xbb\x90*\x1b!\xeeB\x16', | |||
|
349 | '\xb1zk\xd3g=\x9a\xb8\xce\xd5\x81\xa2\t\xf6/=\xa5\xccEx', | |||
|
350 | '\xc5\xb1\xf9\xd3n\x1c\xc18\xbf\xb6\xef\xb3\xde\xb7]\x8c\xcad\x94\xc3' | |||
|
351 | ], | |||
|
352 | 'path': 'b' | |||
|
353 | } | |||
|
354 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |||
|
355 | received frame(size=249; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
356 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |||
|
357 | received frame(size=11; request=3; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
358 | received frame(size=389; request=3; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
359 | received frame(size=0; request=3; stream=2; streamflags=; type=command-response; flags=eos) | |||
271 | updating the branch cache |
|
360 | updating the branch cache | |
272 | new changesets cd2534766bec:caa2a465451d (3 drafts) |
|
361 | new changesets cd2534766bec:caa2a465451d (3 drafts) | |
273 | (run 'hg update' to get a working copy) |
|
362 | (run 'hg update' to get a working copy) | |
@@ -421,6 +510,37 b' Bookmarks are transferred on clone' | |||||
421 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) |
|
510 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |
422 | received frame(size=922; request=1; stream=2; streamflags=; type=command-response; flags=continuation) |
|
511 | received frame(size=922; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |
423 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) |
|
512 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |
|
513 | sending 2 commands | |||
|
514 | sending command filedata: { | |||
|
515 | 'fields': set([ | |||
|
516 | 'parents', | |||
|
517 | 'revision' | |||
|
518 | ]), | |||
|
519 | 'nodes': [ | |||
|
520 | '+N\xb0s\x19\xbf\xa0w\xa4\n/\x04\x916Y\xae\xf0\xdaB\xda', | |||
|
521 | '\x9a8\x12)\x97\xb3\xac\x97\xbe*\x9a\xa2\xe5V\x83\x83A\xfd\xf2\xcc', | |||
|
522 | '\xc2\xa2\x05\xc8\xb2\xad\xe2J\xf2`b\xe5<\xd5\xbc8\x01\xd6`\xda' | |||
|
523 | ], | |||
|
524 | 'path': 'a' | |||
|
525 | } | |||
|
526 | sending command filedata: { | |||
|
527 | 'fields': set([ | |||
|
528 | 'parents', | |||
|
529 | 'revision' | |||
|
530 | ]), | |||
|
531 | 'nodes': [ | |||
|
532 | '\x81\x9e%\x8d1\xa5\xe1`f)\xf3e\xbb\x90*\x1b!\xeeB\x16', | |||
|
533 | '\xb1zk\xd3g=\x9a\xb8\xce\xd5\x81\xa2\t\xf6/=\xa5\xccEx', | |||
|
534 | '\xc5\xb1\xf9\xd3n\x1c\xc18\xbf\xb6\xef\xb3\xde\xb7]\x8c\xcad\x94\xc3' | |||
|
535 | ], | |||
|
536 | 'path': 'b' | |||
|
537 | } | |||
|
538 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |||
|
539 | received frame(size=389; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
540 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |||
|
541 | received frame(size=11; request=3; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
542 | received frame(size=389; request=3; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
543 | received frame(size=0; request=3; stream=2; streamflags=; type=command-response; flags=eos) | |||
424 | updating the branch cache |
|
544 | updating the branch cache | |
425 | new changesets 3390ef850073:caa2a465451d (1 drafts) |
|
545 | new changesets 3390ef850073:caa2a465451d (1 drafts) | |
426 |
|
546 |
General Comments 0
You need to be logged in to leave comments.
Login now