Show More
@@ -16,6 +16,7 from .node import ( | |||||
16 | ) |
|
16 | ) | |
17 | from . import ( |
|
17 | from . import ( | |
18 | bookmarks, |
|
18 | bookmarks, | |
|
19 | error, | |||
19 | mdiff, |
|
20 | mdiff, | |
20 | phases, |
|
21 | phases, | |
21 | pycompat, |
|
22 | pycompat, | |
@@ -57,6 +58,8 def pull(pullop): | |||||
57 | remote.url(), pullop.gettransaction, |
|
58 | remote.url(), pullop.gettransaction, | |
58 | explicit=pullop.explicitbookmarks) |
|
59 | explicit=pullop.explicitbookmarks) | |
59 |
|
60 | |||
|
61 | _fetchmanifests(repo, tr, remote, csetres['manifestnodes']) | |||
|
62 | ||||
60 | def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True): |
|
63 | def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True): | |
61 | """Determine which changesets need to be pulled.""" |
|
64 | """Determine which changesets need to be pulled.""" | |
62 |
|
65 | |||
@@ -121,6 +124,8 def _processchangesetdata(repo, tr, objs | |||||
121 | unit=_('chunks'), |
|
124 | unit=_('chunks'), | |
122 | total=meta.get(b'totalitems')) |
|
125 | total=meta.get(b'totalitems')) | |
123 |
|
126 | |||
|
127 | manifestnodes = {} | |||
|
128 | ||||
124 | def linkrev(node): |
|
129 | def linkrev(node): | |
125 | repo.ui.debug('add changeset %s\n' % short(node)) |
|
130 | repo.ui.debug('add changeset %s\n' % short(node)) | |
126 | # Linkrev for changelog is always self. |
|
131 | # Linkrev for changelog is always self. | |
@@ -129,6 +134,12 def _processchangesetdata(repo, tr, objs | |||||
129 | def onchangeset(cl, node): |
|
134 | def onchangeset(cl, node): | |
130 | progress.increment() |
|
135 | progress.increment() | |
131 |
|
136 | |||
|
137 | revision = cl.changelogrevision(node) | |||
|
138 | ||||
|
139 | # We need to preserve the mapping of changelog revision to node | |||
|
140 | # so we can set the linkrev accordingly when manifests are added. | |||
|
141 | manifestnodes[cl.rev(node)] = revision.manifest | |||
|
142 | ||||
132 | nodesbyphase = {phase: set() for phase in phases.phasenames} |
|
143 | nodesbyphase = {phase: set() for phase in phases.phasenames} | |
133 | remotebookmarks = {} |
|
144 | remotebookmarks = {} | |
134 |
|
145 | |||
@@ -178,4 +189,106 def _processchangesetdata(repo, tr, objs | |||||
178 | 'added': added, |
|
189 | 'added': added, | |
179 | 'nodesbyphase': nodesbyphase, |
|
190 | 'nodesbyphase': nodesbyphase, | |
180 | 'bookmarks': remotebookmarks, |
|
191 | 'bookmarks': remotebookmarks, | |
|
192 | 'manifestnodes': manifestnodes, | |||
181 | } |
|
193 | } | |
|
194 | ||||
|
195 | def _fetchmanifests(repo, tr, remote, manifestnodes): | |||
|
196 | rootmanifest = repo.manifestlog.getstorage(b'') | |||
|
197 | ||||
|
198 | # Some manifests can be shared between changesets. Filter out revisions | |||
|
199 | # we already know about. | |||
|
200 | fetchnodes = [] | |||
|
201 | linkrevs = {} | |||
|
202 | seen = set() | |||
|
203 | ||||
|
204 | for clrev, node in sorted(manifestnodes.iteritems()): | |||
|
205 | if node in seen: | |||
|
206 | continue | |||
|
207 | ||||
|
208 | try: | |||
|
209 | rootmanifest.rev(node) | |||
|
210 | except error.LookupError: | |||
|
211 | fetchnodes.append(node) | |||
|
212 | linkrevs[node] = clrev | |||
|
213 | ||||
|
214 | seen.add(node) | |||
|
215 | ||||
|
216 | # TODO handle tree manifests | |||
|
217 | ||||
|
218 | # addgroup() expects 7-tuple describing revisions. This normalizes | |||
|
219 | # the wire data to that format. | |||
|
220 | def iterrevisions(objs, progress): | |||
|
221 | for manifest in objs: | |||
|
222 | node = manifest[b'node'] | |||
|
223 | ||||
|
224 | if b'deltasize' in manifest: | |||
|
225 | basenode = manifest[b'deltabasenode'] | |||
|
226 | delta = next(objs) | |||
|
227 | elif b'revisionsize' in manifest: | |||
|
228 | basenode = nullid | |||
|
229 | revision = next(objs) | |||
|
230 | delta = mdiff.trivialdiffheader(len(revision)) + revision | |||
|
231 | else: | |||
|
232 | continue | |||
|
233 | ||||
|
234 | yield ( | |||
|
235 | node, | |||
|
236 | manifest[b'parents'][0], | |||
|
237 | manifest[b'parents'][1], | |||
|
238 | # The value passed in is passed to the lookup function passed | |||
|
239 | # to addgroup(). We already have a map of manifest node to | |||
|
240 | # changelog revision number. So we just pass in the | |||
|
241 | # manifest node here and use linkrevs.__getitem__ as the | |||
|
242 | # resolution function. | |||
|
243 | node, | |||
|
244 | basenode, | |||
|
245 | delta, | |||
|
246 | # Flags not yet supported. | |||
|
247 | 0 | |||
|
248 | ) | |||
|
249 | ||||
|
250 | progress.increment() | |||
|
251 | ||||
|
252 | progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'), | |||
|
253 | total=len(fetchnodes)) | |||
|
254 | ||||
|
255 | # Fetch manifests 10,000 per command. | |||
|
256 | # TODO have server advertise preferences? | |||
|
257 | # TODO make size configurable on client? | |||
|
258 | batchsize = 10000 | |||
|
259 | ||||
|
260 | # We send commands 1 at a time to the remote. This is not the most | |||
|
261 | # efficient because we incur a round trip at the end of each batch. | |||
|
262 | # However, the existing frame-based reactor keeps consuming server | |||
|
263 | # data in the background. And this results in response data buffering | |||
|
264 | # in memory. This can consume gigabytes of memory. | |||
|
265 | # TODO send multiple commands in a request once background buffering | |||
|
266 | # issues are resolved. | |||
|
267 | ||||
|
268 | added = [] | |||
|
269 | ||||
|
270 | for i in pycompat.xrange(0, len(fetchnodes), batchsize): | |||
|
271 | batch = [node for node in fetchnodes[i:i + batchsize]] | |||
|
272 | if not batch: | |||
|
273 | continue | |||
|
274 | ||||
|
275 | with remote.commandexecutor() as e: | |||
|
276 | objs = e.callcommand(b'manifestdata', { | |||
|
277 | b'tree': b'', | |||
|
278 | b'nodes': batch, | |||
|
279 | b'fields': {b'parents', b'revision'}, | |||
|
280 | }).result() | |||
|
281 | ||||
|
282 | # Chomp off header object. | |||
|
283 | next(objs) | |||
|
284 | ||||
|
285 | added.extend(rootmanifest.addgroup( | |||
|
286 | iterrevisions(objs, progress), | |||
|
287 | linkrevs.__getitem__, | |||
|
288 | weakref.proxy(tr))) | |||
|
289 | ||||
|
290 | progress.complete() | |||
|
291 | ||||
|
292 | return { | |||
|
293 | 'added': added, | |||
|
294 | } |
@@ -76,6 +76,24 Test basic clone | |||||
76 | add changeset e96ae20f4188 |
|
76 | add changeset e96ae20f4188 | |
77 | add changeset caa2a465451d |
|
77 | add changeset caa2a465451d | |
78 | checking for updated bookmarks |
|
78 | checking for updated bookmarks | |
|
79 | sending 1 commands | |||
|
80 | sending command manifestdata: { | |||
|
81 | 'fields': set([ | |||
|
82 | 'parents', | |||
|
83 | 'revision' | |||
|
84 | ]), | |||
|
85 | 'nodes': [ | |||
|
86 | '\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&A', | |||
|
87 | '\xa9\x88\xfbCX>\x87\x1d\x1e\xd5u\x0e\xe0t\xc6\xd8@\xbb\xbf\xc8', | |||
|
88 | '\xec\x80NH\x8c \x88\xc25\t\x9a\x10 u\x13\xbe\xcd\xc3\xdd\xa5', | |||
|
89 | '\x04\\\x7f9\'\xda\x13\xe7Z\xf8\xf0\xe4\xf0HI\xe4a\xa9x\x0f', | |||
|
90 | '7\x9c\xb0\xc2\xe6d\\y\xdd\xc5\x9a\x1dG\'\xa9\xfb\x83\n\xeb&' | |||
|
91 | ], | |||
|
92 | 'tree': '' | |||
|
93 | } | |||
|
94 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |||
|
95 | received frame(size=922; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
96 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |||
79 | updating the branch cache |
|
97 | updating the branch cache | |
80 | new changesets 3390ef850073:caa2a465451d (3 drafts) |
|
98 | new changesets 3390ef850073:caa2a465451d (3 drafts) | |
81 |
|
99 | |||
@@ -101,6 +119,16 All changesets should have been transfer | |||||
101 | o 0 3390ef850073fbc2f0dfff2244342c8e9229013a public |
|
119 | o 0 3390ef850073fbc2f0dfff2244342c8e9229013a public | |
102 |
|
120 | |||
103 |
|
121 | |||
|
122 | All manifests should have been transferred | |||
|
123 | ||||
|
124 | $ hg -R client-simple debugindex -m | |||
|
125 | rev linkrev nodeid p1 p2 | |||
|
126 | 0 0 992f4779029a 000000000000 000000000000 | |||
|
127 | 1 1 a988fb43583e 992f4779029a 000000000000 | |||
|
128 | 2 2 ec804e488c20 a988fb43583e 000000000000 | |||
|
129 | 3 3 045c7f3927da 992f4779029a 000000000000 | |||
|
130 | 4 4 379cb0c2e664 045c7f3927da 000000000000 | |||
|
131 | ||||
104 | Cloning only a specific revision works |
|
132 | Cloning only a specific revision works | |
105 |
|
133 | |||
106 | $ hg --debug clone -U -r 4432d83626e8 http://localhost:$HGPORT client-singlehead |
|
134 | $ hg --debug clone -U -r 4432d83626e8 http://localhost:$HGPORT client-singlehead | |
@@ -146,6 +174,21 Cloning only a specific revision works | |||||
146 | add changeset 3390ef850073 |
|
174 | add changeset 3390ef850073 | |
147 | add changeset 4432d83626e8 |
|
175 | add changeset 4432d83626e8 | |
148 | checking for updated bookmarks |
|
176 | checking for updated bookmarks | |
|
177 | sending 1 commands | |||
|
178 | sending command manifestdata: { | |||
|
179 | 'fields': set([ | |||
|
180 | 'parents', | |||
|
181 | 'revision' | |||
|
182 | ]), | |||
|
183 | 'nodes': [ | |||
|
184 | '\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&A', | |||
|
185 | '\xa9\x88\xfbCX>\x87\x1d\x1e\xd5u\x0e\xe0t\xc6\xd8@\xbb\xbf\xc8' | |||
|
186 | ], | |||
|
187 | 'tree': '' | |||
|
188 | } | |||
|
189 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |||
|
190 | received frame(size=376; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
191 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |||
149 | updating the branch cache |
|
192 | updating the branch cache | |
150 | new changesets 3390ef850073:4432d83626e8 |
|
193 | new changesets 3390ef850073:4432d83626e8 | |
151 |
|
194 | |||
@@ -157,6 +200,11 Cloning only a specific revision works | |||||
157 | o 0 3390ef850073fbc2f0dfff2244342c8e9229013a public |
|
200 | o 0 3390ef850073fbc2f0dfff2244342c8e9229013a public | |
158 |
|
201 | |||
159 |
|
202 | |||
|
203 | $ hg debugindex -m | |||
|
204 | rev linkrev nodeid p1 p2 | |||
|
205 | 0 0 992f4779029a 000000000000 000000000000 | |||
|
206 | 1 1 a988fb43583e 992f4779029a 000000000000 | |||
|
207 | ||||
160 | Incremental pull works |
|
208 | Incremental pull works | |
161 |
|
209 | |||
162 | $ hg --debug pull |
|
210 | $ hg --debug pull | |
@@ -204,6 +252,22 Incremental pull works | |||||
204 | add changeset e96ae20f4188 |
|
252 | add changeset e96ae20f4188 | |
205 | add changeset caa2a465451d |
|
253 | add changeset caa2a465451d | |
206 | checking for updated bookmarks |
|
254 | checking for updated bookmarks | |
|
255 | sending 1 commands | |||
|
256 | sending command manifestdata: { | |||
|
257 | 'fields': set([ | |||
|
258 | 'parents', | |||
|
259 | 'revision' | |||
|
260 | ]), | |||
|
261 | 'nodes': [ | |||
|
262 | '\xec\x80NH\x8c \x88\xc25\t\x9a\x10 u\x13\xbe\xcd\xc3\xdd\xa5', | |||
|
263 | '\x04\\\x7f9\'\xda\x13\xe7Z\xf8\xf0\xe4\xf0HI\xe4a\xa9x\x0f', | |||
|
264 | '7\x9c\xb0\xc2\xe6d\\y\xdd\xc5\x9a\x1dG\'\xa9\xfb\x83\n\xeb&' | |||
|
265 | ], | |||
|
266 | 'tree': '' | |||
|
267 | } | |||
|
268 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |||
|
269 | received frame(size=559; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
270 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |||
207 | updating the branch cache |
|
271 | updating the branch cache | |
208 | new changesets cd2534766bec:caa2a465451d (3 drafts) |
|
272 | new changesets cd2534766bec:caa2a465451d (3 drafts) | |
209 | (run 'hg update' to get a working copy) |
|
273 | (run 'hg update' to get a working copy) | |
@@ -220,6 +284,14 Incremental pull works | |||||
220 | o 0 3390ef850073fbc2f0dfff2244342c8e9229013a public |
|
284 | o 0 3390ef850073fbc2f0dfff2244342c8e9229013a public | |
221 |
|
285 | |||
222 |
|
286 | |||
|
287 | $ hg debugindex -m | |||
|
288 | rev linkrev nodeid p1 p2 | |||
|
289 | 0 0 992f4779029a 000000000000 000000000000 | |||
|
290 | 1 1 a988fb43583e 992f4779029a 000000000000 | |||
|
291 | 2 2 ec804e488c20 a988fb43583e 000000000000 | |||
|
292 | 3 3 045c7f3927da 992f4779029a 000000000000 | |||
|
293 | 4 4 379cb0c2e664 045c7f3927da 000000000000 | |||
|
294 | ||||
223 | Phase-only update works |
|
295 | Phase-only update works | |
224 |
|
296 | |||
225 | $ hg -R ../server-simple phase --public -r caa2a465451dd |
|
297 | $ hg -R ../server-simple phase --public -r caa2a465451dd | |
@@ -331,6 +403,24 Bookmarks are transferred on clone | |||||
331 | checking for updated bookmarks |
|
403 | checking for updated bookmarks | |
332 | adding remote bookmark book-1 |
|
404 | adding remote bookmark book-1 | |
333 | adding remote bookmark book-2 |
|
405 | adding remote bookmark book-2 | |
|
406 | sending 1 commands | |||
|
407 | sending command manifestdata: { | |||
|
408 | 'fields': set([ | |||
|
409 | 'parents', | |||
|
410 | 'revision' | |||
|
411 | ]), | |||
|
412 | 'nodes': [ | |||
|
413 | '\x99/Gy\x02\x9a=\xf8\xd0fm\x00\xbb\x92OicN&A', | |||
|
414 | '\xa9\x88\xfbCX>\x87\x1d\x1e\xd5u\x0e\xe0t\xc6\xd8@\xbb\xbf\xc8', | |||
|
415 | '\xec\x80NH\x8c \x88\xc25\t\x9a\x10 u\x13\xbe\xcd\xc3\xdd\xa5', | |||
|
416 | '\x04\\\x7f9\'\xda\x13\xe7Z\xf8\xf0\xe4\xf0HI\xe4a\xa9x\x0f', | |||
|
417 | '7\x9c\xb0\xc2\xe6d\\y\xdd\xc5\x9a\x1dG\'\xa9\xfb\x83\n\xeb&' | |||
|
418 | ], | |||
|
419 | 'tree': '' | |||
|
420 | } | |||
|
421 | received frame(size=11; request=1; stream=2; streamflags=stream-begin; type=command-response; flags=continuation) | |||
|
422 | received frame(size=922; request=1; stream=2; streamflags=; type=command-response; flags=continuation) | |||
|
423 | received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos) | |||
334 | updating the branch cache |
|
424 | updating the branch cache | |
335 | new changesets 3390ef850073:caa2a465451d (1 drafts) |
|
425 | new changesets 3390ef850073:caa2a465451d (1 drafts) | |
336 |
|
426 |
General Comments 0
You need to be logged in to leave comments.
Login now