##// END OF EJS Templates
changegroup: clean up changelog callback...
Gregory Szorc -
r39273:3634ed95 default
parent child Browse files
Show More
@@ -1,1394 +1,1394 b''
1 1 # changegroup.py - Mercurial changegroup manipulation functions
2 2 #
3 3 # Copyright 2006 Matt Mackall <mpm@selenic.com>
4 4 #
5 5 # This software may be used and distributed according to the terms of the
6 6 # GNU General Public License version 2 or any later version.
7 7
8 8 from __future__ import absolute_import
9 9
10 10 import os
11 11 import struct
12 12 import weakref
13 13
14 14 from .i18n import _
15 15 from .node import (
16 16 hex,
17 17 nullid,
18 18 nullrev,
19 19 short,
20 20 )
21 21
22 22 from .thirdparty import (
23 23 attr,
24 24 )
25 25
26 26 from . import (
27 27 dagop,
28 28 error,
29 29 match as matchmod,
30 30 mdiff,
31 31 phases,
32 32 pycompat,
33 33 repository,
34 34 util,
35 35 )
36 36
37 37 from .utils import (
38 38 interfaceutil,
39 39 stringutil,
40 40 )
41 41
42 42 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s")
43 43 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s")
44 44 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH")
45 45
46 46 LFS_REQUIREMENT = 'lfs'
47 47
48 48 readexactly = util.readexactly
49 49
50 50 def getchunk(stream):
51 51 """return the next chunk from stream as a string"""
52 52 d = readexactly(stream, 4)
53 53 l = struct.unpack(">l", d)[0]
54 54 if l <= 4:
55 55 if l:
56 56 raise error.Abort(_("invalid chunk length %d") % l)
57 57 return ""
58 58 return readexactly(stream, l - 4)
59 59
60 60 def chunkheader(length):
61 61 """return a changegroup chunk header (string)"""
62 62 return struct.pack(">l", length + 4)
63 63
64 64 def closechunk():
65 65 """return a changegroup chunk header (string) for a zero-length chunk"""
66 66 return struct.pack(">l", 0)
67 67
68 68 def _fileheader(path):
69 69 """Obtain a changegroup chunk header for a named path."""
70 70 return chunkheader(len(path)) + path
71 71
72 72 def writechunks(ui, chunks, filename, vfs=None):
73 73 """Write chunks to a file and return its filename.
74 74
75 75 The stream is assumed to be a bundle file.
76 76 Existing files will not be overwritten.
77 77 If no filename is specified, a temporary file is created.
78 78 """
79 79 fh = None
80 80 cleanup = None
81 81 try:
82 82 if filename:
83 83 if vfs:
84 84 fh = vfs.open(filename, "wb")
85 85 else:
86 86 # Increase default buffer size because default is usually
87 87 # small (4k is common on Linux).
88 88 fh = open(filename, "wb", 131072)
89 89 else:
90 90 fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg")
91 91 fh = os.fdopen(fd, r"wb")
92 92 cleanup = filename
93 93 for c in chunks:
94 94 fh.write(c)
95 95 cleanup = None
96 96 return filename
97 97 finally:
98 98 if fh is not None:
99 99 fh.close()
100 100 if cleanup is not None:
101 101 if filename and vfs:
102 102 vfs.unlink(cleanup)
103 103 else:
104 104 os.unlink(cleanup)
105 105
106 106 class cg1unpacker(object):
107 107 """Unpacker for cg1 changegroup streams.
108 108
109 109 A changegroup unpacker handles the framing of the revision data in
110 110 the wire format. Most consumers will want to use the apply()
111 111 method to add the changes from the changegroup to a repository.
112 112
113 113 If you're forwarding a changegroup unmodified to another consumer,
114 114 use getchunks(), which returns an iterator of changegroup
115 115 chunks. This is mostly useful for cases where you need to know the
116 116 data stream has ended by observing the end of the changegroup.
117 117
118 118 deltachunk() is useful only if you're applying delta data. Most
119 119 consumers should prefer apply() instead.
120 120
121 121 A few other public methods exist. Those are used only for
122 122 bundlerepo and some debug commands - their use is discouraged.
123 123 """
124 124 deltaheader = _CHANGEGROUPV1_DELTA_HEADER
125 125 deltaheadersize = deltaheader.size
126 126 version = '01'
127 127 _grouplistcount = 1 # One list of files after the manifests
128 128
129 129 def __init__(self, fh, alg, extras=None):
130 130 if alg is None:
131 131 alg = 'UN'
132 132 if alg not in util.compengines.supportedbundletypes:
133 133 raise error.Abort(_('unknown stream compression type: %s')
134 134 % alg)
135 135 if alg == 'BZ':
136 136 alg = '_truncatedBZ'
137 137
138 138 compengine = util.compengines.forbundletype(alg)
139 139 self._stream = compengine.decompressorreader(fh)
140 140 self._type = alg
141 141 self.extras = extras or {}
142 142 self.callback = None
143 143
144 144 # These methods (compressed, read, seek, tell) all appear to only
145 145 # be used by bundlerepo, but it's a little hard to tell.
146 146 def compressed(self):
147 147 return self._type is not None and self._type != 'UN'
148 148 def read(self, l):
149 149 return self._stream.read(l)
150 150 def seek(self, pos):
151 151 return self._stream.seek(pos)
152 152 def tell(self):
153 153 return self._stream.tell()
154 154 def close(self):
155 155 return self._stream.close()
156 156
157 157 def _chunklength(self):
158 158 d = readexactly(self._stream, 4)
159 159 l = struct.unpack(">l", d)[0]
160 160 if l <= 4:
161 161 if l:
162 162 raise error.Abort(_("invalid chunk length %d") % l)
163 163 return 0
164 164 if self.callback:
165 165 self.callback()
166 166 return l - 4
167 167
168 168 def changelogheader(self):
169 169 """v10 does not have a changelog header chunk"""
170 170 return {}
171 171
172 172 def manifestheader(self):
173 173 """v10 does not have a manifest header chunk"""
174 174 return {}
175 175
176 176 def filelogheader(self):
177 177 """return the header of the filelogs chunk, v10 only has the filename"""
178 178 l = self._chunklength()
179 179 if not l:
180 180 return {}
181 181 fname = readexactly(self._stream, l)
182 182 return {'filename': fname}
183 183
184 184 def _deltaheader(self, headertuple, prevnode):
185 185 node, p1, p2, cs = headertuple
186 186 if prevnode is None:
187 187 deltabase = p1
188 188 else:
189 189 deltabase = prevnode
190 190 flags = 0
191 191 return node, p1, p2, deltabase, cs, flags
192 192
193 193 def deltachunk(self, prevnode):
194 194 l = self._chunklength()
195 195 if not l:
196 196 return {}
197 197 headerdata = readexactly(self._stream, self.deltaheadersize)
198 198 header = self.deltaheader.unpack(headerdata)
199 199 delta = readexactly(self._stream, l - self.deltaheadersize)
200 200 node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
201 201 return (node, p1, p2, cs, deltabase, delta, flags)
202 202
203 203 def getchunks(self):
204 204 """returns all the chunks contains in the bundle
205 205
206 206 Used when you need to forward the binary stream to a file or another
207 207 network API. To do so, it parse the changegroup data, otherwise it will
208 208 block in case of sshrepo because it don't know the end of the stream.
209 209 """
210 210 # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog,
211 211 # and a list of filelogs. For changegroup 3, we expect 4 parts:
212 212 # changelog, manifestlog, a list of tree manifestlogs, and a list of
213 213 # filelogs.
214 214 #
215 215 # Changelog and manifestlog parts are terminated with empty chunks. The
216 216 # tree and file parts are a list of entry sections. Each entry section
217 217 # is a series of chunks terminating in an empty chunk. The list of these
218 218 # entry sections is terminated in yet another empty chunk, so we know
219 219 # we've reached the end of the tree/file list when we reach an empty
220 220 # chunk that was proceeded by no non-empty chunks.
221 221
222 222 parts = 0
223 223 while parts < 2 + self._grouplistcount:
224 224 noentries = True
225 225 while True:
226 226 chunk = getchunk(self)
227 227 if not chunk:
228 228 # The first two empty chunks represent the end of the
229 229 # changelog and the manifestlog portions. The remaining
230 230 # empty chunks represent either A) the end of individual
231 231 # tree or file entries in the file list, or B) the end of
232 232 # the entire list. It's the end of the entire list if there
233 233 # were no entries (i.e. noentries is True).
234 234 if parts < 2:
235 235 parts += 1
236 236 elif noentries:
237 237 parts += 1
238 238 break
239 239 noentries = False
240 240 yield chunkheader(len(chunk))
241 241 pos = 0
242 242 while pos < len(chunk):
243 243 next = pos + 2**20
244 244 yield chunk[pos:next]
245 245 pos = next
246 246 yield closechunk()
247 247
248 248 def _unpackmanifests(self, repo, revmap, trp, prog):
249 249 self.callback = prog.increment
250 250 # no need to check for empty manifest group here:
251 251 # if the result of the merge of 1 and 2 is the same in 3 and 4,
252 252 # no new manifest will be created and the manifest group will
253 253 # be empty during the pull
254 254 self.manifestheader()
255 255 deltas = self.deltaiter()
256 256 repo.manifestlog.addgroup(deltas, revmap, trp)
257 257 prog.complete()
258 258 self.callback = None
259 259
260 260 def apply(self, repo, tr, srctype, url, targetphase=phases.draft,
261 261 expectedtotal=None):
262 262 """Add the changegroup returned by source.read() to this repo.
263 263 srctype is a string like 'push', 'pull', or 'unbundle'. url is
264 264 the URL of the repo where this changegroup is coming from.
265 265
266 266 Return an integer summarizing the change to this repo:
267 267 - nothing changed or no source: 0
268 268 - more heads than before: 1+added heads (2..n)
269 269 - fewer heads than before: -1-removed heads (-2..-n)
270 270 - number of heads stays the same: 1
271 271 """
272 272 repo = repo.unfiltered()
273 273 def csmap(x):
274 274 repo.ui.debug("add changeset %s\n" % short(x))
275 275 return len(cl)
276 276
277 277 def revmap(x):
278 278 return cl.rev(x)
279 279
280 280 changesets = files = revisions = 0
281 281
282 282 try:
283 283 # The transaction may already carry source information. In this
284 284 # case we use the top level data. We overwrite the argument
285 285 # because we need to use the top level value (if they exist)
286 286 # in this function.
287 287 srctype = tr.hookargs.setdefault('source', srctype)
288 288 url = tr.hookargs.setdefault('url', url)
289 289 repo.hook('prechangegroup',
290 290 throw=True, **pycompat.strkwargs(tr.hookargs))
291 291
292 292 # write changelog data to temp files so concurrent readers
293 293 # will not see an inconsistent view
294 294 cl = repo.changelog
295 295 cl.delayupdate(tr)
296 296 oldheads = set(cl.heads())
297 297
298 298 trp = weakref.proxy(tr)
299 299 # pull off the changeset group
300 300 repo.ui.status(_("adding changesets\n"))
301 301 clstart = len(cl)
302 302 progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'),
303 303 total=expectedtotal)
304 304 self.callback = progress.increment
305 305
306 306 efiles = set()
307 307 def onchangelog(cl, node):
308 308 efiles.update(cl.readfiles(node))
309 309
310 310 self.changelogheader()
311 311 deltas = self.deltaiter()
312 312 cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog)
313 313 efiles = len(efiles)
314 314
315 315 if not cgnodes:
316 316 repo.ui.develwarn('applied empty changegroup',
317 317 config='warn-empty-changegroup')
318 318 clend = len(cl)
319 319 changesets = clend - clstart
320 320 progress.complete()
321 321 self.callback = None
322 322
323 323 # pull off the manifest group
324 324 repo.ui.status(_("adding manifests\n"))
325 325 # We know that we'll never have more manifests than we had
326 326 # changesets.
327 327 progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'),
328 328 total=changesets)
329 329 self._unpackmanifests(repo, revmap, trp, progress)
330 330
331 331 needfiles = {}
332 332 if repo.ui.configbool('server', 'validate'):
333 333 cl = repo.changelog
334 334 ml = repo.manifestlog
335 335 # validate incoming csets have their manifests
336 336 for cset in pycompat.xrange(clstart, clend):
337 337 mfnode = cl.changelogrevision(cset).manifest
338 338 mfest = ml[mfnode].readdelta()
339 339 # store file cgnodes we must see
340 340 for f, n in mfest.iteritems():
341 341 needfiles.setdefault(f, set()).add(n)
342 342
343 343 # process the files
344 344 repo.ui.status(_("adding file changes\n"))
345 345 newrevs, newfiles = _addchangegroupfiles(
346 346 repo, self, revmap, trp, efiles, needfiles)
347 347 revisions += newrevs
348 348 files += newfiles
349 349
350 350 deltaheads = 0
351 351 if oldheads:
352 352 heads = cl.heads()
353 353 deltaheads = len(heads) - len(oldheads)
354 354 for h in heads:
355 355 if h not in oldheads and repo[h].closesbranch():
356 356 deltaheads -= 1
357 357 htext = ""
358 358 if deltaheads:
359 359 htext = _(" (%+d heads)") % deltaheads
360 360
361 361 repo.ui.status(_("added %d changesets"
362 362 " with %d changes to %d files%s\n")
363 363 % (changesets, revisions, files, htext))
364 364 repo.invalidatevolatilesets()
365 365
366 366 if changesets > 0:
367 367 if 'node' not in tr.hookargs:
368 368 tr.hookargs['node'] = hex(cl.node(clstart))
369 369 tr.hookargs['node_last'] = hex(cl.node(clend - 1))
370 370 hookargs = dict(tr.hookargs)
371 371 else:
372 372 hookargs = dict(tr.hookargs)
373 373 hookargs['node'] = hex(cl.node(clstart))
374 374 hookargs['node_last'] = hex(cl.node(clend - 1))
375 375 repo.hook('pretxnchangegroup',
376 376 throw=True, **pycompat.strkwargs(hookargs))
377 377
378 378 added = [cl.node(r) for r in pycompat.xrange(clstart, clend)]
379 379 phaseall = None
380 380 if srctype in ('push', 'serve'):
381 381 # Old servers can not push the boundary themselves.
382 382 # New servers won't push the boundary if changeset already
383 383 # exists locally as secret
384 384 #
385 385 # We should not use added here but the list of all change in
386 386 # the bundle
387 387 if repo.publishing():
388 388 targetphase = phaseall = phases.public
389 389 else:
390 390 # closer target phase computation
391 391
392 392 # Those changesets have been pushed from the
393 393 # outside, their phases are going to be pushed
394 394 # alongside. Therefor `targetphase` is
395 395 # ignored.
396 396 targetphase = phaseall = phases.draft
397 397 if added:
398 398 phases.registernew(repo, tr, targetphase, added)
399 399 if phaseall is not None:
400 400 phases.advanceboundary(repo, tr, phaseall, cgnodes)
401 401
402 402 if changesets > 0:
403 403
404 404 def runhooks():
405 405 # These hooks run when the lock releases, not when the
406 406 # transaction closes. So it's possible for the changelog
407 407 # to have changed since we last saw it.
408 408 if clstart >= len(repo):
409 409 return
410 410
411 411 repo.hook("changegroup", **pycompat.strkwargs(hookargs))
412 412
413 413 for n in added:
414 414 args = hookargs.copy()
415 415 args['node'] = hex(n)
416 416 del args['node_last']
417 417 repo.hook("incoming", **pycompat.strkwargs(args))
418 418
419 419 newheads = [h for h in repo.heads()
420 420 if h not in oldheads]
421 421 repo.ui.log("incoming",
422 422 "%d incoming changes - new heads: %s\n",
423 423 len(added),
424 424 ', '.join([hex(c[:6]) for c in newheads]))
425 425
426 426 tr.addpostclose('changegroup-runhooks-%020i' % clstart,
427 427 lambda tr: repo._afterlock(runhooks))
428 428 finally:
429 429 repo.ui.flush()
430 430 # never return 0 here:
431 431 if deltaheads < 0:
432 432 ret = deltaheads - 1
433 433 else:
434 434 ret = deltaheads + 1
435 435 return ret
436 436
437 437 def deltaiter(self):
438 438 """
439 439 returns an iterator of the deltas in this changegroup
440 440
441 441 Useful for passing to the underlying storage system to be stored.
442 442 """
443 443 chain = None
444 444 for chunkdata in iter(lambda: self.deltachunk(chain), {}):
445 445 # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags)
446 446 yield chunkdata
447 447 chain = chunkdata[0]
448 448
449 449 class cg2unpacker(cg1unpacker):
450 450 """Unpacker for cg2 streams.
451 451
452 452 cg2 streams add support for generaldelta, so the delta header
453 453 format is slightly different. All other features about the data
454 454 remain the same.
455 455 """
456 456 deltaheader = _CHANGEGROUPV2_DELTA_HEADER
457 457 deltaheadersize = deltaheader.size
458 458 version = '02'
459 459
460 460 def _deltaheader(self, headertuple, prevnode):
461 461 node, p1, p2, deltabase, cs = headertuple
462 462 flags = 0
463 463 return node, p1, p2, deltabase, cs, flags
464 464
465 465 class cg3unpacker(cg2unpacker):
466 466 """Unpacker for cg3 streams.
467 467
468 468 cg3 streams add support for exchanging treemanifests and revlog
469 469 flags. It adds the revlog flags to the delta header and an empty chunk
470 470 separating manifests and files.
471 471 """
472 472 deltaheader = _CHANGEGROUPV3_DELTA_HEADER
473 473 deltaheadersize = deltaheader.size
474 474 version = '03'
475 475 _grouplistcount = 2 # One list of manifests and one list of files
476 476
477 477 def _deltaheader(self, headertuple, prevnode):
478 478 node, p1, p2, deltabase, cs, flags = headertuple
479 479 return node, p1, p2, deltabase, cs, flags
480 480
481 481 def _unpackmanifests(self, repo, revmap, trp, prog):
482 482 super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog)
483 483 for chunkdata in iter(self.filelogheader, {}):
484 484 # If we get here, there are directory manifests in the changegroup
485 485 d = chunkdata["filename"]
486 486 repo.ui.debug("adding %s revisions\n" % d)
487 487 dirlog = repo.manifestlog._revlog.dirlog(d)
488 488 deltas = self.deltaiter()
489 489 if not dirlog.addgroup(deltas, revmap, trp):
490 490 raise error.Abort(_("received dir revlog group is empty"))
491 491
492 492 class headerlessfixup(object):
493 493 def __init__(self, fh, h):
494 494 self._h = h
495 495 self._fh = fh
496 496 def read(self, n):
497 497 if self._h:
498 498 d, self._h = self._h[:n], self._h[n:]
499 499 if len(d) < n:
500 500 d += readexactly(self._fh, n - len(d))
501 501 return d
502 502 return readexactly(self._fh, n)
503 503
504 504 @interfaceutil.implementer(repository.irevisiondeltarequest)
505 505 @attr.s(slots=True, frozen=True)
506 506 class revisiondeltarequest(object):
507 507 node = attr.ib()
508 508 linknode = attr.ib()
509 509 p1node = attr.ib()
510 510 p2node = attr.ib()
511 511 basenode = attr.ib()
512 512 ellipsis = attr.ib(default=False)
513 513
514 514 def _revisiondeltatochunks(delta, headerfn):
515 515 """Serialize a revisiondelta to changegroup chunks."""
516 516
517 517 # The captured revision delta may be encoded as a delta against
518 518 # a base revision or as a full revision. The changegroup format
519 519 # requires that everything on the wire be deltas. So for full
520 520 # revisions, we need to invent a header that says to rewrite
521 521 # data.
522 522
523 523 if delta.delta is not None:
524 524 prefix, data = b'', delta.delta
525 525 elif delta.basenode == nullid:
526 526 data = delta.revision
527 527 prefix = mdiff.trivialdiffheader(len(data))
528 528 else:
529 529 data = delta.revision
530 530 prefix = mdiff.replacediffheader(delta.baserevisionsize,
531 531 len(data))
532 532
533 533 meta = headerfn(delta)
534 534
535 535 yield chunkheader(len(meta) + len(prefix) + len(data))
536 536 yield meta
537 537 if prefix:
538 538 yield prefix
539 539 yield data
540 540
541 541 def _sortnodesnormal(store, nodes, reorder):
542 542 """Sort nodes for changegroup generation and turn into revnums."""
543 543 # for generaldelta revlogs, we linearize the revs; this will both be
544 544 # much quicker and generate a much smaller bundle
545 545 if (store._generaldelta and reorder is None) or reorder:
546 546 revs = set(store.rev(n) for n in nodes)
547 547 return dagop.linearize(revs, store.parentrevs)
548 548 else:
549 549 return sorted([store.rev(n) for n in nodes])
550 550
551 551 def _sortnodesellipsis(store, nodes, cl, lookup):
552 552 """Sort nodes for changegroup generation and turn into revnums."""
553 553 # Ellipses serving mode.
554 554 #
555 555 # In a perfect world, we'd generate better ellipsis-ified graphs
556 556 # for non-changelog revlogs. In practice, we haven't started doing
557 557 # that yet, so the resulting DAGs for the manifestlog and filelogs
558 558 # are actually full of bogus parentage on all the ellipsis
559 559 # nodes. This has the side effect that, while the contents are
560 560 # correct, the individual DAGs might be completely out of whack in
561 561 # a case like 882681bc3166 and its ancestors (back about 10
562 562 # revisions or so) in the main hg repo.
563 563 #
564 564 # The one invariant we *know* holds is that the new (potentially
565 565 # bogus) DAG shape will be valid if we order the nodes in the
566 566 # order that they're introduced in dramatis personae by the
567 567 # changelog, so what we do is we sort the non-changelog histories
568 568 # by the order in which they are used by the changelog.
569 569 key = lambda n: cl.rev(lookup(n))
570 570 return [store.rev(n) for n in sorted(nodes, key=key)]
571 571
572 572 def _makenarrowdeltarequest(cl, store, ischangelog, rev, node, linkrev,
573 573 linknode, clrevtolocalrev, fullclnodes,
574 574 precomputedellipsis):
575 575 linkparents = precomputedellipsis[linkrev]
576 576 def local(clrev):
577 577 """Turn a changelog revnum into a local revnum.
578 578
579 579 The ellipsis dag is stored as revnums on the changelog,
580 580 but when we're producing ellipsis entries for
581 581 non-changelog revlogs, we need to turn those numbers into
582 582 something local. This does that for us, and during the
583 583 changelog sending phase will also expand the stored
584 584 mappings as needed.
585 585 """
586 586 if clrev == nullrev:
587 587 return nullrev
588 588
589 589 if ischangelog:
590 590 return clrev
591 591
592 592 # Walk the ellipsis-ized changelog breadth-first looking for a
593 593 # change that has been linked from the current revlog.
594 594 #
595 595 # For a flat manifest revlog only a single step should be necessary
596 596 # as all relevant changelog entries are relevant to the flat
597 597 # manifest.
598 598 #
599 599 # For a filelog or tree manifest dirlog however not every changelog
600 600 # entry will have been relevant, so we need to skip some changelog
601 601 # nodes even after ellipsis-izing.
602 602 walk = [clrev]
603 603 while walk:
604 604 p = walk[0]
605 605 walk = walk[1:]
606 606 if p in clrevtolocalrev:
607 607 return clrevtolocalrev[p]
608 608 elif p in fullclnodes:
609 609 walk.extend([pp for pp in cl.parentrevs(p)
610 610 if pp != nullrev])
611 611 elif p in precomputedellipsis:
612 612 walk.extend([pp for pp in precomputedellipsis[p]
613 613 if pp != nullrev])
614 614 else:
615 615 # In this case, we've got an ellipsis with parents
616 616 # outside the current bundle (likely an
617 617 # incremental pull). We "know" that we can use the
618 618 # value of this same revlog at whatever revision
619 619 # is pointed to by linknode. "Know" is in scare
620 620 # quotes because I haven't done enough examination
621 621 # of edge cases to convince myself this is really
622 622 # a fact - it works for all the (admittedly
623 623 # thorough) cases in our testsuite, but I would be
624 624 # somewhat unsurprised to find a case in the wild
625 625 # where this breaks down a bit. That said, I don't
626 626 # know if it would hurt anything.
627 627 for i in pycompat.xrange(rev, 0, -1):
628 628 if store.linkrev(i) == clrev:
629 629 return i
630 630 # We failed to resolve a parent for this node, so
631 631 # we crash the changegroup construction.
632 632 raise error.Abort(
633 633 'unable to resolve parent while packing %r %r'
634 634 ' for changeset %r' % (store.indexfile, rev, clrev))
635 635
636 636 return nullrev
637 637
638 638 if not linkparents or (
639 639 store.parentrevs(rev) == (nullrev, nullrev)):
640 640 p1, p2 = nullrev, nullrev
641 641 elif len(linkparents) == 1:
642 642 p1, = sorted(local(p) for p in linkparents)
643 643 p2 = nullrev
644 644 else:
645 645 p1, p2 = sorted(local(p) for p in linkparents)
646 646
647 647 p1node, p2node = store.node(p1), store.node(p2)
648 648
649 649 # TODO: try and actually send deltas for ellipsis data blocks
650 650 return revisiondeltarequest(
651 651 node=node,
652 652 p1node=p1node,
653 653 p2node=p2node,
654 654 linknode=linknode,
655 655 basenode=nullid,
656 656 ellipsis=True,
657 657 )
658 658
659 659 def deltagroup(repo, store, nodes, ischangelog, lookup, forcedeltaparentprev,
660 660 allowreorder,
661 661 units=None,
662 662 ellipses=False, clrevtolocalrev=None, fullclnodes=None,
663 663 precomputedellipsis=None):
664 664 """Calculate deltas for a set of revisions.
665 665
666 666 Is a generator of ``revisiondelta`` instances.
667 667
668 668 If units is not None, progress detail will be generated, units specifies
669 669 the type of revlog that is touched (changelog, manifest, etc.).
670 670 """
671 671 if not nodes:
672 672 return
673 673
674 674 # We perform two passes over the revisions whose data we will emit.
675 675 #
676 676 # In the first pass, we obtain information about the deltas that will
677 677 # be generated. This involves computing linknodes and adjusting the
678 678 # request to take shallow fetching into account. The end result of
679 679 # this pass is a list of "request" objects stating which deltas
680 680 # to obtain.
681 681 #
682 682 # The second pass is simply resolving the requested deltas.
683 683
684 684 cl = repo.changelog
685 685
686 686 if ischangelog:
687 687 # Changelog doesn't benefit from reordering revisions. So send
688 688 # out revisions in store order.
689 689 # TODO the API would be cleaner if this were controlled by the
690 690 # store producing the deltas.
691 691 revs = sorted(cl.rev(n) for n in nodes)
692 692 elif ellipses:
693 693 revs = _sortnodesellipsis(store, nodes, cl, lookup)
694 694 else:
695 695 revs = _sortnodesnormal(store, nodes, allowreorder)
696 696
697 697 # In the first pass, collect info about the deltas we'll be
698 698 # generating.
699 699 requests = []
700 700
701 701 # Add the parent of the first rev.
702 702 revs.insert(0, store.parentrevs(revs[0])[0])
703 703
704 704 for i in pycompat.xrange(len(revs) - 1):
705 705 prev = revs[i]
706 706 curr = revs[i + 1]
707 707
708 708 node = store.node(curr)
709 709 linknode = lookup(node)
710 710 p1node, p2node = store.parents(node)
711 711
712 712 if ellipses:
713 713 linkrev = cl.rev(linknode)
714 714 clrevtolocalrev[linkrev] = curr
715 715
716 716 # This is a node to send in full, because the changeset it
717 717 # corresponds to was a full changeset.
718 718 if linknode in fullclnodes:
719 719 requests.append(revisiondeltarequest(
720 720 node=node,
721 721 p1node=p1node,
722 722 p2node=p2node,
723 723 linknode=linknode,
724 724 basenode=None,
725 725 ))
726 726
727 727 elif linkrev not in precomputedellipsis:
728 728 pass
729 729 else:
730 730 requests.append(_makenarrowdeltarequest(
731 731 cl, store, ischangelog, curr, node, linkrev, linknode,
732 732 clrevtolocalrev, fullclnodes,
733 733 precomputedellipsis))
734 734 else:
735 735 requests.append(revisiondeltarequest(
736 736 node=node,
737 737 p1node=p1node,
738 738 p2node=p2node,
739 739 linknode=linknode,
740 740 basenode=store.node(prev) if forcedeltaparentprev else None,
741 741 ))
742 742
743 743 # We expect the first pass to be fast, so we only engage the progress
744 744 # meter for constructing the revision deltas.
745 745 progress = None
746 746 if units is not None:
747 747 progress = repo.ui.makeprogress(_('bundling'), unit=units,
748 748 total=len(requests))
749 749
750 750 for i, delta in enumerate(store.emitrevisiondeltas(requests)):
751 751 if progress:
752 752 progress.update(i + 1)
753 753
754 754 yield delta
755 755
756 756 if progress:
757 757 progress.complete()
758 758
759 759 class cgpacker(object):
760 760 def __init__(self, repo, filematcher, version, allowreorder,
761 761 builddeltaheader, manifestsend,
762 762 forcedeltaparentprev=False,
763 763 bundlecaps=None, ellipses=False,
764 764 shallow=False, ellipsisroots=None, fullnodes=None):
765 765 """Given a source repo, construct a bundler.
766 766
767 767 filematcher is a matcher that matches on files to include in the
768 768 changegroup. Used to facilitate sparse changegroups.
769 769
770 770 allowreorder controls whether reordering of revisions is allowed.
771 771 This value is used when ``bundle.reorder`` is ``auto`` or isn't
772 772 set.
773 773
774 774 forcedeltaparentprev indicates whether delta parents must be against
775 775 the previous revision in a delta group. This should only be used for
776 776 compatibility with changegroup version 1.
777 777
778 778 builddeltaheader is a callable that constructs the header for a group
779 779 delta.
780 780
781 781 manifestsend is a chunk to send after manifests have been fully emitted.
782 782
783 783 ellipses indicates whether ellipsis serving mode is enabled.
784 784
785 785 bundlecaps is optional and can be used to specify the set of
786 786 capabilities which can be used to build the bundle. While bundlecaps is
787 787 unused in core Mercurial, extensions rely on this feature to communicate
788 788 capabilities to customize the changegroup packer.
789 789
790 790 shallow indicates whether shallow data might be sent. The packer may
791 791 need to pack file contents not introduced by the changes being packed.
792 792
793 793 fullnodes is the set of changelog nodes which should not be ellipsis
794 794 nodes. We store this rather than the set of nodes that should be
795 795 ellipsis because for very large histories we expect this to be
796 796 significantly smaller.
797 797 """
798 798 assert filematcher
799 799 self._filematcher = filematcher
800 800
801 801 self.version = version
802 802 self._forcedeltaparentprev = forcedeltaparentprev
803 803 self._builddeltaheader = builddeltaheader
804 804 self._manifestsend = manifestsend
805 805 self._ellipses = ellipses
806 806
807 807 # Set of capabilities we can use to build the bundle.
808 808 if bundlecaps is None:
809 809 bundlecaps = set()
810 810 self._bundlecaps = bundlecaps
811 811 self._isshallow = shallow
812 812 self._fullclnodes = fullnodes
813 813
814 814 # Maps ellipsis revs to their roots at the changelog level.
815 815 self._precomputedellipsis = ellipsisroots
816 816
817 817 # experimental config: bundle.reorder
818 818 reorder = repo.ui.config('bundle', 'reorder')
819 819 if reorder == 'auto':
820 820 self._reorder = allowreorder
821 821 else:
822 822 self._reorder = stringutil.parsebool(reorder)
823 823
824 824 self._repo = repo
825 825
826 826 if self._repo.ui.verbose and not self._repo.ui.debugflag:
827 827 self._verbosenote = self._repo.ui.note
828 828 else:
829 829 self._verbosenote = lambda s: None
830 830
831 831 def generate(self, commonrevs, clnodes, fastpathlinkrev, source):
832 832 """Yield a sequence of changegroup byte chunks."""
833 833
834 834 repo = self._repo
835 835 cl = repo.changelog
836 836
837 837 self._verbosenote(_('uncompressed size of bundle content:\n'))
838 838 size = 0
839 839
840 840 clstate, deltas = self._generatechangelog(cl, clnodes)
841 841 for delta in deltas:
842 842 for chunk in _revisiondeltatochunks(delta, self._builddeltaheader):
843 843 size += len(chunk)
844 844 yield chunk
845 845
846 846 close = closechunk()
847 847 size += len(close)
848 848 yield closechunk()
849 849
850 850 self._verbosenote(_('%8.i (changelog)\n') % size)
851 851
852 852 clrevorder = clstate['clrevorder']
853 853 mfs = clstate['mfs']
854 854 changedfiles = clstate['changedfiles']
855 855
856 856 # We need to make sure that the linkrev in the changegroup refers to
857 857 # the first changeset that introduced the manifest or file revision.
858 858 # The fastpath is usually safer than the slowpath, because the filelogs
859 859 # are walked in revlog order.
860 860 #
861 861 # When taking the slowpath with reorder=None and the manifest revlog
862 862 # uses generaldelta, the manifest may be walked in the "wrong" order.
863 863 # Without 'clrevorder', we would get an incorrect linkrev (see fix in
864 864 # cc0ff93d0c0c).
865 865 #
866 866 # When taking the fastpath, we are only vulnerable to reordering
867 867 # of the changelog itself. The changelog never uses generaldelta, so
868 868 # it is only reordered when reorder=True. To handle this case, we
869 869 # simply take the slowpath, which already has the 'clrevorder' logic.
870 870 # This was also fixed in cc0ff93d0c0c.
871 871 fastpathlinkrev = fastpathlinkrev and not self._reorder
872 872 # Treemanifests don't work correctly with fastpathlinkrev
873 873 # either, because we don't discover which directory nodes to
874 874 # send along with files. This could probably be fixed.
875 875 fastpathlinkrev = fastpathlinkrev and (
876 876 'treemanifest' not in repo.requirements)
877 877
878 878 fnodes = {} # needed file nodes
879 879
880 880 size = 0
881 881 it = self.generatemanifests(
882 882 commonrevs, clrevorder, fastpathlinkrev, mfs, fnodes, source,
883 883 clstate['clrevtomanifestrev'])
884 884
885 885 for tree, deltas in it:
886 886 if tree:
887 887 assert self.version == b'03'
888 888 chunk = _fileheader(tree)
889 889 size += len(chunk)
890 890 yield chunk
891 891
892 892 for delta in deltas:
893 893 chunks = _revisiondeltatochunks(delta, self._builddeltaheader)
894 894 for chunk in chunks:
895 895 size += len(chunk)
896 896 yield chunk
897 897
898 898 close = closechunk()
899 899 size += len(close)
900 900 yield close
901 901
902 902 self._verbosenote(_('%8.i (manifests)\n') % size)
903 903 yield self._manifestsend
904 904
905 905 mfdicts = None
906 906 if self._ellipses and self._isshallow:
907 907 mfdicts = [(self._repo.manifestlog[n].read(), lr)
908 908 for (n, lr) in mfs.iteritems()]
909 909
910 910 mfs.clear()
911 911 clrevs = set(cl.rev(x) for x in clnodes)
912 912
913 913 it = self.generatefiles(changedfiles, commonrevs,
914 914 source, mfdicts, fastpathlinkrev,
915 915 fnodes, clrevs)
916 916
917 917 for path, deltas in it:
918 918 h = _fileheader(path)
919 919 size = len(h)
920 920 yield h
921 921
922 922 for delta in deltas:
923 923 chunks = _revisiondeltatochunks(delta, self._builddeltaheader)
924 924 for chunk in chunks:
925 925 size += len(chunk)
926 926 yield chunk
927 927
928 928 close = closechunk()
929 929 size += len(close)
930 930 yield close
931 931
932 932 self._verbosenote(_('%8.i %s\n') % (size, path))
933 933
934 934 yield closechunk()
935 935
936 936 if clnodes:
937 937 repo.hook('outgoing', node=hex(clnodes[0]), source=source)
938 938
939 939 def _generatechangelog(self, cl, nodes):
940 940 """Generate data for changelog chunks.
941 941
942 942 Returns a 2-tuple of a dict containing state and an iterable of
943 943 byte chunks. The state will not be fully populated until the
944 944 chunk stream has been fully consumed.
945 945 """
946 946 clrevorder = {}
947 947 mfs = {} # needed manifests
948 948 mfl = self._repo.manifestlog
949 949 changedfiles = set()
950 950 clrevtomanifestrev = {}
951 951
952 952 # Callback for the changelog, used to collect changed files and
953 953 # manifest nodes.
954 954 # Returns the linkrev node (identity in the changelog case).
955 955 def lookupcl(x):
956 c = cl.read(x)
956 c = cl.changelogrevision(x)
957 957 clrevorder[x] = len(clrevorder)
958 958
959 959 if self._ellipses:
960 960 # Only update mfs if x is going to be sent. Otherwise we
961 961 # end up with bogus linkrevs specified for manifests and
962 962 # we skip some manifest nodes that we should otherwise
963 963 # have sent.
964 964 if (x in self._fullclnodes
965 965 or cl.rev(x) in self._precomputedellipsis):
966 n = c[0]
966
967 manifestnode = c.manifest
967 968 # Record the first changeset introducing this manifest
968 969 # version.
969 mfs.setdefault(n, x)
970 mfs.setdefault(manifestnode, x)
970 971 # Set this narrow-specific dict so we have the lowest
971 972 # manifest revnum to look up for this cl revnum. (Part of
972 973 # mapping changelog ellipsis parents to manifest ellipsis
973 974 # parents)
974 clrevtomanifestrev.setdefault(cl.rev(x), mfl.rev(n))
975 clrevtomanifestrev.setdefault(
976 cl.rev(x), mfl.rev(manifestnode))
975 977 # We can't trust the changed files list in the changeset if the
976 978 # client requested a shallow clone.
977 979 if self._isshallow:
978 changedfiles.update(mfl[c[0]].read().keys())
980 changedfiles.update(mfl[c.manifest].read().keys())
979 981 else:
980 changedfiles.update(c[3])
982 changedfiles.update(c.files)
981 983 else:
982
983 n = c[0]
984 984 # record the first changeset introducing this manifest version
985 mfs.setdefault(n, x)
985 mfs.setdefault(c.manifest, x)
986 986 # Record a complete list of potentially-changed files in
987 987 # this manifest.
988 changedfiles.update(c[3])
988 changedfiles.update(c.files)
989 989
990 990 return x
991 991
992 992 state = {
993 993 'clrevorder': clrevorder,
994 994 'mfs': mfs,
995 995 'changedfiles': changedfiles,
996 996 'clrevtomanifestrev': clrevtomanifestrev,
997 997 }
998 998
999 999 gen = deltagroup(
1000 1000 self._repo, cl, nodes, True, lookupcl,
1001 1001 self._forcedeltaparentprev,
1002 1002 # Reorder settings are currently ignored for changelog.
1003 1003 True,
1004 1004 ellipses=self._ellipses,
1005 1005 units=_('changesets'),
1006 1006 clrevtolocalrev={},
1007 1007 fullclnodes=self._fullclnodes,
1008 1008 precomputedellipsis=self._precomputedellipsis)
1009 1009
1010 1010 return state, gen
1011 1011
1012 1012 def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs,
1013 1013 fnodes, source, clrevtolocalrev):
1014 1014 """Returns an iterator of changegroup chunks containing manifests.
1015 1015
1016 1016 `source` is unused here, but is used by extensions like remotefilelog to
1017 1017 change what is sent based in pulls vs pushes, etc.
1018 1018 """
1019 1019 repo = self._repo
1020 1020 mfl = repo.manifestlog
1021 1021 dirlog = mfl._revlog.dirlog
1022 1022 tmfnodes = {'': mfs}
1023 1023
1024 1024 # Callback for the manifest, used to collect linkrevs for filelog
1025 1025 # revisions.
1026 1026 # Returns the linkrev node (collected in lookupcl).
1027 1027 def makelookupmflinknode(tree, nodes):
1028 1028 if fastpathlinkrev:
1029 1029 assert not tree
1030 1030 return mfs.__getitem__
1031 1031
1032 1032 def lookupmflinknode(x):
1033 1033 """Callback for looking up the linknode for manifests.
1034 1034
1035 1035 Returns the linkrev node for the specified manifest.
1036 1036
1037 1037 SIDE EFFECT:
1038 1038
1039 1039 1) fclnodes gets populated with the list of relevant
1040 1040 file nodes if we're not using fastpathlinkrev
1041 1041 2) When treemanifests are in use, collects treemanifest nodes
1042 1042 to send
1043 1043
1044 1044 Note that this means manifests must be completely sent to
1045 1045 the client before you can trust the list of files and
1046 1046 treemanifests to send.
1047 1047 """
1048 1048 clnode = nodes[x]
1049 1049 mdata = mfl.get(tree, x).readfast(shallow=True)
1050 1050 for p, n, fl in mdata.iterentries():
1051 1051 if fl == 't': # subdirectory manifest
1052 1052 subtree = tree + p + '/'
1053 1053 tmfclnodes = tmfnodes.setdefault(subtree, {})
1054 1054 tmfclnode = tmfclnodes.setdefault(n, clnode)
1055 1055 if clrevorder[clnode] < clrevorder[tmfclnode]:
1056 1056 tmfclnodes[n] = clnode
1057 1057 else:
1058 1058 f = tree + p
1059 1059 fclnodes = fnodes.setdefault(f, {})
1060 1060 fclnode = fclnodes.setdefault(n, clnode)
1061 1061 if clrevorder[clnode] < clrevorder[fclnode]:
1062 1062 fclnodes[n] = clnode
1063 1063 return clnode
1064 1064 return lookupmflinknode
1065 1065
1066 1066 while tmfnodes:
1067 1067 tree, nodes = tmfnodes.popitem()
1068 1068 store = dirlog(tree)
1069 1069
1070 1070 if not self._filematcher.visitdir(store._dir[:-1] or '.'):
1071 1071 prunednodes = []
1072 1072 else:
1073 1073 frev, flr = store.rev, store.linkrev
1074 1074 prunednodes = [n for n in nodes
1075 1075 if flr(frev(n)) not in commonrevs]
1076 1076
1077 1077 if tree and not prunednodes:
1078 1078 continue
1079 1079
1080 1080 lookupfn = makelookupmflinknode(tree, nodes)
1081 1081
1082 1082 deltas = deltagroup(
1083 1083 self._repo, store, prunednodes, False, lookupfn,
1084 1084 self._forcedeltaparentprev, self._reorder,
1085 1085 ellipses=self._ellipses,
1086 1086 units=_('manifests'),
1087 1087 clrevtolocalrev=clrevtolocalrev,
1088 1088 fullclnodes=self._fullclnodes,
1089 1089 precomputedellipsis=self._precomputedellipsis)
1090 1090
1091 1091 yield tree, deltas
1092 1092
1093 1093 # The 'source' parameter is useful for extensions
1094 1094 def generatefiles(self, changedfiles, commonrevs, source,
1095 1095 mfdicts, fastpathlinkrev, fnodes, clrevs):
1096 1096 changedfiles = list(filter(self._filematcher, changedfiles))
1097 1097
1098 1098 if not fastpathlinkrev:
1099 1099 def normallinknodes(unused, fname):
1100 1100 return fnodes.get(fname, {})
1101 1101 else:
1102 1102 cln = self._repo.changelog.node
1103 1103
1104 1104 def normallinknodes(store, fname):
1105 1105 flinkrev = store.linkrev
1106 1106 fnode = store.node
1107 1107 revs = ((r, flinkrev(r)) for r in store)
1108 1108 return dict((fnode(r), cln(lr))
1109 1109 for r, lr in revs if lr in clrevs)
1110 1110
1111 1111 clrevtolocalrev = {}
1112 1112
1113 1113 if self._isshallow:
1114 1114 # In a shallow clone, the linknodes callback needs to also include
1115 1115 # those file nodes that are in the manifests we sent but weren't
1116 1116 # introduced by those manifests.
1117 1117 commonctxs = [self._repo[c] for c in commonrevs]
1118 1118 clrev = self._repo.changelog.rev
1119 1119
1120 1120 # Defining this function has a side-effect of overriding the
1121 1121 # function of the same name that was passed in as an argument.
1122 1122 # TODO have caller pass in appropriate function.
1123 1123 def linknodes(flog, fname):
1124 1124 for c in commonctxs:
1125 1125 try:
1126 1126 fnode = c.filenode(fname)
1127 1127 clrevtolocalrev[c.rev()] = flog.rev(fnode)
1128 1128 except error.ManifestLookupError:
1129 1129 pass
1130 1130 links = normallinknodes(flog, fname)
1131 1131 if len(links) != len(mfdicts):
1132 1132 for mf, lr in mfdicts:
1133 1133 fnode = mf.get(fname, None)
1134 1134 if fnode in links:
1135 1135 links[fnode] = min(links[fnode], lr, key=clrev)
1136 1136 elif fnode:
1137 1137 links[fnode] = lr
1138 1138 return links
1139 1139 else:
1140 1140 linknodes = normallinknodes
1141 1141
1142 1142 repo = self._repo
1143 1143 progress = repo.ui.makeprogress(_('bundling'), unit=_('files'),
1144 1144 total=len(changedfiles))
1145 1145 for i, fname in enumerate(sorted(changedfiles)):
1146 1146 filerevlog = repo.file(fname)
1147 1147 if not filerevlog:
1148 1148 raise error.Abort(_("empty or missing file data for %s") %
1149 1149 fname)
1150 1150
1151 1151 clrevtolocalrev.clear()
1152 1152
1153 1153 linkrevnodes = linknodes(filerevlog, fname)
1154 1154 # Lookup for filenodes, we collected the linkrev nodes above in the
1155 1155 # fastpath case and with lookupmf in the slowpath case.
1156 1156 def lookupfilelog(x):
1157 1157 return linkrevnodes[x]
1158 1158
1159 1159 frev, flr = filerevlog.rev, filerevlog.linkrev
1160 1160 filenodes = [n for n in linkrevnodes
1161 1161 if flr(frev(n)) not in commonrevs]
1162 1162
1163 1163 if not filenodes:
1164 1164 continue
1165 1165
1166 1166 progress.update(i + 1, item=fname)
1167 1167
1168 1168 deltas = deltagroup(
1169 1169 self._repo, filerevlog, filenodes, False, lookupfilelog,
1170 1170 self._forcedeltaparentprev, self._reorder,
1171 1171 ellipses=self._ellipses,
1172 1172 clrevtolocalrev=clrevtolocalrev,
1173 1173 fullclnodes=self._fullclnodes,
1174 1174 precomputedellipsis=self._precomputedellipsis)
1175 1175
1176 1176 yield fname, deltas
1177 1177
1178 1178 progress.complete()
1179 1179
1180 1180 def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False,
1181 1181 shallow=False, ellipsisroots=None, fullnodes=None):
1182 1182 builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
1183 1183 d.node, d.p1node, d.p2node, d.linknode)
1184 1184
1185 1185 return cgpacker(repo, filematcher, b'01',
1186 1186 allowreorder=None,
1187 1187 builddeltaheader=builddeltaheader,
1188 1188 manifestsend=b'',
1189 1189 forcedeltaparentprev=True,
1190 1190 bundlecaps=bundlecaps,
1191 1191 ellipses=ellipses,
1192 1192 shallow=shallow,
1193 1193 ellipsisroots=ellipsisroots,
1194 1194 fullnodes=fullnodes)
1195 1195
1196 1196 def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False,
1197 1197 shallow=False, ellipsisroots=None, fullnodes=None):
1198 1198 builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
1199 1199 d.node, d.p1node, d.p2node, d.basenode, d.linknode)
1200 1200
1201 1201 # Since generaldelta is directly supported by cg2, reordering
1202 1202 # generally doesn't help, so we disable it by default (treating
1203 1203 # bundle.reorder=auto just like bundle.reorder=False).
1204 1204 return cgpacker(repo, filematcher, b'02',
1205 1205 allowreorder=False,
1206 1206 builddeltaheader=builddeltaheader,
1207 1207 manifestsend=b'',
1208 1208 bundlecaps=bundlecaps,
1209 1209 ellipses=ellipses,
1210 1210 shallow=shallow,
1211 1211 ellipsisroots=ellipsisroots,
1212 1212 fullnodes=fullnodes)
1213 1213
1214 1214 def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False,
1215 1215 shallow=False, ellipsisroots=None, fullnodes=None):
1216 1216 builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
1217 1217 d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags)
1218 1218
1219 1219 return cgpacker(repo, filematcher, b'03',
1220 1220 allowreorder=False,
1221 1221 builddeltaheader=builddeltaheader,
1222 1222 manifestsend=closechunk(),
1223 1223 bundlecaps=bundlecaps,
1224 1224 ellipses=ellipses,
1225 1225 shallow=shallow,
1226 1226 ellipsisroots=ellipsisroots,
1227 1227 fullnodes=fullnodes)
1228 1228
1229 1229 _packermap = {'01': (_makecg1packer, cg1unpacker),
1230 1230 # cg2 adds support for exchanging generaldelta
1231 1231 '02': (_makecg2packer, cg2unpacker),
1232 1232 # cg3 adds support for exchanging revlog flags and treemanifests
1233 1233 '03': (_makecg3packer, cg3unpacker),
1234 1234 }
1235 1235
1236 1236 def allsupportedversions(repo):
1237 1237 versions = set(_packermap.keys())
1238 1238 if not (repo.ui.configbool('experimental', 'changegroup3') or
1239 1239 repo.ui.configbool('experimental', 'treemanifest') or
1240 1240 'treemanifest' in repo.requirements):
1241 1241 versions.discard('03')
1242 1242 return versions
1243 1243
1244 1244 # Changegroup versions that can be applied to the repo
1245 1245 def supportedincomingversions(repo):
1246 1246 return allsupportedversions(repo)
1247 1247
1248 1248 # Changegroup versions that can be created from the repo
1249 1249 def supportedoutgoingversions(repo):
1250 1250 versions = allsupportedversions(repo)
1251 1251 if 'treemanifest' in repo.requirements:
1252 1252 # Versions 01 and 02 support only flat manifests and it's just too
1253 1253 # expensive to convert between the flat manifest and tree manifest on
1254 1254 # the fly. Since tree manifests are hashed differently, all of history
1255 1255 # would have to be converted. Instead, we simply don't even pretend to
1256 1256 # support versions 01 and 02.
1257 1257 versions.discard('01')
1258 1258 versions.discard('02')
1259 1259 if repository.NARROW_REQUIREMENT in repo.requirements:
1260 1260 # Versions 01 and 02 don't support revlog flags, and we need to
1261 1261 # support that for stripping and unbundling to work.
1262 1262 versions.discard('01')
1263 1263 versions.discard('02')
1264 1264 if LFS_REQUIREMENT in repo.requirements:
1265 1265 # Versions 01 and 02 don't support revlog flags, and we need to
1266 1266 # mark LFS entries with REVIDX_EXTSTORED.
1267 1267 versions.discard('01')
1268 1268 versions.discard('02')
1269 1269
1270 1270 return versions
1271 1271
1272 1272 def localversion(repo):
1273 1273 # Finds the best version to use for bundles that are meant to be used
1274 1274 # locally, such as those from strip and shelve, and temporary bundles.
1275 1275 return max(supportedoutgoingversions(repo))
1276 1276
1277 1277 def safeversion(repo):
1278 1278 # Finds the smallest version that it's safe to assume clients of the repo
1279 1279 # will support. For example, all hg versions that support generaldelta also
1280 1280 # support changegroup 02.
1281 1281 versions = supportedoutgoingversions(repo)
1282 1282 if 'generaldelta' in repo.requirements:
1283 1283 versions.discard('01')
1284 1284 assert versions
1285 1285 return min(versions)
1286 1286
1287 1287 def getbundler(version, repo, bundlecaps=None, filematcher=None,
1288 1288 ellipses=False, shallow=False, ellipsisroots=None,
1289 1289 fullnodes=None):
1290 1290 assert version in supportedoutgoingversions(repo)
1291 1291
1292 1292 if filematcher is None:
1293 1293 filematcher = matchmod.alwaysmatcher(repo.root, '')
1294 1294
1295 1295 if version == '01' and not filematcher.always():
1296 1296 raise error.ProgrammingError('version 01 changegroups do not support '
1297 1297 'sparse file matchers')
1298 1298
1299 1299 if ellipses and version in (b'01', b'02'):
1300 1300 raise error.Abort(
1301 1301 _('ellipsis nodes require at least cg3 on client and server, '
1302 1302 'but negotiated version %s') % version)
1303 1303
1304 1304 # Requested files could include files not in the local store. So
1305 1305 # filter those out.
1306 1306 filematcher = matchmod.intersectmatchers(repo.narrowmatch(),
1307 1307 filematcher)
1308 1308
1309 1309 fn = _packermap[version][0]
1310 1310 return fn(repo, filematcher, bundlecaps, ellipses=ellipses,
1311 1311 shallow=shallow, ellipsisroots=ellipsisroots,
1312 1312 fullnodes=fullnodes)
1313 1313
1314 1314 def getunbundler(version, fh, alg, extras=None):
1315 1315 return _packermap[version][1](fh, alg, extras=extras)
1316 1316
1317 1317 def _changegroupinfo(repo, nodes, source):
1318 1318 if repo.ui.verbose or source == 'bundle':
1319 1319 repo.ui.status(_("%d changesets found\n") % len(nodes))
1320 1320 if repo.ui.debugflag:
1321 1321 repo.ui.debug("list of changesets:\n")
1322 1322 for node in nodes:
1323 1323 repo.ui.debug("%s\n" % hex(node))
1324 1324
1325 1325 def makechangegroup(repo, outgoing, version, source, fastpath=False,
1326 1326 bundlecaps=None):
1327 1327 cgstream = makestream(repo, outgoing, version, source,
1328 1328 fastpath=fastpath, bundlecaps=bundlecaps)
1329 1329 return getunbundler(version, util.chunkbuffer(cgstream), None,
1330 1330 {'clcount': len(outgoing.missing) })
1331 1331
1332 1332 def makestream(repo, outgoing, version, source, fastpath=False,
1333 1333 bundlecaps=None, filematcher=None):
1334 1334 bundler = getbundler(version, repo, bundlecaps=bundlecaps,
1335 1335 filematcher=filematcher)
1336 1336
1337 1337 repo = repo.unfiltered()
1338 1338 commonrevs = outgoing.common
1339 1339 csets = outgoing.missing
1340 1340 heads = outgoing.missingheads
1341 1341 # We go through the fast path if we get told to, or if all (unfiltered
1342 1342 # heads have been requested (since we then know there all linkrevs will
1343 1343 # be pulled by the client).
1344 1344 heads.sort()
1345 1345 fastpathlinkrev = fastpath or (
1346 1346 repo.filtername is None and heads == sorted(repo.heads()))
1347 1347
1348 1348 repo.hook('preoutgoing', throw=True, source=source)
1349 1349 _changegroupinfo(repo, csets, source)
1350 1350 return bundler.generate(commonrevs, csets, fastpathlinkrev, source)
1351 1351
1352 1352 def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles):
1353 1353 revisions = 0
1354 1354 files = 0
1355 1355 progress = repo.ui.makeprogress(_('files'), unit=_('files'),
1356 1356 total=expectedfiles)
1357 1357 for chunkdata in iter(source.filelogheader, {}):
1358 1358 files += 1
1359 1359 f = chunkdata["filename"]
1360 1360 repo.ui.debug("adding %s revisions\n" % f)
1361 1361 progress.increment()
1362 1362 fl = repo.file(f)
1363 1363 o = len(fl)
1364 1364 try:
1365 1365 deltas = source.deltaiter()
1366 1366 if not fl.addgroup(deltas, revmap, trp):
1367 1367 raise error.Abort(_("received file revlog group is empty"))
1368 1368 except error.CensoredBaseError as e:
1369 1369 raise error.Abort(_("received delta base is censored: %s") % e)
1370 1370 revisions += len(fl) - o
1371 1371 if f in needfiles:
1372 1372 needs = needfiles[f]
1373 1373 for new in pycompat.xrange(o, len(fl)):
1374 1374 n = fl.node(new)
1375 1375 if n in needs:
1376 1376 needs.remove(n)
1377 1377 else:
1378 1378 raise error.Abort(
1379 1379 _("received spurious file revlog entry"))
1380 1380 if not needs:
1381 1381 del needfiles[f]
1382 1382 progress.complete()
1383 1383
1384 1384 for f, needs in needfiles.iteritems():
1385 1385 fl = repo.file(f)
1386 1386 for n in needs:
1387 1387 try:
1388 1388 fl.rev(n)
1389 1389 except error.LookupError:
1390 1390 raise error.Abort(
1391 1391 _('missing file data for %s:%s - run hg verify') %
1392 1392 (f, hex(n)))
1393 1393
1394 1394 return revisions, files
General Comments 0
You need to be logged in to leave comments. Login now