changegroup.py
1691 lines
| 56.6 KiB
| text/x-python
|
PythonLexer
/ mercurial / changegroup.py
|
r8226 | # changegroup.py - Mercurial changegroup manipulation functions | ||
# | ||||
# Copyright 2006 Matt Mackall <mpm@selenic.com> | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r3877 | |||
|
r25921 | from __future__ import absolute_import | ||
import os | ||||
import struct | ||||
|
r20933 | import weakref | ||
|
r25921 | |||
from .i18n import _ | ||||
from .node import ( | ||||
hex, | ||||
|
r38919 | nullid, | ||
|
r25921 | nullrev, | ||
short, | ||||
) | ||||
|
r43355 | from .pycompat import open | ||
|
r25921 | |||
from . import ( | ||||
error, | ||||
|
r38830 | match as matchmod, | ||
|
r25921 | mdiff, | ||
phases, | ||||
|
r30925 | pycompat, | ||
|
r43078 | util, | ||
) | ||||
|
r43346 | from .interfaces import repository | ||
|
r1981 | |||
|
r43347 | _CHANGEGROUPV1_DELTA_HEADER = struct.Struct(b"20s20s20s20s") | ||
_CHANGEGROUPV2_DELTA_HEADER = struct.Struct(b"20s20s20s20s20s") | ||||
_CHANGEGROUPV3_DELTA_HEADER = struct.Struct(b">20s20s20s20s20sH") | ||||
|
r14141 | |||
|
r43347 | LFS_REQUIREMENT = b'lfs' | ||
|
r37150 | |||
|
r35772 | readexactly = util.readexactly | ||
|
r13457 | |||
|
r43346 | |||
|
r13457 | def getchunk(stream): | ||
"""return the next chunk from stream as a string""" | ||||
d = readexactly(stream, 4) | ||||
|
r43347 | l = struct.unpack(b">l", d)[0] | ||
|
r1981 | if l <= 4: | ||
|
r13458 | if l: | ||
|
r43347 | raise error.Abort(_(b"invalid chunk length %d") % l) | ||
return b"" | ||||
|
r13457 | return readexactly(stream, l - 4) | ||
|
r1981 | |||
|
r43346 | |||
|
r5368 | def chunkheader(length): | ||
|
r9437 | """return a changegroup chunk header (string)""" | ||
|
r43347 | return struct.pack(b">l", length + 4) | ||
|
r1981 | |||
|
r43346 | |||
|
r1981 | def closechunk(): | ||
|
r9437 | """return a changegroup chunk header (string) for a zero-length chunk""" | ||
|
r43347 | return struct.pack(b">l", 0) | ||
|
r1981 | |||
|
r43346 | |||
|
r39017 | def _fileheader(path): | ||
"""Obtain a changegroup chunk header for a named path.""" | ||||
return chunkheader(len(path)) + path | ||||
|
r43346 | |||
|
r26540 | def writechunks(ui, chunks, filename, vfs=None): | ||
"""Write chunks to a file and return its filename. | ||||
|
r3659 | |||
|
r26540 | The stream is assumed to be a bundle file. | ||
|
r3659 | Existing files will not be overwritten. | ||
If no filename is specified, a temporary file is created. | ||||
""" | ||||
fh = None | ||||
cleanup = None | ||||
try: | ||||
if filename: | ||||
|
r20976 | if vfs: | ||
|
r43347 | fh = vfs.open(filename, b"wb") | ||
|
r20976 | else: | ||
|
r30212 | # Increase default buffer size because default is usually | ||
# small (4k is common on Linux). | ||||
|
r43347 | fh = open(filename, b"wb", 131072) | ||
|
r3659 | else: | ||
|
r43347 | fd, filename = pycompat.mkstemp(prefix=b"hg-bundle-", suffix=b".hg") | ||
|
r43809 | fh = os.fdopen(fd, "wb") | ||
|
r3659 | cleanup = filename | ||
|
r26540 | for c in chunks: | ||
fh.write(c) | ||||
|
r3659 | cleanup = None | ||
return filename | ||||
finally: | ||||
if fh is not None: | ||||
fh.close() | ||||
if cleanup is not None: | ||||
|
r20976 | if filename and vfs: | ||
vfs.unlink(cleanup) | ||||
else: | ||||
os.unlink(cleanup) | ||||
|
r3660 | |||
|
r43346 | |||
|
r22390 | class cg1unpacker(object): | ||
|
r26708 | """Unpacker for cg1 changegroup streams. | ||
A changegroup unpacker handles the framing of the revision data in | ||||
the wire format. Most consumers will want to use the apply() | ||||
method to add the changes from the changegroup to a repository. | ||||
If you're forwarding a changegroup unmodified to another consumer, | ||||
use getchunks(), which returns an iterator of changegroup | ||||
chunks. This is mostly useful for cases where you need to know the | ||||
data stream has ended by observing the end of the changegroup. | ||||
deltachunk() is useful only if you're applying delta data. Most | ||||
consumers should prefer apply() instead. | ||||
A few other public methods exist. Those are used only for | ||||
bundlerepo and some debug commands - their use is discouraged. | ||||
""" | ||||
|
r43346 | |||
|
r22390 | deltaheader = _CHANGEGROUPV1_DELTA_HEADER | ||
|
r38932 | deltaheadersize = deltaheader.size | ||
|
r43347 | version = b'01' | ||
|
r43346 | _grouplistcount = 1 # One list of files after the manifests | ||
|
r27920 | |||
|
r29593 | def __init__(self, fh, alg, extras=None): | ||
|
r30354 | if alg is None: | ||
|
r43347 | alg = b'UN' | ||
|
r30354 | if alg not in util.compengines.supportedbundletypes: | ||
|
r43347 | raise error.Abort(_(b'unknown stream compression type: %s') % alg) | ||
if alg == b'BZ': | ||||
alg = b'_truncatedBZ' | ||||
|
r30354 | |||
compengine = util.compengines.forbundletype(alg) | ||||
self._stream = compengine.decompressorreader(fh) | ||||
|
r12044 | self._type = alg | ||
|
r29593 | self.extras = extras or {} | ||
|
r12334 | self.callback = None | ||
|
r26706 | |||
# These methods (compressed, read, seek, tell) all appear to only | ||||
# be used by bundlerepo, but it's a little hard to tell. | ||||
|
r12044 | def compressed(self): | ||
|
r43347 | return self._type is not None and self._type != b'UN' | ||
|
r43346 | |||
|
r12043 | def read(self, l): | ||
return self._stream.read(l) | ||||
|
r43346 | |||
|
r12330 | def seek(self, pos): | ||
return self._stream.seek(pos) | ||||
|
r43346 | |||
|
r12330 | def tell(self): | ||
|
r12332 | return self._stream.tell() | ||
|
r43346 | |||
|
r12347 | def close(self): | ||
return self._stream.close() | ||||
|
r12334 | |||
|
r26707 | def _chunklength(self): | ||
|
r13459 | d = readexactly(self._stream, 4) | ||
|
r43347 | l = struct.unpack(b">l", d)[0] | ||
|
r13458 | if l <= 4: | ||
if l: | ||||
|
r43347 | raise error.Abort(_(b"invalid chunk length %d") % l) | ||
|
r13458 | return 0 | ||
if self.callback: | ||||
|
r12334 | self.callback() | ||
|
r13458 | return l - 4 | ||
|
r12334 | |||
|
r14144 | def changelogheader(self): | ||
"""v10 does not have a changelog header chunk""" | ||||
return {} | ||||
def manifestheader(self): | ||||
"""v10 does not have a manifest header chunk""" | ||||
return {} | ||||
def filelogheader(self): | ||||
"""return the header of the filelogs chunk, v10 only has the filename""" | ||||
|
r26707 | l = self._chunklength() | ||
|
r14144 | if not l: | ||
return {} | ||||
fname = readexactly(self._stream, l) | ||||
|
r43347 | return {b'filename': fname} | ||
|
r12334 | |||
|
r14141 | def _deltaheader(self, headertuple, prevnode): | ||
node, p1, p2, cs = headertuple | ||||
if prevnode is None: | ||||
deltabase = p1 | ||||
else: | ||||
deltabase = prevnode | ||||
|
r27433 | flags = 0 | ||
return node, p1, p2, deltabase, cs, flags | ||||
|
r14141 | |||
|
r14144 | def deltachunk(self, prevnode): | ||
|
r26707 | l = self._chunklength() | ||
|
r12336 | if not l: | ||
return {} | ||||
|
r14141 | headerdata = readexactly(self._stream, self.deltaheadersize) | ||
|
r38932 | header = self.deltaheader.unpack(headerdata) | ||
|
r14141 | delta = readexactly(self._stream, l - self.deltaheadersize) | ||
|
r27433 | node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode) | ||
|
r34295 | return (node, p1, p2, cs, deltabase, delta, flags) | ||
|
r12336 | |||
|
r20999 | def getchunks(self): | ||
"""returns all the chunks contains in the bundle | ||||
Used when you need to forward the binary stream to a file or another | ||||
network API. To do so, it parse the changegroup data, otherwise it will | ||||
block in case of sshrepo because it don't know the end of the stream. | ||||
""" | ||||
|
r34093 | # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog, | ||
# and a list of filelogs. For changegroup 3, we expect 4 parts: | ||||
# changelog, manifestlog, a list of tree manifestlogs, and a list of | ||||
# filelogs. | ||||
# | ||||
# Changelog and manifestlog parts are terminated with empty chunks. The | ||||
# tree and file parts are a list of entry sections. Each entry section | ||||
# is a series of chunks terminating in an empty chunk. The list of these | ||||
# entry sections is terminated in yet another empty chunk, so we know | ||||
# we've reached the end of the tree/file list when we reach an empty | ||||
# chunk that was proceeded by no non-empty chunks. | ||||
parts = 0 | ||||
while parts < 2 + self._grouplistcount: | ||||
noentries = True | ||||
|
r20999 | while True: | ||
chunk = getchunk(self) | ||||
if not chunk: | ||||
|
r34093 | # The first two empty chunks represent the end of the | ||
# changelog and the manifestlog portions. The remaining | ||||
# empty chunks represent either A) the end of individual | ||||
# tree or file entries in the file list, or B) the end of | ||||
# the entire list. It's the end of the entire list if there | ||||
# were no entries (i.e. noentries is True). | ||||
if parts < 2: | ||||
parts += 1 | ||||
elif noentries: | ||||
parts += 1 | ||||
|
r20999 | break | ||
|
r34093 | noentries = False | ||
|
r20999 | yield chunkheader(len(chunk)) | ||
pos = 0 | ||||
while pos < len(chunk): | ||||
|
r43346 | next = pos + 2 ** 20 | ||
|
r20999 | yield chunk[pos:next] | ||
pos = next | ||||
yield closechunk() | ||||
|
r38365 | def _unpackmanifests(self, repo, revmap, trp, prog): | ||
self.callback = prog.increment | ||||
|
r26712 | # no need to check for empty manifest group here: | ||
# if the result of the merge of 1 and 2 is the same in 3 and 4, | ||||
# no new manifest will be created and the manifest group will | ||||
# be empty during the pull | ||||
self.manifestheader() | ||||
|
r34292 | deltas = self.deltaiter() | ||
|
r39280 | repo.manifestlog.getstorage(b'').addgroup(deltas, revmap, trp) | ||
|
r38392 | prog.complete() | ||
|
r28360 | self.callback = None | ||
|
r26712 | |||
|
r43346 | def apply( | ||
self, | ||||
repo, | ||||
tr, | ||||
srctype, | ||||
url, | ||||
targetphase=phases.draft, | ||||
expectedtotal=None, | ||||
): | ||||
|
r26695 | """Add the changegroup returned by source.read() to this repo. | ||
srctype is a string like 'push', 'pull', or 'unbundle'. url is | ||||
the URL of the repo where this changegroup is coming from. | ||||
Return an integer summarizing the change to this repo: | ||||
- nothing changed or no source: 0 | ||||
- more heads than before: 1+added heads (2..n) | ||||
- fewer heads than before: -1-removed heads (-2..-n) | ||||
- number of heads stays the same: 1 | ||||
""" | ||||
repo = repo.unfiltered() | ||||
|
r43346 | |||
|
r26695 | def csmap(x): | ||
|
r43347 | repo.ui.debug(b"add changeset %s\n" % short(x)) | ||
|
r26695 | return len(cl) | ||
def revmap(x): | ||||
return cl.rev(x) | ||||
|
r26880 | try: | ||
|
r32931 | # The transaction may already carry source information. In this | ||
# case we use the top level data. We overwrite the argument | ||||
# because we need to use the top level value (if they exist) | ||||
# in this function. | ||||
|
r43347 | srctype = tr.hookargs.setdefault(b'source', srctype) | ||
tr.hookargs.setdefault(b'url', url) | ||||
|
r43346 | repo.hook( | ||
|
r43347 | b'prechangegroup', throw=True, **pycompat.strkwargs(tr.hookargs) | ||
|
r43346 | ) | ||
|
r26695 | |||
|
r32931 | # write changelog data to temp files so concurrent readers | ||
# will not see an inconsistent view | ||||
cl = repo.changelog | ||||
cl.delayupdate(tr) | ||||
oldheads = set(cl.heads()) | ||||
|
r26695 | |||
|
r32931 | trp = weakref.proxy(tr) | ||
# pull off the changeset group | ||||
|
r43347 | repo.ui.status(_(b"adding changesets\n")) | ||
|
r32931 | clstart = len(cl) | ||
|
r43346 | progress = repo.ui.makeprogress( | ||
|
r43347 | _(b'changesets'), unit=_(b'chunks'), total=expectedtotal | ||
|
r43346 | ) | ||
|
r38365 | self.callback = progress.increment | ||
|
r26695 | |||
|
r43790 | efilesset = set() | ||
|
r43346 | |||
|
r32931 | def onchangelog(cl, node): | ||
|
r43790 | efilesset.update(cl.readfiles(node)) | ||
|
r26695 | |||
|
r32931 | self.changelogheader() | ||
|
r34292 | deltas = self.deltaiter() | ||
cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog) | ||||
|
r43790 | efiles = len(efilesset) | ||
|
r26695 | |||
|
r33308 | if not cgnodes: | ||
|
r43346 | repo.ui.develwarn( | ||
|
r43347 | b'applied empty changelog from changegroup', | ||
config=b'warn-empty-changegroup', | ||||
|
r43346 | ) | ||
|
r32931 | clend = len(cl) | ||
changesets = clend - clstart | ||||
|
r38392 | progress.complete() | ||
|
r32931 | self.callback = None | ||
|
r26695 | |||
|
r32931 | # pull off the manifest group | ||
|
r43347 | repo.ui.status(_(b"adding manifests\n")) | ||
|
r38365 | # We know that we'll never have more manifests than we had | ||
# changesets. | ||||
|
r43346 | progress = repo.ui.makeprogress( | ||
|
r43347 | _(b'manifests'), unit=_(b'chunks'), total=changesets | ||
|
r43346 | ) | ||
|
r38365 | self._unpackmanifests(repo, revmap, trp, progress) | ||
|
r26695 | |||
|
r32931 | needfiles = {} | ||
|
r43347 | if repo.ui.configbool(b'server', b'validate'): | ||
|
r32931 | cl = repo.changelog | ||
ml = repo.manifestlog | ||||
# validate incoming csets have their manifests | ||||
|
r38806 | for cset in pycompat.xrange(clstart, clend): | ||
|
r32931 | mfnode = cl.changelogrevision(cset).manifest | ||
mfest = ml[mfnode].readdelta() | ||||
# store file cgnodes we must see | ||||
|
r43376 | for f, n in pycompat.iteritems(mfest): | ||
|
r32931 | needfiles.setdefault(f, set()).add(n) | ||
|
r26695 | |||
|
r32931 | # process the files | ||
|
r43347 | repo.ui.status(_(b"adding file changes\n")) | ||
|
r32931 | newrevs, newfiles = _addchangegroupfiles( | ||
|
r43346 | repo, self, revmap, trp, efiles, needfiles | ||
) | ||||
r43167 | ||||
# making sure the value exists | ||||
|
r43347 | tr.changes.setdefault(b'changegroup-count-changesets', 0) | ||
tr.changes.setdefault(b'changegroup-count-revisions', 0) | ||||
tr.changes.setdefault(b'changegroup-count-files', 0) | ||||
tr.changes.setdefault(b'changegroup-count-heads', 0) | ||||
r43167 | ||||
# some code use bundle operation for internal purpose. They usually | ||||
# set `ui.quiet` to do this outside of user sight. Size the report | ||||
# of such operation now happens at the end of the transaction, that | ||||
# ui.quiet has not direct effect on the output. | ||||
# | ||||
# To preserve this intend use an inelegant hack, we fail to report | ||||
# the change if `quiet` is set. We should probably move to | ||||
# something better, but this is a good first step to allow the "end | ||||
# of transaction report" to pass tests. | ||||
if not repo.ui.quiet: | ||||
|
r43347 | tr.changes[b'changegroup-count-changesets'] += changesets | ||
tr.changes[b'changegroup-count-revisions'] += newrevs | ||||
tr.changes[b'changegroup-count-files'] += newfiles | ||||
|
r26695 | |||
|
r32931 | deltaheads = 0 | ||
if oldheads: | ||||
heads = cl.heads() | ||||
r43167 | deltaheads += len(heads) - len(oldheads) | |||
|
r32931 | for h in heads: | ||
if h not in oldheads and repo[h].closesbranch(): | ||||
deltaheads -= 1 | ||||
|
r26695 | |||
r43167 | # see previous comment about checking ui.quiet | |||
if not repo.ui.quiet: | ||||
|
r43347 | tr.changes[b'changegroup-count-heads'] += deltaheads | ||
|
r32931 | repo.invalidatevolatilesets() | ||
|
r26695 | |||
|
r32931 | if changesets > 0: | ||
|
r43347 | if b'node' not in tr.hookargs: | ||
tr.hookargs[b'node'] = hex(cl.node(clstart)) | ||||
tr.hookargs[b'node_last'] = hex(cl.node(clend - 1)) | ||||
|
r32931 | hookargs = dict(tr.hookargs) | ||
else: | ||||
hookargs = dict(tr.hookargs) | ||||
|
r43347 | hookargs[b'node'] = hex(cl.node(clstart)) | ||
hookargs[b'node_last'] = hex(cl.node(clend - 1)) | ||||
|
r43346 | repo.hook( | ||
|
r43347 | b'pretxnchangegroup', | ||
|
r43346 | throw=True, | ||
**pycompat.strkwargs(hookargs) | ||||
) | ||||
|
r26695 | |||
|
r38806 | added = [cl.node(r) for r in pycompat.xrange(clstart, clend)] | ||
|
r33456 | phaseall = None | ||
|
r43347 | if srctype in (b'push', b'serve'): | ||
|
r32931 | # Old servers can not push the boundary themselves. | ||
# New servers won't push the boundary if changeset already | ||||
# exists locally as secret | ||||
# | ||||
# We should not use added here but the list of all change in | ||||
# the bundle | ||||
if repo.publishing(): | ||||
|
r33456 | targetphase = phaseall = phases.public | ||
|
r32931 | else: | ||
|
r33456 | # closer target phase computation | ||
|
r32931 | # Those changesets have been pushed from the | ||
# outside, their phases are going to be pushed | ||||
# alongside. Therefor `targetphase` is | ||||
# ignored. | ||||
|
r33456 | targetphase = phaseall = phases.draft | ||
if added: | ||||
phases.registernew(repo, tr, targetphase, added) | ||||
if phaseall is not None: | ||||
phases.advanceboundary(repo, tr, phaseall, cgnodes) | ||||
|
r26695 | |||
|
r32931 | if changesets > 0: | ||
|
r26695 | |||
|
r44217 | def runhooks(unused_success): | ||
|
r32931 | # These hooks run when the lock releases, not when the | ||
# transaction closes. So it's possible for the changelog | ||||
# to have changed since we last saw it. | ||||
if clstart >= len(repo): | ||||
return | ||||
|
r26695 | |||
|
r43347 | repo.hook(b"changegroup", **pycompat.strkwargs(hookargs)) | ||
|
r27867 | |||
|
r32931 | for n in added: | ||
args = hookargs.copy() | ||||
|
r43347 | args[b'node'] = hex(n) | ||
del args[b'node_last'] | ||||
repo.hook(b"incoming", **pycompat.strkwargs(args)) | ||||
|
r26695 | |||
|
r43346 | newheads = [h for h in repo.heads() if h not in oldheads] | ||
repo.ui.log( | ||||
|
r43347 | b"incoming", | ||
b"%d incoming changes - new heads: %s\n", | ||||
|
r43346 | len(added), | ||
|
r43347 | b', '.join([hex(c[:6]) for c in newheads]), | ||
|
r43346 | ) | ||
|
r26695 | |||
|
r43346 | tr.addpostclose( | ||
|
r43347 | b'changegroup-runhooks-%020i' % clstart, | ||
|
r43346 | lambda tr: repo._afterlock(runhooks), | ||
) | ||||
|
r26695 | finally: | ||
repo.ui.flush() | ||||
# never return 0 here: | ||||
|
r32870 | if deltaheads < 0: | ||
|
r33030 | ret = deltaheads - 1 | ||
|
r26695 | else: | ||
|
r33030 | ret = deltaheads + 1 | ||
|
r33461 | return ret | ||
|
r26695 | |||
|
r34292 | def deltaiter(self): | ||
|
r34147 | """ | ||
returns an iterator of the deltas in this changegroup | ||||
Useful for passing to the underlying storage system to be stored. | ||||
""" | ||||
chain = None | ||||
for chunkdata in iter(lambda: self.deltachunk(chain), {}): | ||||
|
r34295 | # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags) | ||
yield chunkdata | ||||
chain = chunkdata[0] | ||||
|
r34147 | |||
|
r43346 | |||
|
r23181 | class cg2unpacker(cg1unpacker): | ||
|
r26708 | """Unpacker for cg2 streams. | ||
cg2 streams add support for generaldelta, so the delta header | ||||
format is slightly different. All other features about the data | ||||
remain the same. | ||||
""" | ||||
|
r43346 | |||
|
r23181 | deltaheader = _CHANGEGROUPV2_DELTA_HEADER | ||
|
r38932 | deltaheadersize = deltaheader.size | ||
|
r43347 | version = b'02' | ||
|
r23181 | |||
def _deltaheader(self, headertuple, prevnode): | ||||
node, p1, p2, deltabase, cs = headertuple | ||||
|
r27433 | flags = 0 | ||
return node, p1, p2, deltabase, cs, flags | ||||
|
r23181 | |||
|
r43346 | |||
|
r27432 | class cg3unpacker(cg2unpacker): | ||
"""Unpacker for cg3 streams. | ||||
|
r27433 | cg3 streams add support for exchanging treemanifests and revlog | ||
|
r27753 | flags. It adds the revlog flags to the delta header and an empty chunk | ||
separating manifests and files. | ||||
|
r27432 | """ | ||
|
r43346 | |||
|
r27433 | deltaheader = _CHANGEGROUPV3_DELTA_HEADER | ||
|
r38932 | deltaheadersize = deltaheader.size | ||
|
r43347 | version = b'03' | ||
|
r43346 | _grouplistcount = 2 # One list of manifests and one list of files | ||
|
r27432 | |||
|
r27433 | def _deltaheader(self, headertuple, prevnode): | ||
node, p1, p2, deltabase, cs, flags = headertuple | ||||
return node, p1, p2, deltabase, cs, flags | ||||
|
r38365 | def _unpackmanifests(self, repo, revmap, trp, prog): | ||
super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog) | ||||
|
r29724 | for chunkdata in iter(self.filelogheader, {}): | ||
|
r27754 | # If we get here, there are directory manifests in the changegroup | ||
|
r43347 | d = chunkdata[b"filename"] | ||
repo.ui.debug(b"adding %s revisions\n" % d) | ||||
|
r34292 | deltas = self.deltaiter() | ||
|
r39280 | if not repo.manifestlog.getstorage(d).addgroup(deltas, revmap, trp): | ||
|
r43347 | raise error.Abort(_(b"received dir revlog group is empty")) | ||
|
r27754 | |||
|
r43346 | |||
|
r12329 | class headerlessfixup(object): | ||
def __init__(self, fh, h): | ||||
self._h = h | ||||
self._fh = fh | ||||
|
r43346 | |||
|
r12329 | def read(self, n): | ||
if self._h: | ||||
d, self._h = self._h[:n], self._h[n:] | ||||
if len(d) < n: | ||||
|
r13457 | d += readexactly(self._fh, n - len(d)) | ||
|
r12329 | return d | ||
|
r13457 | return readexactly(self._fh, n) | ||
|
r12329 | |||
|
r43346 | |||
|
r39050 | def _revisiondeltatochunks(delta, headerfn): | ||
"""Serialize a revisiondelta to changegroup chunks.""" | ||||
|
r39052 | |||
# The captured revision delta may be encoded as a delta against | ||||
# a base revision or as a full revision. The changegroup format | ||||
# requires that everything on the wire be deltas. So for full | ||||
# revisions, we need to invent a header that says to rewrite | ||||
# data. | ||||
if delta.delta is not None: | ||||
prefix, data = b'', delta.delta | ||||
elif delta.basenode == nullid: | ||||
data = delta.revision | ||||
prefix = mdiff.trivialdiffheader(len(data)) | ||||
else: | ||||
data = delta.revision | ||||
|
r43346 | prefix = mdiff.replacediffheader(delta.baserevisionsize, len(data)) | ||
|
r39052 | |||
|
r39050 | meta = headerfn(delta) | ||
|
r39052 | |||
yield chunkheader(len(meta) + len(prefix) + len(data)) | ||||
|
r39050 | yield meta | ||
|
r39052 | if prefix: | ||
yield prefix | ||||
yield data | ||||
|
r39050 | |||
|
r43346 | |||
|
r39033 | def _sortnodesellipsis(store, nodes, cl, lookup): | ||
|
r39901 | """Sort nodes for changegroup generation.""" | ||
|
r39018 | # Ellipses serving mode. | ||
# | ||||
# In a perfect world, we'd generate better ellipsis-ified graphs | ||||
# for non-changelog revlogs. In practice, we haven't started doing | ||||
# that yet, so the resulting DAGs for the manifestlog and filelogs | ||||
# are actually full of bogus parentage on all the ellipsis | ||||
# nodes. This has the side effect that, while the contents are | ||||
# correct, the individual DAGs might be completely out of whack in | ||||
# a case like 882681bc3166 and its ancestors (back about 10 | ||||
# revisions or so) in the main hg repo. | ||||
# | ||||
# The one invariant we *know* holds is that the new (potentially | ||||
# bogus) DAG shape will be valid if we order the nodes in the | ||||
# order that they're introduced in dramatis personae by the | ||||
# changelog, so what we do is we sort the non-changelog histories | ||||
# by the order in which they are used by the changelog. | ||||
|
r39033 | key = lambda n: cl.rev(lookup(n)) | ||
|
r39901 | return sorted(nodes, key=key) | ||
|
r39018 | |||
|
r43346 | |||
def _resolvenarrowrevisioninfo( | ||||
cl, | ||||
store, | ||||
ischangelog, | ||||
rev, | ||||
linkrev, | ||||
linknode, | ||||
clrevtolocalrev, | ||||
fullclnodes, | ||||
precomputedellipsis, | ||||
): | ||||
|
r39040 | linkparents = precomputedellipsis[linkrev] | ||
|
r43346 | |||
|
r39040 | def local(clrev): | ||
"""Turn a changelog revnum into a local revnum. | ||||
The ellipsis dag is stored as revnums on the changelog, | ||||
but when we're producing ellipsis entries for | ||||
non-changelog revlogs, we need to turn those numbers into | ||||
something local. This does that for us, and during the | ||||
changelog sending phase will also expand the stored | ||||
mappings as needed. | ||||
""" | ||||
if clrev == nullrev: | ||||
return nullrev | ||||
if ischangelog: | ||||
return clrev | ||||
# Walk the ellipsis-ized changelog breadth-first looking for a | ||||
# change that has been linked from the current revlog. | ||||
# | ||||
# For a flat manifest revlog only a single step should be necessary | ||||
# as all relevant changelog entries are relevant to the flat | ||||
# manifest. | ||||
# | ||||
# For a filelog or tree manifest dirlog however not every changelog | ||||
# entry will have been relevant, so we need to skip some changelog | ||||
# nodes even after ellipsis-izing. | ||||
walk = [clrev] | ||||
while walk: | ||||
p = walk[0] | ||||
walk = walk[1:] | ||||
if p in clrevtolocalrev: | ||||
return clrevtolocalrev[p] | ||||
elif p in fullclnodes: | ||||
|
r43346 | walk.extend([pp for pp in cl.parentrevs(p) if pp != nullrev]) | ||
|
r39040 | elif p in precomputedellipsis: | ||
|
r43346 | walk.extend( | ||
[pp for pp in precomputedellipsis[p] if pp != nullrev] | ||||
) | ||||
|
r39040 | else: | ||
# In this case, we've got an ellipsis with parents | ||||
# outside the current bundle (likely an | ||||
# incremental pull). We "know" that we can use the | ||||
# value of this same revlog at whatever revision | ||||
# is pointed to by linknode. "Know" is in scare | ||||
# quotes because I haven't done enough examination | ||||
# of edge cases to convince myself this is really | ||||
# a fact - it works for all the (admittedly | ||||
# thorough) cases in our testsuite, but I would be | ||||
# somewhat unsurprised to find a case in the wild | ||||
# where this breaks down a bit. That said, I don't | ||||
# know if it would hurt anything. | ||||
for i in pycompat.xrange(rev, 0, -1): | ||||
if store.linkrev(i) == clrev: | ||||
return i | ||||
# We failed to resolve a parent for this node, so | ||||
# we crash the changegroup construction. | ||||
raise error.Abort( | ||||
|
r43347 | b'unable to resolve parent while packing %r %r' | ||
b' for changeset %r' % (store.indexfile, rev, clrev) | ||||
|
r43346 | ) | ||
|
r39040 | |||
return nullrev | ||||
|
r43346 | if not linkparents or (store.parentrevs(rev) == (nullrev, nullrev)): | ||
|
r39040 | p1, p2 = nullrev, nullrev | ||
elif len(linkparents) == 1: | ||||
|
r43346 | (p1,) = sorted(local(p) for p in linkparents) | ||
|
r39040 | p2 = nullrev | ||
else: | ||||
p1, p2 = sorted(local(p) for p in linkparents) | ||||
|
r39054 | p1node, p2node = store.node(p1), store.node(p2) | ||
|
r39040 | |||
|
r39901 | return p1node, p2node, linknode | ||
|
r39040 | |||
|
r43346 | |||
def deltagroup( | ||||
repo, | ||||
store, | ||||
nodes, | ||||
ischangelog, | ||||
lookup, | ||||
forcedeltaparentprev, | ||||
topic=None, | ||||
ellipses=False, | ||||
clrevtolocalrev=None, | ||||
fullclnodes=None, | ||||
precomputedellipsis=None, | ||||
): | ||||
|
r39050 | """Calculate deltas for a set of revisions. | ||
|
r39045 | |||
|
r39050 | Is a generator of ``revisiondelta`` instances. | ||
|
r39045 | |||
|
r39275 | If topic is not None, progress detail will be generated using this | ||
topic name (e.g. changesets, manifests, etc). | ||||
|
r39045 | """ | ||
|
r39265 | if not nodes: | ||
|
r39045 | return | ||
cl = repo.changelog | ||||
|
r39265 | if ischangelog: | ||
|
r39901 | # `hg log` shows changesets in storage order. To preserve order | ||
# across clones, send out changesets in storage order. | ||||
|
r43347 | nodesorder = b'storage' | ||
|
r39265 | elif ellipses: | ||
|
r39901 | nodes = _sortnodesellipsis(store, nodes, cl, lookup) | ||
|
r43347 | nodesorder = b'nodes' | ||
|
r39265 | else: | ||
|
r39901 | nodesorder = None | ||
|
r39265 | |||
|
r39901 | # Perform ellipses filtering and revision massaging. We do this before | ||
# emitrevisions() because a) filtering out revisions creates less work | ||||
# for emitrevisions() b) dropping revisions would break emitrevisions()'s | ||||
# assumptions about delta choices and we would possibly send a delta | ||||
# referencing a missing base revision. | ||||
# | ||||
# Also, calling lookup() has side-effects with regards to populating | ||||
# data structures. If we don't call lookup() for each node or if we call | ||||
# lookup() after the first pass through each node, things can break - | ||||
# possibly intermittently depending on the python hash seed! For that | ||||
# reason, we store a mapping of all linknodes during the initial node | ||||
# pass rather than use lookup() on the output side. | ||||
if ellipses: | ||||
filtered = [] | ||||
adjustedparents = {} | ||||
linknodes = {} | ||||
|
r39045 | |||
|
r39901 | for node in nodes: | ||
rev = store.rev(node) | ||||
linknode = lookup(node) | ||||
|
r39045 | linkrev = cl.rev(linknode) | ||
|
r39901 | clrevtolocalrev[linkrev] = rev | ||
|
r39045 | |||
|
r39901 | # If linknode is in fullclnodes, it means the corresponding | ||
# changeset was a full changeset and is being sent unaltered. | ||||
|
r39045 | if linknode in fullclnodes: | ||
|
r39901 | linknodes[node] = linknode | ||
|
r39054 | |||
|
r39901 | # If the corresponding changeset wasn't in the set computed | ||
# as relevant to us, it should be dropped outright. | ||||
|
r39045 | elif linkrev not in precomputedellipsis: | ||
|
r39901 | continue | ||
|
r39045 | else: | ||
|
r39901 | # We could probably do this later and avoid the dict | ||
# holding state. But it likely doesn't matter. | ||||
p1node, p2node, linknode = _resolvenarrowrevisioninfo( | ||||
|
r43346 | cl, | ||
store, | ||||
ischangelog, | ||||
rev, | ||||
linkrev, | ||||
linknode, | ||||
clrevtolocalrev, | ||||
fullclnodes, | ||||
precomputedellipsis, | ||||
) | ||||
|
r39901 | |||
adjustedparents[node] = (p1node, p2node) | ||||
linknodes[node] = linknode | ||||
filtered.append(node) | ||||
nodes = filtered | ||||
|
r39045 | |||
|
r39054 | # We expect the first pass to be fast, so we only engage the progress | ||
# meter for constructing the revision deltas. | ||||
progress = None | ||||
|
r39275 | if topic is not None: | ||
|
r43346 | progress = repo.ui.makeprogress( | ||
|
r43347 | topic, unit=_(b'chunks'), total=len(nodes) | ||
|
r43346 | ) | ||
|
r39054 | |||
|
r43347 | configtarget = repo.ui.config(b'devel', b'bundle.delta') | ||
if configtarget not in (b'', b'p1', b'full'): | ||||
|
r40458 | msg = _("""config "devel.bundle.delta" as unknown value: %s""") | ||
repo.ui.warn(msg % configtarget) | ||||
|
r40457 | deltamode = repository.CG_DELTAMODE_STD | ||
if forcedeltaparentprev: | ||||
deltamode = repository.CG_DELTAMODE_PREV | ||||
|
r43347 | elif configtarget == b'p1': | ||
|
r40458 | deltamode = repository.CG_DELTAMODE_P1 | ||
|
r43347 | elif configtarget == b'full': | ||
|
r40459 | deltamode = repository.CG_DELTAMODE_FULL | ||
|
r40457 | |||
|
r39901 | revisions = store.emitrevisions( | ||
nodes, | ||||
nodesorder=nodesorder, | ||||
revisiondata=True, | ||||
assumehaveparentrevisions=not ellipses, | ||||
|
r43346 | deltamode=deltamode, | ||
) | ||||
|
r39901 | |||
for i, revision in enumerate(revisions): | ||||
|
r39054 | if progress: | ||
progress.update(i + 1) | ||||
|
r39901 | if ellipses: | ||
linknode = linknodes[revision.node] | ||||
if revision.node in adjustedparents: | ||||
p1node, p2node = adjustedparents[revision.node] | ||||
revision.p1node = p1node | ||||
revision.p2node = p2node | ||||
|
r40083 | revision.flags |= repository.REVISION_FLAG_ELLIPSIS | ||
|
r39901 | |||
else: | ||||
linknode = lookup(revision.node) | ||||
revision.linknode = linknode | ||||
yield revision | ||||
|
r39054 | |||
|
r39045 | if progress: | ||
progress.complete() | ||||
|
r43346 | |||
|
r38938 | class cgpacker(object): | ||
|
r43346 | def __init__( | ||
self, | ||||
repo, | ||||
oldmatcher, | ||||
matcher, | ||||
version, | ||||
builddeltaheader, | ||||
manifestsend, | ||||
forcedeltaparentprev=False, | ||||
bundlecaps=None, | ||||
ellipses=False, | ||||
shallow=False, | ||||
ellipsisroots=None, | ||||
fullnodes=None, | ||||
): | ||||
|
r19202 | """Given a source repo, construct a bundler. | ||
|
r32287 | |||
|
r40380 | oldmatcher is a matcher that matches on files the client already has. | ||
These will not be included in the changegroup. | ||||
matcher is a matcher that matches on files to include in the | ||||
|
r38830 | changegroup. Used to facilitate sparse changegroups. | ||
|
r39053 | forcedeltaparentprev indicates whether delta parents must be against | ||
the previous revision in a delta group. This should only be used for | ||||
compatibility with changegroup version 1. | ||||
|
r38937 | |||
|
r38933 | builddeltaheader is a callable that constructs the header for a group | ||
delta. | ||||
|
r38934 | manifestsend is a chunk to send after manifests have been fully emitted. | ||
|
r38944 | ellipses indicates whether ellipsis serving mode is enabled. | ||
|
r32287 | bundlecaps is optional and can be used to specify the set of | ||
capabilities which can be used to build the bundle. While bundlecaps is | ||||
unused in core Mercurial, extensions rely on this feature to communicate | ||||
capabilities to customize the changegroup packer. | ||||
|
r38940 | |||
shallow indicates whether shallow data might be sent. The packer may | ||||
need to pack file contents not introduced by the changes being packed. | ||||
|
r38945 | |||
|
r39032 | fullnodes is the set of changelog nodes which should not be ellipsis | ||
nodes. We store this rather than the set of nodes that should be | ||||
ellipsis because for very large histories we expect this to be | ||||
significantly smaller. | ||||
|
r19202 | """ | ||
|
r40380 | assert oldmatcher | ||
assert matcher | ||||
self._oldmatcher = oldmatcher | ||||
self._matcher = matcher | ||||
|
r38830 | |||
|
r38931 | self.version = version | ||
|
r39053 | self._forcedeltaparentprev = forcedeltaparentprev | ||
|
r38933 | self._builddeltaheader = builddeltaheader | ||
|
r38934 | self._manifestsend = manifestsend | ||
|
r38944 | self._ellipses = ellipses | ||
|
r38931 | |||
|
r32287 | # Set of capabilities we can use to build the bundle. | ||
if bundlecaps is None: | ||||
bundlecaps = set() | ||||
self._bundlecaps = bundlecaps | ||||
|
r38940 | self._isshallow = shallow | ||
|
r39032 | self._fullclnodes = fullnodes | ||
|
r38936 | |||
|
r38943 | # Maps ellipsis revs to their roots at the changelog level. | ||
self._precomputedellipsis = ellipsisroots | ||||
|
r19202 | self._repo = repo | ||
|
r38936 | |||
|
r23748 | if self._repo.ui.verbose and not self._repo.ui.debugflag: | ||
self._verbosenote = self._repo.ui.note | ||||
else: | ||||
self._verbosenote = lambda s: None | ||||
|
r43346 | def generate( | ||
self, commonrevs, clnodes, fastpathlinkrev, source, changelog=True | ||||
): | ||||
|
r39708 | """Yield a sequence of changegroup byte chunks. | ||
If changelog is False, changelog data won't be added to changegroup | ||||
""" | ||||
|
r39012 | |||
|
r19202 | repo = self._repo | ||
|
r24978 | cl = repo.changelog | ||
|
r19204 | |||
|
r43347 | self._verbosenote(_(b'uncompressed size of bundle content:\n')) | ||
|
r39012 | size = 0 | ||
|
r43346 | clstate, deltas = self._generatechangelog( | ||
cl, clnodes, generate=changelog | ||||
) | ||||
|
r39050 | for delta in deltas: | ||
|
r43346 | for chunk in _revisiondeltatochunks(delta, self._builddeltaheader): | ||
|
r41491 | size += len(chunk) | ||
yield chunk | ||||
|
r39012 | |||
|
r39046 | close = closechunk() | ||
size += len(close) | ||||
yield closechunk() | ||||
|
r43347 | self._verbosenote(_(b'%8.i (changelog)\n') % size) | ||
|
r39012 | |||
|
r43347 | clrevorder = clstate[b'clrevorder'] | ||
manifests = clstate[b'manifests'] | ||||
changedfiles = clstate[b'changedfiles'] | ||||
|
r39012 | |||
# We need to make sure that the linkrev in the changegroup refers to | ||||
# the first changeset that introduced the manifest or file revision. | ||||
# The fastpath is usually safer than the slowpath, because the filelogs | ||||
# are walked in revlog order. | ||||
# | ||||
|
r39897 | # When taking the slowpath when the manifest revlog uses generaldelta, | ||
# the manifest may be walked in the "wrong" order. Without 'clrevorder', | ||||
# we would get an incorrect linkrev (see fix in cc0ff93d0c0c). | ||||
|
r39012 | # | ||
# When taking the fastpath, we are only vulnerable to reordering | ||||
|
r39897 | # of the changelog itself. The changelog never uses generaldelta and is | ||
# never reordered. To handle this case, we simply take the slowpath, | ||||
# which already has the 'clrevorder' logic. This was also fixed in | ||||
# cc0ff93d0c0c. | ||||
|
r39012 | # Treemanifests don't work correctly with fastpathlinkrev | ||
# either, because we don't discover which directory nodes to | ||||
# send along with files. This could probably be fixed. | ||||
fastpathlinkrev = fastpathlinkrev and ( | ||||
|
r43347 | b'treemanifest' not in repo.requirements | ||
|
r43346 | ) | ||
|
r39012 | |||
fnodes = {} # needed file nodes | ||||
|
r39047 | size = 0 | ||
|
r39048 | it = self.generatemanifests( | ||
|
r43346 | commonrevs, | ||
clrevorder, | ||||
fastpathlinkrev, | ||||
manifests, | ||||
fnodes, | ||||
source, | ||||
|
r43347 | clstate[b'clrevtomanifestrev'], | ||
|
r43346 | ) | ||
|
r39048 | |||
|
r39269 | for tree, deltas in it: | ||
if tree: | ||||
|
r39048 | assert self.version == b'03' | ||
|
r39269 | chunk = _fileheader(tree) | ||
|
r39048 | size += len(chunk) | ||
yield chunk | ||||
|
r39050 | for delta in deltas: | ||
chunks = _revisiondeltatochunks(delta, self._builddeltaheader) | ||||
for chunk in chunks: | ||||
size += len(chunk) | ||||
yield chunk | ||||
|
r39048 | |||
close = closechunk() | ||||
size += len(close) | ||||
yield close | ||||
|
r39012 | |||
|
r43347 | self._verbosenote(_(b'%8.i (manifests)\n') % size) | ||
|
r39047 | yield self._manifestsend | ||
|
r39019 | mfdicts = None | ||
if self._ellipses and self._isshallow: | ||||
|
r43346 | mfdicts = [ | ||
(self._repo.manifestlog[n].read(), lr) | ||||
|
r43376 | for (n, lr) in pycompat.iteritems(manifests) | ||
|
r43346 | ] | ||
|
r39012 | |||
|
r39274 | manifests.clear() | ||
|
r39012 | clrevs = set(cl.rev(x) for x in clnodes) | ||
|
r43346 | it = self.generatefiles( | ||
changedfiles, | ||||
commonrevs, | ||||
source, | ||||
mfdicts, | ||||
fastpathlinkrev, | ||||
fnodes, | ||||
clrevs, | ||||
) | ||||
|
r39049 | |||
|
r39050 | for path, deltas in it: | ||
|
r39049 | h = _fileheader(path) | ||
size = len(h) | ||||
yield h | ||||
|
r39050 | for delta in deltas: | ||
chunks = _revisiondeltatochunks(delta, self._builddeltaheader) | ||||
for chunk in chunks: | ||||
size += len(chunk) | ||||
yield chunk | ||||
|
r39049 | |||
close = closechunk() | ||||
size += len(close) | ||||
yield close | ||||
|
r43347 | self._verbosenote(_(b'%8.i %s\n') % (size, path)) | ||
|
r39012 | |||
|
r39038 | yield closechunk() | ||
|
r39012 | |||
if clnodes: | ||||
|
r43347 | repo.hook(b'outgoing', node=hex(clnodes[0]), source=source) | ||
|
r39012 | |||
|
r41491 | def _generatechangelog(self, cl, nodes, generate=True): | ||
|
r39012 | """Generate data for changelog chunks. | ||
Returns a 2-tuple of a dict containing state and an iterable of | ||||
byte chunks. The state will not be fully populated until the | ||||
chunk stream has been fully consumed. | ||||
|
r41491 | |||
if generate is False, the state will be fully populated and no chunk | ||||
stream will be yielded | ||||
|
r39012 | """ | ||
|
r23381 | clrevorder = {} | ||
|
r39274 | manifests = {} | ||
|
r39012 | mfl = self._repo.manifestlog | ||
|
r28241 | changedfiles = set() | ||
|
r39034 | clrevtomanifestrev = {} | ||
|
r19204 | |||
|
r41490 | state = { | ||
|
r43347 | b'clrevorder': clrevorder, | ||
b'manifests': manifests, | ||||
b'changedfiles': changedfiles, | ||||
b'clrevtomanifestrev': clrevtomanifestrev, | ||||
|
r41490 | } | ||
|
r41491 | if not (generate or self._ellipses): | ||
# sort the nodes in storage order | ||||
nodes = sorted(nodes, key=cl.rev) | ||||
for node in nodes: | ||||
c = cl.changelogrevision(node) | ||||
clrevorder[node] = len(clrevorder) | ||||
# record the first changeset introducing this manifest version | ||||
manifests.setdefault(c.manifest, node) | ||||
# Record a complete list of potentially-changed files in | ||||
# this manifest. | ||||
changedfiles.update(c.files) | ||||
return state, () | ||||
|
r38926 | # Callback for the changelog, used to collect changed files and | ||
# manifest nodes. | ||||
|
r19207 | # Returns the linkrev node (identity in the changelog case). | ||
def lookupcl(x): | ||||
|
r39273 | c = cl.changelogrevision(x) | ||
|
r23381 | clrevorder[x] = len(clrevorder) | ||
|
r38926 | |||
|
r38944 | if self._ellipses: | ||
|
r39274 | # Only update manifests if x is going to be sent. Otherwise we | ||
|
r38926 | # end up with bogus linkrevs specified for manifests and | ||
# we skip some manifest nodes that we should otherwise | ||||
# have sent. | ||||
|
r43346 | if ( | ||
x in self._fullclnodes | ||||
or cl.rev(x) in self._precomputedellipsis | ||||
): | ||||
|
r39273 | |||
manifestnode = c.manifest | ||||
|
r38926 | # Record the first changeset introducing this manifest | ||
# version. | ||||
|
r39274 | manifests.setdefault(manifestnode, x) | ||
|
r38926 | # Set this narrow-specific dict so we have the lowest | ||
# manifest revnum to look up for this cl revnum. (Part of | ||||
# mapping changelog ellipsis parents to manifest ellipsis | ||||
# parents) | ||||
|
r39273 | clrevtomanifestrev.setdefault( | ||
|
r43346 | cl.rev(x), mfl.rev(manifestnode) | ||
) | ||||
|
r38926 | # We can't trust the changed files list in the changeset if the | ||
# client requested a shallow clone. | ||||
|
r38940 | if self._isshallow: | ||
|
r39273 | changedfiles.update(mfl[c.manifest].read().keys()) | ||
|
r38926 | else: | ||
|
r39273 | changedfiles.update(c.files) | ||
|
r38926 | else: | ||
# record the first changeset introducing this manifest version | ||||
|
r39274 | manifests.setdefault(c.manifest, x) | ||
|
r38926 | # Record a complete list of potentially-changed files in | ||
# this manifest. | ||||
|
r39273 | changedfiles.update(c.files) | ||
|
r38926 | |||
|
r19207 | return x | ||
|
r19204 | |||
|
r39045 | gen = deltagroup( | ||
|
r43346 | self._repo, | ||
cl, | ||||
nodes, | ||||
True, | ||||
lookupcl, | ||||
|
r39053 | self._forcedeltaparentprev, | ||
|
r39045 | ellipses=self._ellipses, | ||
|
r43347 | topic=_(b'changesets'), | ||
|
r39045 | clrevtolocalrev={}, | ||
fullclnodes=self._fullclnodes, | ||||
|
r43346 | precomputedellipsis=self._precomputedellipsis, | ||
) | ||||
|
r28227 | |||
|
r39012 | return state, gen | ||
|
r28227 | |||
|
r43346 | def generatemanifests( | ||
self, | ||||
commonrevs, | ||||
clrevorder, | ||||
fastpathlinkrev, | ||||
manifests, | ||||
fnodes, | ||||
source, | ||||
clrevtolocalrev, | ||||
): | ||||
|
r34148 | """Returns an iterator of changegroup chunks containing manifests. | ||
`source` is unused here, but is used by extensions like remotefilelog to | ||||
change what is sent based in pulls vs pushes, etc. | ||||
""" | ||||
|
r28227 | repo = self._repo | ||
|
r30294 | mfl = repo.manifestlog | ||
|
r43347 | tmfnodes = {b'': manifests} | ||
|
r28227 | |||
|
r19207 | # Callback for the manifest, used to collect linkrevs for filelog | ||
# revisions. | ||||
# Returns the linkrev node (collected in lookupcl). | ||||
|
r39269 | def makelookupmflinknode(tree, nodes): | ||
|
r28231 | if fastpathlinkrev: | ||
|
r39269 | assert not tree | ||
|
r43791 | return ( | ||
|
r44787 | manifests.__getitem__ | ||
) # pytype: disable=unsupported-operands | ||||
|
r28231 | |||
|
r27239 | def lookupmflinknode(x): | ||
"""Callback for looking up the linknode for manifests. | ||||
|
r27219 | |||
|
r27239 | Returns the linkrev node for the specified manifest. | ||
|
r27219 | |||
|
r27239 | SIDE EFFECT: | ||
|
r27432 | 1) fclnodes gets populated with the list of relevant | ||
file nodes if we're not using fastpathlinkrev | ||||
2) When treemanifests are in use, collects treemanifest nodes | ||||
to send | ||||
|
r27219 | |||
|
r27432 | Note that this means manifests must be completely sent to | ||
the client before you can trust the list of files and | ||||
treemanifests to send. | ||||
|
r27239 | """ | ||
|
r35013 | clnode = nodes[x] | ||
|
r39269 | mdata = mfl.get(tree, x).readfast(shallow=True) | ||
|
r28241 | for p, n, fl in mdata.iterentries(): | ||
|
r43347 | if fl == b't': # subdirectory manifest | ||
subtree = tree + p + b'/' | ||||
|
r39269 | tmfclnodes = tmfnodes.setdefault(subtree, {}) | ||
|
r28241 | tmfclnode = tmfclnodes.setdefault(n, clnode) | ||
if clrevorder[clnode] < clrevorder[tmfclnode]: | ||||
tmfclnodes[n] = clnode | ||||
else: | ||||
|
r39269 | f = tree + p | ||
|
r28240 | fclnodes = fnodes.setdefault(f, {}) | ||
fclnode = fclnodes.setdefault(n, clnode) | ||||
if clrevorder[clnode] < clrevorder[fclnode]: | ||||
fclnodes[n] = clnode | ||||
|
r27239 | return clnode | ||
|
r43346 | |||
|
r28231 | return lookupmflinknode | ||
|
r19206 | |||
|
r28232 | while tmfnodes: | ||
|
r39269 | tree, nodes = tmfnodes.popitem() | ||
|
r40700 | |||
|
r42528 | should_visit = self._matcher.visitdir(tree[:-1]) | ||
|
r40700 | if tree and not should_visit: | ||
continue | ||||
|
r39280 | store = mfl.getstorage(tree) | ||
|
r39043 | |||
|
r40700 | if not should_visit: | ||
|
r39769 | # No nodes to send because this directory is out of | ||
# the client's view of the repository (probably | ||||
|
r40700 | # because of narrow clones). Do this even for the root | ||
# directory (tree=='') | ||||
|
r39043 | prunednodes = [] | ||
else: | ||||
|
r39769 | # Avoid sending any manifest nodes we can prove the | ||
# client already has by checking linkrevs. See the | ||||
# related comment in generatefiles(). | ||||
|
r39768 | prunednodes = self._prunemanifests(store, nodes, commonrevs) | ||
|
r40700 | |||
|
r39269 | if tree and not prunednodes: | ||
|
r39041 | continue | ||
|
r39269 | lookupfn = makelookupmflinknode(tree, nodes) | ||
|
r39018 | |||
|
r39050 | deltas = deltagroup( | ||
|
r43346 | self._repo, | ||
store, | ||||
prunednodes, | ||||
False, | ||||
lookupfn, | ||||
|
r39897 | self._forcedeltaparentprev, | ||
|
r39044 | ellipses=self._ellipses, | ||
|
r43347 | topic=_(b'manifests'), | ||
|
r39044 | clrevtolocalrev=clrevtolocalrev, | ||
fullclnodes=self._fullclnodes, | ||||
|
r43346 | precomputedellipsis=self._precomputedellipsis, | ||
) | ||||
|
r39044 | |||
|
r42528 | if not self._oldmatcher.visitdir(store.tree[:-1]): | ||
|
r40380 | yield tree, deltas | ||
else: | ||||
# 'deltas' is a generator and we need to consume it even if | ||||
# we are not going to send it because a side-effect is that | ||||
# it updates tmdnodes (via lookupfn) | ||||
for d in deltas: | ||||
pass | ||||
if not tree: | ||||
yield tree, [] | ||||
|
r39046 | |||
|
r39768 | def _prunemanifests(self, store, nodes, commonrevs): | ||
|
r41933 | if not self._ellipses: | ||
# In non-ellipses case and large repositories, it is better to | ||||
# prevent calling of store.rev and store.linkrev on a lot of | ||||
# nodes as compared to sending some extra data | ||||
return nodes.copy() | ||||
|
r39768 | # This is split out as a separate method to allow filtering | ||
# commonrevs in extension code. | ||||
# | ||||
# TODO(augie): this shouldn't be required, instead we should | ||||
# make filtering of revisions to send delegated to the store | ||||
# layer. | ||||
frev, flr = store.rev, store.linkrev | ||||
return [n for n in nodes if flr(frev(n)) not in commonrevs] | ||||
|
r24897 | # The 'source' parameter is useful for extensions | ||
|
r43346 | def generatefiles( | ||
self, | ||||
changedfiles, | ||||
commonrevs, | ||||
source, | ||||
mfdicts, | ||||
fastpathlinkrev, | ||||
fnodes, | ||||
clrevs, | ||||
): | ||||
changedfiles = [ | ||||
f | ||||
for f in changedfiles | ||||
if self._matcher(f) and not self._oldmatcher(f) | ||||
] | ||||
|
r38925 | |||
|
r39035 | if not fastpathlinkrev: | ||
|
r43346 | |||
|
r39035 | def normallinknodes(unused, fname): | ||
return fnodes.get(fname, {}) | ||||
|
r43346 | |||
|
r39035 | else: | ||
cln = self._repo.changelog.node | ||||
def normallinknodes(store, fname): | ||||
flinkrev = store.linkrev | ||||
fnode = store.node | ||||
revs = ((r, flinkrev(r)) for r in store) | ||||
|
r43346 | return dict( | ||
(fnode(r), cln(lr)) for r, lr in revs if lr in clrevs | ||||
) | ||||
|
r39035 | |||
|
r39037 | clrevtolocalrev = {} | ||
|
r38940 | if self._isshallow: | ||
|
r38925 | # In a shallow clone, the linknodes callback needs to also include | ||
# those file nodes that are in the manifests we sent but weren't | ||||
# introduced by those manifests. | ||||
commonctxs = [self._repo[c] for c in commonrevs] | ||||
clrev = self._repo.changelog.rev | ||||
def linknodes(flog, fname): | ||||
for c in commonctxs: | ||||
try: | ||||
fnode = c.filenode(fname) | ||||
|
r39037 | clrevtolocalrev[c.rev()] = flog.rev(fnode) | ||
|
r38925 | except error.ManifestLookupError: | ||
pass | ||||
|
r39035 | links = normallinknodes(flog, fname) | ||
|
r38925 | if len(links) != len(mfdicts): | ||
for mf, lr in mfdicts: | ||||
fnode = mf.get(fname, None) | ||||
if fnode in links: | ||||
links[fnode] = min(links[fnode], lr, key=clrev) | ||||
elif fnode: | ||||
links[fnode] = lr | ||||
return links | ||||
|
r43346 | |||
|
r39035 | else: | ||
linknodes = normallinknodes | ||||
|
r38925 | |||
|
r19334 | repo = self._repo | ||
|
r43346 | progress = repo.ui.makeprogress( | ||
|
r43347 | _(b'files'), unit=_(b'files'), total=len(changedfiles) | ||
|
r43346 | ) | ||
|
r19334 | for i, fname in enumerate(sorted(changedfiles)): | ||
filerevlog = repo.file(fname) | ||||
if not filerevlog: | ||||
|
r43346 | raise error.Abort( | ||
|
r43347 | _(b"empty or missing file data for %s") % fname | ||
|
r43346 | ) | ||
|
r19334 | |||
|
r39037 | clrevtolocalrev.clear() | ||
|
r19334 | linkrevnodes = linknodes(filerevlog, fname) | ||
|
r19207 | # Lookup for filenodes, we collected the linkrev nodes above in the | ||
# fastpath case and with lookupmf in the slowpath case. | ||||
def lookupfilelog(x): | ||||
return linkrevnodes[x] | ||||
|
r39043 | frev, flr = filerevlog.rev, filerevlog.linkrev | ||
|
r39769 | # Skip sending any filenode we know the client already | ||
# has. This avoids over-sending files relatively | ||||
# inexpensively, so it's not a problem if we under-filter | ||||
# here. | ||||
|
r43346 | filenodes = [ | ||
n for n in linkrevnodes if flr(frev(n)) not in commonrevs | ||||
] | ||||
|
r39043 | |||
|
r39056 | if not filenodes: | ||
continue | ||||
|
r39018 | |||
|
r39056 | progress.update(i + 1, item=fname) | ||
|
r39044 | |||
|
r39056 | deltas = deltagroup( | ||
|
r43346 | self._repo, | ||
filerevlog, | ||||
filenodes, | ||||
False, | ||||
lookupfilelog, | ||||
|
r39897 | self._forcedeltaparentprev, | ||
|
r39056 | ellipses=self._ellipses, | ||
clrevtolocalrev=clrevtolocalrev, | ||||
fullclnodes=self._fullclnodes, | ||||
|
r43346 | precomputedellipsis=self._precomputedellipsis, | ||
) | ||||
|
r39056 | |||
yield fname, deltas | ||||
|
r39046 | |||
|
r38429 | progress.complete() | ||
|
r19200 | |||
|
r43346 | |||
def _makecg1packer( | ||||
repo, | ||||
oldmatcher, | ||||
matcher, | ||||
bundlecaps, | ||||
ellipses=False, | ||||
shallow=False, | ||||
ellipsisroots=None, | ||||
fullnodes=None, | ||||
): | ||||
|
r38933 | builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( | ||
|
r43346 | d.node, d.p1node, d.p2node, d.linknode | ||
) | ||||
|
r27432 | |||
|
r43346 | return cgpacker( | ||
repo, | ||||
oldmatcher, | ||||
matcher, | ||||
b'01', | ||||
builddeltaheader=builddeltaheader, | ||||
manifestsend=b'', | ||||
forcedeltaparentprev=True, | ||||
bundlecaps=bundlecaps, | ||||
ellipses=ellipses, | ||||
shallow=shallow, | ||||
ellipsisroots=ellipsisroots, | ||||
fullnodes=fullnodes, | ||||
) | ||||
|
r38930 | |||
|
r43346 | |||
def _makecg2packer( | ||||
repo, | ||||
oldmatcher, | ||||
matcher, | ||||
bundlecaps, | ||||
ellipses=False, | ||||
shallow=False, | ||||
ellipsisroots=None, | ||||
fullnodes=None, | ||||
): | ||||
|
r38933 | builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( | ||
|
r43346 | d.node, d.p1node, d.p2node, d.basenode, d.linknode | ||
) | ||||
|
r38933 | |||
|
r43346 | return cgpacker( | ||
repo, | ||||
oldmatcher, | ||||
matcher, | ||||
b'02', | ||||
builddeltaheader=builddeltaheader, | ||||
manifestsend=b'', | ||||
bundlecaps=bundlecaps, | ||||
ellipses=ellipses, | ||||
shallow=shallow, | ||||
ellipsisroots=ellipsisroots, | ||||
fullnodes=fullnodes, | ||||
) | ||||
|
r38930 | |||
|
r38933 | |||
|
r43346 | def _makecg3packer( | ||
repo, | ||||
oldmatcher, | ||||
matcher, | ||||
bundlecaps, | ||||
ellipses=False, | ||||
shallow=False, | ||||
ellipsisroots=None, | ||||
fullnodes=None, | ||||
): | ||||
builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( | ||||
d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags | ||||
) | ||||
|
r38930 | |||
|
r43346 | return cgpacker( | ||
repo, | ||||
oldmatcher, | ||||
matcher, | ||||
b'03', | ||||
builddeltaheader=builddeltaheader, | ||||
manifestsend=closechunk(), | ||||
bundlecaps=bundlecaps, | ||||
ellipses=ellipses, | ||||
shallow=shallow, | ||||
ellipsisroots=ellipsisroots, | ||||
fullnodes=fullnodes, | ||||
) | ||||
_packermap = { | ||||
|
r43347 | b'01': (_makecg1packer, cg1unpacker), | ||
|
r43346 | # cg2 adds support for exchanging generaldelta | ||
|
r43347 | b'02': (_makecg2packer, cg2unpacker), | ||
|
r43346 | # cg3 adds support for exchanging revlog flags and treemanifests | ||
|
r43347 | b'03': (_makecg3packer, cg3unpacker), | ||
|
r26709 | } | ||
|
r23168 | |||
|
r43346 | |||
|
r30627 | def allsupportedversions(repo): | ||
|
r27928 | versions = set(_packermap.keys()) | ||
r43329 | needv03 = False | |||
|
r43346 | if ( | ||
|
r43347 | repo.ui.configbool(b'experimental', b'changegroup3') | ||
or repo.ui.configbool(b'experimental', b'treemanifest') | ||||
or b'treemanifest' in repo.requirements | ||||
|
r43346 | ): | ||
r43329 | # we keep version 03 because we need to to exchange treemanifest data | |||
# | ||||
# we also keep vresion 01 and 02, because it is possible for repo to | ||||
# contains both normal and tree manifest at the same time. so using | ||||
# older version to pull data is viable | ||||
# | ||||
# (or even to push subset of history) | ||||
needv03 = True | ||||
r43402 | if b'exp-sidedata-flag' in repo.requirements: | |||
needv03 = True | ||||
# don't attempt to use 01/02 until we do sidedata cleaning | ||||
versions.discard(b'01') | ||||
versions.discard(b'02') | ||||
r43329 | if not needv03: | |||
|
r43347 | versions.discard(b'03') | ||
|
r27953 | return versions | ||
|
r43346 | |||
|
r27953 | # Changegroup versions that can be applied to the repo | ||
def supportedincomingversions(repo): | ||||
|
r30628 | return allsupportedversions(repo) | ||
|
r27953 | |||
|
r43346 | |||
|
r27953 | # Changegroup versions that can be created from the repo | ||
def supportedoutgoingversions(repo): | ||||
|
r30627 | versions = allsupportedversions(repo) | ||
|
r43347 | if b'treemanifest' in repo.requirements: | ||
|
r27928 | # Versions 01 and 02 support only flat manifests and it's just too | ||
# expensive to convert between the flat manifest and tree manifest on | ||||
# the fly. Since tree manifests are hashed differently, all of history | ||||
# would have to be converted. Instead, we simply don't even pretend to | ||||
# support versions 01 and 02. | ||||
|
r43347 | versions.discard(b'01') | ||
versions.discard(b'02') | ||||
|
r38871 | if repository.NARROW_REQUIREMENT in repo.requirements: | ||
|
r36483 | # Versions 01 and 02 don't support revlog flags, and we need to | ||
# support that for stripping and unbundling to work. | ||||
|
r43347 | versions.discard(b'01') | ||
versions.discard(b'02') | ||||
|
r37150 | if LFS_REQUIREMENT in repo.requirements: | ||
# Versions 01 and 02 don't support revlog flags, and we need to | ||||
# mark LFS entries with REVIDX_EXTSTORED. | ||||
|
r43347 | versions.discard(b'01') | ||
versions.discard(b'02') | ||||
|
r37150 | |||
|
r27752 | return versions | ||
|
r27751 | |||
|
r43346 | |||
|
r34179 | def localversion(repo): | ||
# Finds the best version to use for bundles that are meant to be used | ||||
# locally, such as those from strip and shelve, and temporary bundles. | ||||
return max(supportedoutgoingversions(repo)) | ||||
|
r43346 | |||
|
r27929 | def safeversion(repo): | ||
# Finds the smallest version that it's safe to assume clients of the repo | ||||
|
r27931 | # will support. For example, all hg versions that support generaldelta also | ||
# support changegroup 02. | ||||
|
r27953 | versions = supportedoutgoingversions(repo) | ||
|
r43347 | if b'generaldelta' in repo.requirements: | ||
versions.discard(b'01') | ||||
|
r27929 | assert versions | ||
return min(versions) | ||||
|
r43346 | |||
def getbundler( | ||||
version, | ||||
repo, | ||||
bundlecaps=None, | ||||
oldmatcher=None, | ||||
matcher=None, | ||||
ellipses=False, | ||||
shallow=False, | ||||
ellipsisroots=None, | ||||
fullnodes=None, | ||||
): | ||||
|
r27953 | assert version in supportedoutgoingversions(repo) | ||
|
r38830 | |||
|
r40380 | if matcher is None: | ||
|
r41825 | matcher = matchmod.always() | ||
|
r40380 | if oldmatcher is None: | ||
|
r41825 | oldmatcher = matchmod.never() | ||
|
r38830 | |||
|
r43347 | if version == b'01' and not matcher.always(): | ||
|
r43346 | raise error.ProgrammingError( | ||
|
r43387 | b'version 01 changegroups do not support sparse file matchers' | ||
|
r43346 | ) | ||
|
r38830 | |||
|
r38944 | if ellipses and version in (b'01', b'02'): | ||
raise error.Abort( | ||||
|
r43346 | _( | ||
|
r43347 | b'ellipsis nodes require at least cg3 on client and server, ' | ||
b'but negotiated version %s' | ||||
|
r43346 | ) | ||
% version | ||||
) | ||||
|
r38944 | |||
|
r38830 | # Requested files could include files not in the local store. So | ||
# filter those out. | ||||
|
r40380 | matcher = repo.narrowmatch(matcher) | ||
|
r38830 | |||
|
r38930 | fn = _packermap[version][0] | ||
|
r43346 | return fn( | ||
repo, | ||||
oldmatcher, | ||||
matcher, | ||||
bundlecaps, | ||||
ellipses=ellipses, | ||||
shallow=shallow, | ||||
ellipsisroots=ellipsisroots, | ||||
fullnodes=fullnodes, | ||||
) | ||||
|
r27751 | |||
|
r29593 | def getunbundler(version, fh, alg, extras=None): | ||
return _packermap[version][1](fh, alg, extras=extras) | ||||
|
r27751 | |||
|
r43346 | |||
|
r20926 | def _changegroupinfo(repo, nodes, source): | ||
|
r43347 | if repo.ui.verbose or source == b'bundle': | ||
repo.ui.status(_(b"%d changesets found\n") % len(nodes)) | ||||
|
r20926 | if repo.ui.debugflag: | ||
|
r43347 | repo.ui.debug(b"list of changesets:\n") | ||
|
r20926 | for node in nodes: | ||
|
r43347 | repo.ui.debug(b"%s\n" % hex(node)) | ||
|
r20926 | |||
|
r34098 | |||
|
r43346 | def makechangegroup( | ||
repo, outgoing, version, source, fastpath=False, bundlecaps=None | ||||
): | ||||
cgstream = makestream( | ||||
repo, | ||||
outgoing, | ||||
version, | ||||
source, | ||||
fastpath=fastpath, | ||||
bundlecaps=bundlecaps, | ||||
) | ||||
return getunbundler( | ||||
version, | ||||
util.chunkbuffer(cgstream), | ||||
None, | ||||
|
r43347 | {b'clcount': len(outgoing.missing)}, | ||
|
r43346 | ) | ||
def makestream( | ||||
repo, | ||||
outgoing, | ||||
version, | ||||
source, | ||||
fastpath=False, | ||||
bundlecaps=None, | ||||
matcher=None, | ||||
): | ||||
bundler = getbundler(version, repo, bundlecaps=bundlecaps, matcher=matcher) | ||||
|
r34105 | |||
|
r20925 | repo = repo.unfiltered() | ||
commonrevs = outgoing.common | ||||
csets = outgoing.missing | ||||
heads = outgoing.missingheads | ||||
# We go through the fast path if we get told to, or if all (unfiltered | ||||
# heads have been requested (since we then know there all linkrevs will | ||||
# be pulled by the client). | ||||
heads.sort() | ||||
fastpathlinkrev = fastpath or ( | ||||
|
r43346 | repo.filtername is None and heads == sorted(repo.heads()) | ||
) | ||||
|
r20925 | |||
|
r43347 | repo.hook(b'preoutgoing', throw=True, source=source) | ||
|
r20926 | _changegroupinfo(repo, csets, source) | ||
|
r23177 | return bundler.generate(commonrevs, csets, fastpathlinkrev, source) | ||
|
r43346 | |||
|
r28361 | def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles): | ||
|
r20932 | revisions = 0 | ||
files = 0 | ||||
|
r43346 | progress = repo.ui.makeprogress( | ||
|
r43347 | _(b'files'), unit=_(b'files'), total=expectedfiles | ||
|
r43346 | ) | ||
|
r29724 | for chunkdata in iter(source.filelogheader, {}): | ||
|
r28361 | files += 1 | ||
|
r43347 | f = chunkdata[b"filename"] | ||
repo.ui.debug(b"adding %s revisions\n" % f) | ||||
|
r38401 | progress.increment() | ||
|
r27754 | fl = repo.file(f) | ||
|
r20932 | o = len(fl) | ||
|
r24120 | try: | ||
|
r34292 | deltas = source.deltaiter() | ||
if not fl.addgroup(deltas, revmap, trp): | ||||
|
r43347 | raise error.Abort(_(b"received file revlog group is empty")) | ||
|
r25660 | except error.CensoredBaseError as e: | ||
|
r43347 | raise error.Abort(_(b"received delta base is censored: %s") % e) | ||
|
r27754 | revisions += len(fl) - o | ||
|
r20932 | if f in needfiles: | ||
needs = needfiles[f] | ||||
|
r38806 | for new in pycompat.xrange(o, len(fl)): | ||
|
r20932 | n = fl.node(new) | ||
if n in needs: | ||||
needs.remove(n) | ||||
else: | ||||
|
r43347 | raise error.Abort(_(b"received spurious file revlog entry")) | ||
|
r20932 | if not needs: | ||
del needfiles[f] | ||||
|
r38401 | progress.complete() | ||
|
r20932 | |||
|
r43376 | for f, needs in pycompat.iteritems(needfiles): | ||
|
r20932 | fl = repo.file(f) | ||
for n in needs: | ||||
try: | ||||
fl.rev(n) | ||||
except error.LookupError: | ||||
|
r26587 | raise error.Abort( | ||
|
r43347 | _(b'missing file data for %s:%s - run hg verify') | ||
|
r43346 | % (f, hex(n)) | ||
) | ||||
|
r20932 | |||
return revisions, files | ||||