changegroup.py
1436 lines
| 55.7 KiB
| text/x-python
|
PythonLexer
/ mercurial / changegroup.py
|
r8226 | # changegroup.py - Mercurial changegroup manipulation functions | ||
# | ||||
# Copyright 2006 Matt Mackall <mpm@selenic.com> | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r3877 | |||
|
r25921 | from __future__ import absolute_import | ||
import os | ||||
import struct | ||||
|
r20933 | import weakref | ||
|
r25921 | |||
from .i18n import _ | ||||
from .node import ( | ||||
hex, | ||||
|
r38919 | nullid, | ||
|
r25921 | nullrev, | ||
short, | ||||
) | ||||
|
r38929 | from .thirdparty import ( | ||
attr, | ||||
) | ||||
|
r25921 | from . import ( | ||
dagutil, | ||||
error, | ||||
|
r38842 | manifest, | ||
|
r38830 | match as matchmod, | ||
|
r25921 | mdiff, | ||
phases, | ||||
|
r30925 | pycompat, | ||
|
r38871 | repository, | ||
|
r38919 | revlog, | ||
|
r25921 | util, | ||
) | ||||
|
r1981 | |||
|
r37102 | from .utils import ( | ||
stringutil, | ||||
) | ||||
|
r38932 | _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s") | ||
_CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s") | ||||
_CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH") | ||||
|
r14141 | |||
|
r37150 | LFS_REQUIREMENT = 'lfs' | ||
|
r35772 | readexactly = util.readexactly | ||
|
r13457 | |||
def getchunk(stream): | ||||
"""return the next chunk from stream as a string""" | ||||
d = readexactly(stream, 4) | ||||
|
r1981 | l = struct.unpack(">l", d)[0] | ||
if l <= 4: | ||||
|
r13458 | if l: | ||
|
r26587 | raise error.Abort(_("invalid chunk length %d") % l) | ||
|
r1981 | return "" | ||
|
r13457 | return readexactly(stream, l - 4) | ||
|
r1981 | |||
|
r5368 | def chunkheader(length): | ||
|
r9437 | """return a changegroup chunk header (string)""" | ||
|
r5368 | return struct.pack(">l", length + 4) | ||
|
r1981 | |||
def closechunk(): | ||||
|
r9437 | """return a changegroup chunk header (string) for a zero-length chunk""" | ||
|
r1981 | return struct.pack(">l", 0) | ||
|
r39017 | def _fileheader(path): | ||
"""Obtain a changegroup chunk header for a named path.""" | ||||
return chunkheader(len(path)) + path | ||||
|
r26540 | def writechunks(ui, chunks, filename, vfs=None): | ||
"""Write chunks to a file and return its filename. | ||||
|
r3659 | |||
|
r26540 | The stream is assumed to be a bundle file. | ||
|
r3659 | Existing files will not be overwritten. | ||
If no filename is specified, a temporary file is created. | ||||
""" | ||||
fh = None | ||||
cleanup = None | ||||
try: | ||||
if filename: | ||||
|
r20976 | if vfs: | ||
fh = vfs.open(filename, "wb") | ||||
else: | ||||
|
r30212 | # Increase default buffer size because default is usually | ||
# small (4k is common on Linux). | ||||
fh = open(filename, "wb", 131072) | ||||
|
r3659 | else: | ||
|
r38182 | fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg") | ||
|
r36853 | fh = os.fdopen(fd, r"wb") | ||
|
r3659 | cleanup = filename | ||
|
r26540 | for c in chunks: | ||
fh.write(c) | ||||
|
r3659 | cleanup = None | ||
return filename | ||||
finally: | ||||
if fh is not None: | ||||
fh.close() | ||||
if cleanup is not None: | ||||
|
r20976 | if filename and vfs: | ||
vfs.unlink(cleanup) | ||||
else: | ||||
os.unlink(cleanup) | ||||
|
r3660 | |||
|
r22390 | class cg1unpacker(object): | ||
|
r26708 | """Unpacker for cg1 changegroup streams. | ||
A changegroup unpacker handles the framing of the revision data in | ||||
the wire format. Most consumers will want to use the apply() | ||||
method to add the changes from the changegroup to a repository. | ||||
If you're forwarding a changegroup unmodified to another consumer, | ||||
use getchunks(), which returns an iterator of changegroup | ||||
chunks. This is mostly useful for cases where you need to know the | ||||
data stream has ended by observing the end of the changegroup. | ||||
deltachunk() is useful only if you're applying delta data. Most | ||||
consumers should prefer apply() instead. | ||||
A few other public methods exist. Those are used only for | ||||
bundlerepo and some debug commands - their use is discouraged. | ||||
""" | ||||
|
r22390 | deltaheader = _CHANGEGROUPV1_DELTA_HEADER | ||
|
r38932 | deltaheadersize = deltaheader.size | ||
|
r23896 | version = '01' | ||
|
r27920 | _grouplistcount = 1 # One list of files after the manifests | ||
|
r29593 | def __init__(self, fh, alg, extras=None): | ||
|
r30354 | if alg is None: | ||
alg = 'UN' | ||||
if alg not in util.compengines.supportedbundletypes: | ||||
|
r26587 | raise error.Abort(_('unknown stream compression type: %s') | ||
|
r26266 | % alg) | ||
|
r26392 | if alg == 'BZ': | ||
alg = '_truncatedBZ' | ||||
|
r30354 | |||
compengine = util.compengines.forbundletype(alg) | ||||
self._stream = compengine.decompressorreader(fh) | ||||
|
r12044 | self._type = alg | ||
|
r29593 | self.extras = extras or {} | ||
|
r12334 | self.callback = None | ||
|
r26706 | |||
# These methods (compressed, read, seek, tell) all appear to only | ||||
# be used by bundlerepo, but it's a little hard to tell. | ||||
|
r12044 | def compressed(self): | ||
|
r30589 | return self._type is not None and self._type != 'UN' | ||
|
r12043 | def read(self, l): | ||
return self._stream.read(l) | ||||
|
r12330 | def seek(self, pos): | ||
return self._stream.seek(pos) | ||||
def tell(self): | ||||
|
r12332 | return self._stream.tell() | ||
|
r12347 | def close(self): | ||
return self._stream.close() | ||||
|
r12334 | |||
|
r26707 | def _chunklength(self): | ||
|
r13459 | d = readexactly(self._stream, 4) | ||
|
r13458 | l = struct.unpack(">l", d)[0] | ||
if l <= 4: | ||||
if l: | ||||
|
r26587 | raise error.Abort(_("invalid chunk length %d") % l) | ||
|
r13458 | return 0 | ||
if self.callback: | ||||
|
r12334 | self.callback() | ||
|
r13458 | return l - 4 | ||
|
r12334 | |||
|
r14144 | def changelogheader(self): | ||
"""v10 does not have a changelog header chunk""" | ||||
return {} | ||||
def manifestheader(self): | ||||
"""v10 does not have a manifest header chunk""" | ||||
return {} | ||||
def filelogheader(self): | ||||
"""return the header of the filelogs chunk, v10 only has the filename""" | ||||
|
r26707 | l = self._chunklength() | ||
|
r14144 | if not l: | ||
return {} | ||||
fname = readexactly(self._stream, l) | ||||
|
r20675 | return {'filename': fname} | ||
|
r12334 | |||
|
r14141 | def _deltaheader(self, headertuple, prevnode): | ||
node, p1, p2, cs = headertuple | ||||
if prevnode is None: | ||||
deltabase = p1 | ||||
else: | ||||
deltabase = prevnode | ||||
|
r27433 | flags = 0 | ||
return node, p1, p2, deltabase, cs, flags | ||||
|
r14141 | |||
|
r14144 | def deltachunk(self, prevnode): | ||
|
r26707 | l = self._chunklength() | ||
|
r12336 | if not l: | ||
return {} | ||||
|
r14141 | headerdata = readexactly(self._stream, self.deltaheadersize) | ||
|
r38932 | header = self.deltaheader.unpack(headerdata) | ||
|
r14141 | delta = readexactly(self._stream, l - self.deltaheadersize) | ||
|
r27433 | node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode) | ||
|
r34295 | return (node, p1, p2, cs, deltabase, delta, flags) | ||
|
r12336 | |||
|
r20999 | def getchunks(self): | ||
"""returns all the chunks contains in the bundle | ||||
Used when you need to forward the binary stream to a file or another | ||||
network API. To do so, it parse the changegroup data, otherwise it will | ||||
block in case of sshrepo because it don't know the end of the stream. | ||||
""" | ||||
|
r34093 | # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog, | ||
# and a list of filelogs. For changegroup 3, we expect 4 parts: | ||||
# changelog, manifestlog, a list of tree manifestlogs, and a list of | ||||
# filelogs. | ||||
# | ||||
# Changelog and manifestlog parts are terminated with empty chunks. The | ||||
# tree and file parts are a list of entry sections. Each entry section | ||||
# is a series of chunks terminating in an empty chunk. The list of these | ||||
# entry sections is terminated in yet another empty chunk, so we know | ||||
# we've reached the end of the tree/file list when we reach an empty | ||||
# chunk that was proceeded by no non-empty chunks. | ||||
parts = 0 | ||||
while parts < 2 + self._grouplistcount: | ||||
noentries = True | ||||
|
r20999 | while True: | ||
chunk = getchunk(self) | ||||
if not chunk: | ||||
|
r34093 | # The first two empty chunks represent the end of the | ||
# changelog and the manifestlog portions. The remaining | ||||
# empty chunks represent either A) the end of individual | ||||
# tree or file entries in the file list, or B) the end of | ||||
# the entire list. It's the end of the entire list if there | ||||
# were no entries (i.e. noentries is True). | ||||
if parts < 2: | ||||
parts += 1 | ||||
elif noentries: | ||||
parts += 1 | ||||
|
r20999 | break | ||
|
r34093 | noentries = False | ||
|
r20999 | yield chunkheader(len(chunk)) | ||
pos = 0 | ||||
while pos < len(chunk): | ||||
next = pos + 2**20 | ||||
yield chunk[pos:next] | ||||
pos = next | ||||
yield closechunk() | ||||
|
r38365 | def _unpackmanifests(self, repo, revmap, trp, prog): | ||
self.callback = prog.increment | ||||
|
r26712 | # no need to check for empty manifest group here: | ||
# if the result of the merge of 1 and 2 is the same in 3 and 4, | ||||
# no new manifest will be created and the manifest group will | ||||
# be empty during the pull | ||||
self.manifestheader() | ||||
|
r34292 | deltas = self.deltaiter() | ||
|
r38574 | repo.manifestlog.addgroup(deltas, revmap, trp) | ||
|
r38392 | prog.complete() | ||
|
r28360 | self.callback = None | ||
|
r26712 | |||
|
r33308 | def apply(self, repo, tr, srctype, url, targetphase=phases.draft, | ||
expectedtotal=None): | ||||
|
r26695 | """Add the changegroup returned by source.read() to this repo. | ||
srctype is a string like 'push', 'pull', or 'unbundle'. url is | ||||
the URL of the repo where this changegroup is coming from. | ||||
Return an integer summarizing the change to this repo: | ||||
- nothing changed or no source: 0 | ||||
- more heads than before: 1+added heads (2..n) | ||||
- fewer heads than before: -1-removed heads (-2..-n) | ||||
- number of heads stays the same: 1 | ||||
""" | ||||
repo = repo.unfiltered() | ||||
def csmap(x): | ||||
repo.ui.debug("add changeset %s\n" % short(x)) | ||||
return len(cl) | ||||
def revmap(x): | ||||
return cl.rev(x) | ||||
changesets = files = revisions = 0 | ||||
|
r26880 | try: | ||
|
r32931 | # The transaction may already carry source information. In this | ||
# case we use the top level data. We overwrite the argument | ||||
# because we need to use the top level value (if they exist) | ||||
# in this function. | ||||
srctype = tr.hookargs.setdefault('source', srctype) | ||||
url = tr.hookargs.setdefault('url', url) | ||||
|
r33634 | repo.hook('prechangegroup', | ||
throw=True, **pycompat.strkwargs(tr.hookargs)) | ||||
|
r26695 | |||
|
r32931 | # write changelog data to temp files so concurrent readers | ||
# will not see an inconsistent view | ||||
cl = repo.changelog | ||||
cl.delayupdate(tr) | ||||
oldheads = set(cl.heads()) | ||||
|
r26695 | |||
|
r32931 | trp = weakref.proxy(tr) | ||
# pull off the changeset group | ||||
repo.ui.status(_("adding changesets\n")) | ||||
clstart = len(cl) | ||||
|
r38365 | progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'), | ||
total=expectedtotal) | ||||
self.callback = progress.increment | ||||
|
r26695 | |||
|
r32931 | efiles = set() | ||
def onchangelog(cl, node): | ||||
efiles.update(cl.readfiles(node)) | ||||
|
r26695 | |||
|
r32931 | self.changelogheader() | ||
|
r34292 | deltas = self.deltaiter() | ||
cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog) | ||||
|
r32931 | efiles = len(efiles) | ||
|
r26695 | |||
|
r33308 | if not cgnodes: | ||
|
r33309 | repo.ui.develwarn('applied empty changegroup', | ||
|
r34735 | config='warn-empty-changegroup') | ||
|
r32931 | clend = len(cl) | ||
changesets = clend - clstart | ||||
|
r38392 | progress.complete() | ||
|
r32931 | self.callback = None | ||
|
r26695 | |||
|
r32931 | # pull off the manifest group | ||
repo.ui.status(_("adding manifests\n")) | ||||
|
r38365 | # We know that we'll never have more manifests than we had | ||
# changesets. | ||||
progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'), | ||||
total=changesets) | ||||
self._unpackmanifests(repo, revmap, trp, progress) | ||||
|
r26695 | |||
|
r32931 | needfiles = {} | ||
r33224 | if repo.ui.configbool('server', 'validate'): | |||
|
r32931 | cl = repo.changelog | ||
ml = repo.manifestlog | ||||
# validate incoming csets have their manifests | ||||
|
r38806 | for cset in pycompat.xrange(clstart, clend): | ||
|
r32931 | mfnode = cl.changelogrevision(cset).manifest | ||
mfest = ml[mfnode].readdelta() | ||||
# store file cgnodes we must see | ||||
for f, n in mfest.iteritems(): | ||||
needfiles.setdefault(f, set()).add(n) | ||||
|
r26695 | |||
|
r32931 | # process the files | ||
repo.ui.status(_("adding file changes\n")) | ||||
newrevs, newfiles = _addchangegroupfiles( | ||||
repo, self, revmap, trp, efiles, needfiles) | ||||
revisions += newrevs | ||||
files += newfiles | ||||
|
r26695 | |||
|
r32931 | deltaheads = 0 | ||
if oldheads: | ||||
heads = cl.heads() | ||||
deltaheads = len(heads) - len(oldheads) | ||||
for h in heads: | ||||
if h not in oldheads and repo[h].closesbranch(): | ||||
deltaheads -= 1 | ||||
htext = "" | ||||
if deltaheads: | ||||
htext = _(" (%+d heads)") % deltaheads | ||||
|
r26695 | |||
|
r32931 | repo.ui.status(_("added %d changesets" | ||
" with %d changes to %d files%s\n") | ||||
% (changesets, revisions, files, htext)) | ||||
repo.invalidatevolatilesets() | ||||
|
r26695 | |||
|
r32931 | if changesets > 0: | ||
if 'node' not in tr.hookargs: | ||||
tr.hookargs['node'] = hex(cl.node(clstart)) | ||||
tr.hookargs['node_last'] = hex(cl.node(clend - 1)) | ||||
hookargs = dict(tr.hookargs) | ||||
else: | ||||
hookargs = dict(tr.hookargs) | ||||
hookargs['node'] = hex(cl.node(clstart)) | ||||
hookargs['node_last'] = hex(cl.node(clend - 1)) | ||||
|
r33634 | repo.hook('pretxnchangegroup', | ||
throw=True, **pycompat.strkwargs(hookargs)) | ||||
|
r26695 | |||
|
r38806 | added = [cl.node(r) for r in pycompat.xrange(clstart, clend)] | ||
|
r33456 | phaseall = None | ||
|
r32931 | if srctype in ('push', 'serve'): | ||
# Old servers can not push the boundary themselves. | ||||
# New servers won't push the boundary if changeset already | ||||
# exists locally as secret | ||||
# | ||||
# We should not use added here but the list of all change in | ||||
# the bundle | ||||
if repo.publishing(): | ||||
|
r33456 | targetphase = phaseall = phases.public | ||
|
r32931 | else: | ||
|
r33456 | # closer target phase computation | ||
|
r32931 | # Those changesets have been pushed from the | ||
# outside, their phases are going to be pushed | ||||
# alongside. Therefor `targetphase` is | ||||
# ignored. | ||||
|
r33456 | targetphase = phaseall = phases.draft | ||
if added: | ||||
phases.registernew(repo, tr, targetphase, added) | ||||
if phaseall is not None: | ||||
phases.advanceboundary(repo, tr, phaseall, cgnodes) | ||||
|
r26695 | |||
|
r32931 | if changesets > 0: | ||
|
r26695 | |||
|
r32931 | def runhooks(): | ||
# These hooks run when the lock releases, not when the | ||||
# transaction closes. So it's possible for the changelog | ||||
# to have changed since we last saw it. | ||||
if clstart >= len(repo): | ||||
return | ||||
|
r26695 | |||
|
r33678 | repo.hook("changegroup", **pycompat.strkwargs(hookargs)) | ||
|
r27867 | |||
|
r32931 | for n in added: | ||
args = hookargs.copy() | ||||
args['node'] = hex(n) | ||||
del args['node_last'] | ||||
|
r33678 | repo.hook("incoming", **pycompat.strkwargs(args)) | ||
|
r26695 | |||
|
r32931 | newheads = [h for h in repo.heads() | ||
if h not in oldheads] | ||||
repo.ui.log("incoming", | ||||
|
r36755 | "%d incoming changes - new heads: %s\n", | ||
|
r32931 | len(added), | ||
', '.join([hex(c[:6]) for c in newheads])) | ||||
|
r26695 | |||
|
r32931 | tr.addpostclose('changegroup-runhooks-%020i' % clstart, | ||
lambda tr: repo._afterlock(runhooks)) | ||||
|
r26695 | finally: | ||
repo.ui.flush() | ||||
# never return 0 here: | ||||
|
r32870 | if deltaheads < 0: | ||
|
r33030 | ret = deltaheads - 1 | ||
|
r26695 | else: | ||
|
r33030 | ret = deltaheads + 1 | ||
|
r33461 | return ret | ||
|
r26695 | |||
|
r34292 | def deltaiter(self): | ||
|
r34147 | """ | ||
returns an iterator of the deltas in this changegroup | ||||
Useful for passing to the underlying storage system to be stored. | ||||
""" | ||||
chain = None | ||||
for chunkdata in iter(lambda: self.deltachunk(chain), {}): | ||||
|
r34295 | # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags) | ||
yield chunkdata | ||||
chain = chunkdata[0] | ||||
|
r34147 | |||
|
r23181 | class cg2unpacker(cg1unpacker): | ||
|
r26708 | """Unpacker for cg2 streams. | ||
cg2 streams add support for generaldelta, so the delta header | ||||
format is slightly different. All other features about the data | ||||
remain the same. | ||||
""" | ||||
|
r23181 | deltaheader = _CHANGEGROUPV2_DELTA_HEADER | ||
|
r38932 | deltaheadersize = deltaheader.size | ||
|
r23896 | version = '02' | ||
|
r23181 | |||
def _deltaheader(self, headertuple, prevnode): | ||||
node, p1, p2, deltabase, cs = headertuple | ||||
|
r27433 | flags = 0 | ||
return node, p1, p2, deltabase, cs, flags | ||||
|
r23181 | |||
|
r27432 | class cg3unpacker(cg2unpacker): | ||
"""Unpacker for cg3 streams. | ||||
|
r27433 | cg3 streams add support for exchanging treemanifests and revlog | ||
|
r27753 | flags. It adds the revlog flags to the delta header and an empty chunk | ||
separating manifests and files. | ||||
|
r27432 | """ | ||
|
r27433 | deltaheader = _CHANGEGROUPV3_DELTA_HEADER | ||
|
r38932 | deltaheadersize = deltaheader.size | ||
|
r27432 | version = '03' | ||
|
r27920 | _grouplistcount = 2 # One list of manifests and one list of files | ||
|
r27432 | |||
|
r27433 | def _deltaheader(self, headertuple, prevnode): | ||
node, p1, p2, deltabase, cs, flags = headertuple | ||||
return node, p1, p2, deltabase, cs, flags | ||||
|
r38365 | def _unpackmanifests(self, repo, revmap, trp, prog): | ||
super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog) | ||||
|
r29724 | for chunkdata in iter(self.filelogheader, {}): | ||
|
r27754 | # If we get here, there are directory manifests in the changegroup | ||
d = chunkdata["filename"] | ||||
repo.ui.debug("adding %s revisions\n" % d) | ||||
|
r30339 | dirlog = repo.manifestlog._revlog.dirlog(d) | ||
|
r34292 | deltas = self.deltaiter() | ||
if not dirlog.addgroup(deltas, revmap, trp): | ||||
|
r27754 | raise error.Abort(_("received dir revlog group is empty")) | ||
|
r12329 | class headerlessfixup(object): | ||
def __init__(self, fh, h): | ||||
self._h = h | ||||
self._fh = fh | ||||
def read(self, n): | ||||
if self._h: | ||||
d, self._h = self._h[:n], self._h[n:] | ||||
if len(d) < n: | ||||
|
r13457 | d += readexactly(self._fh, n - len(d)) | ||
|
r12329 | return d | ||
|
r13457 | return readexactly(self._fh, n) | ||
|
r12329 | |||
|
r38929 | @attr.s(slots=True, frozen=True) | ||
class revisiondelta(object): | ||||
"""Describes a delta entry in a changegroup. | ||||
Captured data is sufficient to serialize the delta into multiple | ||||
formats. | ||||
""" | ||||
# 20 byte node of this revision. | ||||
node = attr.ib() | ||||
# 20 byte nodes of parent revisions. | ||||
p1node = attr.ib() | ||||
p2node = attr.ib() | ||||
# 20 byte node of node this delta is against. | ||||
basenode = attr.ib() | ||||
# 20 byte node of changeset revision this delta is associated with. | ||||
linknode = attr.ib() | ||||
# 2 bytes of flags to apply to revision data. | ||||
flags = attr.ib() | ||||
# Iterable of chunks holding raw delta data. | ||||
deltachunks = attr.ib() | ||||
|
r38919 | |||
|
r39018 | def _sortnodesnormal(store, nodes, reorder): | ||
"""Sort nodes for changegroup generation and turn into revnums.""" | ||||
# for generaldelta revlogs, we linearize the revs; this will both be | ||||
# much quicker and generate a much smaller bundle | ||||
if (store._generaldelta and reorder is None) or reorder: | ||||
dag = dagutil.revlogdag(store) | ||||
return dag.linearize(set(store.rev(n) for n in nodes)) | ||||
else: | ||||
return sorted([store.rev(n) for n in nodes]) | ||||
def _sortnodesellipsis(store, nodes, clnodetorev, lookup): | ||||
"""Sort nodes for changegroup generation and turn into revnums.""" | ||||
# Ellipses serving mode. | ||||
# | ||||
# In a perfect world, we'd generate better ellipsis-ified graphs | ||||
# for non-changelog revlogs. In practice, we haven't started doing | ||||
# that yet, so the resulting DAGs for the manifestlog and filelogs | ||||
# are actually full of bogus parentage on all the ellipsis | ||||
# nodes. This has the side effect that, while the contents are | ||||
# correct, the individual DAGs might be completely out of whack in | ||||
# a case like 882681bc3166 and its ancestors (back about 10 | ||||
# revisions or so) in the main hg repo. | ||||
# | ||||
# The one invariant we *know* holds is that the new (potentially | ||||
# bogus) DAG shape will be valid if we order the nodes in the | ||||
# order that they're introduced in dramatis personae by the | ||||
# changelog, so what we do is we sort the non-changelog histories | ||||
# by the order in which they are used by the changelog. | ||||
key = lambda n: clnodetorev[lookup(n)] | ||||
return [store.rev(n) for n in sorted(nodes, key=key)] | ||||
|
r39021 | def _revisiondeltanormal(store, rev, prev, linknode, deltaparentfn): | ||
"""Construct a revision delta for non-ellipses changegroup generation.""" | ||||
node = store.node(rev) | ||||
p1, p2 = store.parentrevs(rev) | ||||
base = deltaparentfn(store, rev, p1, p2, prev) | ||||
prefix = '' | ||||
if store.iscensored(base) or store.iscensored(rev): | ||||
try: | ||||
delta = store.revision(node, raw=True) | ||||
except error.CensoredNodeError as e: | ||||
delta = e.tombstone | ||||
if base == nullrev: | ||||
prefix = mdiff.trivialdiffheader(len(delta)) | ||||
else: | ||||
baselen = store.rawsize(base) | ||||
prefix = mdiff.replacediffheader(baselen, len(delta)) | ||||
elif base == nullrev: | ||||
delta = store.revision(node, raw=True) | ||||
prefix = mdiff.trivialdiffheader(len(delta)) | ||||
else: | ||||
delta = store.revdiff(base, rev) | ||||
p1n, p2n = store.parents(node) | ||||
return revisiondelta( | ||||
node=node, | ||||
p1node=p1n, | ||||
p2node=p2n, | ||||
basenode=store.node(base), | ||||
linknode=linknode, | ||||
flags=store.flags(rev), | ||||
deltachunks=(prefix, delta), | ||||
) | ||||
|
r38938 | class cgpacker(object): | ||
|
r38936 | def __init__(self, repo, filematcher, version, allowreorder, | ||
|
r39011 | deltaparentfn, builddeltaheader, manifestsend, | ||
|
r38961 | bundlecaps=None, ellipses=False, | ||
|
r38945 | shallow=False, ellipsisroots=None, fullnodes=None): | ||
|
r19202 | """Given a source repo, construct a bundler. | ||
|
r32287 | |||
|
r38830 | filematcher is a matcher that matches on files to include in the | ||
changegroup. Used to facilitate sparse changegroups. | ||||
|
r38936 | allowreorder controls whether reordering of revisions is allowed. | ||
This value is used when ``bundle.reorder`` is ``auto`` or isn't | ||||
set. | ||||
|
r39011 | deltaparentfn is a callable that resolves the delta parent for | ||
a specific revision. | ||||
|
r38937 | |||
|
r38933 | builddeltaheader is a callable that constructs the header for a group | ||
delta. | ||||
|
r38934 | manifestsend is a chunk to send after manifests have been fully emitted. | ||
|
r38944 | ellipses indicates whether ellipsis serving mode is enabled. | ||
|
r32287 | bundlecaps is optional and can be used to specify the set of | ||
capabilities which can be used to build the bundle. While bundlecaps is | ||||
unused in core Mercurial, extensions rely on this feature to communicate | ||||
capabilities to customize the changegroup packer. | ||||
|
r38940 | |||
shallow indicates whether shallow data might be sent. The packer may | ||||
need to pack file contents not introduced by the changes being packed. | ||||
|
r38945 | |||
fullnodes is the list of nodes which should not be ellipsis nodes. We | ||||
store this rather than the set of nodes that should be ellipsis because | ||||
for very large histories we expect this to be significantly smaller. | ||||
|
r19202 | """ | ||
|
r38830 | assert filematcher | ||
self._filematcher = filematcher | ||||
|
r38931 | self.version = version | ||
|
r39011 | self._deltaparentfn = deltaparentfn | ||
|
r38933 | self._builddeltaheader = builddeltaheader | ||
|
r38934 | self._manifestsend = manifestsend | ||
|
r38944 | self._ellipses = ellipses | ||
|
r38931 | |||
|
r32287 | # Set of capabilities we can use to build the bundle. | ||
if bundlecaps is None: | ||||
bundlecaps = set() | ||||
self._bundlecaps = bundlecaps | ||||
|
r38940 | self._isshallow = shallow | ||
|
r38945 | self._fullnodes = fullnodes | ||
|
r38936 | |||
|
r38943 | # Maps ellipsis revs to their roots at the changelog level. | ||
self._precomputedellipsis = ellipsisroots | ||||
|
r25831 | # experimental config: bundle.reorder | ||
r33180 | reorder = repo.ui.config('bundle', 'reorder') | |||
|
r19202 | if reorder == 'auto': | ||
|
r38936 | self._reorder = allowreorder | ||
|
r19202 | else: | ||
|
r38936 | self._reorder = stringutil.parsebool(reorder) | ||
|
r19202 | self._repo = repo | ||
|
r38936 | |||
|
r23748 | if self._repo.ui.verbose and not self._repo.ui.debugflag: | ||
self._verbosenote = self._repo.ui.note | ||||
else: | ||||
self._verbosenote = lambda s: None | ||||
|
r38942 | # Maps CL revs to per-revlog revisions. Cleared in close() at | ||
# the end of each group. | ||||
self._clrevtolocalrev = {} | ||||
self._nextclrevtolocalrev = {} | ||||
# Maps changelog nodes to changelog revs. Filled in once | ||||
# during changelog stage and then left unmodified. | ||||
self._clnodetorev = {} | ||||
|
r38939 | def _close(self): | ||
|
r38923 | # Ellipses serving mode. | ||
|
r38942 | self._clrevtolocalrev.clear() | ||
|
r39010 | if self._nextclrevtolocalrev is not None: | ||
|
r38968 | self._clrevtolocalrev = self._nextclrevtolocalrev | ||
|
r39010 | self._nextclrevtolocalrev = None | ||
|
r38923 | |||
|
r13831 | return closechunk() | ||
|
r19200 | |||
|
r39018 | def group(self, revs, store, ischangelog, lookup, units=None): | ||
|
r19200 | """Calculate a delta group, yielding a sequence of changegroup chunks | ||
(strings). | ||||
Given a list of changeset revs, return a set of deltas and | ||||
metadata corresponding to nodes. The first delta is | ||||
first parent(nodelist[0]) -> nodelist[0], the receiver is | ||||
guaranteed to have this parent as it has all history before | ||||
these changesets. In the case firstparent is nullrev the | ||||
changegroup starts with a full revision. | ||||
|
r19208 | |||
If units is not None, progress detail will be generated, units specifies | ||||
the type of revlog that is touched (changelog, manifest, etc.). | ||||
|
r19200 | """ | ||
# if we don't have any revisions touched by these changesets, bail | ||||
|
r39018 | if len(revs) == 0: | ||
|
r38939 | yield self._close() | ||
|
r19200 | return | ||
# add the parent of the first rev | ||||
|
r38927 | p = store.parentrevs(revs[0])[0] | ||
|
r19200 | revs.insert(0, p) | ||
# build deltas | ||||
|
r38429 | progress = None | ||
if units is not None: | ||||
progress = self._repo.ui.makeprogress(_('bundling'), unit=units, | ||||
total=(len(revs) - 1)) | ||||
|
r38806 | for r in pycompat.xrange(len(revs) - 1): | ||
|
r38429 | if progress: | ||
progress.update(r + 1) | ||||
|
r19200 | prev, curr = revs[r], revs[r + 1] | ||
|
r38927 | linknode = lookup(store.node(curr)) | ||
|
r39020 | |||
if self._ellipses: | ||||
delta = self._revisiondeltanarrow(store, ischangelog, | ||||
curr, prev, linknode) | ||||
else: | ||||
|
r39021 | delta = _revisiondeltanormal(store, curr, prev, linknode, | ||
self._deltaparentfn) | ||||
|
r39020 | |||
if not delta: | ||||
continue | ||||
meta = self._builddeltaheader(delta) | ||||
l = len(meta) + sum(len(x) for x in delta.deltachunks) | ||||
yield chunkheader(l) | ||||
yield meta | ||||
for x in delta.deltachunks: | ||||
yield x | ||||
|
r19200 | |||
|
r38429 | if progress: | ||
progress.complete() | ||||
|
r38939 | yield self._close() | ||
|
r19200 | |||
|
r19289 | # filter any nodes that claim to be part of the known set | ||
|
r38939 | def _prune(self, store, missing, commonrevs): | ||
|
r38842 | # TODO this violates storage abstraction for manifests. | ||
|
r38927 | if isinstance(store, manifest.manifestrevlog): | ||
if not self._filematcher.visitdir(store._dir[:-1] or '.'): | ||||
|
r38842 | return [] | ||
|
r38927 | rr, rl = store.rev, store.linkrev | ||
|
r19289 | return [n for n in missing if rl(rr(n)) not in commonrevs] | ||
|
r39018 | def _packmanifests(self, dir, dirlog, revs, lookuplinknode): | ||
|
r38961 | """Pack manifests into a changegroup stream. | ||
|
r38935 | |||
Encodes the directory name in the output so multiple manifests | ||||
|
r38961 | can be sent. Multiple manifests is not supported by cg1 and cg2. | ||
|
r38935 | """ | ||
if dir: | ||||
|
r38961 | assert self.version == b'03' | ||
|
r39017 | yield _fileheader(dir) | ||
|
r38935 | |||
|
r39018 | for chunk in self.group(revs, dirlog, False, lookuplinknode, | ||
|
r38935 | units=_('manifests')): | ||
yield chunk | ||||
|
r19204 | def generate(self, commonrevs, clnodes, fastpathlinkrev, source): | ||
|
r39012 | """Yield a sequence of changegroup byte chunks.""" | ||
|
r19202 | repo = self._repo | ||
|
r24978 | cl = repo.changelog | ||
|
r19204 | |||
|
r39012 | self._verbosenote(_('uncompressed size of bundle content:\n')) | ||
size = 0 | ||||
clstate, chunks = self._generatechangelog(cl, clnodes) | ||||
for chunk in chunks: | ||||
size += len(chunk) | ||||
yield chunk | ||||
self._verbosenote(_('%8.i (changelog)\n') % size) | ||||
clrevorder = clstate['clrevorder'] | ||||
mfs = clstate['mfs'] | ||||
changedfiles = clstate['changedfiles'] | ||||
# We need to make sure that the linkrev in the changegroup refers to | ||||
# the first changeset that introduced the manifest or file revision. | ||||
# The fastpath is usually safer than the slowpath, because the filelogs | ||||
# are walked in revlog order. | ||||
# | ||||
# When taking the slowpath with reorder=None and the manifest revlog | ||||
# uses generaldelta, the manifest may be walked in the "wrong" order. | ||||
# Without 'clrevorder', we would get an incorrect linkrev (see fix in | ||||
# cc0ff93d0c0c). | ||||
# | ||||
# When taking the fastpath, we are only vulnerable to reordering | ||||
# of the changelog itself. The changelog never uses generaldelta, so | ||||
# it is only reordered when reorder=True. To handle this case, we | ||||
# simply take the slowpath, which already has the 'clrevorder' logic. | ||||
# This was also fixed in cc0ff93d0c0c. | ||||
fastpathlinkrev = fastpathlinkrev and not self._reorder | ||||
# Treemanifests don't work correctly with fastpathlinkrev | ||||
# either, because we don't discover which directory nodes to | ||||
# send along with files. This could probably be fixed. | ||||
fastpathlinkrev = fastpathlinkrev and ( | ||||
'treemanifest' not in repo.requirements) | ||||
fnodes = {} # needed file nodes | ||||
for chunk in self.generatemanifests(commonrevs, clrevorder, | ||||
fastpathlinkrev, mfs, fnodes, source): | ||||
yield chunk | ||||
|
r39019 | mfdicts = None | ||
if self._ellipses and self._isshallow: | ||||
mfdicts = [(self._repo.manifestlog[n].read(), lr) | ||||
for (n, lr) in mfs.iteritems()] | ||||
|
r39012 | |||
mfs.clear() | ||||
clrevs = set(cl.rev(x) for x in clnodes) | ||||
if not fastpathlinkrev: | ||||
def linknodes(unused, fname): | ||||
return fnodes.get(fname, {}) | ||||
else: | ||||
cln = cl.node | ||||
def linknodes(filerevlog, fname): | ||||
llr = filerevlog.linkrev | ||||
fln = filerevlog.node | ||||
revs = ((r, llr(r)) for r in filerevlog) | ||||
return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs) | ||||
for chunk in self.generatefiles(changedfiles, linknodes, commonrevs, | ||||
|
r39019 | source, mfdicts): | ||
|
r39012 | yield chunk | ||
yield self._close() | ||||
if clnodes: | ||||
repo.hook('outgoing', node=hex(clnodes[0]), source=source) | ||||
def _generatechangelog(self, cl, nodes): | ||||
"""Generate data for changelog chunks. | ||||
Returns a 2-tuple of a dict containing state and an iterable of | ||||
byte chunks. The state will not be fully populated until the | ||||
chunk stream has been fully consumed. | ||||
""" | ||||
|
r23381 | clrevorder = {} | ||
|
r19204 | mfs = {} # needed manifests | ||
|
r39012 | mfl = self._repo.manifestlog | ||
|
r38926 | # TODO violates storage abstraction. | ||
mfrevlog = mfl._revlog | ||||
|
r28241 | changedfiles = set() | ||
|
r19204 | |||
|
r38926 | # Callback for the changelog, used to collect changed files and | ||
# manifest nodes. | ||||
|
r19207 | # Returns the linkrev node (identity in the changelog case). | ||
def lookupcl(x): | ||||
c = cl.read(x) | ||||
|
r23381 | clrevorder[x] = len(clrevorder) | ||
|
r38926 | |||
|
r38944 | if self._ellipses: | ||
|
r38926 | # Only update mfs if x is going to be sent. Otherwise we | ||
# end up with bogus linkrevs specified for manifests and | ||||
# we skip some manifest nodes that we should otherwise | ||||
# have sent. | ||||
|
r38945 | if (x in self._fullnodes | ||
|
r38943 | or cl.rev(x) in self._precomputedellipsis): | ||
|
r38926 | n = c[0] | ||
# Record the first changeset introducing this manifest | ||||
# version. | ||||
mfs.setdefault(n, x) | ||||
# Set this narrow-specific dict so we have the lowest | ||||
# manifest revnum to look up for this cl revnum. (Part of | ||||
# mapping changelog ellipsis parents to manifest ellipsis | ||||
# parents) | ||||
|
r38942 | self._nextclrevtolocalrev.setdefault(cl.rev(x), | ||
mfrevlog.rev(n)) | ||||
|
r38926 | # We can't trust the changed files list in the changeset if the | ||
# client requested a shallow clone. | ||||
|
r38940 | if self._isshallow: | ||
|
r38926 | changedfiles.update(mfl[c[0]].read().keys()) | ||
else: | ||||
changedfiles.update(c[3]) | ||||
else: | ||||
n = c[0] | ||||
# record the first changeset introducing this manifest version | ||||
mfs.setdefault(n, x) | ||||
# Record a complete list of potentially-changed files in | ||||
# this manifest. | ||||
changedfiles.update(c[3]) | ||||
|
r19207 | return x | ||
|
r19204 | |||
|
r39018 | # Changelog doesn't benefit from reordering revisions. So send out | ||
# revisions in store order. | ||||
revs = sorted(cl.rev(n) for n in nodes) | ||||
|
r39012 | state = { | ||
'clrevorder': clrevorder, | ||||
'mfs': mfs, | ||||
'changedfiles': changedfiles, | ||||
} | ||||
|
r38926 | |||
|
r39018 | gen = self.group(revs, cl, True, lookupcl, units=_('changesets')) | ||
|
r28227 | |||
|
r39012 | return state, gen | ||
|
r28227 | |||
def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs, | ||||
|
r34148 | fnodes, source): | ||
"""Returns an iterator of changegroup chunks containing manifests. | ||||
`source` is unused here, but is used by extensions like remotefilelog to | ||||
change what is sent based in pulls vs pushes, etc. | ||||
""" | ||||
|
r28227 | repo = self._repo | ||
|
r30294 | mfl = repo.manifestlog | ||
dirlog = mfl._revlog.dirlog | ||||
|
r28232 | tmfnodes = {'': mfs} | ||
|
r28227 | |||
|
r19207 | # Callback for the manifest, used to collect linkrevs for filelog | ||
# revisions. | ||||
# Returns the linkrev node (collected in lookupcl). | ||||
|
r35013 | def makelookupmflinknode(dir, nodes): | ||
|
r28231 | if fastpathlinkrev: | ||
assert not dir | ||||
return mfs.__getitem__ | ||||
|
r27239 | def lookupmflinknode(x): | ||
"""Callback for looking up the linknode for manifests. | ||||
|
r27219 | |||
|
r27239 | Returns the linkrev node for the specified manifest. | ||
|
r27219 | |||
|
r27239 | SIDE EFFECT: | ||
|
r27432 | 1) fclnodes gets populated with the list of relevant | ||
file nodes if we're not using fastpathlinkrev | ||||
2) When treemanifests are in use, collects treemanifest nodes | ||||
to send | ||||
|
r27219 | |||
|
r27432 | Note that this means manifests must be completely sent to | ||
the client before you can trust the list of files and | ||||
treemanifests to send. | ||||
|
r27239 | """ | ||
|
r35013 | clnode = nodes[x] | ||
|
r30294 | mdata = mfl.get(dir, x).readfast(shallow=True) | ||
|
r28241 | for p, n, fl in mdata.iterentries(): | ||
if fl == 't': # subdirectory manifest | ||||
subdir = dir + p + '/' | ||||
tmfclnodes = tmfnodes.setdefault(subdir, {}) | ||||
tmfclnode = tmfclnodes.setdefault(n, clnode) | ||||
if clrevorder[clnode] < clrevorder[tmfclnode]: | ||||
tmfclnodes[n] = clnode | ||||
else: | ||||
f = dir + p | ||||
|
r28240 | fclnodes = fnodes.setdefault(f, {}) | ||
fclnode = fclnodes.setdefault(n, clnode) | ||||
if clrevorder[clnode] < clrevorder[fclnode]: | ||||
fclnodes[n] = clnode | ||||
|
r27239 | return clnode | ||
|
r28231 | return lookupmflinknode | ||
|
r19206 | |||
|
r28228 | size = 0 | ||
|
r28232 | while tmfnodes: | ||
|
r35013 | dir, nodes = tmfnodes.popitem() | ||
|
r39018 | store = dirlog(dir) | ||
prunednodes = self._prune(store, nodes, commonrevs) | ||||
|
r29371 | if not dir or prunednodes: | ||
|
r39018 | lookupfn = makelookupmflinknode(dir, nodes) | ||
if self._ellipses: | ||||
revs = _sortnodesellipsis(store, prunednodes, | ||||
self._clnodetorev, lookupfn) | ||||
else: | ||||
revs = _sortnodesnormal(store, prunednodes, | ||||
self._reorder) | ||||
for x in self._packmanifests(dir, store, revs, lookupfn): | ||||
|
r29371 | size += len(x) | ||
yield x | ||||
|
r28229 | self._verbosenote(_('%8.i (manifests)\n') % size) | ||
|
r38934 | yield self._manifestsend | ||
|
r19206 | |||
|
r24897 | # The 'source' parameter is useful for extensions | ||
|
r39019 | def generatefiles(self, changedfiles, linknodes, commonrevs, source, | ||
mfdicts): | ||||
|
r38925 | changedfiles = list(filter(self._filematcher, changedfiles)) | ||
|
r38940 | if self._isshallow: | ||
|
r38925 | # In a shallow clone, the linknodes callback needs to also include | ||
# those file nodes that are in the manifests we sent but weren't | ||||
# introduced by those manifests. | ||||
commonctxs = [self._repo[c] for c in commonrevs] | ||||
oldlinknodes = linknodes | ||||
clrev = self._repo.changelog.rev | ||||
# Defining this function has a side-effect of overriding the | ||||
# function of the same name that was passed in as an argument. | ||||
# TODO have caller pass in appropriate function. | ||||
def linknodes(flog, fname): | ||||
for c in commonctxs: | ||||
try: | ||||
fnode = c.filenode(fname) | ||||
|
r38942 | self._clrevtolocalrev[c.rev()] = flog.rev(fnode) | ||
|
r38925 | except error.ManifestLookupError: | ||
pass | ||||
links = oldlinknodes(flog, fname) | ||||
if len(links) != len(mfdicts): | ||||
for mf, lr in mfdicts: | ||||
fnode = mf.get(fname, None) | ||||
if fnode in links: | ||||
links[fnode] = min(links[fnode], lr, key=clrev) | ||||
elif fnode: | ||||
links[fnode] = lr | ||||
return links | ||||
return self._generatefiles(changedfiles, linknodes, commonrevs, source) | ||||
def _generatefiles(self, changedfiles, linknodes, commonrevs, source): | ||||
|
r19334 | repo = self._repo | ||
|
r38429 | progress = repo.ui.makeprogress(_('bundling'), unit=_('files'), | ||
total=len(changedfiles)) | ||||
|
r19334 | for i, fname in enumerate(sorted(changedfiles)): | ||
filerevlog = repo.file(fname) | ||||
if not filerevlog: | ||||
|
r37357 | raise error.Abort(_("empty or missing file data for %s") % | ||
fname) | ||||
|
r19334 | |||
linkrevnodes = linknodes(filerevlog, fname) | ||||
|
r19207 | # Lookup for filenodes, we collected the linkrev nodes above in the | ||
# fastpath case and with lookupmf in the slowpath case. | ||||
def lookupfilelog(x): | ||||
return linkrevnodes[x] | ||||
|
r38939 | filenodes = self._prune(filerevlog, linkrevnodes, commonrevs) | ||
|
r19206 | if filenodes: | ||
|
r39018 | if self._ellipses: | ||
revs = _sortnodesellipsis(filerevlog, filenodes, | ||||
self._clnodetorev, lookupfilelog) | ||||
else: | ||||
revs = _sortnodesnormal(filerevlog, filenodes, | ||||
self._reorder) | ||||
|
r38429 | progress.update(i + 1, item=fname) | ||
|
r39017 | h = _fileheader(fname) | ||
|
r23748 | size = len(h) | ||
yield h | ||||
|
r39018 | for chunk in self.group(revs, filerevlog, False, lookupfilelog): | ||
|
r23748 | size += len(chunk) | ||
|
r19202 | yield chunk | ||
|
r23748 | self._verbosenote(_('%8.i %s\n') % (size, fname)) | ||
|
r38429 | progress.complete() | ||
|
r19200 | |||
|
r39016 | def _revisiondeltanarrow(self, store, ischangelog, rev, prev, linknode): | ||
|
r38922 | # build up some mapping information that's useful later. See | ||
# the local() nested function below. | ||||
|
r39016 | if ischangelog: | ||
|
r38942 | self._clnodetorev[linknode] = rev | ||
|
r38922 | linkrev = rev | ||
|
r38942 | self._clrevtolocalrev[linkrev] = rev | ||
|
r38922 | else: | ||
|
r38942 | linkrev = self._clnodetorev[linknode] | ||
self._clrevtolocalrev[linkrev] = rev | ||||
|
r38922 | |||
# This is a node to send in full, because the changeset it | ||||
# corresponds to was a full changeset. | ||||
|
r38945 | if linknode in self._fullnodes: | ||
|
r39021 | return _revisiondeltanormal(store, rev, prev, linknode, | ||
self._deltaparentfn) | ||||
|
r38922 | |||
# At this point, a node can either be one we should skip or an | ||||
# ellipsis. If it's not an ellipsis, bail immediately. | ||||
|
r38943 | if linkrev not in self._precomputedellipsis: | ||
|
r38922 | return | ||
|
r38943 | linkparents = self._precomputedellipsis[linkrev] | ||
|
r38922 | def local(clrev): | ||
"""Turn a changelog revnum into a local revnum. | ||||
The ellipsis dag is stored as revnums on the changelog, | ||||
but when we're producing ellipsis entries for | ||||
non-changelog revlogs, we need to turn those numbers into | ||||
something local. This does that for us, and during the | ||||
changelog sending phase will also expand the stored | ||||
mappings as needed. | ||||
""" | ||||
if clrev == nullrev: | ||||
return nullrev | ||||
|
r39016 | if ischangelog: | ||
|
r38922 | # If we're doing the changelog, it's possible that we | ||
# have a parent that is already on the client, and we | ||||
# need to store some extra mapping information so that | ||||
# our contained ellipsis nodes will be able to resolve | ||||
# their parents. | ||||
|
r38942 | if clrev not in self._clrevtolocalrev: | ||
|
r38927 | clnode = store.node(clrev) | ||
|
r38942 | self._clnodetorev[clnode] = clrev | ||
|
r38922 | return clrev | ||
# Walk the ellipsis-ized changelog breadth-first looking for a | ||||
# change that has been linked from the current revlog. | ||||
# | ||||
# For a flat manifest revlog only a single step should be necessary | ||||
# as all relevant changelog entries are relevant to the flat | ||||
# manifest. | ||||
# | ||||
# For a filelog or tree manifest dirlog however not every changelog | ||||
# entry will have been relevant, so we need to skip some changelog | ||||
# nodes even after ellipsis-izing. | ||||
walk = [clrev] | ||||
while walk: | ||||
p = walk[0] | ||||
walk = walk[1:] | ||||
|
r38942 | if p in self._clrevtolocalrev: | ||
return self._clrevtolocalrev[p] | ||||
|
r38945 | elif p in self._fullnodes: | ||
|
r38922 | walk.extend([pp for pp in self._repo.changelog.parentrevs(p) | ||
if pp != nullrev]) | ||||
|
r38943 | elif p in self._precomputedellipsis: | ||
walk.extend([pp for pp in self._precomputedellipsis[p] | ||||
|
r38922 | if pp != nullrev]) | ||
else: | ||||
# In this case, we've got an ellipsis with parents | ||||
# outside the current bundle (likely an | ||||
# incremental pull). We "know" that we can use the | ||||
# value of this same revlog at whatever revision | ||||
# is pointed to by linknode. "Know" is in scare | ||||
# quotes because I haven't done enough examination | ||||
# of edge cases to convince myself this is really | ||||
# a fact - it works for all the (admittedly | ||||
# thorough) cases in our testsuite, but I would be | ||||
# somewhat unsurprised to find a case in the wild | ||||
# where this breaks down a bit. That said, I don't | ||||
# know if it would hurt anything. | ||||
for i in pycompat.xrange(rev, 0, -1): | ||||
|
r38927 | if store.linkrev(i) == clrev: | ||
|
r38922 | return i | ||
# We failed to resolve a parent for this node, so | ||||
# we crash the changegroup construction. | ||||
raise error.Abort( | ||||
'unable to resolve parent while packing %r %r' | ||||
|
r38927 | ' for changeset %r' % (store.indexfile, rev, clrev)) | ||
|
r38922 | |||
return nullrev | ||||
if not linkparents or ( | ||||
|
r38927 | store.parentrevs(rev) == (nullrev, nullrev)): | ||
|
r38922 | p1, p2 = nullrev, nullrev | ||
elif len(linkparents) == 1: | ||||
p1, = sorted(local(p) for p in linkparents) | ||||
p2 = nullrev | ||||
else: | ||||
p1, p2 = sorted(local(p) for p in linkparents) | ||||
|
r38928 | |||
|
r38927 | n = store.node(rev) | ||
|
r38928 | p1n, p2n = store.node(p1), store.node(p2) | ||
flags = store.flags(rev) | ||||
flags |= revlog.REVIDX_ELLIPSIS | ||||
|
r38929 | |||
|
r38928 | # TODO: try and actually send deltas for ellipsis data blocks | ||
data = store.revision(n) | ||||
diffheader = mdiff.trivialdiffheader(len(data)) | ||||
|
r38929 | |||
return revisiondelta( | ||||
node=n, | ||||
p1node=p1n, | ||||
p2node=p2n, | ||||
basenode=nullid, | ||||
linknode=linknode, | ||||
flags=flags, | ||||
deltachunks=(diffheader, data), | ||||
) | ||||
|
r38922 | |||
|
r39011 | def _deltaparentprev(store, rev, p1, p2, prev): | ||
"""Resolve a delta parent to the previous revision. | ||||
Used for version 1 changegroups, which don't support generaldelta. | ||||
""" | ||||
return prev | ||||
def _deltaparentgeneraldelta(store, rev, p1, p2, prev): | ||||
"""Resolve a delta parent when general deltas are supported.""" | ||||
dp = store.deltaparent(rev) | ||||
if dp == nullrev and store.storedeltachains: | ||||
# Avoid sending full revisions when delta parent is null. Pick prev | ||||
# in that case. It's tempting to pick p1 in this case, as p1 will | ||||
# be smaller in the common case. However, computing a delta against | ||||
# p1 may require resolving the raw text of p1, which could be | ||||
# expensive. The revlog caches should have prev cached, meaning | ||||
# less CPU for changegroup generation. There is likely room to add | ||||
# a flag and/or config option to control this behavior. | ||||
base = prev | ||||
elif dp == nullrev: | ||||
# revlog is configured to use full snapshot for a reason, | ||||
# stick to full snapshot. | ||||
base = nullrev | ||||
elif dp not in (p1, p2, prev): | ||||
# Pick prev when we can't be sure remote has the base revision. | ||||
return prev | ||||
else: | ||||
base = dp | ||||
if base != nullrev and not store.candelta(base, rev): | ||||
base = nullrev | ||||
return base | ||||
def _deltaparentellipses(store, rev, p1, p2, prev): | ||||
"""Resolve a delta parent when in ellipses mode.""" | ||||
# TODO: send better deltas when in narrow mode. | ||||
# | ||||
# changegroup.group() loops over revisions to send, | ||||
# including revisions we'll skip. What this means is that | ||||
# `prev` will be a potentially useless delta base for all | ||||
# ellipsis nodes, as the client likely won't have it. In | ||||
# the future we should do bookkeeping about which nodes | ||||
# have been sent to the client, and try to be | ||||
# significantly smarter about delta bases. This is | ||||
# slightly tricky because this same code has to work for | ||||
# all revlogs, and we don't have the linkrev/linknode here. | ||||
return p1 | ||||
|
r38944 | def _makecg1packer(repo, filematcher, bundlecaps, ellipses=False, | ||
|
r38945 | shallow=False, ellipsisroots=None, fullnodes=None): | ||
|
r38933 | builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( | ||
d.node, d.p1node, d.p2node, d.linknode) | ||||
|
r27432 | |||
|
r38938 | return cgpacker(repo, filematcher, b'01', | ||
|
r39011 | deltaparentfn=_deltaparentprev, | ||
|
r38938 | allowreorder=None, | ||
builddeltaheader=builddeltaheader, | ||||
manifestsend=b'', | ||||
|
r38940 | bundlecaps=bundlecaps, | ||
|
r38944 | ellipses=ellipses, | ||
|
r38943 | shallow=shallow, | ||
|
r38945 | ellipsisroots=ellipsisroots, | ||
fullnodes=fullnodes) | ||||
|
r38930 | |||
|
r38944 | def _makecg2packer(repo, filematcher, bundlecaps, ellipses=False, | ||
|
r38945 | shallow=False, ellipsisroots=None, fullnodes=None): | ||
|
r38933 | builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( | ||
d.node, d.p1node, d.p2node, d.basenode, d.linknode) | ||||
|
r38936 | # Since generaldelta is directly supported by cg2, reordering | ||
# generally doesn't help, so we disable it by default (treating | ||||
# bundle.reorder=auto just like bundle.reorder=False). | ||||
|
r38938 | return cgpacker(repo, filematcher, b'02', | ||
|
r39011 | deltaparentfn=_deltaparentgeneraldelta, | ||
|
r38938 | allowreorder=False, | ||
builddeltaheader=builddeltaheader, | ||||
manifestsend=b'', | ||||
|
r38940 | bundlecaps=bundlecaps, | ||
|
r38944 | ellipses=ellipses, | ||
|
r38943 | shallow=shallow, | ||
|
r38945 | ellipsisroots=ellipsisroots, | ||
fullnodes=fullnodes) | ||||
|
r38930 | |||
|
r38944 | def _makecg3packer(repo, filematcher, bundlecaps, ellipses=False, | ||
|
r38945 | shallow=False, ellipsisroots=None, fullnodes=None): | ||
|
r38933 | builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( | ||
d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags) | ||||
|
r39011 | deltaparentfn = (_deltaparentellipses if ellipses | ||
else _deltaparentgeneraldelta) | ||||
|
r38938 | return cgpacker(repo, filematcher, b'03', | ||
|
r39011 | deltaparentfn=deltaparentfn, | ||
|
r38938 | allowreorder=False, | ||
builddeltaheader=builddeltaheader, | ||||
manifestsend=closechunk(), | ||||
|
r38940 | bundlecaps=bundlecaps, | ||
|
r38944 | ellipses=ellipses, | ||
|
r38943 | shallow=shallow, | ||
|
r38945 | ellipsisroots=ellipsisroots, | ||
fullnodes=fullnodes) | ||||
|
r38930 | |||
_packermap = {'01': (_makecg1packer, cg1unpacker), | ||||
|
r26709 | # cg2 adds support for exchanging generaldelta | ||
|
r38930 | '02': (_makecg2packer, cg2unpacker), | ||
|
r27753 | # cg3 adds support for exchanging revlog flags and treemanifests | ||
|
r38930 | '03': (_makecg3packer, cg3unpacker), | ||
|
r26709 | } | ||
|
r23168 | |||
|
r30627 | def allsupportedversions(repo): | ||
|
r27928 | versions = set(_packermap.keys()) | ||
|
r30627 | if not (repo.ui.configbool('experimental', 'changegroup3') or | ||
|
r30628 | repo.ui.configbool('experimental', 'treemanifest') or | ||
'treemanifest' in repo.requirements): | ||||
|
r30626 | versions.discard('03') | ||
|
r27953 | return versions | ||
# Changegroup versions that can be applied to the repo | ||||
def supportedincomingversions(repo): | ||||
|
r30628 | return allsupportedversions(repo) | ||
|
r27953 | |||
# Changegroup versions that can be created from the repo | ||||
def supportedoutgoingversions(repo): | ||||
|
r30627 | versions = allsupportedversions(repo) | ||
|
r27953 | if 'treemanifest' in repo.requirements: | ||
|
r27928 | # Versions 01 and 02 support only flat manifests and it's just too | ||
# expensive to convert between the flat manifest and tree manifest on | ||||
# the fly. Since tree manifests are hashed differently, all of history | ||||
# would have to be converted. Instead, we simply don't even pretend to | ||||
# support versions 01 and 02. | ||||
versions.discard('01') | ||||
versions.discard('02') | ||||
|
r38871 | if repository.NARROW_REQUIREMENT in repo.requirements: | ||
|
r36483 | # Versions 01 and 02 don't support revlog flags, and we need to | ||
# support that for stripping and unbundling to work. | ||||
versions.discard('01') | ||||
versions.discard('02') | ||||
|
r37150 | if LFS_REQUIREMENT in repo.requirements: | ||
# Versions 01 and 02 don't support revlog flags, and we need to | ||||
# mark LFS entries with REVIDX_EXTSTORED. | ||||
versions.discard('01') | ||||
versions.discard('02') | ||||
|
r27752 | return versions | ||
|
r27751 | |||
|
r34179 | def localversion(repo): | ||
# Finds the best version to use for bundles that are meant to be used | ||||
# locally, such as those from strip and shelve, and temporary bundles. | ||||
return max(supportedoutgoingversions(repo)) | ||||
|
r27929 | def safeversion(repo): | ||
# Finds the smallest version that it's safe to assume clients of the repo | ||||
|
r27931 | # will support. For example, all hg versions that support generaldelta also | ||
# support changegroup 02. | ||||
|
r27953 | versions = supportedoutgoingversions(repo) | ||
|
r27929 | if 'generaldelta' in repo.requirements: | ||
versions.discard('01') | ||||
assert versions | ||||
return min(versions) | ||||
|
r38940 | def getbundler(version, repo, bundlecaps=None, filematcher=None, | ||
|
r38945 | ellipses=False, shallow=False, ellipsisroots=None, | ||
fullnodes=None): | ||||
|
r27953 | assert version in supportedoutgoingversions(repo) | ||
|
r38830 | |||
if filematcher is None: | ||||
filematcher = matchmod.alwaysmatcher(repo.root, '') | ||||
if version == '01' and not filematcher.always(): | ||||
raise error.ProgrammingError('version 01 changegroups do not support ' | ||||
'sparse file matchers') | ||||
|
r38944 | if ellipses and version in (b'01', b'02'): | ||
raise error.Abort( | ||||
_('ellipsis nodes require at least cg3 on client and server, ' | ||||
'but negotiated version %s') % version) | ||||
|
r38830 | # Requested files could include files not in the local store. So | ||
# filter those out. | ||||
filematcher = matchmod.intersectmatchers(repo.narrowmatch(), | ||||
filematcher) | ||||
|
r38930 | fn = _packermap[version][0] | ||
|
r38944 | return fn(repo, filematcher, bundlecaps, ellipses=ellipses, | ||
|
r38945 | shallow=shallow, ellipsisroots=ellipsisroots, | ||
fullnodes=fullnodes) | ||||
|
r27751 | |||
|
r29593 | def getunbundler(version, fh, alg, extras=None): | ||
return _packermap[version][1](fh, alg, extras=extras) | ||||
|
r27751 | |||
|
r20926 | def _changegroupinfo(repo, nodes, source): | ||
if repo.ui.verbose or source == 'bundle': | ||||
repo.ui.status(_("%d changesets found\n") % len(nodes)) | ||||
if repo.ui.debugflag: | ||||
repo.ui.debug("list of changesets:\n") | ||||
for node in nodes: | ||||
repo.ui.debug("%s\n" % hex(node)) | ||||
|
r34098 | def makechangegroup(repo, outgoing, version, source, fastpath=False, | ||
bundlecaps=None): | ||||
cgstream = makestream(repo, outgoing, version, source, | ||||
fastpath=fastpath, bundlecaps=bundlecaps) | ||||
return getunbundler(version, util.chunkbuffer(cgstream), None, | ||||
{'clcount': len(outgoing.missing) }) | ||||
|
r34105 | def makestream(repo, outgoing, version, source, fastpath=False, | ||
|
r38830 | bundlecaps=None, filematcher=None): | ||
bundler = getbundler(version, repo, bundlecaps=bundlecaps, | ||||
filematcher=filematcher) | ||||
|
r34105 | |||
|
r20925 | repo = repo.unfiltered() | ||
commonrevs = outgoing.common | ||||
csets = outgoing.missing | ||||
heads = outgoing.missingheads | ||||
# We go through the fast path if we get told to, or if all (unfiltered | ||||
# heads have been requested (since we then know there all linkrevs will | ||||
# be pulled by the client). | ||||
heads.sort() | ||||
fastpathlinkrev = fastpath or ( | ||||
repo.filtername is None and heads == sorted(repo.heads())) | ||||
repo.hook('preoutgoing', throw=True, source=source) | ||||
|
r20926 | _changegroupinfo(repo, csets, source) | ||
|
r23177 | return bundler.generate(commonrevs, csets, fastpathlinkrev, source) | ||
|
r28361 | def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles): | ||
|
r20932 | revisions = 0 | ||
files = 0 | ||||
|
r38401 | progress = repo.ui.makeprogress(_('files'), unit=_('files'), | ||
total=expectedfiles) | ||||
|
r29724 | for chunkdata in iter(source.filelogheader, {}): | ||
|
r28361 | files += 1 | ||
|
r20932 | f = chunkdata["filename"] | ||
repo.ui.debug("adding %s revisions\n" % f) | ||||
|
r38401 | progress.increment() | ||
|
r27754 | fl = repo.file(f) | ||
|
r20932 | o = len(fl) | ||
|
r24120 | try: | ||
|
r34292 | deltas = source.deltaiter() | ||
if not fl.addgroup(deltas, revmap, trp): | ||||
|
r26587 | raise error.Abort(_("received file revlog group is empty")) | ||
|
r25660 | except error.CensoredBaseError as e: | ||
|
r26587 | raise error.Abort(_("received delta base is censored: %s") % e) | ||
|
r27754 | revisions += len(fl) - o | ||
|
r20932 | if f in needfiles: | ||
needs = needfiles[f] | ||||
|
r38806 | for new in pycompat.xrange(o, len(fl)): | ||
|
r20932 | n = fl.node(new) | ||
if n in needs: | ||||
needs.remove(n) | ||||
else: | ||||
|
r26587 | raise error.Abort( | ||
|
r20932 | _("received spurious file revlog entry")) | ||
if not needs: | ||||
del needfiles[f] | ||||
|
r38401 | progress.complete() | ||
|
r20932 | |||
for f, needs in needfiles.iteritems(): | ||||
fl = repo.file(f) | ||||
for n in needs: | ||||
try: | ||||
fl.rev(n) | ||||
except error.LookupError: | ||||
|
r26587 | raise error.Abort( | ||
|
r20932 | _('missing file data for %s:%s - run hg verify') % | ||
(f, hex(n))) | ||||
return revisions, files | ||||