changegroup.py
1394 lines
| 53.1 KiB
| text/x-python
|
PythonLexer
/ mercurial / changegroup.py
|
r8226 | # changegroup.py - Mercurial changegroup manipulation functions | ||
# | ||||
# Copyright 2006 Matt Mackall <mpm@selenic.com> | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r3877 | |||
|
r25921 | from __future__ import absolute_import | ||
import os | ||||
import struct | ||||
|
r20933 | import weakref | ||
|
r25921 | |||
from .i18n import _ | ||||
from .node import ( | ||||
hex, | ||||
|
r38919 | nullid, | ||
|
r25921 | nullrev, | ||
short, | ||||
) | ||||
from . import ( | ||||
error, | ||||
|
r38830 | match as matchmod, | ||
|
r25921 | mdiff, | ||
phases, | ||||
|
r30925 | pycompat, | ||
|
r38871 | repository, | ||
|
r25921 | util, | ||
) | ||||
|
r1981 | |||
|
r38932 | _CHANGEGROUPV1_DELTA_HEADER = struct.Struct("20s20s20s20s") | ||
_CHANGEGROUPV2_DELTA_HEADER = struct.Struct("20s20s20s20s20s") | ||||
_CHANGEGROUPV3_DELTA_HEADER = struct.Struct(">20s20s20s20s20sH") | ||||
|
r14141 | |||
|
r37150 | LFS_REQUIREMENT = 'lfs' | ||
|
r35772 | readexactly = util.readexactly | ||
|
r13457 | |||
def getchunk(stream): | ||||
"""return the next chunk from stream as a string""" | ||||
d = readexactly(stream, 4) | ||||
|
r1981 | l = struct.unpack(">l", d)[0] | ||
if l <= 4: | ||||
|
r13458 | if l: | ||
|
r26587 | raise error.Abort(_("invalid chunk length %d") % l) | ||
|
r1981 | return "" | ||
|
r13457 | return readexactly(stream, l - 4) | ||
|
r1981 | |||
|
r5368 | def chunkheader(length): | ||
|
r9437 | """return a changegroup chunk header (string)""" | ||
|
r5368 | return struct.pack(">l", length + 4) | ||
|
r1981 | |||
def closechunk(): | ||||
|
r9437 | """return a changegroup chunk header (string) for a zero-length chunk""" | ||
|
r1981 | return struct.pack(">l", 0) | ||
|
r39017 | def _fileheader(path): | ||
"""Obtain a changegroup chunk header for a named path.""" | ||||
return chunkheader(len(path)) + path | ||||
|
r26540 | def writechunks(ui, chunks, filename, vfs=None): | ||
"""Write chunks to a file and return its filename. | ||||
|
r3659 | |||
|
r26540 | The stream is assumed to be a bundle file. | ||
|
r3659 | Existing files will not be overwritten. | ||
If no filename is specified, a temporary file is created. | ||||
""" | ||||
fh = None | ||||
cleanup = None | ||||
try: | ||||
if filename: | ||||
|
r20976 | if vfs: | ||
fh = vfs.open(filename, "wb") | ||||
else: | ||||
|
r30212 | # Increase default buffer size because default is usually | ||
# small (4k is common on Linux). | ||||
fh = open(filename, "wb", 131072) | ||||
|
r3659 | else: | ||
|
r38182 | fd, filename = pycompat.mkstemp(prefix="hg-bundle-", suffix=".hg") | ||
|
r36853 | fh = os.fdopen(fd, r"wb") | ||
|
r3659 | cleanup = filename | ||
|
r26540 | for c in chunks: | ||
fh.write(c) | ||||
|
r3659 | cleanup = None | ||
return filename | ||||
finally: | ||||
if fh is not None: | ||||
fh.close() | ||||
if cleanup is not None: | ||||
|
r20976 | if filename and vfs: | ||
vfs.unlink(cleanup) | ||||
else: | ||||
os.unlink(cleanup) | ||||
|
r3660 | |||
|
r22390 | class cg1unpacker(object): | ||
|
r26708 | """Unpacker for cg1 changegroup streams. | ||
A changegroup unpacker handles the framing of the revision data in | ||||
the wire format. Most consumers will want to use the apply() | ||||
method to add the changes from the changegroup to a repository. | ||||
If you're forwarding a changegroup unmodified to another consumer, | ||||
use getchunks(), which returns an iterator of changegroup | ||||
chunks. This is mostly useful for cases where you need to know the | ||||
data stream has ended by observing the end of the changegroup. | ||||
deltachunk() is useful only if you're applying delta data. Most | ||||
consumers should prefer apply() instead. | ||||
A few other public methods exist. Those are used only for | ||||
bundlerepo and some debug commands - their use is discouraged. | ||||
""" | ||||
|
r22390 | deltaheader = _CHANGEGROUPV1_DELTA_HEADER | ||
|
r38932 | deltaheadersize = deltaheader.size | ||
|
r23896 | version = '01' | ||
|
r27920 | _grouplistcount = 1 # One list of files after the manifests | ||
|
r29593 | def __init__(self, fh, alg, extras=None): | ||
|
r30354 | if alg is None: | ||
alg = 'UN' | ||||
if alg not in util.compengines.supportedbundletypes: | ||||
|
r26587 | raise error.Abort(_('unknown stream compression type: %s') | ||
|
r26266 | % alg) | ||
|
r26392 | if alg == 'BZ': | ||
alg = '_truncatedBZ' | ||||
|
r30354 | |||
compengine = util.compengines.forbundletype(alg) | ||||
self._stream = compengine.decompressorreader(fh) | ||||
|
r12044 | self._type = alg | ||
|
r29593 | self.extras = extras or {} | ||
|
r12334 | self.callback = None | ||
|
r26706 | |||
# These methods (compressed, read, seek, tell) all appear to only | ||||
# be used by bundlerepo, but it's a little hard to tell. | ||||
|
r12044 | def compressed(self): | ||
|
r30589 | return self._type is not None and self._type != 'UN' | ||
|
r12043 | def read(self, l): | ||
return self._stream.read(l) | ||||
|
r12330 | def seek(self, pos): | ||
return self._stream.seek(pos) | ||||
def tell(self): | ||||
|
r12332 | return self._stream.tell() | ||
|
r12347 | def close(self): | ||
return self._stream.close() | ||||
|
r12334 | |||
|
r26707 | def _chunklength(self): | ||
|
r13459 | d = readexactly(self._stream, 4) | ||
|
r13458 | l = struct.unpack(">l", d)[0] | ||
if l <= 4: | ||||
if l: | ||||
|
r26587 | raise error.Abort(_("invalid chunk length %d") % l) | ||
|
r13458 | return 0 | ||
if self.callback: | ||||
|
r12334 | self.callback() | ||
|
r13458 | return l - 4 | ||
|
r12334 | |||
|
r14144 | def changelogheader(self): | ||
"""v10 does not have a changelog header chunk""" | ||||
return {} | ||||
def manifestheader(self): | ||||
"""v10 does not have a manifest header chunk""" | ||||
return {} | ||||
def filelogheader(self): | ||||
"""return the header of the filelogs chunk, v10 only has the filename""" | ||||
|
r26707 | l = self._chunklength() | ||
|
r14144 | if not l: | ||
return {} | ||||
fname = readexactly(self._stream, l) | ||||
|
r20675 | return {'filename': fname} | ||
|
r12334 | |||
|
r14141 | def _deltaheader(self, headertuple, prevnode): | ||
node, p1, p2, cs = headertuple | ||||
if prevnode is None: | ||||
deltabase = p1 | ||||
else: | ||||
deltabase = prevnode | ||||
|
r27433 | flags = 0 | ||
return node, p1, p2, deltabase, cs, flags | ||||
|
r14141 | |||
|
r14144 | def deltachunk(self, prevnode): | ||
|
r26707 | l = self._chunklength() | ||
|
r12336 | if not l: | ||
return {} | ||||
|
r14141 | headerdata = readexactly(self._stream, self.deltaheadersize) | ||
|
r38932 | header = self.deltaheader.unpack(headerdata) | ||
|
r14141 | delta = readexactly(self._stream, l - self.deltaheadersize) | ||
|
r27433 | node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode) | ||
|
r34295 | return (node, p1, p2, cs, deltabase, delta, flags) | ||
|
r12336 | |||
|
r20999 | def getchunks(self): | ||
"""returns all the chunks contains in the bundle | ||||
Used when you need to forward the binary stream to a file or another | ||||
network API. To do so, it parse the changegroup data, otherwise it will | ||||
block in case of sshrepo because it don't know the end of the stream. | ||||
""" | ||||
|
r34093 | # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog, | ||
# and a list of filelogs. For changegroup 3, we expect 4 parts: | ||||
# changelog, manifestlog, a list of tree manifestlogs, and a list of | ||||
# filelogs. | ||||
# | ||||
# Changelog and manifestlog parts are terminated with empty chunks. The | ||||
# tree and file parts are a list of entry sections. Each entry section | ||||
# is a series of chunks terminating in an empty chunk. The list of these | ||||
# entry sections is terminated in yet another empty chunk, so we know | ||||
# we've reached the end of the tree/file list when we reach an empty | ||||
# chunk that was proceeded by no non-empty chunks. | ||||
parts = 0 | ||||
while parts < 2 + self._grouplistcount: | ||||
noentries = True | ||||
|
r20999 | while True: | ||
chunk = getchunk(self) | ||||
if not chunk: | ||||
|
r34093 | # The first two empty chunks represent the end of the | ||
# changelog and the manifestlog portions. The remaining | ||||
# empty chunks represent either A) the end of individual | ||||
# tree or file entries in the file list, or B) the end of | ||||
# the entire list. It's the end of the entire list if there | ||||
# were no entries (i.e. noentries is True). | ||||
if parts < 2: | ||||
parts += 1 | ||||
elif noentries: | ||||
parts += 1 | ||||
|
r20999 | break | ||
|
r34093 | noentries = False | ||
|
r20999 | yield chunkheader(len(chunk)) | ||
pos = 0 | ||||
while pos < len(chunk): | ||||
next = pos + 2**20 | ||||
yield chunk[pos:next] | ||||
pos = next | ||||
yield closechunk() | ||||
|
r38365 | def _unpackmanifests(self, repo, revmap, trp, prog): | ||
self.callback = prog.increment | ||||
|
r26712 | # no need to check for empty manifest group here: | ||
# if the result of the merge of 1 and 2 is the same in 3 and 4, | ||||
# no new manifest will be created and the manifest group will | ||||
# be empty during the pull | ||||
self.manifestheader() | ||||
|
r34292 | deltas = self.deltaiter() | ||
|
r39280 | repo.manifestlog.getstorage(b'').addgroup(deltas, revmap, trp) | ||
|
r38392 | prog.complete() | ||
|
r28360 | self.callback = None | ||
|
r26712 | |||
|
r33308 | def apply(self, repo, tr, srctype, url, targetphase=phases.draft, | ||
expectedtotal=None): | ||||
|
r26695 | """Add the changegroup returned by source.read() to this repo. | ||
srctype is a string like 'push', 'pull', or 'unbundle'. url is | ||||
the URL of the repo where this changegroup is coming from. | ||||
Return an integer summarizing the change to this repo: | ||||
- nothing changed or no source: 0 | ||||
- more heads than before: 1+added heads (2..n) | ||||
- fewer heads than before: -1-removed heads (-2..-n) | ||||
- number of heads stays the same: 1 | ||||
""" | ||||
repo = repo.unfiltered() | ||||
def csmap(x): | ||||
repo.ui.debug("add changeset %s\n" % short(x)) | ||||
return len(cl) | ||||
def revmap(x): | ||||
return cl.rev(x) | ||||
changesets = files = revisions = 0 | ||||
|
r26880 | try: | ||
|
r32931 | # The transaction may already carry source information. In this | ||
# case we use the top level data. We overwrite the argument | ||||
# because we need to use the top level value (if they exist) | ||||
# in this function. | ||||
srctype = tr.hookargs.setdefault('source', srctype) | ||||
url = tr.hookargs.setdefault('url', url) | ||||
|
r33634 | repo.hook('prechangegroup', | ||
throw=True, **pycompat.strkwargs(tr.hookargs)) | ||||
|
r26695 | |||
|
r32931 | # write changelog data to temp files so concurrent readers | ||
# will not see an inconsistent view | ||||
cl = repo.changelog | ||||
cl.delayupdate(tr) | ||||
oldheads = set(cl.heads()) | ||||
|
r26695 | |||
|
r32931 | trp = weakref.proxy(tr) | ||
# pull off the changeset group | ||||
repo.ui.status(_("adding changesets\n")) | ||||
clstart = len(cl) | ||||
|
r38365 | progress = repo.ui.makeprogress(_('changesets'), unit=_('chunks'), | ||
total=expectedtotal) | ||||
self.callback = progress.increment | ||||
|
r26695 | |||
|
r32931 | efiles = set() | ||
def onchangelog(cl, node): | ||||
efiles.update(cl.readfiles(node)) | ||||
|
r26695 | |||
|
r32931 | self.changelogheader() | ||
|
r34292 | deltas = self.deltaiter() | ||
cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog) | ||||
|
r32931 | efiles = len(efiles) | ||
|
r26695 | |||
|
r33308 | if not cgnodes: | ||
|
r39706 | repo.ui.develwarn('applied empty changelog from changegroup', | ||
|
r34735 | config='warn-empty-changegroup') | ||
|
r32931 | clend = len(cl) | ||
changesets = clend - clstart | ||||
|
r38392 | progress.complete() | ||
|
r32931 | self.callback = None | ||
|
r26695 | |||
|
r32931 | # pull off the manifest group | ||
repo.ui.status(_("adding manifests\n")) | ||||
|
r38365 | # We know that we'll never have more manifests than we had | ||
# changesets. | ||||
progress = repo.ui.makeprogress(_('manifests'), unit=_('chunks'), | ||||
total=changesets) | ||||
self._unpackmanifests(repo, revmap, trp, progress) | ||||
|
r26695 | |||
|
r32931 | needfiles = {} | ||
r33224 | if repo.ui.configbool('server', 'validate'): | |||
|
r32931 | cl = repo.changelog | ||
ml = repo.manifestlog | ||||
# validate incoming csets have their manifests | ||||
|
r38806 | for cset in pycompat.xrange(clstart, clend): | ||
|
r32931 | mfnode = cl.changelogrevision(cset).manifest | ||
mfest = ml[mfnode].readdelta() | ||||
# store file cgnodes we must see | ||||
for f, n in mfest.iteritems(): | ||||
needfiles.setdefault(f, set()).add(n) | ||||
|
r26695 | |||
|
r32931 | # process the files | ||
repo.ui.status(_("adding file changes\n")) | ||||
newrevs, newfiles = _addchangegroupfiles( | ||||
repo, self, revmap, trp, efiles, needfiles) | ||||
revisions += newrevs | ||||
files += newfiles | ||||
|
r26695 | |||
|
r32931 | deltaheads = 0 | ||
if oldheads: | ||||
heads = cl.heads() | ||||
deltaheads = len(heads) - len(oldheads) | ||||
for h in heads: | ||||
if h not in oldheads and repo[h].closesbranch(): | ||||
deltaheads -= 1 | ||||
htext = "" | ||||
if deltaheads: | ||||
htext = _(" (%+d heads)") % deltaheads | ||||
|
r26695 | |||
|
r32931 | repo.ui.status(_("added %d changesets" | ||
" with %d changes to %d files%s\n") | ||||
% (changesets, revisions, files, htext)) | ||||
repo.invalidatevolatilesets() | ||||
|
r26695 | |||
|
r32931 | if changesets > 0: | ||
if 'node' not in tr.hookargs: | ||||
tr.hookargs['node'] = hex(cl.node(clstart)) | ||||
tr.hookargs['node_last'] = hex(cl.node(clend - 1)) | ||||
hookargs = dict(tr.hookargs) | ||||
else: | ||||
hookargs = dict(tr.hookargs) | ||||
hookargs['node'] = hex(cl.node(clstart)) | ||||
hookargs['node_last'] = hex(cl.node(clend - 1)) | ||||
|
r33634 | repo.hook('pretxnchangegroup', | ||
throw=True, **pycompat.strkwargs(hookargs)) | ||||
|
r26695 | |||
|
r38806 | added = [cl.node(r) for r in pycompat.xrange(clstart, clend)] | ||
|
r33456 | phaseall = None | ||
|
r32931 | if srctype in ('push', 'serve'): | ||
# Old servers can not push the boundary themselves. | ||||
# New servers won't push the boundary if changeset already | ||||
# exists locally as secret | ||||
# | ||||
# We should not use added here but the list of all change in | ||||
# the bundle | ||||
if repo.publishing(): | ||||
|
r33456 | targetphase = phaseall = phases.public | ||
|
r32931 | else: | ||
|
r33456 | # closer target phase computation | ||
|
r32931 | # Those changesets have been pushed from the | ||
# outside, their phases are going to be pushed | ||||
# alongside. Therefor `targetphase` is | ||||
# ignored. | ||||
|
r33456 | targetphase = phaseall = phases.draft | ||
if added: | ||||
phases.registernew(repo, tr, targetphase, added) | ||||
if phaseall is not None: | ||||
phases.advanceboundary(repo, tr, phaseall, cgnodes) | ||||
|
r26695 | |||
|
r32931 | if changesets > 0: | ||
|
r26695 | |||
|
r32931 | def runhooks(): | ||
# These hooks run when the lock releases, not when the | ||||
# transaction closes. So it's possible for the changelog | ||||
# to have changed since we last saw it. | ||||
if clstart >= len(repo): | ||||
return | ||||
|
r26695 | |||
|
r33678 | repo.hook("changegroup", **pycompat.strkwargs(hookargs)) | ||
|
r27867 | |||
|
r32931 | for n in added: | ||
args = hookargs.copy() | ||||
args['node'] = hex(n) | ||||
del args['node_last'] | ||||
|
r33678 | repo.hook("incoming", **pycompat.strkwargs(args)) | ||
|
r26695 | |||
|
r32931 | newheads = [h for h in repo.heads() | ||
if h not in oldheads] | ||||
repo.ui.log("incoming", | ||||
|
r36755 | "%d incoming changes - new heads: %s\n", | ||
|
r32931 | len(added), | ||
', '.join([hex(c[:6]) for c in newheads])) | ||||
|
r26695 | |||
|
r32931 | tr.addpostclose('changegroup-runhooks-%020i' % clstart, | ||
lambda tr: repo._afterlock(runhooks)) | ||||
|
r26695 | finally: | ||
repo.ui.flush() | ||||
# never return 0 here: | ||||
|
r32870 | if deltaheads < 0: | ||
|
r33030 | ret = deltaheads - 1 | ||
|
r26695 | else: | ||
|
r33030 | ret = deltaheads + 1 | ||
|
r33461 | return ret | ||
|
r26695 | |||
|
r34292 | def deltaiter(self): | ||
|
r34147 | """ | ||
returns an iterator of the deltas in this changegroup | ||||
Useful for passing to the underlying storage system to be stored. | ||||
""" | ||||
chain = None | ||||
for chunkdata in iter(lambda: self.deltachunk(chain), {}): | ||||
|
r34295 | # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags) | ||
yield chunkdata | ||||
chain = chunkdata[0] | ||||
|
r34147 | |||
|
r23181 | class cg2unpacker(cg1unpacker): | ||
|
r26708 | """Unpacker for cg2 streams. | ||
cg2 streams add support for generaldelta, so the delta header | ||||
format is slightly different. All other features about the data | ||||
remain the same. | ||||
""" | ||||
|
r23181 | deltaheader = _CHANGEGROUPV2_DELTA_HEADER | ||
|
r38932 | deltaheadersize = deltaheader.size | ||
|
r23896 | version = '02' | ||
|
r23181 | |||
def _deltaheader(self, headertuple, prevnode): | ||||
node, p1, p2, deltabase, cs = headertuple | ||||
|
r27433 | flags = 0 | ||
return node, p1, p2, deltabase, cs, flags | ||||
|
r23181 | |||
|
r27432 | class cg3unpacker(cg2unpacker): | ||
"""Unpacker for cg3 streams. | ||||
|
r27433 | cg3 streams add support for exchanging treemanifests and revlog | ||
|
r27753 | flags. It adds the revlog flags to the delta header and an empty chunk | ||
separating manifests and files. | ||||
|
r27432 | """ | ||
|
r27433 | deltaheader = _CHANGEGROUPV3_DELTA_HEADER | ||
|
r38932 | deltaheadersize = deltaheader.size | ||
|
r27432 | version = '03' | ||
|
r27920 | _grouplistcount = 2 # One list of manifests and one list of files | ||
|
r27432 | |||
|
r27433 | def _deltaheader(self, headertuple, prevnode): | ||
node, p1, p2, deltabase, cs, flags = headertuple | ||||
return node, p1, p2, deltabase, cs, flags | ||||
|
r38365 | def _unpackmanifests(self, repo, revmap, trp, prog): | ||
super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog) | ||||
|
r29724 | for chunkdata in iter(self.filelogheader, {}): | ||
|
r27754 | # If we get here, there are directory manifests in the changegroup | ||
d = chunkdata["filename"] | ||||
repo.ui.debug("adding %s revisions\n" % d) | ||||
|
r34292 | deltas = self.deltaiter() | ||
|
r39280 | if not repo.manifestlog.getstorage(d).addgroup(deltas, revmap, trp): | ||
|
r27754 | raise error.Abort(_("received dir revlog group is empty")) | ||
|
r12329 | class headerlessfixup(object): | ||
def __init__(self, fh, h): | ||||
self._h = h | ||||
self._fh = fh | ||||
def read(self, n): | ||||
if self._h: | ||||
d, self._h = self._h[:n], self._h[n:] | ||||
if len(d) < n: | ||||
|
r13457 | d += readexactly(self._fh, n - len(d)) | ||
|
r12329 | return d | ||
|
r13457 | return readexactly(self._fh, n) | ||
|
r12329 | |||
|
r39050 | def _revisiondeltatochunks(delta, headerfn): | ||
"""Serialize a revisiondelta to changegroup chunks.""" | ||||
|
r39052 | |||
# The captured revision delta may be encoded as a delta against | ||||
# a base revision or as a full revision. The changegroup format | ||||
# requires that everything on the wire be deltas. So for full | ||||
# revisions, we need to invent a header that says to rewrite | ||||
# data. | ||||
if delta.delta is not None: | ||||
prefix, data = b'', delta.delta | ||||
elif delta.basenode == nullid: | ||||
data = delta.revision | ||||
prefix = mdiff.trivialdiffheader(len(data)) | ||||
else: | ||||
data = delta.revision | ||||
prefix = mdiff.replacediffheader(delta.baserevisionsize, | ||||
len(data)) | ||||
|
r39050 | meta = headerfn(delta) | ||
|
r39052 | |||
yield chunkheader(len(meta) + len(prefix) + len(data)) | ||||
|
r39050 | yield meta | ||
|
r39052 | if prefix: | ||
yield prefix | ||||
yield data | ||||
|
r39050 | |||
|
r39033 | def _sortnodesellipsis(store, nodes, cl, lookup): | ||
|
r39901 | """Sort nodes for changegroup generation.""" | ||
|
r39018 | # Ellipses serving mode. | ||
# | ||||
# In a perfect world, we'd generate better ellipsis-ified graphs | ||||
# for non-changelog revlogs. In practice, we haven't started doing | ||||
# that yet, so the resulting DAGs for the manifestlog and filelogs | ||||
# are actually full of bogus parentage on all the ellipsis | ||||
# nodes. This has the side effect that, while the contents are | ||||
# correct, the individual DAGs might be completely out of whack in | ||||
# a case like 882681bc3166 and its ancestors (back about 10 | ||||
# revisions or so) in the main hg repo. | ||||
# | ||||
# The one invariant we *know* holds is that the new (potentially | ||||
# bogus) DAG shape will be valid if we order the nodes in the | ||||
# order that they're introduced in dramatis personae by the | ||||
# changelog, so what we do is we sort the non-changelog histories | ||||
# by the order in which they are used by the changelog. | ||||
|
r39033 | key = lambda n: cl.rev(lookup(n)) | ||
|
r39901 | return sorted(nodes, key=key) | ||
|
r39018 | |||
|
r39901 | def _resolvenarrowrevisioninfo(cl, store, ischangelog, rev, linkrev, | ||
linknode, clrevtolocalrev, fullclnodes, | ||||
precomputedellipsis): | ||||
|
r39040 | linkparents = precomputedellipsis[linkrev] | ||
def local(clrev): | ||||
"""Turn a changelog revnum into a local revnum. | ||||
The ellipsis dag is stored as revnums on the changelog, | ||||
but when we're producing ellipsis entries for | ||||
non-changelog revlogs, we need to turn those numbers into | ||||
something local. This does that for us, and during the | ||||
changelog sending phase will also expand the stored | ||||
mappings as needed. | ||||
""" | ||||
if clrev == nullrev: | ||||
return nullrev | ||||
if ischangelog: | ||||
return clrev | ||||
# Walk the ellipsis-ized changelog breadth-first looking for a | ||||
# change that has been linked from the current revlog. | ||||
# | ||||
# For a flat manifest revlog only a single step should be necessary | ||||
# as all relevant changelog entries are relevant to the flat | ||||
# manifest. | ||||
# | ||||
# For a filelog or tree manifest dirlog however not every changelog | ||||
# entry will have been relevant, so we need to skip some changelog | ||||
# nodes even after ellipsis-izing. | ||||
walk = [clrev] | ||||
while walk: | ||||
p = walk[0] | ||||
walk = walk[1:] | ||||
if p in clrevtolocalrev: | ||||
return clrevtolocalrev[p] | ||||
elif p in fullclnodes: | ||||
walk.extend([pp for pp in cl.parentrevs(p) | ||||
if pp != nullrev]) | ||||
elif p in precomputedellipsis: | ||||
walk.extend([pp for pp in precomputedellipsis[p] | ||||
if pp != nullrev]) | ||||
else: | ||||
# In this case, we've got an ellipsis with parents | ||||
# outside the current bundle (likely an | ||||
# incremental pull). We "know" that we can use the | ||||
# value of this same revlog at whatever revision | ||||
# is pointed to by linknode. "Know" is in scare | ||||
# quotes because I haven't done enough examination | ||||
# of edge cases to convince myself this is really | ||||
# a fact - it works for all the (admittedly | ||||
# thorough) cases in our testsuite, but I would be | ||||
# somewhat unsurprised to find a case in the wild | ||||
# where this breaks down a bit. That said, I don't | ||||
# know if it would hurt anything. | ||||
for i in pycompat.xrange(rev, 0, -1): | ||||
if store.linkrev(i) == clrev: | ||||
return i | ||||
# We failed to resolve a parent for this node, so | ||||
# we crash the changegroup construction. | ||||
raise error.Abort( | ||||
'unable to resolve parent while packing %r %r' | ||||
' for changeset %r' % (store.indexfile, rev, clrev)) | ||||
return nullrev | ||||
if not linkparents or ( | ||||
store.parentrevs(rev) == (nullrev, nullrev)): | ||||
p1, p2 = nullrev, nullrev | ||||
elif len(linkparents) == 1: | ||||
p1, = sorted(local(p) for p in linkparents) | ||||
p2 = nullrev | ||||
else: | ||||
p1, p2 = sorted(local(p) for p in linkparents) | ||||
|
r39054 | p1node, p2node = store.node(p1), store.node(p2) | ||
|
r39040 | |||
|
r39901 | return p1node, p2node, linknode | ||
|
r39040 | |||
|
r39265 | def deltagroup(repo, store, nodes, ischangelog, lookup, forcedeltaparentprev, | ||
|
r39275 | topic=None, | ||
|
r39045 | ellipses=False, clrevtolocalrev=None, fullclnodes=None, | ||
precomputedellipsis=None): | ||||
|
r39050 | """Calculate deltas for a set of revisions. | ||
|
r39045 | |||
|
r39050 | Is a generator of ``revisiondelta`` instances. | ||
|
r39045 | |||
|
r39275 | If topic is not None, progress detail will be generated using this | ||
topic name (e.g. changesets, manifests, etc). | ||||
|
r39045 | """ | ||
|
r39265 | if not nodes: | ||
|
r39045 | return | ||
cl = repo.changelog | ||||
|
r39265 | if ischangelog: | ||
|
r39901 | # `hg log` shows changesets in storage order. To preserve order | ||
# across clones, send out changesets in storage order. | ||||
nodesorder = 'storage' | ||||
|
r39265 | elif ellipses: | ||
|
r39901 | nodes = _sortnodesellipsis(store, nodes, cl, lookup) | ||
nodesorder = 'nodes' | ||||
|
r39265 | else: | ||
|
r39901 | nodesorder = None | ||
|
r39265 | |||
|
r39901 | # Perform ellipses filtering and revision massaging. We do this before | ||
# emitrevisions() because a) filtering out revisions creates less work | ||||
# for emitrevisions() b) dropping revisions would break emitrevisions()'s | ||||
# assumptions about delta choices and we would possibly send a delta | ||||
# referencing a missing base revision. | ||||
# | ||||
# Also, calling lookup() has side-effects with regards to populating | ||||
# data structures. If we don't call lookup() for each node or if we call | ||||
# lookup() after the first pass through each node, things can break - | ||||
# possibly intermittently depending on the python hash seed! For that | ||||
# reason, we store a mapping of all linknodes during the initial node | ||||
# pass rather than use lookup() on the output side. | ||||
if ellipses: | ||||
filtered = [] | ||||
adjustedparents = {} | ||||
linknodes = {} | ||||
|
r39045 | |||
|
r39901 | for node in nodes: | ||
rev = store.rev(node) | ||||
linknode = lookup(node) | ||||
|
r39045 | linkrev = cl.rev(linknode) | ||
|
r39901 | clrevtolocalrev[linkrev] = rev | ||
|
r39045 | |||
|
r39901 | # If linknode is in fullclnodes, it means the corresponding | ||
# changeset was a full changeset and is being sent unaltered. | ||||
|
r39045 | if linknode in fullclnodes: | ||
|
r39901 | linknodes[node] = linknode | ||
|
r39054 | |||
|
r39901 | # If the corresponding changeset wasn't in the set computed | ||
# as relevant to us, it should be dropped outright. | ||||
|
r39045 | elif linkrev not in precomputedellipsis: | ||
|
r39901 | continue | ||
|
r39045 | else: | ||
|
r39901 | # We could probably do this later and avoid the dict | ||
# holding state. But it likely doesn't matter. | ||||
p1node, p2node, linknode = _resolvenarrowrevisioninfo( | ||||
cl, store, ischangelog, rev, linkrev, linknode, | ||||
clrevtolocalrev, fullclnodes, precomputedellipsis) | ||||
adjustedparents[node] = (p1node, p2node) | ||||
linknodes[node] = linknode | ||||
filtered.append(node) | ||||
nodes = filtered | ||||
|
r39045 | |||
|
r39054 | # We expect the first pass to be fast, so we only engage the progress | ||
# meter for constructing the revision deltas. | ||||
progress = None | ||||
|
r39275 | if topic is not None: | ||
progress = repo.ui.makeprogress(topic, unit=_('chunks'), | ||||
|
r39901 | total=len(nodes)) | ||
|
r39054 | |||
|
r40458 | configtarget = repo.ui.config('devel', 'bundle.delta') | ||
|
r40459 | if configtarget not in ('', 'p1', 'full'): | ||
|
r40458 | msg = _("""config "devel.bundle.delta" as unknown value: %s""") | ||
repo.ui.warn(msg % configtarget) | ||||
|
r40457 | deltamode = repository.CG_DELTAMODE_STD | ||
if forcedeltaparentprev: | ||||
deltamode = repository.CG_DELTAMODE_PREV | ||||
|
r40458 | elif configtarget == 'p1': | ||
deltamode = repository.CG_DELTAMODE_P1 | ||||
|
r40459 | elif configtarget == 'full': | ||
deltamode = repository.CG_DELTAMODE_FULL | ||||
|
r40457 | |||
|
r39901 | revisions = store.emitrevisions( | ||
nodes, | ||||
nodesorder=nodesorder, | ||||
revisiondata=True, | ||||
assumehaveparentrevisions=not ellipses, | ||||
|
r40457 | deltamode=deltamode) | ||
|
r39901 | |||
for i, revision in enumerate(revisions): | ||||
|
r39054 | if progress: | ||
progress.update(i + 1) | ||||
|
r39901 | if ellipses: | ||
linknode = linknodes[revision.node] | ||||
if revision.node in adjustedparents: | ||||
p1node, p2node = adjustedparents[revision.node] | ||||
revision.p1node = p1node | ||||
revision.p2node = p2node | ||||
|
r40083 | revision.flags |= repository.REVISION_FLAG_ELLIPSIS | ||
|
r39901 | |||
else: | ||||
linknode = lookup(revision.node) | ||||
revision.linknode = linknode | ||||
yield revision | ||||
|
r39054 | |||
|
r39045 | if progress: | ||
progress.complete() | ||||
|
r38938 | class cgpacker(object): | ||
|
r40380 | def __init__(self, repo, oldmatcher, matcher, version, | ||
|
r39053 | builddeltaheader, manifestsend, | ||
forcedeltaparentprev=False, | ||||
|
r38961 | bundlecaps=None, ellipses=False, | ||
|
r38945 | shallow=False, ellipsisroots=None, fullnodes=None): | ||
|
r19202 | """Given a source repo, construct a bundler. | ||
|
r32287 | |||
|
r40380 | oldmatcher is a matcher that matches on files the client already has. | ||
These will not be included in the changegroup. | ||||
matcher is a matcher that matches on files to include in the | ||||
|
r38830 | changegroup. Used to facilitate sparse changegroups. | ||
|
r39053 | forcedeltaparentprev indicates whether delta parents must be against | ||
the previous revision in a delta group. This should only be used for | ||||
compatibility with changegroup version 1. | ||||
|
r38937 | |||
|
r38933 | builddeltaheader is a callable that constructs the header for a group | ||
delta. | ||||
|
r38934 | manifestsend is a chunk to send after manifests have been fully emitted. | ||
|
r38944 | ellipses indicates whether ellipsis serving mode is enabled. | ||
|
r32287 | bundlecaps is optional and can be used to specify the set of | ||
capabilities which can be used to build the bundle. While bundlecaps is | ||||
unused in core Mercurial, extensions rely on this feature to communicate | ||||
capabilities to customize the changegroup packer. | ||||
|
r38940 | |||
shallow indicates whether shallow data might be sent. The packer may | ||||
need to pack file contents not introduced by the changes being packed. | ||||
|
r38945 | |||
|
r39032 | fullnodes is the set of changelog nodes which should not be ellipsis | ||
nodes. We store this rather than the set of nodes that should be | ||||
ellipsis because for very large histories we expect this to be | ||||
significantly smaller. | ||||
|
r19202 | """ | ||
|
r40380 | assert oldmatcher | ||
assert matcher | ||||
self._oldmatcher = oldmatcher | ||||
self._matcher = matcher | ||||
|
r38830 | |||
|
r38931 | self.version = version | ||
|
r39053 | self._forcedeltaparentprev = forcedeltaparentprev | ||
|
r38933 | self._builddeltaheader = builddeltaheader | ||
|
r38934 | self._manifestsend = manifestsend | ||
|
r38944 | self._ellipses = ellipses | ||
|
r38931 | |||
|
r32287 | # Set of capabilities we can use to build the bundle. | ||
if bundlecaps is None: | ||||
bundlecaps = set() | ||||
self._bundlecaps = bundlecaps | ||||
|
r38940 | self._isshallow = shallow | ||
|
r39032 | self._fullclnodes = fullnodes | ||
|
r38936 | |||
|
r38943 | # Maps ellipsis revs to their roots at the changelog level. | ||
self._precomputedellipsis = ellipsisroots | ||||
|
r19202 | self._repo = repo | ||
|
r38936 | |||
|
r23748 | if self._repo.ui.verbose and not self._repo.ui.debugflag: | ||
self._verbosenote = self._repo.ui.note | ||||
else: | ||||
self._verbosenote = lambda s: None | ||||
|
r39708 | def generate(self, commonrevs, clnodes, fastpathlinkrev, source, | ||
changelog=True): | ||||
"""Yield a sequence of changegroup byte chunks. | ||||
If changelog is False, changelog data won't be added to changegroup | ||||
""" | ||||
|
r39012 | |||
|
r19202 | repo = self._repo | ||
|
r24978 | cl = repo.changelog | ||
|
r19204 | |||
|
r39012 | self._verbosenote(_('uncompressed size of bundle content:\n')) | ||
size = 0 | ||||
|
r39050 | clstate, deltas = self._generatechangelog(cl, clnodes) | ||
for delta in deltas: | ||||
|
r39708 | if changelog: | ||
for chunk in _revisiondeltatochunks(delta, | ||||
self._builddeltaheader): | ||||
size += len(chunk) | ||||
yield chunk | ||||
|
r39012 | |||
|
r39046 | close = closechunk() | ||
size += len(close) | ||||
yield closechunk() | ||||
|
r39012 | self._verbosenote(_('%8.i (changelog)\n') % size) | ||
clrevorder = clstate['clrevorder'] | ||||
|
r39274 | manifests = clstate['manifests'] | ||
|
r39012 | changedfiles = clstate['changedfiles'] | ||
# We need to make sure that the linkrev in the changegroup refers to | ||||
# the first changeset that introduced the manifest or file revision. | ||||
# The fastpath is usually safer than the slowpath, because the filelogs | ||||
# are walked in revlog order. | ||||
# | ||||
|
r39897 | # When taking the slowpath when the manifest revlog uses generaldelta, | ||
# the manifest may be walked in the "wrong" order. Without 'clrevorder', | ||||
# we would get an incorrect linkrev (see fix in cc0ff93d0c0c). | ||||
|
r39012 | # | ||
# When taking the fastpath, we are only vulnerable to reordering | ||||
|
r39897 | # of the changelog itself. The changelog never uses generaldelta and is | ||
# never reordered. To handle this case, we simply take the slowpath, | ||||
# which already has the 'clrevorder' logic. This was also fixed in | ||||
# cc0ff93d0c0c. | ||||
|
r39012 | # Treemanifests don't work correctly with fastpathlinkrev | ||
# either, because we don't discover which directory nodes to | ||||
# send along with files. This could probably be fixed. | ||||
fastpathlinkrev = fastpathlinkrev and ( | ||||
'treemanifest' not in repo.requirements) | ||||
fnodes = {} # needed file nodes | ||||
|
r39047 | size = 0 | ||
|
r39048 | it = self.generatemanifests( | ||
|
r39274 | commonrevs, clrevorder, fastpathlinkrev, manifests, fnodes, source, | ||
|
r39048 | clstate['clrevtomanifestrev']) | ||
|
r39269 | for tree, deltas in it: | ||
if tree: | ||||
|
r39048 | assert self.version == b'03' | ||
|
r39269 | chunk = _fileheader(tree) | ||
|
r39048 | size += len(chunk) | ||
yield chunk | ||||
|
r39050 | for delta in deltas: | ||
chunks = _revisiondeltatochunks(delta, self._builddeltaheader) | ||||
for chunk in chunks: | ||||
size += len(chunk) | ||||
yield chunk | ||||
|
r39048 | |||
close = closechunk() | ||||
size += len(close) | ||||
yield close | ||||
|
r39012 | |||
|
r39047 | self._verbosenote(_('%8.i (manifests)\n') % size) | ||
yield self._manifestsend | ||||
|
r39019 | mfdicts = None | ||
if self._ellipses and self._isshallow: | ||||
mfdicts = [(self._repo.manifestlog[n].read(), lr) | ||||
|
r39274 | for (n, lr) in manifests.iteritems()] | ||
|
r39012 | |||
|
r39274 | manifests.clear() | ||
|
r39012 | clrevs = set(cl.rev(x) for x in clnodes) | ||
|
r39049 | it = self.generatefiles(changedfiles, commonrevs, | ||
source, mfdicts, fastpathlinkrev, | ||||
fnodes, clrevs) | ||||
|
r39050 | for path, deltas in it: | ||
|
r39049 | h = _fileheader(path) | ||
size = len(h) | ||||
yield h | ||||
|
r39050 | for delta in deltas: | ||
chunks = _revisiondeltatochunks(delta, self._builddeltaheader) | ||||
for chunk in chunks: | ||||
size += len(chunk) | ||||
yield chunk | ||||
|
r39049 | |||
close = closechunk() | ||||
size += len(close) | ||||
yield close | ||||
self._verbosenote(_('%8.i %s\n') % (size, path)) | ||||
|
r39012 | |||
|
r39038 | yield closechunk() | ||
|
r39012 | |||
if clnodes: | ||||
repo.hook('outgoing', node=hex(clnodes[0]), source=source) | ||||
def _generatechangelog(self, cl, nodes): | ||||
"""Generate data for changelog chunks. | ||||
Returns a 2-tuple of a dict containing state and an iterable of | ||||
byte chunks. The state will not be fully populated until the | ||||
chunk stream has been fully consumed. | ||||
""" | ||||
|
r23381 | clrevorder = {} | ||
|
r39274 | manifests = {} | ||
|
r39012 | mfl = self._repo.manifestlog | ||
|
r28241 | changedfiles = set() | ||
|
r39034 | clrevtomanifestrev = {} | ||
|
r19204 | |||
|
r38926 | # Callback for the changelog, used to collect changed files and | ||
# manifest nodes. | ||||
|
r19207 | # Returns the linkrev node (identity in the changelog case). | ||
def lookupcl(x): | ||||
|
r39273 | c = cl.changelogrevision(x) | ||
|
r23381 | clrevorder[x] = len(clrevorder) | ||
|
r38926 | |||
|
r38944 | if self._ellipses: | ||
|
r39274 | # Only update manifests if x is going to be sent. Otherwise we | ||
|
r38926 | # end up with bogus linkrevs specified for manifests and | ||
# we skip some manifest nodes that we should otherwise | ||||
# have sent. | ||||
|
r39032 | if (x in self._fullclnodes | ||
|
r38943 | or cl.rev(x) in self._precomputedellipsis): | ||
|
r39273 | |||
manifestnode = c.manifest | ||||
|
r38926 | # Record the first changeset introducing this manifest | ||
# version. | ||||
|
r39274 | manifests.setdefault(manifestnode, x) | ||
|
r38926 | # Set this narrow-specific dict so we have the lowest | ||
# manifest revnum to look up for this cl revnum. (Part of | ||||
# mapping changelog ellipsis parents to manifest ellipsis | ||||
# parents) | ||||
|
r39273 | clrevtomanifestrev.setdefault( | ||
cl.rev(x), mfl.rev(manifestnode)) | ||||
|
r38926 | # We can't trust the changed files list in the changeset if the | ||
# client requested a shallow clone. | ||||
|
r38940 | if self._isshallow: | ||
|
r39273 | changedfiles.update(mfl[c.manifest].read().keys()) | ||
|
r38926 | else: | ||
|
r39273 | changedfiles.update(c.files) | ||
|
r38926 | else: | ||
# record the first changeset introducing this manifest version | ||||
|
r39274 | manifests.setdefault(c.manifest, x) | ||
|
r38926 | # Record a complete list of potentially-changed files in | ||
# this manifest. | ||||
|
r39273 | changedfiles.update(c.files) | ||
|
r38926 | |||
|
r19207 | return x | ||
|
r19204 | |||
|
r39012 | state = { | ||
'clrevorder': clrevorder, | ||||
|
r39274 | 'manifests': manifests, | ||
|
r39012 | 'changedfiles': changedfiles, | ||
|
r39034 | 'clrevtomanifestrev': clrevtomanifestrev, | ||
|
r39012 | } | ||
|
r38926 | |||
|
r39045 | gen = deltagroup( | ||
|
r39265 | self._repo, cl, nodes, True, lookupcl, | ||
|
r39053 | self._forcedeltaparentprev, | ||
|
r39045 | ellipses=self._ellipses, | ||
|
r39275 | topic=_('changesets'), | ||
|
r39045 | clrevtolocalrev={}, | ||
fullclnodes=self._fullclnodes, | ||||
precomputedellipsis=self._precomputedellipsis) | ||||
|
r28227 | |||
|
r39012 | return state, gen | ||
|
r28227 | |||
|
r39274 | def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, | ||
manifests, fnodes, source, clrevtolocalrev): | ||||
|
r34148 | """Returns an iterator of changegroup chunks containing manifests. | ||
`source` is unused here, but is used by extensions like remotefilelog to | ||||
change what is sent based in pulls vs pushes, etc. | ||||
""" | ||||
|
r28227 | repo = self._repo | ||
|
r30294 | mfl = repo.manifestlog | ||
|
r39274 | tmfnodes = {'': manifests} | ||
|
r28227 | |||
|
r19207 | # Callback for the manifest, used to collect linkrevs for filelog | ||
# revisions. | ||||
# Returns the linkrev node (collected in lookupcl). | ||||
|
r39269 | def makelookupmflinknode(tree, nodes): | ||
|
r28231 | if fastpathlinkrev: | ||
|
r39269 | assert not tree | ||
|
r39274 | return manifests.__getitem__ | ||
|
r28231 | |||
|
r27239 | def lookupmflinknode(x): | ||
"""Callback for looking up the linknode for manifests. | ||||
|
r27219 | |||
|
r27239 | Returns the linkrev node for the specified manifest. | ||
|
r27219 | |||
|
r27239 | SIDE EFFECT: | ||
|
r27432 | 1) fclnodes gets populated with the list of relevant | ||
file nodes if we're not using fastpathlinkrev | ||||
2) When treemanifests are in use, collects treemanifest nodes | ||||
to send | ||||
|
r27219 | |||
|
r27432 | Note that this means manifests must be completely sent to | ||
the client before you can trust the list of files and | ||||
treemanifests to send. | ||||
|
r27239 | """ | ||
|
r35013 | clnode = nodes[x] | ||
|
r39269 | mdata = mfl.get(tree, x).readfast(shallow=True) | ||
|
r28241 | for p, n, fl in mdata.iterentries(): | ||
if fl == 't': # subdirectory manifest | ||||
|
r39269 | subtree = tree + p + '/' | ||
tmfclnodes = tmfnodes.setdefault(subtree, {}) | ||||
|
r28241 | tmfclnode = tmfclnodes.setdefault(n, clnode) | ||
if clrevorder[clnode] < clrevorder[tmfclnode]: | ||||
tmfclnodes[n] = clnode | ||||
else: | ||||
|
r39269 | f = tree + p | ||
|
r28240 | fclnodes = fnodes.setdefault(f, {}) | ||
fclnode = fclnodes.setdefault(n, clnode) | ||||
if clrevorder[clnode] < clrevorder[fclnode]: | ||||
fclnodes[n] = clnode | ||||
|
r27239 | return clnode | ||
|
r28231 | return lookupmflinknode | ||
|
r19206 | |||
|
r28232 | while tmfnodes: | ||
|
r39269 | tree, nodes = tmfnodes.popitem() | ||
|
r39280 | store = mfl.getstorage(tree) | ||
|
r39043 | |||
|
r40380 | if not self._matcher.visitdir(store.tree[:-1] or '.'): | ||
|
r39769 | # No nodes to send because this directory is out of | ||
# the client's view of the repository (probably | ||||
# because of narrow clones). | ||||
|
r39043 | prunednodes = [] | ||
else: | ||||
|
r39769 | # Avoid sending any manifest nodes we can prove the | ||
# client already has by checking linkrevs. See the | ||||
# related comment in generatefiles(). | ||||
|
r39768 | prunednodes = self._prunemanifests(store, nodes, commonrevs) | ||
|
r39269 | if tree and not prunednodes: | ||
|
r39041 | continue | ||
|
r39269 | lookupfn = makelookupmflinknode(tree, nodes) | ||
|
r39018 | |||
|
r39050 | deltas = deltagroup( | ||
|
r39265 | self._repo, store, prunednodes, False, lookupfn, | ||
|
r39897 | self._forcedeltaparentprev, | ||
|
r39044 | ellipses=self._ellipses, | ||
|
r39275 | topic=_('manifests'), | ||
|
r39044 | clrevtolocalrev=clrevtolocalrev, | ||
fullclnodes=self._fullclnodes, | ||||
precomputedellipsis=self._precomputedellipsis) | ||||
|
r40380 | if not self._oldmatcher.visitdir(store.tree[:-1] or '.'): | ||
yield tree, deltas | ||||
else: | ||||
# 'deltas' is a generator and we need to consume it even if | ||||
# we are not going to send it because a side-effect is that | ||||
# it updates tmdnodes (via lookupfn) | ||||
for d in deltas: | ||||
pass | ||||
if not tree: | ||||
yield tree, [] | ||||
|
r39046 | |||
|
r39768 | def _prunemanifests(self, store, nodes, commonrevs): | ||
# This is split out as a separate method to allow filtering | ||||
# commonrevs in extension code. | ||||
# | ||||
# TODO(augie): this shouldn't be required, instead we should | ||||
# make filtering of revisions to send delegated to the store | ||||
# layer. | ||||
frev, flr = store.rev, store.linkrev | ||||
return [n for n in nodes if flr(frev(n)) not in commonrevs] | ||||
|
r24897 | # The 'source' parameter is useful for extensions | ||
|
r39035 | def generatefiles(self, changedfiles, commonrevs, source, | ||
mfdicts, fastpathlinkrev, fnodes, clrevs): | ||||
|
r40380 | changedfiles = [f for f in changedfiles | ||
if self._matcher(f) and not self._oldmatcher(f)] | ||||
|
r38925 | |||
|
r39035 | if not fastpathlinkrev: | ||
def normallinknodes(unused, fname): | ||||
return fnodes.get(fname, {}) | ||||
else: | ||||
cln = self._repo.changelog.node | ||||
def normallinknodes(store, fname): | ||||
flinkrev = store.linkrev | ||||
fnode = store.node | ||||
revs = ((r, flinkrev(r)) for r in store) | ||||
return dict((fnode(r), cln(lr)) | ||||
for r, lr in revs if lr in clrevs) | ||||
|
r39037 | clrevtolocalrev = {} | ||
|
r38940 | if self._isshallow: | ||
|
r38925 | # In a shallow clone, the linknodes callback needs to also include | ||
# those file nodes that are in the manifests we sent but weren't | ||||
# introduced by those manifests. | ||||
commonctxs = [self._repo[c] for c in commonrevs] | ||||
clrev = self._repo.changelog.rev | ||||
def linknodes(flog, fname): | ||||
for c in commonctxs: | ||||
try: | ||||
fnode = c.filenode(fname) | ||||
|
r39037 | clrevtolocalrev[c.rev()] = flog.rev(fnode) | ||
|
r38925 | except error.ManifestLookupError: | ||
pass | ||||
|
r39035 | links = normallinknodes(flog, fname) | ||
|
r38925 | if len(links) != len(mfdicts): | ||
for mf, lr in mfdicts: | ||||
fnode = mf.get(fname, None) | ||||
if fnode in links: | ||||
links[fnode] = min(links[fnode], lr, key=clrev) | ||||
elif fnode: | ||||
links[fnode] = lr | ||||
return links | ||||
|
r39035 | else: | ||
linknodes = normallinknodes | ||||
|
r38925 | |||
|
r19334 | repo = self._repo | ||
|
r39275 | progress = repo.ui.makeprogress(_('files'), unit=_('files'), | ||
|
r38429 | total=len(changedfiles)) | ||
|
r19334 | for i, fname in enumerate(sorted(changedfiles)): | ||
filerevlog = repo.file(fname) | ||||
if not filerevlog: | ||||
|
r37357 | raise error.Abort(_("empty or missing file data for %s") % | ||
fname) | ||||
|
r19334 | |||
|
r39037 | clrevtolocalrev.clear() | ||
|
r19334 | linkrevnodes = linknodes(filerevlog, fname) | ||
|
r19207 | # Lookup for filenodes, we collected the linkrev nodes above in the | ||
# fastpath case and with lookupmf in the slowpath case. | ||||
def lookupfilelog(x): | ||||
return linkrevnodes[x] | ||||
|
r39043 | frev, flr = filerevlog.rev, filerevlog.linkrev | ||
|
r39769 | # Skip sending any filenode we know the client already | ||
# has. This avoids over-sending files relatively | ||||
# inexpensively, so it's not a problem if we under-filter | ||||
# here. | ||||
|
r39043 | filenodes = [n for n in linkrevnodes | ||
if flr(frev(n)) not in commonrevs] | ||||
|
r39056 | if not filenodes: | ||
continue | ||||
|
r39018 | |||
|
r39056 | progress.update(i + 1, item=fname) | ||
|
r39044 | |||
|
r39056 | deltas = deltagroup( | ||
|
r39265 | self._repo, filerevlog, filenodes, False, lookupfilelog, | ||
|
r39897 | self._forcedeltaparentprev, | ||
|
r39056 | ellipses=self._ellipses, | ||
clrevtolocalrev=clrevtolocalrev, | ||||
fullclnodes=self._fullclnodes, | ||||
precomputedellipsis=self._precomputedellipsis) | ||||
yield fname, deltas | ||||
|
r39046 | |||
|
r38429 | progress.complete() | ||
|
r19200 | |||
|
r40380 | def _makecg1packer(repo, oldmatcher, matcher, bundlecaps, | ||
ellipses=False, shallow=False, ellipsisroots=None, | ||||
fullnodes=None): | ||||
|
r38933 | builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack( | ||
d.node, d.p1node, d.p2node, d.linknode) | ||||
|
r27432 | |||
|
r40380 | return cgpacker(repo, oldmatcher, matcher, b'01', | ||
|
r38938 | builddeltaheader=builddeltaheader, | ||
manifestsend=b'', | ||||
|
r39053 | forcedeltaparentprev=True, | ||
|
r38940 | bundlecaps=bundlecaps, | ||
|
r38944 | ellipses=ellipses, | ||
|
r38943 | shallow=shallow, | ||
|
r38945 | ellipsisroots=ellipsisroots, | ||
fullnodes=fullnodes) | ||||
|
r38930 | |||
|
r40380 | def _makecg2packer(repo, oldmatcher, matcher, bundlecaps, | ||
ellipses=False, shallow=False, ellipsisroots=None, | ||||
fullnodes=None): | ||||
|
r38933 | builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack( | ||
d.node, d.p1node, d.p2node, d.basenode, d.linknode) | ||||
|
r40380 | return cgpacker(repo, oldmatcher, matcher, b'02', | ||
|
r38938 | builddeltaheader=builddeltaheader, | ||
manifestsend=b'', | ||||
|
r38940 | bundlecaps=bundlecaps, | ||
|
r38944 | ellipses=ellipses, | ||
|
r38943 | shallow=shallow, | ||
|
r38945 | ellipsisroots=ellipsisroots, | ||
fullnodes=fullnodes) | ||||
|
r38930 | |||
|
r40380 | def _makecg3packer(repo, oldmatcher, matcher, bundlecaps, | ||
ellipses=False, shallow=False, ellipsisroots=None, | ||||
fullnodes=None): | ||||
|
r38933 | builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack( | ||
d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags) | ||||
|
r40380 | return cgpacker(repo, oldmatcher, matcher, b'03', | ||
|
r38938 | builddeltaheader=builddeltaheader, | ||
manifestsend=closechunk(), | ||||
|
r38940 | bundlecaps=bundlecaps, | ||
|
r38944 | ellipses=ellipses, | ||
|
r38943 | shallow=shallow, | ||
|
r38945 | ellipsisroots=ellipsisroots, | ||
fullnodes=fullnodes) | ||||
|
r38930 | |||
_packermap = {'01': (_makecg1packer, cg1unpacker), | ||||
|
r26709 | # cg2 adds support for exchanging generaldelta | ||
|
r38930 | '02': (_makecg2packer, cg2unpacker), | ||
|
r27753 | # cg3 adds support for exchanging revlog flags and treemanifests | ||
|
r38930 | '03': (_makecg3packer, cg3unpacker), | ||
|
r26709 | } | ||
|
r23168 | |||
|
r30627 | def allsupportedversions(repo): | ||
|
r27928 | versions = set(_packermap.keys()) | ||
|
r30627 | if not (repo.ui.configbool('experimental', 'changegroup3') or | ||
|
r30628 | repo.ui.configbool('experimental', 'treemanifest') or | ||
'treemanifest' in repo.requirements): | ||||
|
r30626 | versions.discard('03') | ||
|
r27953 | return versions | ||
# Changegroup versions that can be applied to the repo | ||||
def supportedincomingversions(repo): | ||||
|
r30628 | return allsupportedversions(repo) | ||
|
r27953 | |||
# Changegroup versions that can be created from the repo | ||||
def supportedoutgoingversions(repo): | ||||
|
r30627 | versions = allsupportedversions(repo) | ||
|
r27953 | if 'treemanifest' in repo.requirements: | ||
|
r27928 | # Versions 01 and 02 support only flat manifests and it's just too | ||
# expensive to convert between the flat manifest and tree manifest on | ||||
# the fly. Since tree manifests are hashed differently, all of history | ||||
# would have to be converted. Instead, we simply don't even pretend to | ||||
# support versions 01 and 02. | ||||
versions.discard('01') | ||||
versions.discard('02') | ||||
|
r38871 | if repository.NARROW_REQUIREMENT in repo.requirements: | ||
|
r36483 | # Versions 01 and 02 don't support revlog flags, and we need to | ||
# support that for stripping and unbundling to work. | ||||
versions.discard('01') | ||||
versions.discard('02') | ||||
|
r37150 | if LFS_REQUIREMENT in repo.requirements: | ||
# Versions 01 and 02 don't support revlog flags, and we need to | ||||
# mark LFS entries with REVIDX_EXTSTORED. | ||||
versions.discard('01') | ||||
versions.discard('02') | ||||
|
r27752 | return versions | ||
|
r27751 | |||
|
r34179 | def localversion(repo): | ||
# Finds the best version to use for bundles that are meant to be used | ||||
# locally, such as those from strip and shelve, and temporary bundles. | ||||
return max(supportedoutgoingversions(repo)) | ||||
|
r27929 | def safeversion(repo): | ||
# Finds the smallest version that it's safe to assume clients of the repo | ||||
|
r27931 | # will support. For example, all hg versions that support generaldelta also | ||
# support changegroup 02. | ||||
|
r27953 | versions = supportedoutgoingversions(repo) | ||
|
r27929 | if 'generaldelta' in repo.requirements: | ||
versions.discard('01') | ||||
assert versions | ||||
return min(versions) | ||||
|
r40380 | def getbundler(version, repo, bundlecaps=None, oldmatcher=None, | ||
matcher=None, ellipses=False, shallow=False, | ||||
ellipsisroots=None, fullnodes=None): | ||||
|
r27953 | assert version in supportedoutgoingversions(repo) | ||
|
r38830 | |||
|
r40380 | if matcher is None: | ||
matcher = matchmod.alwaysmatcher(repo.root, '') | ||||
if oldmatcher is None: | ||||
oldmatcher = matchmod.nevermatcher(repo.root, '') | ||||
|
r38830 | |||
|
r40380 | if version == '01' and not matcher.always(): | ||
|
r38830 | raise error.ProgrammingError('version 01 changegroups do not support ' | ||
'sparse file matchers') | ||||
|
r38944 | if ellipses and version in (b'01', b'02'): | ||
raise error.Abort( | ||||
_('ellipsis nodes require at least cg3 on client and server, ' | ||||
'but negotiated version %s') % version) | ||||
|
r38830 | # Requested files could include files not in the local store. So | ||
# filter those out. | ||||
|
r40380 | matcher = repo.narrowmatch(matcher) | ||
|
r38830 | |||
|
r38930 | fn = _packermap[version][0] | ||
|
r40380 | return fn(repo, oldmatcher, matcher, bundlecaps, ellipses=ellipses, | ||
|
r38945 | shallow=shallow, ellipsisroots=ellipsisroots, | ||
fullnodes=fullnodes) | ||||
|
r27751 | |||
|
r29593 | def getunbundler(version, fh, alg, extras=None): | ||
return _packermap[version][1](fh, alg, extras=extras) | ||||
|
r27751 | |||
|
r20926 | def _changegroupinfo(repo, nodes, source): | ||
if repo.ui.verbose or source == 'bundle': | ||||
repo.ui.status(_("%d changesets found\n") % len(nodes)) | ||||
if repo.ui.debugflag: | ||||
repo.ui.debug("list of changesets:\n") | ||||
for node in nodes: | ||||
repo.ui.debug("%s\n" % hex(node)) | ||||
|
r34098 | def makechangegroup(repo, outgoing, version, source, fastpath=False, | ||
bundlecaps=None): | ||||
cgstream = makestream(repo, outgoing, version, source, | ||||
fastpath=fastpath, bundlecaps=bundlecaps) | ||||
return getunbundler(version, util.chunkbuffer(cgstream), None, | ||||
{'clcount': len(outgoing.missing) }) | ||||
|
r34105 | def makestream(repo, outgoing, version, source, fastpath=False, | ||
|
r40380 | bundlecaps=None, matcher=None): | ||
|
r38830 | bundler = getbundler(version, repo, bundlecaps=bundlecaps, | ||
|
r40380 | matcher=matcher) | ||
|
r34105 | |||
|
r20925 | repo = repo.unfiltered() | ||
commonrevs = outgoing.common | ||||
csets = outgoing.missing | ||||
heads = outgoing.missingheads | ||||
# We go through the fast path if we get told to, or if all (unfiltered | ||||
# heads have been requested (since we then know there all linkrevs will | ||||
# be pulled by the client). | ||||
heads.sort() | ||||
fastpathlinkrev = fastpath or ( | ||||
repo.filtername is None and heads == sorted(repo.heads())) | ||||
repo.hook('preoutgoing', throw=True, source=source) | ||||
|
r20926 | _changegroupinfo(repo, csets, source) | ||
|
r23177 | return bundler.generate(commonrevs, csets, fastpathlinkrev, source) | ||
|
r28361 | def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles): | ||
|
r20932 | revisions = 0 | ||
files = 0 | ||||
|
r38401 | progress = repo.ui.makeprogress(_('files'), unit=_('files'), | ||
total=expectedfiles) | ||||
|
r29724 | for chunkdata in iter(source.filelogheader, {}): | ||
|
r28361 | files += 1 | ||
|
r20932 | f = chunkdata["filename"] | ||
repo.ui.debug("adding %s revisions\n" % f) | ||||
|
r38401 | progress.increment() | ||
|
r27754 | fl = repo.file(f) | ||
|
r20932 | o = len(fl) | ||
|
r24120 | try: | ||
|
r34292 | deltas = source.deltaiter() | ||
if not fl.addgroup(deltas, revmap, trp): | ||||
|
r26587 | raise error.Abort(_("received file revlog group is empty")) | ||
|
r25660 | except error.CensoredBaseError as e: | ||
|
r26587 | raise error.Abort(_("received delta base is censored: %s") % e) | ||
|
r27754 | revisions += len(fl) - o | ||
|
r20932 | if f in needfiles: | ||
needs = needfiles[f] | ||||
|
r38806 | for new in pycompat.xrange(o, len(fl)): | ||
|
r20932 | n = fl.node(new) | ||
if n in needs: | ||||
needs.remove(n) | ||||
else: | ||||
|
r26587 | raise error.Abort( | ||
|
r20932 | _("received spurious file revlog entry")) | ||
if not needs: | ||||
del needfiles[f] | ||||
|
r38401 | progress.complete() | ||
|
r20932 | |||
for f, needs in needfiles.iteritems(): | ||||
fl = repo.file(f) | ||||
for n in needs: | ||||
try: | ||||
fl.rev(n) | ||||
except error.LookupError: | ||||
|
r26587 | raise error.Abort( | ||
|
r20932 | _('missing file data for %s:%s - run hg verify') % | ||
(f, hex(n))) | ||||
return revisions, files | ||||