changegroup.py
1003 lines
| 38.5 KiB
| text/x-python
|
PythonLexer
/ mercurial / changegroup.py
|
r8226 | # changegroup.py - Mercurial changegroup manipulation functions | ||
# | ||||
# Copyright 2006 Matt Mackall <mpm@selenic.com> | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
|
r10263 | # GNU General Public License version 2 or any later version. | ||
|
r3877 | |||
|
r25921 | from __future__ import absolute_import | ||
import os | ||||
import struct | ||||
import tempfile | ||||
|
r20933 | import weakref | ||
|
r25921 | |||
from .i18n import _ | ||||
from .node import ( | ||||
hex, | ||||
nullrev, | ||||
short, | ||||
) | ||||
from . import ( | ||||
dagutil, | ||||
error, | ||||
mdiff, | ||||
phases, | ||||
|
r30925 | pycompat, | ||
|
r25921 | util, | ||
) | ||||
|
r1981 | |||
|
r22390 | _CHANGEGROUPV1_DELTA_HEADER = "20s20s20s20s" | ||
|
r23181 | _CHANGEGROUPV2_DELTA_HEADER = "20s20s20s20s20s" | ||
|
r27433 | _CHANGEGROUPV3_DELTA_HEADER = ">20s20s20s20s20sH" | ||
|
r14141 | |||
|
r13457 | def readexactly(stream, n): | ||
'''read n bytes from stream.read and abort if less was available''' | ||||
s = stream.read(n) | ||||
if len(s) < n: | ||||
|
r26587 | raise error.Abort(_("stream ended unexpectedly" | ||
|
r13457 | " (got %d bytes, expected %d)") | ||
% (len(s), n)) | ||||
return s | ||||
def getchunk(stream): | ||||
"""return the next chunk from stream as a string""" | ||||
d = readexactly(stream, 4) | ||||
|
r1981 | l = struct.unpack(">l", d)[0] | ||
if l <= 4: | ||||
|
r13458 | if l: | ||
|
r26587 | raise error.Abort(_("invalid chunk length %d") % l) | ||
|
r1981 | return "" | ||
|
r13457 | return readexactly(stream, l - 4) | ||
|
r1981 | |||
|
r5368 | def chunkheader(length): | ||
|
r9437 | """return a changegroup chunk header (string)""" | ||
|
r5368 | return struct.pack(">l", length + 4) | ||
|
r1981 | |||
def closechunk(): | ||||
|
r9437 | """return a changegroup chunk header (string) for a zero-length chunk""" | ||
|
r1981 | return struct.pack(">l", 0) | ||
|
r26540 | def writechunks(ui, chunks, filename, vfs=None): | ||
"""Write chunks to a file and return its filename. | ||||
|
r3659 | |||
|
r26540 | The stream is assumed to be a bundle file. | ||
|
r3659 | Existing files will not be overwritten. | ||
If no filename is specified, a temporary file is created. | ||||
""" | ||||
fh = None | ||||
cleanup = None | ||||
try: | ||||
if filename: | ||||
|
r20976 | if vfs: | ||
fh = vfs.open(filename, "wb") | ||||
else: | ||||
|
r30212 | # Increase default buffer size because default is usually | ||
# small (4k is common on Linux). | ||||
fh = open(filename, "wb", 131072) | ||||
|
r3659 | else: | ||
fd, filename = tempfile.mkstemp(prefix="hg-bundle-", suffix=".hg") | ||||
|
r30925 | fh = os.fdopen(fd, pycompat.sysstr("wb")) | ||
|
r3659 | cleanup = filename | ||
|
r26540 | for c in chunks: | ||
fh.write(c) | ||||
|
r3659 | cleanup = None | ||
return filename | ||||
finally: | ||||
if fh is not None: | ||||
fh.close() | ||||
if cleanup is not None: | ||||
|
r20976 | if filename and vfs: | ||
vfs.unlink(cleanup) | ||||
else: | ||||
os.unlink(cleanup) | ||||
|
r3660 | |||
|
r22390 | class cg1unpacker(object): | ||
|
r26708 | """Unpacker for cg1 changegroup streams. | ||
A changegroup unpacker handles the framing of the revision data in | ||||
the wire format. Most consumers will want to use the apply() | ||||
method to add the changes from the changegroup to a repository. | ||||
If you're forwarding a changegroup unmodified to another consumer, | ||||
use getchunks(), which returns an iterator of changegroup | ||||
chunks. This is mostly useful for cases where you need to know the | ||||
data stream has ended by observing the end of the changegroup. | ||||
deltachunk() is useful only if you're applying delta data. Most | ||||
consumers should prefer apply() instead. | ||||
A few other public methods exist. Those are used only for | ||||
bundlerepo and some debug commands - their use is discouraged. | ||||
""" | ||||
|
r22390 | deltaheader = _CHANGEGROUPV1_DELTA_HEADER | ||
|
r14141 | deltaheadersize = struct.calcsize(deltaheader) | ||
|
r23896 | version = '01' | ||
|
r27920 | _grouplistcount = 1 # One list of files after the manifests | ||
|
r29593 | def __init__(self, fh, alg, extras=None): | ||
|
r30354 | if alg is None: | ||
alg = 'UN' | ||||
if alg not in util.compengines.supportedbundletypes: | ||||
|
r26587 | raise error.Abort(_('unknown stream compression type: %s') | ||
|
r26266 | % alg) | ||
|
r26392 | if alg == 'BZ': | ||
alg = '_truncatedBZ' | ||||
|
r30354 | |||
compengine = util.compengines.forbundletype(alg) | ||||
self._stream = compengine.decompressorreader(fh) | ||||
|
r12044 | self._type = alg | ||
|
r29593 | self.extras = extras or {} | ||
|
r12334 | self.callback = None | ||
|
r26706 | |||
# These methods (compressed, read, seek, tell) all appear to only | ||||
# be used by bundlerepo, but it's a little hard to tell. | ||||
|
r12044 | def compressed(self): | ||
|
r30589 | return self._type is not None and self._type != 'UN' | ||
|
r12043 | def read(self, l): | ||
return self._stream.read(l) | ||||
|
r12330 | def seek(self, pos): | ||
return self._stream.seek(pos) | ||||
def tell(self): | ||||
|
r12332 | return self._stream.tell() | ||
|
r12347 | def close(self): | ||
return self._stream.close() | ||||
|
r12334 | |||
|
r26707 | def _chunklength(self): | ||
|
r13459 | d = readexactly(self._stream, 4) | ||
|
r13458 | l = struct.unpack(">l", d)[0] | ||
if l <= 4: | ||||
if l: | ||||
|
r26587 | raise error.Abort(_("invalid chunk length %d") % l) | ||
|
r13458 | return 0 | ||
if self.callback: | ||||
|
r12334 | self.callback() | ||
|
r13458 | return l - 4 | ||
|
r12334 | |||
|
r14144 | def changelogheader(self): | ||
"""v10 does not have a changelog header chunk""" | ||||
return {} | ||||
def manifestheader(self): | ||||
"""v10 does not have a manifest header chunk""" | ||||
return {} | ||||
def filelogheader(self): | ||||
"""return the header of the filelogs chunk, v10 only has the filename""" | ||||
|
r26707 | l = self._chunklength() | ||
|
r14144 | if not l: | ||
return {} | ||||
fname = readexactly(self._stream, l) | ||||
|
r20675 | return {'filename': fname} | ||
|
r12334 | |||
|
r14141 | def _deltaheader(self, headertuple, prevnode): | ||
node, p1, p2, cs = headertuple | ||||
if prevnode is None: | ||||
deltabase = p1 | ||||
else: | ||||
deltabase = prevnode | ||||
|
r27433 | flags = 0 | ||
return node, p1, p2, deltabase, cs, flags | ||||
|
r14141 | |||
|
r14144 | def deltachunk(self, prevnode): | ||
|
r26707 | l = self._chunklength() | ||
|
r12336 | if not l: | ||
return {} | ||||
|
r14141 | headerdata = readexactly(self._stream, self.deltaheadersize) | ||
header = struct.unpack(self.deltaheader, headerdata) | ||||
delta = readexactly(self._stream, l - self.deltaheadersize) | ||||
|
r27433 | node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode) | ||
|
r34295 | return (node, p1, p2, cs, deltabase, delta, flags) | ||
|
r12336 | |||
|
r20999 | def getchunks(self): | ||
"""returns all the chunks contains in the bundle | ||||
Used when you need to forward the binary stream to a file or another | ||||
network API. To do so, it parse the changegroup data, otherwise it will | ||||
block in case of sshrepo because it don't know the end of the stream. | ||||
""" | ||||
|
r34093 | # For changegroup 1 and 2, we expect 3 parts: changelog, manifestlog, | ||
# and a list of filelogs. For changegroup 3, we expect 4 parts: | ||||
# changelog, manifestlog, a list of tree manifestlogs, and a list of | ||||
# filelogs. | ||||
# | ||||
# Changelog and manifestlog parts are terminated with empty chunks. The | ||||
# tree and file parts are a list of entry sections. Each entry section | ||||
# is a series of chunks terminating in an empty chunk. The list of these | ||||
# entry sections is terminated in yet another empty chunk, so we know | ||||
# we've reached the end of the tree/file list when we reach an empty | ||||
# chunk that was proceeded by no non-empty chunks. | ||||
parts = 0 | ||||
while parts < 2 + self._grouplistcount: | ||||
noentries = True | ||||
|
r20999 | while True: | ||
chunk = getchunk(self) | ||||
if not chunk: | ||||
|
r34093 | # The first two empty chunks represent the end of the | ||
# changelog and the manifestlog portions. The remaining | ||||
# empty chunks represent either A) the end of individual | ||||
# tree or file entries in the file list, or B) the end of | ||||
# the entire list. It's the end of the entire list if there | ||||
# were no entries (i.e. noentries is True). | ||||
if parts < 2: | ||||
parts += 1 | ||||
elif noentries: | ||||
parts += 1 | ||||
|
r20999 | break | ||
|
r34093 | noentries = False | ||
|
r20999 | yield chunkheader(len(chunk)) | ||
pos = 0 | ||||
while pos < len(chunk): | ||||
next = pos + 2**20 | ||||
yield chunk[pos:next] | ||||
pos = next | ||||
yield closechunk() | ||||
|
r26712 | def _unpackmanifests(self, repo, revmap, trp, prog, numchanges): | ||
# We know that we'll never have more manifests than we had | ||||
# changesets. | ||||
self.callback = prog(_('manifests'), numchanges) | ||||
# no need to check for empty manifest group here: | ||||
# if the result of the merge of 1 and 2 is the same in 3 and 4, | ||||
# no new manifest will be created and the manifest group will | ||||
# be empty during the pull | ||||
self.manifestheader() | ||||
|
r34292 | deltas = self.deltaiter() | ||
repo.manifestlog._revlog.addgroup(deltas, revmap, trp) | ||||
|
r26712 | repo.ui.progress(_('manifests'), None) | ||
|
r28360 | self.callback = None | ||
|
r26712 | |||
|
r33308 | def apply(self, repo, tr, srctype, url, targetphase=phases.draft, | ||
expectedtotal=None): | ||||
|
r26695 | """Add the changegroup returned by source.read() to this repo. | ||
srctype is a string like 'push', 'pull', or 'unbundle'. url is | ||||
the URL of the repo where this changegroup is coming from. | ||||
Return an integer summarizing the change to this repo: | ||||
- nothing changed or no source: 0 | ||||
- more heads than before: 1+added heads (2..n) | ||||
- fewer heads than before: -1-removed heads (-2..-n) | ||||
- number of heads stays the same: 1 | ||||
""" | ||||
repo = repo.unfiltered() | ||||
def csmap(x): | ||||
repo.ui.debug("add changeset %s\n" % short(x)) | ||||
return len(cl) | ||||
def revmap(x): | ||||
return cl.rev(x) | ||||
changesets = files = revisions = 0 | ||||
|
r26880 | try: | ||
|
r32931 | # The transaction may already carry source information. In this | ||
# case we use the top level data. We overwrite the argument | ||||
# because we need to use the top level value (if they exist) | ||||
# in this function. | ||||
srctype = tr.hookargs.setdefault('source', srctype) | ||||
url = tr.hookargs.setdefault('url', url) | ||||
|
r33634 | repo.hook('prechangegroup', | ||
throw=True, **pycompat.strkwargs(tr.hookargs)) | ||||
|
r26695 | |||
|
r32931 | # write changelog data to temp files so concurrent readers | ||
# will not see an inconsistent view | ||||
cl = repo.changelog | ||||
cl.delayupdate(tr) | ||||
oldheads = set(cl.heads()) | ||||
|
r26695 | |||
|
r32931 | trp = weakref.proxy(tr) | ||
# pull off the changeset group | ||||
repo.ui.status(_("adding changesets\n")) | ||||
clstart = len(cl) | ||||
class prog(object): | ||||
def __init__(self, step, total): | ||||
self._step = step | ||||
self._total = total | ||||
self._count = 1 | ||||
def __call__(self): | ||||
repo.ui.progress(self._step, self._count, unit=_('chunks'), | ||||
total=self._total) | ||||
self._count += 1 | ||||
self.callback = prog(_('changesets'), expectedtotal) | ||||
|
r26695 | |||
|
r32931 | efiles = set() | ||
def onchangelog(cl, node): | ||||
efiles.update(cl.readfiles(node)) | ||||
|
r26695 | |||
|
r32931 | self.changelogheader() | ||
|
r34292 | deltas = self.deltaiter() | ||
cgnodes = cl.addgroup(deltas, csmap, trp, addrevisioncb=onchangelog) | ||||
|
r32931 | efiles = len(efiles) | ||
|
r26695 | |||
|
r33308 | if not cgnodes: | ||
|
r33309 | repo.ui.develwarn('applied empty changegroup', | ||
|
r34735 | config='warn-empty-changegroup') | ||
|
r32931 | clend = len(cl) | ||
changesets = clend - clstart | ||||
repo.ui.progress(_('changesets'), None) | ||||
self.callback = None | ||||
|
r26695 | |||
|
r32931 | # pull off the manifest group | ||
repo.ui.status(_("adding manifests\n")) | ||||
self._unpackmanifests(repo, revmap, trp, prog, changesets) | ||||
|
r26695 | |||
|
r32931 | needfiles = {} | ||
r33224 | if repo.ui.configbool('server', 'validate'): | |||
|
r32931 | cl = repo.changelog | ||
ml = repo.manifestlog | ||||
# validate incoming csets have their manifests | ||||
for cset in xrange(clstart, clend): | ||||
mfnode = cl.changelogrevision(cset).manifest | ||||
mfest = ml[mfnode].readdelta() | ||||
# store file cgnodes we must see | ||||
for f, n in mfest.iteritems(): | ||||
needfiles.setdefault(f, set()).add(n) | ||||
|
r26695 | |||
|
r32931 | # process the files | ||
repo.ui.status(_("adding file changes\n")) | ||||
newrevs, newfiles = _addchangegroupfiles( | ||||
repo, self, revmap, trp, efiles, needfiles) | ||||
revisions += newrevs | ||||
files += newfiles | ||||
|
r26695 | |||
|
r32931 | deltaheads = 0 | ||
if oldheads: | ||||
heads = cl.heads() | ||||
deltaheads = len(heads) - len(oldheads) | ||||
for h in heads: | ||||
if h not in oldheads and repo[h].closesbranch(): | ||||
deltaheads -= 1 | ||||
htext = "" | ||||
if deltaheads: | ||||
htext = _(" (%+d heads)") % deltaheads | ||||
|
r26695 | |||
|
r32931 | repo.ui.status(_("added %d changesets" | ||
" with %d changes to %d files%s\n") | ||||
% (changesets, revisions, files, htext)) | ||||
repo.invalidatevolatilesets() | ||||
|
r26695 | |||
|
r32931 | if changesets > 0: | ||
if 'node' not in tr.hookargs: | ||||
tr.hookargs['node'] = hex(cl.node(clstart)) | ||||
tr.hookargs['node_last'] = hex(cl.node(clend - 1)) | ||||
hookargs = dict(tr.hookargs) | ||||
else: | ||||
hookargs = dict(tr.hookargs) | ||||
hookargs['node'] = hex(cl.node(clstart)) | ||||
hookargs['node_last'] = hex(cl.node(clend - 1)) | ||||
|
r33634 | repo.hook('pretxnchangegroup', | ||
throw=True, **pycompat.strkwargs(hookargs)) | ||||
|
r26695 | |||
|
r32931 | added = [cl.node(r) for r in xrange(clstart, clend)] | ||
|
r33456 | phaseall = None | ||
|
r32931 | if srctype in ('push', 'serve'): | ||
# Old servers can not push the boundary themselves. | ||||
# New servers won't push the boundary if changeset already | ||||
# exists locally as secret | ||||
# | ||||
# We should not use added here but the list of all change in | ||||
# the bundle | ||||
if repo.publishing(): | ||||
|
r33456 | targetphase = phaseall = phases.public | ||
|
r32931 | else: | ||
|
r33456 | # closer target phase computation | ||
|
r32931 | # Those changesets have been pushed from the | ||
# outside, their phases are going to be pushed | ||||
# alongside. Therefor `targetphase` is | ||||
# ignored. | ||||
|
r33456 | targetphase = phaseall = phases.draft | ||
if added: | ||||
phases.registernew(repo, tr, targetphase, added) | ||||
if phaseall is not None: | ||||
phases.advanceboundary(repo, tr, phaseall, cgnodes) | ||||
|
r26695 | |||
|
r32931 | if changesets > 0: | ||
|
r26695 | |||
|
r32931 | def runhooks(): | ||
# These hooks run when the lock releases, not when the | ||||
# transaction closes. So it's possible for the changelog | ||||
# to have changed since we last saw it. | ||||
if clstart >= len(repo): | ||||
return | ||||
|
r26695 | |||
|
r33678 | repo.hook("changegroup", **pycompat.strkwargs(hookargs)) | ||
|
r27867 | |||
|
r32931 | for n in added: | ||
args = hookargs.copy() | ||||
args['node'] = hex(n) | ||||
del args['node_last'] | ||||
|
r33678 | repo.hook("incoming", **pycompat.strkwargs(args)) | ||
|
r26695 | |||
|
r32931 | newheads = [h for h in repo.heads() | ||
if h not in oldheads] | ||||
repo.ui.log("incoming", | ||||
"%s incoming changes - new heads: %s\n", | ||||
len(added), | ||||
', '.join([hex(c[:6]) for c in newheads])) | ||||
|
r26695 | |||
|
r32931 | tr.addpostclose('changegroup-runhooks-%020i' % clstart, | ||
lambda tr: repo._afterlock(runhooks)) | ||||
|
r26695 | finally: | ||
repo.ui.flush() | ||||
# never return 0 here: | ||||
|
r32870 | if deltaheads < 0: | ||
|
r33030 | ret = deltaheads - 1 | ||
|
r26695 | else: | ||
|
r33030 | ret = deltaheads + 1 | ||
|
r33461 | return ret | ||
|
r26695 | |||
|
r34292 | def deltaiter(self): | ||
|
r34147 | """ | ||
returns an iterator of the deltas in this changegroup | ||||
Useful for passing to the underlying storage system to be stored. | ||||
""" | ||||
chain = None | ||||
for chunkdata in iter(lambda: self.deltachunk(chain), {}): | ||||
|
r34295 | # Chunkdata: (node, p1, p2, cs, deltabase, delta, flags) | ||
yield chunkdata | ||||
chain = chunkdata[0] | ||||
|
r34147 | |||
|
r23181 | class cg2unpacker(cg1unpacker): | ||
|
r26708 | """Unpacker for cg2 streams. | ||
cg2 streams add support for generaldelta, so the delta header | ||||
format is slightly different. All other features about the data | ||||
remain the same. | ||||
""" | ||||
|
r23181 | deltaheader = _CHANGEGROUPV2_DELTA_HEADER | ||
deltaheadersize = struct.calcsize(deltaheader) | ||||
|
r23896 | version = '02' | ||
|
r23181 | |||
def _deltaheader(self, headertuple, prevnode): | ||||
node, p1, p2, deltabase, cs = headertuple | ||||
|
r27433 | flags = 0 | ||
return node, p1, p2, deltabase, cs, flags | ||||
|
r23181 | |||
|
r27432 | class cg3unpacker(cg2unpacker): | ||
"""Unpacker for cg3 streams. | ||||
|
r27433 | cg3 streams add support for exchanging treemanifests and revlog | ||
|
r27753 | flags. It adds the revlog flags to the delta header and an empty chunk | ||
separating manifests and files. | ||||
|
r27432 | """ | ||
|
r27433 | deltaheader = _CHANGEGROUPV3_DELTA_HEADER | ||
deltaheadersize = struct.calcsize(deltaheader) | ||||
|
r27432 | version = '03' | ||
|
r27920 | _grouplistcount = 2 # One list of manifests and one list of files | ||
|
r27432 | |||
|
r27433 | def _deltaheader(self, headertuple, prevnode): | ||
node, p1, p2, deltabase, cs, flags = headertuple | ||||
return node, p1, p2, deltabase, cs, flags | ||||
|
r27754 | def _unpackmanifests(self, repo, revmap, trp, prog, numchanges): | ||
super(cg3unpacker, self)._unpackmanifests(repo, revmap, trp, prog, | ||||
numchanges) | ||||
|
r29724 | for chunkdata in iter(self.filelogheader, {}): | ||
|
r27754 | # If we get here, there are directory manifests in the changegroup | ||
d = chunkdata["filename"] | ||||
repo.ui.debug("adding %s revisions\n" % d) | ||||
|
r30339 | dirlog = repo.manifestlog._revlog.dirlog(d) | ||
|
r34292 | deltas = self.deltaiter() | ||
if not dirlog.addgroup(deltas, revmap, trp): | ||||
|
r27754 | raise error.Abort(_("received dir revlog group is empty")) | ||
|
r12329 | class headerlessfixup(object): | ||
def __init__(self, fh, h): | ||||
self._h = h | ||||
self._fh = fh | ||||
def read(self, n): | ||||
if self._h: | ||||
d, self._h = self._h[:n], self._h[n:] | ||||
if len(d) < n: | ||||
|
r13457 | d += readexactly(self._fh, n - len(d)) | ||
|
r12329 | return d | ||
|
r13457 | return readexactly(self._fh, n) | ||
|
r12329 | |||
|
r22390 | class cg1packer(object): | ||
deltaheader = _CHANGEGROUPV1_DELTA_HEADER | ||||
|
r23896 | version = '01' | ||
|
r32287 | def __init__(self, repo, bundlecaps=None): | ||
|
r19202 | """Given a source repo, construct a bundler. | ||
|
r32287 | |||
bundlecaps is optional and can be used to specify the set of | ||||
capabilities which can be used to build the bundle. While bundlecaps is | ||||
unused in core Mercurial, extensions rely on this feature to communicate | ||||
capabilities to customize the changegroup packer. | ||||
|
r19202 | """ | ||
|
r32287 | # Set of capabilities we can use to build the bundle. | ||
if bundlecaps is None: | ||||
bundlecaps = set() | ||||
self._bundlecaps = bundlecaps | ||||
|
r25831 | # experimental config: bundle.reorder | ||
r33180 | reorder = repo.ui.config('bundle', 'reorder') | |||
|
r19202 | if reorder == 'auto': | ||
reorder = None | ||||
else: | ||||
reorder = util.parsebool(reorder) | ||||
self._repo = repo | ||||
self._reorder = reorder | ||||
|
r19208 | self._progress = repo.ui.progress | ||
|
r23748 | if self._repo.ui.verbose and not self._repo.ui.debugflag: | ||
self._verbosenote = self._repo.ui.note | ||||
else: | ||||
self._verbosenote = lambda s: None | ||||
|
r13831 | def close(self): | ||
return closechunk() | ||||
|
r19200 | |||
|
r13831 | def fileheader(self, fname): | ||
return chunkheader(len(fname)) + fname | ||||
|
r19200 | |||
|
r29236 | # Extracted both for clarity and for overriding in extensions. | ||
def _sortgroup(self, revlog, nodelist, lookup): | ||||
"""Sort nodes for change group and turn them into revnums.""" | ||||
# for generaldelta revlogs, we linearize the revs; this will both be | ||||
# much quicker and generate a much smaller bundle | ||||
if (revlog._generaldelta and self._reorder is None) or self._reorder: | ||||
dag = dagutil.revlogdag(revlog) | ||||
return dag.linearize(set(revlog.rev(n) for n in nodelist)) | ||||
else: | ||||
return sorted([revlog.rev(n) for n in nodelist]) | ||||
|
r24912 | def group(self, nodelist, revlog, lookup, units=None): | ||
|
r19200 | """Calculate a delta group, yielding a sequence of changegroup chunks | ||
(strings). | ||||
Given a list of changeset revs, return a set of deltas and | ||||
metadata corresponding to nodes. The first delta is | ||||
first parent(nodelist[0]) -> nodelist[0], the receiver is | ||||
guaranteed to have this parent as it has all history before | ||||
these changesets. In the case firstparent is nullrev the | ||||
changegroup starts with a full revision. | ||||
|
r19208 | |||
If units is not None, progress detail will be generated, units specifies | ||||
the type of revlog that is touched (changelog, manifest, etc.). | ||||
|
r19200 | """ | ||
# if we don't have any revisions touched by these changesets, bail | ||||
if len(nodelist) == 0: | ||||
yield self.close() | ||||
return | ||||
|
r29236 | revs = self._sortgroup(revlog, nodelist, lookup) | ||
|
r19200 | |||
# add the parent of the first rev | ||||
p = revlog.parentrevs(revs[0])[0] | ||||
revs.insert(0, p) | ||||
# build deltas | ||||
|
r19208 | total = len(revs) - 1 | ||
msgbundling = _('bundling') | ||||
|
r19200 | for r in xrange(len(revs) - 1): | ||
|
r19208 | if units is not None: | ||
self._progress(msgbundling, r + 1, unit=units, total=total) | ||||
|
r19200 | prev, curr = revs[r], revs[r + 1] | ||
|
r19207 | linknode = lookup(revlog.node(curr)) | ||
for c in self.revchunk(revlog, curr, prev, linknode): | ||||
|
r19200 | yield c | ||
|
r24901 | if units is not None: | ||
self._progress(msgbundling, None) | ||||
|
r19200 | yield self.close() | ||
|
r19289 | # filter any nodes that claim to be part of the known set | ||
|
r24896 | def prune(self, revlog, missing, commonrevs): | ||
|
r19289 | rr, rl = revlog.rev, revlog.linkrev | ||
return [n for n in missing if rl(rr(n)) not in commonrevs] | ||||
|
r28228 | def _packmanifests(self, dir, mfnodes, lookuplinknode): | ||
|
r26711 | """Pack flat manifests into a changegroup stream.""" | ||
|
r28228 | assert not dir | ||
|
r30339 | for chunk in self.group(mfnodes, self._repo.manifestlog._revlog, | ||
|
r28228 | lookuplinknode, units=_('manifests')): | ||
|
r26711 | yield chunk | ||
|
r28228 | |||
def _manifestsdone(self): | ||||
return '' | ||||
|
r26711 | |||
|
r19204 | def generate(self, commonrevs, clnodes, fastpathlinkrev, source): | ||
|
r19202 | '''yield a sequence of changegroup chunks (strings)''' | ||
repo = self._repo | ||||
|
r24978 | cl = repo.changelog | ||
|
r19204 | |||
|
r23381 | clrevorder = {} | ||
|
r19204 | mfs = {} # needed manifests | ||
fnodes = {} # needed file nodes | ||||
|
r28241 | changedfiles = set() | ||
|
r19204 | |||
|
r19207 | # Callback for the changelog, used to collect changed files and manifest | ||
# nodes. | ||||
# Returns the linkrev node (identity in the changelog case). | ||||
def lookupcl(x): | ||||
c = cl.read(x) | ||||
|
r23381 | clrevorder[x] = len(clrevorder) | ||
|
r27237 | n = c[0] | ||
|
r19207 | # record the first changeset introducing this manifest version | ||
|
r27237 | mfs.setdefault(n, x) | ||
# Record a complete list of potentially-changed files in | ||||
# this manifest. | ||||
|
r28241 | changedfiles.update(c[3]) | ||
|
r19207 | return x | ||
|
r19204 | |||
|
r23748 | self._verbosenote(_('uncompressed size of bundle content:\n')) | ||
size = 0 | ||||
|
r24912 | for chunk in self.group(clnodes, cl, lookupcl, units=_('changesets')): | ||
|
r23748 | size += len(chunk) | ||
|
r23224 | yield chunk | ||
|
r23748 | self._verbosenote(_('%8.i (changelog)\n') % size) | ||
|
r23224 | |||
|
r24977 | # We need to make sure that the linkrev in the changegroup refers to | ||
# the first changeset that introduced the manifest or file revision. | ||||
# The fastpath is usually safer than the slowpath, because the filelogs | ||||
# are walked in revlog order. | ||||
# | ||||
# When taking the slowpath with reorder=None and the manifest revlog | ||||
# uses generaldelta, the manifest may be walked in the "wrong" order. | ||||
# Without 'clrevorder', we would get an incorrect linkrev (see fix in | ||||
# cc0ff93d0c0c). | ||||
# | ||||
# When taking the fastpath, we are only vulnerable to reordering | ||||
# of the changelog itself. The changelog never uses generaldelta, so | ||||
# it is only reordered when reorder=True. To handle this case, we | ||||
# simply take the slowpath, which already has the 'clrevorder' logic. | ||||
# This was also fixed in cc0ff93d0c0c. | ||||
|
r24976 | fastpathlinkrev = fastpathlinkrev and not self._reorder | ||
|
r27432 | # Treemanifests don't work correctly with fastpathlinkrev | ||
# either, because we don't discover which directory nodes to | ||||
# send along with files. This could probably be fixed. | ||||
fastpathlinkrev = fastpathlinkrev and ( | ||||
'treemanifest' not in repo.requirements) | ||||
|
r28227 | |||
for chunk in self.generatemanifests(commonrevs, clrevorder, | ||||
|
r34148 | fastpathlinkrev, mfs, fnodes, source): | ||
|
r28227 | yield chunk | ||
mfs.clear() | ||||
clrevs = set(cl.rev(x) for x in clnodes) | ||||
if not fastpathlinkrev: | ||||
def linknodes(unused, fname): | ||||
return fnodes.get(fname, {}) | ||||
else: | ||||
cln = cl.node | ||||
def linknodes(filerevlog, fname): | ||||
llr = filerevlog.linkrev | ||||
fln = filerevlog.node | ||||
revs = ((r, llr(r)) for r in filerevlog) | ||||
return dict((fln(r), cln(lr)) for r, lr in revs if lr in clrevs) | ||||
for chunk in self.generatefiles(changedfiles, linknodes, commonrevs, | ||||
source): | ||||
yield chunk | ||||
yield self.close() | ||||
if clnodes: | ||||
repo.hook('outgoing', node=hex(clnodes[0]), source=source) | ||||
def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs, | ||||
|
r34148 | fnodes, source): | ||
"""Returns an iterator of changegroup chunks containing manifests. | ||||
`source` is unused here, but is used by extensions like remotefilelog to | ||||
change what is sent based in pulls vs pushes, etc. | ||||
""" | ||||
|
r28227 | repo = self._repo | ||
|
r30294 | mfl = repo.manifestlog | ||
dirlog = mfl._revlog.dirlog | ||||
|
r28232 | tmfnodes = {'': mfs} | ||
|
r28227 | |||
|
r19207 | # Callback for the manifest, used to collect linkrevs for filelog | ||
# revisions. | ||||
# Returns the linkrev node (collected in lookupcl). | ||||
|
r35013 | def makelookupmflinknode(dir, nodes): | ||
|
r28231 | if fastpathlinkrev: | ||
assert not dir | ||||
return mfs.__getitem__ | ||||
|
r27239 | def lookupmflinknode(x): | ||
"""Callback for looking up the linknode for manifests. | ||||
|
r27219 | |||
|
r27239 | Returns the linkrev node for the specified manifest. | ||
|
r27219 | |||
|
r27239 | SIDE EFFECT: | ||
|
r27432 | 1) fclnodes gets populated with the list of relevant | ||
file nodes if we're not using fastpathlinkrev | ||||
2) When treemanifests are in use, collects treemanifest nodes | ||||
to send | ||||
|
r27219 | |||
|
r27432 | Note that this means manifests must be completely sent to | ||
the client before you can trust the list of files and | ||||
treemanifests to send. | ||||
|
r27239 | """ | ||
|
r35013 | clnode = nodes[x] | ||
|
r30294 | mdata = mfl.get(dir, x).readfast(shallow=True) | ||
|
r28241 | for p, n, fl in mdata.iterentries(): | ||
if fl == 't': # subdirectory manifest | ||||
subdir = dir + p + '/' | ||||
tmfclnodes = tmfnodes.setdefault(subdir, {}) | ||||
tmfclnode = tmfclnodes.setdefault(n, clnode) | ||||
if clrevorder[clnode] < clrevorder[tmfclnode]: | ||||
tmfclnodes[n] = clnode | ||||
else: | ||||
f = dir + p | ||||
|
r28240 | fclnodes = fnodes.setdefault(f, {}) | ||
fclnode = fclnodes.setdefault(n, clnode) | ||||
if clrevorder[clnode] < clrevorder[fclnode]: | ||||
fclnodes[n] = clnode | ||||
|
r27239 | return clnode | ||
|
r28231 | return lookupmflinknode | ||
|
r19206 | |||
|
r28228 | size = 0 | ||
|
r28232 | while tmfnodes: | ||
|
r35013 | dir, nodes = tmfnodes.popitem() | ||
|
r28240 | prunednodes = self.prune(dirlog(dir), nodes, commonrevs) | ||
|
r29371 | if not dir or prunednodes: | ||
for x in self._packmanifests(dir, prunednodes, | ||||
|
r35013 | makelookupmflinknode(dir, nodes)): | ||
|
r29371 | size += len(x) | ||
yield x | ||||
|
r28229 | self._verbosenote(_('%8.i (manifests)\n') % size) | ||
|
r28228 | yield self._manifestsdone() | ||
|
r19206 | |||
|
r24897 | # The 'source' parameter is useful for extensions | ||
|
r19334 | def generatefiles(self, changedfiles, linknodes, commonrevs, source): | ||
repo = self._repo | ||||
progress = self._progress | ||||
msgbundling = _('bundling') | ||||
total = len(changedfiles) | ||||
# for progress output | ||||
msgfiles = _('files') | ||||
for i, fname in enumerate(sorted(changedfiles)): | ||||
filerevlog = repo.file(fname) | ||||
if not filerevlog: | ||||
|
r26587 | raise error.Abort(_("empty or missing revlog for %s") % fname) | ||
|
r19334 | |||
linkrevnodes = linknodes(filerevlog, fname) | ||||
|
r19207 | # Lookup for filenodes, we collected the linkrev nodes above in the | ||
# fastpath case and with lookupmf in the slowpath case. | ||||
def lookupfilelog(x): | ||||
return linkrevnodes[x] | ||||
|
r24896 | filenodes = self.prune(filerevlog, linkrevnodes, commonrevs) | ||
|
r19206 | if filenodes: | ||
|
r19208 | progress(msgbundling, i + 1, item=fname, unit=msgfiles, | ||
total=total) | ||||
|
r23748 | h = self.fileheader(fname) | ||
size = len(h) | ||||
yield h | ||||
|
r24912 | for chunk in self.group(filenodes, filerevlog, lookupfilelog): | ||
|
r23748 | size += len(chunk) | ||
|
r19202 | yield chunk | ||
|
r23748 | self._verbosenote(_('%8.i %s\n') % (size, fname)) | ||
|
r24901 | progress(msgbundling, None) | ||
|
r19200 | |||
|
r23181 | def deltaparent(self, revlog, rev, p1, p2, prev): | ||
return prev | ||||
|
r19207 | def revchunk(self, revlog, rev, prev, linknode): | ||
|
r14143 | node = revlog.node(rev) | ||
p1, p2 = revlog.parentrevs(rev) | ||||
|
r23181 | base = self.deltaparent(revlog, rev, p1, p2, prev) | ||
|
r14143 | |||
prefix = '' | ||||
|
r24190 | if revlog.iscensored(base) or revlog.iscensored(rev): | ||
try: | ||||
|
r30743 | delta = revlog.revision(node, raw=True) | ||
|
r25660 | except error.CensoredNodeError as e: | ||
|
r24190 | delta = e.tombstone | ||
if base == nullrev: | ||||
prefix = mdiff.trivialdiffheader(len(delta)) | ||||
else: | ||||
baselen = revlog.rawsize(base) | ||||
prefix = mdiff.replacediffheader(baselen, len(delta)) | ||||
elif base == nullrev: | ||||
|
r30743 | delta = revlog.revision(node, raw=True) | ||
|
r14143 | prefix = mdiff.trivialdiffheader(len(delta)) | ||
else: | ||||
delta = revlog.revdiff(base, rev) | ||||
p1n, p2n = revlog.parents(node) | ||||
basenode = revlog.node(base) | ||||
|
r27433 | flags = revlog.flags(rev) | ||
meta = self.builddeltaheader(node, p1n, p2n, basenode, linknode, flags) | ||||
|
r14143 | meta += prefix | ||
l = len(meta) + len(delta) | ||||
|
r13831 | yield chunkheader(l) | ||
yield meta | ||||
|
r14143 | yield delta | ||
|
r27433 | def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags): | ||
|
r14143 | # do nothing with basenode, it is implicitly the previous one in HG10 | ||
|
r27433 | # do nothing with flags, it is implicitly 0 for cg1 and cg2 | ||
|
r14143 | return struct.pack(self.deltaheader, node, p1n, p2n, linknode) | ||
|
r20925 | |||
|
r23181 | class cg2packer(cg1packer): | ||
|
r23896 | version = '02' | ||
|
r23181 | deltaheader = _CHANGEGROUPV2_DELTA_HEADER | ||
|
r32287 | def __init__(self, repo, bundlecaps=None): | ||
super(cg2packer, self).__init__(repo, bundlecaps) | ||||
|
r24911 | if self._reorder is None: | ||
# Since generaldelta is directly supported by cg2, reordering | ||||
# generally doesn't help, so we disable it by default (treating | ||||
# bundle.reorder=auto just like bundle.reorder=False). | ||||
self._reorder = False | ||||
|
r23181 | |||
def deltaparent(self, revlog, rev, p1, p2, prev): | ||||
dp = revlog.deltaparent(rev) | ||||
|
r30211 | if dp == nullrev and revlog.storedeltachains: | ||
# Avoid sending full revisions when delta parent is null. Pick prev | ||||
# in that case. It's tempting to pick p1 in this case, as p1 will | ||||
# be smaller in the common case. However, computing a delta against | ||||
# p1 may require resolving the raw text of p1, which could be | ||||
# expensive. The revlog caches should have prev cached, meaning | ||||
# less CPU for changegroup generation. There is likely room to add | ||||
# a flag and/or config option to control this behavior. | ||||
|
r23181 | return prev | ||
|
r30211 | elif dp == nullrev: | ||
# revlog is configured to use full snapshot for a reason, | ||||
# stick to full snapshot. | ||||
return nullrev | ||||
elif dp not in (p1, p2, prev): | ||||
# Pick prev when we can't be sure remote has the base revision. | ||||
return prev | ||||
else: | ||||
return dp | ||||
|
r23181 | |||
|
r27433 | def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags): | ||
# Do nothing with flags, it is implicitly 0 in cg1 and cg2 | ||||
|
r23181 | return struct.pack(self.deltaheader, node, p1n, p2n, basenode, linknode) | ||
|
r27432 | class cg3packer(cg2packer): | ||
version = '03' | ||||
|
r27433 | deltaheader = _CHANGEGROUPV3_DELTA_HEADER | ||
|
r27432 | |||
|
r28228 | def _packmanifests(self, dir, mfnodes, lookuplinknode): | ||
if dir: | ||||
yield self.fileheader(dir) | ||||
|
r30339 | |||
dirlog = self._repo.manifestlog._revlog.dirlog(dir) | ||||
for chunk in self.group(mfnodes, dirlog, lookuplinknode, | ||||
units=_('manifests')): | ||||
|
r28228 | yield chunk | ||
def _manifestsdone(self): | ||||
return self.close() | ||||
|
r27432 | |||
|
r27433 | def builddeltaheader(self, node, p1n, p2n, basenode, linknode, flags): | ||
return struct.pack( | ||||
self.deltaheader, node, p1n, p2n, basenode, linknode, flags) | ||||
|
r27432 | |||
|
r27751 | _packermap = {'01': (cg1packer, cg1unpacker), | ||
|
r26709 | # cg2 adds support for exchanging generaldelta | ||
'02': (cg2packer, cg2unpacker), | ||||
|
r27753 | # cg3 adds support for exchanging revlog flags and treemanifests | ||
|
r27432 | '03': (cg3packer, cg3unpacker), | ||
|
r26709 | } | ||
|
r23168 | |||
|
r30627 | def allsupportedversions(repo): | ||
|
r27928 | versions = set(_packermap.keys()) | ||
|
r30627 | if not (repo.ui.configbool('experimental', 'changegroup3') or | ||
|
r30628 | repo.ui.configbool('experimental', 'treemanifest') or | ||
'treemanifest' in repo.requirements): | ||||
|
r30626 | versions.discard('03') | ||
|
r27953 | return versions | ||
# Changegroup versions that can be applied to the repo | ||||
def supportedincomingversions(repo): | ||||
|
r30628 | return allsupportedversions(repo) | ||
|
r27953 | |||
# Changegroup versions that can be created from the repo | ||||
def supportedoutgoingversions(repo): | ||||
|
r30627 | versions = allsupportedversions(repo) | ||
|
r27953 | if 'treemanifest' in repo.requirements: | ||
|
r27928 | # Versions 01 and 02 support only flat manifests and it's just too | ||
# expensive to convert between the flat manifest and tree manifest on | ||||
# the fly. Since tree manifests are hashed differently, all of history | ||||
# would have to be converted. Instead, we simply don't even pretend to | ||||
# support versions 01 and 02. | ||||
versions.discard('01') | ||||
versions.discard('02') | ||||
|
r27752 | return versions | ||
|
r27751 | |||
|
r34179 | def localversion(repo): | ||
# Finds the best version to use for bundles that are meant to be used | ||||
# locally, such as those from strip and shelve, and temporary bundles. | ||||
return max(supportedoutgoingversions(repo)) | ||||
|
r27929 | def safeversion(repo): | ||
# Finds the smallest version that it's safe to assume clients of the repo | ||||
|
r27931 | # will support. For example, all hg versions that support generaldelta also | ||
# support changegroup 02. | ||||
|
r27953 | versions = supportedoutgoingversions(repo) | ||
|
r27929 | if 'generaldelta' in repo.requirements: | ||
versions.discard('01') | ||||
assert versions | ||||
return min(versions) | ||||
|
r32287 | def getbundler(version, repo, bundlecaps=None): | ||
|
r27953 | assert version in supportedoutgoingversions(repo) | ||
|
r32287 | return _packermap[version][0](repo, bundlecaps) | ||
|
r27751 | |||
|
r29593 | def getunbundler(version, fh, alg, extras=None): | ||
return _packermap[version][1](fh, alg, extras=extras) | ||||
|
r27751 | |||
|
r20926 | def _changegroupinfo(repo, nodes, source): | ||
if repo.ui.verbose or source == 'bundle': | ||||
repo.ui.status(_("%d changesets found\n") % len(nodes)) | ||||
if repo.ui.debugflag: | ||||
repo.ui.debug("list of changesets:\n") | ||||
for node in nodes: | ||||
repo.ui.debug("%s\n" % hex(node)) | ||||
|
r34098 | def makechangegroup(repo, outgoing, version, source, fastpath=False, | ||
bundlecaps=None): | ||||
cgstream = makestream(repo, outgoing, version, source, | ||||
fastpath=fastpath, bundlecaps=bundlecaps) | ||||
return getunbundler(version, util.chunkbuffer(cgstream), None, | ||||
{'clcount': len(outgoing.missing) }) | ||||
|
r34105 | def makestream(repo, outgoing, version, source, fastpath=False, | ||
bundlecaps=None): | ||||
bundler = getbundler(version, repo, bundlecaps=bundlecaps) | ||||
|
r20925 | repo = repo.unfiltered() | ||
commonrevs = outgoing.common | ||||
csets = outgoing.missing | ||||
heads = outgoing.missingheads | ||||
# We go through the fast path if we get told to, or if all (unfiltered | ||||
# heads have been requested (since we then know there all linkrevs will | ||||
# be pulled by the client). | ||||
heads.sort() | ||||
fastpathlinkrev = fastpath or ( | ||||
repo.filtername is None and heads == sorted(repo.heads())) | ||||
repo.hook('preoutgoing', throw=True, source=source) | ||||
|
r20926 | _changegroupinfo(repo, csets, source) | ||
|
r23177 | return bundler.generate(commonrevs, csets, fastpathlinkrev, source) | ||
|
r28361 | def _addchangegroupfiles(repo, source, revmap, trp, expectedfiles, needfiles): | ||
|
r20932 | revisions = 0 | ||
files = 0 | ||||
|
r29724 | for chunkdata in iter(source.filelogheader, {}): | ||
|
r28361 | files += 1 | ||
|
r20932 | f = chunkdata["filename"] | ||
repo.ui.debug("adding %s revisions\n" % f) | ||||
|
r28361 | repo.ui.progress(_('files'), files, unit=_('files'), | ||
total=expectedfiles) | ||||
|
r27754 | fl = repo.file(f) | ||
|
r20932 | o = len(fl) | ||
|
r24120 | try: | ||
|
r34292 | deltas = source.deltaiter() | ||
if not fl.addgroup(deltas, revmap, trp): | ||||
|
r26587 | raise error.Abort(_("received file revlog group is empty")) | ||
|
r25660 | except error.CensoredBaseError as e: | ||
|
r26587 | raise error.Abort(_("received delta base is censored: %s") % e) | ||
|
r27754 | revisions += len(fl) - o | ||
|
r20932 | if f in needfiles: | ||
needs = needfiles[f] | ||||
for new in xrange(o, len(fl)): | ||||
n = fl.node(new) | ||||
if n in needs: | ||||
needs.remove(n) | ||||
else: | ||||
|
r26587 | raise error.Abort( | ||
|
r20932 | _("received spurious file revlog entry")) | ||
if not needs: | ||||
del needfiles[f] | ||||
repo.ui.progress(_('files'), None) | ||||
for f, needs in needfiles.iteritems(): | ||||
fl = repo.file(f) | ||||
for n in needs: | ||||
try: | ||||
fl.rev(n) | ||||
except error.LookupError: | ||||
|
r26587 | raise error.Abort( | ||
|
r20932 | _('missing file data for %s:%s - run hg verify') % | ||
(f, hex(n))) | ||||
return revisions, files | ||||