shallowbundle.py
307 lines
| 10.0 KiB
| text/x-python
|
PythonLexer
|
r40530 | # shallowbundle.py - bundle10 implementation for use with shallow repositories | ||
# | ||||
# Copyright 2013 Facebook, Inc. | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
from __future__ import absolute_import | ||||
from mercurial.i18n import _ | ||||
from mercurial.node import bin, hex, nullid | ||||
from mercurial import ( | ||||
bundlerepo, | ||||
changegroup, | ||||
error, | ||||
match, | ||||
mdiff, | ||||
pycompat, | ||||
) | ||||
from . import ( | ||||
|
r40544 | constants, | ||
|
r40530 | remotefilelog, | ||
shallowutil, | ||||
) | ||||
NoFiles = 0 | ||||
LocalFiles = 1 | ||||
AllFiles = 2 | ||||
|
r43346 | |||
|
r40530 | def shallowgroup(cls, self, nodelist, rlog, lookup, units=None, reorder=None): | ||
if not isinstance(rlog, remotefilelog.remotefilelog): | ||||
|
r43346 | for c in super(cls, self).group(nodelist, rlog, lookup, units=units): | ||
|
r40530 | yield c | ||
return | ||||
if len(nodelist) == 0: | ||||
yield self.close() | ||||
return | ||||
nodelist = shallowutil.sortnodes(nodelist, rlog.parents) | ||||
# add the parent of the first rev | ||||
p = rlog.parents(nodelist[0])[0] | ||||
nodelist.insert(0, p) | ||||
# build deltas | ||||
for i in pycompat.xrange(len(nodelist) - 1): | ||||
prev, curr = nodelist[i], nodelist[i + 1] | ||||
linknode = lookup(curr) | ||||
for c in self.nodechunk(rlog, curr, prev, linknode): | ||||
yield c | ||||
yield self.close() | ||||
|
r43346 | |||
|
r40530 | class shallowcg1packer(changegroup.cgpacker): | ||
|
r44865 | def generate(self, commonrevs, clnodes, fastpathlinkrev, source, **kwargs): | ||
|
r40549 | if shallowutil.isenabled(self._repo): | ||
|
r40530 | fastpathlinkrev = False | ||
|
r43346 | return super(shallowcg1packer, self).generate( | ||
|
r44865 | commonrevs, clnodes, fastpathlinkrev, source, **kwargs | ||
|
r43346 | ) | ||
|
r40530 | |||
def group(self, nodelist, rlog, lookup, units=None, reorder=None): | ||||
|
r43346 | return shallowgroup( | ||
shallowcg1packer, self, nodelist, rlog, lookup, units=units | ||||
) | ||||
|
r40530 | |||
|
r47445 | def generatefiles(self, changedfiles, *args, **kwargs): | ||
|
r40530 | try: | ||
linknodes, commonrevs, source = args | ||||
except ValueError: | ||||
commonrevs, source, mfdicts, fastpathlinkrev, fnodes, clrevs = args | ||||
|
r40549 | if shallowutil.isenabled(self._repo): | ||
|
r40530 | repo = self._repo | ||
if isinstance(repo, bundlerepo.bundlerepository): | ||||
# If the bundle contains filelogs, we can't pull from it, since | ||||
# bundlerepo is heavily tied to revlogs. Instead require that | ||||
# the user use unbundle instead. | ||||
# Force load the filelog data. | ||||
|
r43347 | bundlerepo.bundlerepository.file(repo, b'foo') | ||
|
r40530 | if repo._cgfilespos: | ||
|
r43346 | raise error.Abort( | ||
|
r43347 | b"cannot pull from full bundles", | ||
hint=b"use `hg unbundle` instead", | ||||
|
r43346 | ) | ||
|
r40530 | return [] | ||
filestosend = self.shouldaddfilegroups(source) | ||||
if filestosend == NoFiles: | ||||
|
r43346 | changedfiles = list( | ||
[f for f in changedfiles if not repo.shallowmatch(f)] | ||||
) | ||||
|
r40530 | |||
|
r47445 | return super(shallowcg1packer, self).generatefiles( | ||
changedfiles, *args, **kwargs | ||||
) | ||||
|
r40530 | |||
def shouldaddfilegroups(self, source): | ||||
repo = self._repo | ||||
|
r40549 | if not shallowutil.isenabled(repo): | ||
|
r40530 | return AllFiles | ||
|
r43347 | if source == b"push" or source == b"bundle": | ||
|
r40530 | return AllFiles | ||
caps = self._bundlecaps or [] | ||||
|
r43347 | if source == b"serve" or source == b"pull": | ||
|
r40544 | if constants.BUNDLE2_CAPABLITY in caps: | ||
|
r40530 | return LocalFiles | ||
else: | ||||
# Serving to a full repo requires us to serve everything | ||||
|
r43347 | repo.ui.warn(_(b"pulling from a shallow repo\n")) | ||
|
r40530 | return AllFiles | ||
return NoFiles | ||||
def prune(self, rlog, missing, commonrevs): | ||||
if not isinstance(rlog, remotefilelog.remotefilelog): | ||||
|
r43346 | return super(shallowcg1packer, self).prune( | ||
rlog, missing, commonrevs | ||||
) | ||||
|
r40530 | |||
repo = self._repo | ||||
results = [] | ||||
for fnode in missing: | ||||
fctx = repo.filectx(rlog.filename, fileid=fnode) | ||||
if fctx.linkrev() not in commonrevs: | ||||
results.append(fnode) | ||||
return results | ||||
def nodechunk(self, revlog, node, prevnode, linknode): | ||||
|
r43347 | prefix = b'' | ||
|
r40530 | if prevnode == nullid: | ||
r43050 | delta = revlog.rawdata(node) | |||
|
r40530 | prefix = mdiff.trivialdiffheader(len(delta)) | ||
else: | ||||
# Actually uses remotefilelog.revdiff which works on nodes, not revs | ||||
delta = revlog.revdiff(prevnode, node) | ||||
p1, p2 = revlog.parents(node) | ||||
flags = revlog.flags(node) | ||||
meta = self.builddeltaheader(node, p1, p2, prevnode, linknode, flags) | ||||
meta += prefix | ||||
l = len(meta) + len(delta) | ||||
yield changegroup.chunkheader(l) | ||||
yield meta | ||||
yield delta | ||||
|
r43346 | |||
|
r40530 | def makechangegroup(orig, repo, outgoing, version, source, *args, **kwargs): | ||
|
r40549 | if not shallowutil.isenabled(repo): | ||
|
r40530 | return orig(repo, outgoing, version, source, *args, **kwargs) | ||
original = repo.shallowmatch | ||||
try: | ||||
# if serving, only send files the clients has patterns for | ||||
|
r43347 | if source == b'serve': | ||
|
r43906 | bundlecaps = kwargs.get('bundlecaps') | ||
|
r40530 | includepattern = None | ||
excludepattern = None | ||||
|
r43346 | for cap in bundlecaps or []: | ||
|
r43347 | if cap.startswith(b"includepattern="): | ||
raw = cap[len(b"includepattern=") :] | ||||
|
r40530 | if raw: | ||
|
r43347 | includepattern = raw.split(b'\0') | ||
elif cap.startswith(b"excludepattern="): | ||||
raw = cap[len(b"excludepattern=") :] | ||||
|
r40530 | if raw: | ||
|
r43347 | excludepattern = raw.split(b'\0') | ||
|
r40530 | if includepattern or excludepattern: | ||
|
r43346 | repo.shallowmatch = match.match( | ||
|
r43347 | repo.root, b'', None, includepattern, excludepattern | ||
|
r43346 | ) | ||
|
r40530 | else: | ||
|
r41825 | repo.shallowmatch = match.always() | ||
|
r40530 | return orig(repo, outgoing, version, source, *args, **kwargs) | ||
finally: | ||||
repo.shallowmatch = original | ||||
|
r43346 | |||
|
r47445 | def addchangegroupfiles( | ||
orig, repo, source, revmap, trp, expectedfiles, *args, **kwargs | ||||
): | ||||
|
r40549 | if not shallowutil.isenabled(repo): | ||
|
r47445 | return orig(repo, source, revmap, trp, expectedfiles, *args, **kwargs) | ||
|
r40530 | |||
newfiles = 0 | ||||
visited = set() | ||||
revisiondatas = {} | ||||
queue = [] | ||||
# Normal Mercurial processes each file one at a time, adding all | ||||
# the new revisions for that file at once. In remotefilelog a file | ||||
# revision may depend on a different file's revision (in the case | ||||
# of a rename/copy), so we must lay all revisions down across all | ||||
# files in topological order. | ||||
# read all the file chunks but don't add them | ||||
|
r43347 | progress = repo.ui.makeprogress(_(b'files'), total=expectedfiles) | ||
|
r40530 | while True: | ||
chunkdata = source.filelogheader() | ||||
if not chunkdata: | ||||
break | ||||
|
r43347 | f = chunkdata[b"filename"] | ||
repo.ui.debug(b"adding %s revisions\n" % f) | ||||
|
r40879 | progress.increment() | ||
|
r40530 | |||
if not repo.shallowmatch(f): | ||||
fl = repo.file(f) | ||||
deltas = source.deltaiter() | ||||
fl.addgroup(deltas, revmap, trp) | ||||
continue | ||||
chain = None | ||||
while True: | ||||
# returns: (node, p1, p2, cs, deltabase, delta, flags) or None | ||||
revisiondata = source.deltachunk(chain) | ||||
if not revisiondata: | ||||
break | ||||
chain = revisiondata[0] | ||||
revisiondatas[(f, chain)] = revisiondata | ||||
queue.append((f, chain)) | ||||
if f not in visited: | ||||
newfiles += 1 | ||||
visited.add(f) | ||||
if chain is None: | ||||
|
r43347 | raise error.Abort(_(b"received file revlog group is empty")) | ||
|
r40530 | |||
processed = set() | ||||
|
r43346 | |||
|
r40530 | def available(f, node, depf, depnode): | ||
if depnode != nullid and (depf, depnode) not in processed: | ||||
if not (depf, depnode) in revisiondatas: | ||||
# It's not in the changegroup, assume it's already | ||||
# in the repo | ||||
return True | ||||
# re-add self to queue | ||||
queue.insert(0, (f, node)) | ||||
# add dependency in front | ||||
queue.insert(0, (depf, depnode)) | ||||
return False | ||||
return True | ||||
skipcount = 0 | ||||
# Prefetch the non-bundled revisions that we will need | ||||
prefetchfiles = [] | ||||
for f, node in queue: | ||||
revisiondata = revisiondatas[(f, node)] | ||||
# revisiondata: (node, p1, p2, cs, deltabase, delta, flags) | ||||
dependents = [revisiondata[1], revisiondata[2], revisiondata[4]] | ||||
for dependent in dependents: | ||||
if dependent == nullid or (f, dependent) in revisiondatas: | ||||
continue | ||||
prefetchfiles.append((f, hex(dependent))) | ||||
repo.fileservice.prefetch(prefetchfiles) | ||||
# Apply the revisions in topological order such that a revision | ||||
# is only written once it's deltabase and parents have been written. | ||||
while queue: | ||||
f, node = queue.pop(0) | ||||
if (f, node) in processed: | ||||
continue | ||||
skipcount += 1 | ||||
if skipcount > len(queue) + 1: | ||||
|
r43347 | raise error.Abort(_(b"circular node dependency")) | ||
|
r40530 | |||
fl = repo.file(f) | ||||
revisiondata = revisiondatas[(f, node)] | ||||
# revisiondata: (node, p1, p2, cs, deltabase, delta, flags) | ||||
|
r47445 | node, p1, p2, linknode, deltabase, delta, flags, sidedata = revisiondata | ||
|
r40530 | |||
if not available(f, node, f, deltabase): | ||||
continue | ||||
r43050 | base = fl.rawdata(deltabase) | |||
|
r40530 | text = mdiff.patch(base, delta) | ||
|
r41292 | if not isinstance(text, bytes): | ||
text = bytes(text) | ||||
|
r40530 | |||
meta, text = shallowutil.parsemeta(text) | ||||
|
r43347 | if b'copy' in meta: | ||
copyfrom = meta[b'copy'] | ||||
copynode = bin(meta[b'copyrev']) | ||||
|
r40530 | if not available(f, node, copyfrom, copynode): | ||
continue | ||||
for p in [p1, p2]: | ||||
if p != nullid: | ||||
if not available(f, node, f, p): | ||||
continue | ||||
fl.add(text, meta, trp, linknode, p1, p2) | ||||
processed.add((f, node)) | ||||
skipcount = 0 | ||||
|
r40879 | progress.complete() | ||
|
r40530 | |||
return len(revisiondatas), newfiles | ||||