|
|
# shallowbundle.py - bundle10 implementation for use with shallow repositories
|
|
|
#
|
|
|
# Copyright 2013 Facebook, Inc.
|
|
|
#
|
|
|
# This software may be used and distributed according to the terms of the
|
|
|
# GNU General Public License version 2 or any later version.
|
|
|
|
|
|
from mercurial.i18n import _
|
|
|
from mercurial.node import bin, hex
|
|
|
from mercurial import (
|
|
|
bundlerepo,
|
|
|
changegroup,
|
|
|
error,
|
|
|
match,
|
|
|
mdiff,
|
|
|
)
|
|
|
from . import (
|
|
|
constants,
|
|
|
remotefilelog,
|
|
|
shallowutil,
|
|
|
)
|
|
|
|
|
|
NoFiles = 0
|
|
|
LocalFiles = 1
|
|
|
AllFiles = 2
|
|
|
|
|
|
|
|
|
def shallowgroup(cls, self, nodelist, rlog, lookup, units=None, reorder=None):
|
|
|
if not isinstance(rlog, remotefilelog.remotefilelog):
|
|
|
for c in super(cls, self).group(nodelist, rlog, lookup, units=units):
|
|
|
yield c
|
|
|
return
|
|
|
|
|
|
if len(nodelist) == 0:
|
|
|
yield self.close()
|
|
|
return
|
|
|
|
|
|
nodelist = shallowutil.sortnodes(nodelist, rlog.parents)
|
|
|
|
|
|
# add the parent of the first rev
|
|
|
p = rlog.parents(nodelist[0])[0]
|
|
|
nodelist.insert(0, p)
|
|
|
|
|
|
# build deltas
|
|
|
for i in range(len(nodelist) - 1):
|
|
|
prev, curr = nodelist[i], nodelist[i + 1]
|
|
|
linknode = lookup(curr)
|
|
|
for c in self.nodechunk(rlog, curr, prev, linknode):
|
|
|
yield c
|
|
|
|
|
|
yield self.close()
|
|
|
|
|
|
|
|
|
class shallowcg1packer(changegroup.cgpacker):
|
|
|
def generate(self, commonrevs, clnodes, fastpathlinkrev, source, **kwargs):
|
|
|
if shallowutil.isenabled(self._repo):
|
|
|
fastpathlinkrev = False
|
|
|
|
|
|
return super(shallowcg1packer, self).generate(
|
|
|
commonrevs, clnodes, fastpathlinkrev, source, **kwargs
|
|
|
)
|
|
|
|
|
|
def group(self, nodelist, rlog, lookup, units=None, reorder=None):
|
|
|
return shallowgroup(
|
|
|
shallowcg1packer, self, nodelist, rlog, lookup, units=units
|
|
|
)
|
|
|
|
|
|
def generatefiles(self, changedfiles, *args, **kwargs):
|
|
|
try:
|
|
|
linknodes, commonrevs, source = args
|
|
|
except ValueError:
|
|
|
commonrevs, source, mfdicts, fastpathlinkrev, fnodes, clrevs = args
|
|
|
if shallowutil.isenabled(self._repo):
|
|
|
repo = self._repo
|
|
|
if isinstance(repo, bundlerepo.bundlerepository):
|
|
|
# If the bundle contains filelogs, we can't pull from it, since
|
|
|
# bundlerepo is heavily tied to revlogs. Instead require that
|
|
|
# the user use unbundle instead.
|
|
|
# Force load the filelog data.
|
|
|
bundlerepo.bundlerepository.file(repo, b'foo')
|
|
|
if repo._cgfilespos:
|
|
|
raise error.Abort(
|
|
|
b"cannot pull from full bundles",
|
|
|
hint=b"use `hg unbundle` instead",
|
|
|
)
|
|
|
return []
|
|
|
filestosend = self.shouldaddfilegroups(source)
|
|
|
if filestosend == NoFiles:
|
|
|
changedfiles = list(
|
|
|
[f for f in changedfiles if not repo.shallowmatch(f)]
|
|
|
)
|
|
|
|
|
|
return super(shallowcg1packer, self).generatefiles(
|
|
|
changedfiles, *args, **kwargs
|
|
|
)
|
|
|
|
|
|
def shouldaddfilegroups(self, source):
|
|
|
repo = self._repo
|
|
|
if not shallowutil.isenabled(repo):
|
|
|
return AllFiles
|
|
|
|
|
|
if source == b"push" or source == b"bundle":
|
|
|
return AllFiles
|
|
|
|
|
|
# We won't actually strip the files, but we should put them in any
|
|
|
# backup bundle generated by strip (especially for cases like narrow's
|
|
|
# `hg tracked --removeinclude`, as failing to do so means that the
|
|
|
# "saved" changesets during a strip won't have their files reapplied and
|
|
|
# thus their linknode adjusted, if necessary).
|
|
|
if source == b"strip":
|
|
|
cfg = repo.ui.config(b'remotefilelog', b'strip.includefiles')
|
|
|
if cfg == b'local':
|
|
|
return LocalFiles
|
|
|
elif cfg != b'none':
|
|
|
return AllFiles
|
|
|
|
|
|
caps = self._bundlecaps or []
|
|
|
if source == b"serve" or source == b"pull":
|
|
|
if constants.BUNDLE2_CAPABLITY in caps:
|
|
|
return LocalFiles
|
|
|
else:
|
|
|
# Serving to a full repo requires us to serve everything
|
|
|
repo.ui.warn(_(b"pulling from a shallow repo\n"))
|
|
|
return AllFiles
|
|
|
|
|
|
return NoFiles
|
|
|
|
|
|
def prune(self, rlog, missing, commonrevs):
|
|
|
if not isinstance(rlog, remotefilelog.remotefilelog):
|
|
|
return super(shallowcg1packer, self).prune(
|
|
|
rlog, missing, commonrevs
|
|
|
)
|
|
|
|
|
|
repo = self._repo
|
|
|
results = []
|
|
|
for fnode in missing:
|
|
|
fctx = repo.filectx(rlog.filename, fileid=fnode)
|
|
|
if fctx.linkrev() not in commonrevs:
|
|
|
results.append(fnode)
|
|
|
return results
|
|
|
|
|
|
def nodechunk(self, revlog, node, prevnode, linknode):
|
|
|
prefix = b''
|
|
|
if prevnode == revlog.nullid:
|
|
|
delta = revlog.rawdata(node)
|
|
|
prefix = mdiff.trivialdiffheader(len(delta))
|
|
|
else:
|
|
|
# Actually uses remotefilelog.revdiff which works on nodes, not revs
|
|
|
delta = revlog.revdiff(prevnode, node)
|
|
|
p1, p2 = revlog.parents(node)
|
|
|
flags = revlog.flags(node)
|
|
|
meta = self.builddeltaheader(node, p1, p2, prevnode, linknode, flags)
|
|
|
meta += prefix
|
|
|
l = len(meta) + len(delta)
|
|
|
yield changegroup.chunkheader(l)
|
|
|
yield meta
|
|
|
yield delta
|
|
|
|
|
|
|
|
|
def makechangegroup(orig, repo, outgoing, version, source, *args, **kwargs):
|
|
|
if not shallowutil.isenabled(repo):
|
|
|
return orig(repo, outgoing, version, source, *args, **kwargs)
|
|
|
|
|
|
original = repo.shallowmatch
|
|
|
try:
|
|
|
# if serving, only send files the clients has patterns for
|
|
|
if source == b'serve':
|
|
|
bundlecaps = kwargs.get('bundlecaps')
|
|
|
includepattern = None
|
|
|
excludepattern = None
|
|
|
for cap in bundlecaps or []:
|
|
|
if cap.startswith(b"includepattern="):
|
|
|
raw = cap[len(b"includepattern=") :]
|
|
|
if raw:
|
|
|
includepattern = raw.split(b'\0')
|
|
|
elif cap.startswith(b"excludepattern="):
|
|
|
raw = cap[len(b"excludepattern=") :]
|
|
|
if raw:
|
|
|
excludepattern = raw.split(b'\0')
|
|
|
if includepattern or excludepattern:
|
|
|
repo.shallowmatch = match.match(
|
|
|
repo.root, b'', None, includepattern, excludepattern
|
|
|
)
|
|
|
else:
|
|
|
repo.shallowmatch = match.always()
|
|
|
return orig(repo, outgoing, version, source, *args, **kwargs)
|
|
|
finally:
|
|
|
repo.shallowmatch = original
|
|
|
|
|
|
|
|
|
def addchangegroupfiles(
|
|
|
orig, repo, source, revmap, trp, expectedfiles, *args, **kwargs
|
|
|
):
|
|
|
if not shallowutil.isenabled(repo):
|
|
|
return orig(repo, source, revmap, trp, expectedfiles, *args, **kwargs)
|
|
|
|
|
|
newfiles = 0
|
|
|
visited = set()
|
|
|
revisiondatas = {}
|
|
|
queue = []
|
|
|
|
|
|
# Normal Mercurial processes each file one at a time, adding all
|
|
|
# the new revisions for that file at once. In remotefilelog a file
|
|
|
# revision may depend on a different file's revision (in the case
|
|
|
# of a rename/copy), so we must lay all revisions down across all
|
|
|
# files in topological order.
|
|
|
|
|
|
# read all the file chunks but don't add them
|
|
|
progress = repo.ui.makeprogress(_(b'files'), total=expectedfiles)
|
|
|
while True:
|
|
|
chunkdata = source.filelogheader()
|
|
|
if not chunkdata:
|
|
|
break
|
|
|
f = chunkdata[b"filename"]
|
|
|
repo.ui.debug(b"adding %s revisions\n" % f)
|
|
|
progress.increment()
|
|
|
|
|
|
if not repo.shallowmatch(f):
|
|
|
fl = repo.file(f)
|
|
|
deltas = source.deltaiter()
|
|
|
fl.addgroup(deltas, revmap, trp)
|
|
|
continue
|
|
|
|
|
|
chain = None
|
|
|
while True:
|
|
|
# returns: None or (
|
|
|
# node,
|
|
|
# p1,
|
|
|
# p2,
|
|
|
# cs,
|
|
|
# deltabase,
|
|
|
# delta,
|
|
|
# flags,
|
|
|
# sidedata,
|
|
|
# proto_flags
|
|
|
# )
|
|
|
revisiondata = source.deltachunk(chain)
|
|
|
if not revisiondata:
|
|
|
break
|
|
|
|
|
|
chain = revisiondata[0]
|
|
|
|
|
|
revisiondatas[(f, chain)] = revisiondata
|
|
|
queue.append((f, chain))
|
|
|
|
|
|
if f not in visited:
|
|
|
newfiles += 1
|
|
|
visited.add(f)
|
|
|
|
|
|
if chain is None:
|
|
|
raise error.Abort(_(b"received file revlog group is empty"))
|
|
|
|
|
|
processed = set()
|
|
|
|
|
|
def available(f, node, depf, depnode):
|
|
|
if depnode != repo.nullid and (depf, depnode) not in processed:
|
|
|
if not (depf, depnode) in revisiondatas:
|
|
|
# It's not in the changegroup, assume it's already
|
|
|
# in the repo
|
|
|
return True
|
|
|
# re-add self to queue
|
|
|
queue.insert(0, (f, node))
|
|
|
# add dependency in front
|
|
|
queue.insert(0, (depf, depnode))
|
|
|
return False
|
|
|
return True
|
|
|
|
|
|
skipcount = 0
|
|
|
|
|
|
# Prefetch the non-bundled revisions that we will need
|
|
|
prefetchfiles = []
|
|
|
for f, node in queue:
|
|
|
revisiondata = revisiondatas[(f, node)]
|
|
|
# revisiondata: (node, p1, p2, cs, deltabase, delta, flags, sdata, pfl)
|
|
|
dependents = [revisiondata[1], revisiondata[2], revisiondata[4]]
|
|
|
|
|
|
for dependent in dependents:
|
|
|
if dependent == repo.nullid or (f, dependent) in revisiondatas:
|
|
|
continue
|
|
|
prefetchfiles.append((f, hex(dependent)))
|
|
|
|
|
|
repo.fileservice.prefetch(prefetchfiles)
|
|
|
|
|
|
# Apply the revisions in topological order such that a revision
|
|
|
# is only written once it's deltabase and parents have been written.
|
|
|
while queue:
|
|
|
f, node = queue.pop(0)
|
|
|
if (f, node) in processed:
|
|
|
continue
|
|
|
|
|
|
skipcount += 1
|
|
|
if skipcount > len(queue) + 1:
|
|
|
raise error.Abort(_(b"circular node dependency"))
|
|
|
|
|
|
fl = repo.file(f)
|
|
|
|
|
|
revisiondata = revisiondatas[(f, node)]
|
|
|
# revisiondata: (node, p1, p2, cs, deltabase, delta, flags, sdata, pfl)
|
|
|
(
|
|
|
node,
|
|
|
p1,
|
|
|
p2,
|
|
|
linknode,
|
|
|
deltabase,
|
|
|
delta,
|
|
|
flags,
|
|
|
sidedata,
|
|
|
proto_flags,
|
|
|
) = revisiondata
|
|
|
|
|
|
if not available(f, node, f, deltabase):
|
|
|
continue
|
|
|
|
|
|
base = fl.rawdata(deltabase)
|
|
|
text = mdiff.patch(base, delta)
|
|
|
if not isinstance(text, bytes):
|
|
|
text = bytes(text)
|
|
|
|
|
|
meta, text = shallowutil.parsemeta(text)
|
|
|
if b'copy' in meta:
|
|
|
copyfrom = meta[b'copy']
|
|
|
copynode = bin(meta[b'copyrev'])
|
|
|
if not available(f, node, copyfrom, copynode):
|
|
|
continue
|
|
|
|
|
|
for p in [p1, p2]:
|
|
|
if p != repo.nullid:
|
|
|
if not available(f, node, f, p):
|
|
|
continue
|
|
|
|
|
|
fl.add(text, meta, trp, linknode, p1, p2)
|
|
|
processed.add((f, node))
|
|
|
skipcount = 0
|
|
|
|
|
|
progress.complete()
|
|
|
|
|
|
return len(revisiondatas), newfiles
|
|
|
|
