narrowbundle2.py
343 lines
| 11.3 KiB
| text/x-python
|
PythonLexer
|
r36096 | # narrowbundle2.py - bundle2 extensions for narrow repository support | ||
# | ||||
# Copyright 2017 Google, Inc. | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
from __future__ import absolute_import | ||||
import errno | ||||
import struct | ||||
from mercurial.i18n import _ | ||||
|
r43537 | from mercurial.node import nullid | ||
|
r36096 | from mercurial import ( | ||
bundle2, | ||||
changegroup, | ||||
error, | ||||
exchange, | ||||
|
r40903 | localrepo, | ||
|
r36178 | narrowspec, | ||
|
r36096 | repair, | ||
|
r45932 | requirements, | ||
|
r45666 | scmutil, | ||
|
r36096 | util, | ||
|
r37631 | wireprototypes, | ||
|
r36096 | ) | ||
|
r43346 | from mercurial.utils import stringutil | ||
|
r36096 | |||
|
r43347 | _NARROWACL_SECTION = b'narrowacl' | ||
_CHANGESPECPART = b'narrow:changespec' | ||||
_RESSPECS = b'narrow:responsespec' | ||||
_SPECPART = b'narrow:spec' | ||||
_SPECPART_INCLUDE = b'include' | ||||
_SPECPART_EXCLUDE = b'exclude' | ||||
_KILLNODESIGNAL = b'KILL' | ||||
_DONESIGNAL = b'DONE' | ||||
_ELIDEDCSHEADER = b'>20s20s20sl' # cset id, p1, p2, len(text) | ||||
_ELIDEDMFHEADER = b'>20s20s20s20sl' # manifest id, p1, p2, link id, len(text) | ||||
|
r36104 | _CSHEADERSIZE = struct.calcsize(_ELIDEDCSHEADER) | ||
_MFHEADERSIZE = struct.calcsize(_ELIDEDMFHEADER) | ||||
|
r36096 | |||
# Serve a changegroup for a client with a narrow clone. | ||||
|
r43346 | def getbundlechangegrouppart_narrow( | ||
bundler, | ||||
repo, | ||||
source, | ||||
bundlecaps=None, | ||||
b2caps=None, | ||||
heads=None, | ||||
common=None, | ||||
**kwargs | ||||
): | ||||
|
r43347 | assert repo.ui.configbool(b'experimental', b'narrowservebrokenellipses') | ||
|
r38844 | |||
|
r43347 | cgversions = b2caps.get(b'changegroup') | ||
|
r43346 | cgversions = [ | ||
v | ||||
for v in cgversions | ||||
if v in changegroup.supportedoutgoingversions(repo) | ||||
] | ||||
|
r42557 | if not cgversions: | ||
|
r43347 | raise ValueError(_(b'no common changegroup version')) | ||
|
r42557 | version = max(cgversions) | ||
|
r36096 | |||
|
r43906 | include = sorted(filter(bool, kwargs.get('includepats', []))) | ||
exclude = sorted(filter(bool, kwargs.get('excludepats', []))) | ||||
|
r43346 | generateellipsesbundle2( | ||
bundler, | ||||
repo, | ||||
|
r43537 | include, | ||
exclude, | ||||
|
r43346 | version, | ||
common, | ||||
heads, | ||||
|
r43906 | kwargs.get('depth', None), | ||
|
r43346 | ) | ||
|
r36096 | |||
|
r43346 | def generateellipsesbundle2( | ||
|
r43537 | bundler, repo, include, exclude, version, common, heads, depth, | ||
|
r43346 | ): | ||
|
r43537 | match = narrowspec.match(repo.root, include=include, exclude=exclude) | ||
|
r36096 | if depth is not None: | ||
depth = int(depth) | ||||
if depth < 1: | ||||
|
r43347 | raise error.Abort(_(b'depth must be positive, got %d') % depth) | ||
|
r36096 | |||
heads = set(heads or repo.heads()) | ||||
common = set(common or [nullid]) | ||||
|
r38827 | visitnodes, relevant_nodes, ellipsisroots = exchange._computeellipsis( | ||
|
r43537 | repo, common, heads, set(), match, depth=depth | ||
|
r43346 | ) | ||
|
r36096 | |||
|
r43347 | repo.ui.debug(b'Found %d relevant revs\n' % len(relevant_nodes)) | ||
|
r36096 | if visitnodes: | ||
|
r43346 | packer = changegroup.getbundler( | ||
version, | ||||
repo, | ||||
|
r43537 | matcher=match, | ||
|
r43346 | ellipses=True, | ||
shallow=depth is not None, | ||||
ellipsisroots=ellipsisroots, | ||||
fullnodes=relevant_nodes, | ||||
) | ||||
|
r43347 | cgdata = packer.generate(common, visitnodes, False, b'narrow_widen') | ||
|
r38946 | |||
|
r43347 | part = bundler.newpart(b'changegroup', data=cgdata) | ||
part.addparam(b'version', version) | ||||
|
r46129 | if scmutil.istreemanifest(repo): | ||
|
r43347 | part.addparam(b'treemanifest', b'1') | ||
|
r36096 | |||
|
r43346 | |||
|
r43536 | def generate_ellipses_bundle2_for_widening( | ||
|
r43543 | bundler, repo, oldmatch, newmatch, version, common, known, | ||
|
r43536 | ): | ||
common = set(common or [nullid]) | ||||
|
r43538 | # Steps: | ||
# 1. Send kill for "$known & ::common" | ||||
# | ||||
# 2. Send changegroup for ::common | ||||
# | ||||
# 3. Proceed. | ||||
# | ||||
# In the future, we can send kills for only the specific | ||||
# nodes we know should go away or change shape, and then | ||||
# send a data stream that tells the client something like this: | ||||
# | ||||
# a) apply this changegroup | ||||
# b) apply nodes XXX, YYY, ZZZ that you already have | ||||
# c) goto a | ||||
# | ||||
# until they've built up the full new state. | ||||
|
r43542 | knownrevs = {repo.changelog.rev(n) for n in known} | ||
|
r43538 | # TODO: we could send only roots() of this set, and the | ||
# list of nodes in common, and the client could work out | ||||
# what to strip, instead of us explicitly sending every | ||||
# single node. | ||||
|
r43542 | deadrevs = knownrevs | ||
|
r43536 | |||
|
r43538 | def genkills(): | ||
for r in deadrevs: | ||||
yield _KILLNODESIGNAL | ||||
yield repo.changelog.node(r) | ||||
yield _DONESIGNAL | ||||
|
r43536 | |||
|
r43538 | bundler.newpart(_CHANGESPECPART, data=genkills()) | ||
newvisit, newfull, newellipsis = exchange._computeellipsis( | ||||
|
r43542 | repo, set(), common, knownrevs, newmatch | ||
|
r43538 | ) | ||
if newvisit: | ||||
packer = changegroup.getbundler( | ||||
version, | ||||
repo, | ||||
matcher=newmatch, | ||||
ellipses=True, | ||||
|
r43541 | shallow=False, | ||
|
r43538 | ellipsisroots=newellipsis, | ||
fullnodes=newfull, | ||||
|
r43536 | ) | ||
|
r43538 | cgdata = packer.generate(common, newvisit, False, b'narrow_widen') | ||
|
r43536 | |||
|
r43538 | part = bundler.newpart(b'changegroup', data=cgdata) | ||
part.addparam(b'version', version) | ||||
|
r46129 | if scmutil.istreemanifest(repo): | ||
|
r43538 | part.addparam(b'treemanifest', b'1') | ||
|
r43536 | |||
|
r36104 | @bundle2.parthandler(_SPECPART, (_SPECPART_INCLUDE, _SPECPART_EXCLUDE)) | ||
|
r36096 | def _handlechangespec_2(op, inpart): | ||
|
r42393 | # XXX: This bundle2 handling is buggy and should be removed after hg5.2 is | ||
# released. New servers will send a mandatory bundle2 part named | ||||
# 'Narrowspec' and will send specs as data instead of params. | ||||
# Refer to issue5952 and 6019 | ||||
|
r43347 | includepats = set(inpart.params.get(_SPECPART_INCLUDE, b'').splitlines()) | ||
excludepats = set(inpart.params.get(_SPECPART_EXCLUDE, b'').splitlines()) | ||||
|
r39576 | narrowspec.validatepatterns(includepats) | ||
narrowspec.validatepatterns(excludepats) | ||||
|
r45932 | if not requirements.NARROW_REQUIREMENT in op.repo.requirements: | ||
op.repo.requirements.add(requirements.NARROW_REQUIREMENT) | ||||
|
r45666 | scmutil.writereporequirements(op.repo) | ||
|
r36487 | op.repo.setnarrowpats(includepats, excludepats) | ||
|
r41272 | narrowspec.copytoworkingcopy(op.repo) | ||
|
r36096 | |||
|
r43346 | |||
|
r42393 | @bundle2.parthandler(_RESSPECS) | ||
def _handlenarrowspecs(op, inpart): | ||||
data = inpart.read() | ||||
|
r43347 | inc, exc = data.split(b'\0') | ||
|
r42393 | includepats = set(inc.splitlines()) | ||
excludepats = set(exc.splitlines()) | ||||
narrowspec.validatepatterns(includepats) | ||||
narrowspec.validatepatterns(excludepats) | ||||
|
r45932 | if requirements.NARROW_REQUIREMENT not in op.repo.requirements: | ||
op.repo.requirements.add(requirements.NARROW_REQUIREMENT) | ||||
|
r45666 | scmutil.writereporequirements(op.repo) | ||
|
r42393 | op.repo.setnarrowpats(includepats, excludepats) | ||
narrowspec.copytoworkingcopy(op.repo) | ||||
|
r43346 | |||
|
r36104 | @bundle2.parthandler(_CHANGESPECPART) | ||
|
r36096 | def _handlechangespec(op, inpart): | ||
repo = op.repo | ||||
cl = repo.changelog | ||||
# changesets which need to be stripped entirely. either they're no longer | ||||
# needed in the new narrow spec, or the server is sending a replacement | ||||
# in the changegroup part. | ||||
clkills = set() | ||||
# A changespec part contains all the updates to ellipsis nodes | ||||
# that will happen as a result of widening or narrowing a | ||||
# repo. All the changes that this block encounters are ellipsis | ||||
# nodes or flags to kill an existing ellipsis. | ||||
chunksignal = changegroup.readexactly(inpart, 4) | ||||
|
r36104 | while chunksignal != _DONESIGNAL: | ||
if chunksignal == _KILLNODESIGNAL: | ||||
|
r36096 | # a node used to be an ellipsis but isn't anymore | ||
ck = changegroup.readexactly(inpart, 20) | ||||
if cl.hasnode(ck): | ||||
clkills.add(ck) | ||||
else: | ||||
raise error.Abort( | ||||
|
r43347 | _(b'unexpected changespec node chunk type: %s') % chunksignal | ||
|
r43346 | ) | ||
|
r36096 | chunksignal = changegroup.readexactly(inpart, 4) | ||
if clkills: | ||||
# preserve bookmarks that repair.strip() would otherwise strip | ||||
|
r40903 | op._bookmarksbackup = repo._bookmarks | ||
|
r43346 | |||
|
r36096 | class dummybmstore(dict): | ||
def applychanges(self, repo, tr, changes): | ||||
pass | ||||
|
r43346 | |||
|
r40903 | localrepo.localrepository._bookmarks.set(repo, dummybmstore()) | ||
|
r43346 | chgrpfile = repair.strip( | ||
|
r43347 | op.ui, repo, list(clkills), backup=True, topic=b'widen' | ||
|
r43346 | ) | ||
|
r36096 | if chgrpfile: | ||
|
r41106 | op._widen_uninterr = repo.ui.uninterruptible() | ||
|
r38548 | op._widen_uninterr.__enter__() | ||
|
r36096 | # presence of _widen_bundle attribute activates widen handler later | ||
op._widen_bundle = chgrpfile | ||||
# Set the new narrowspec if we're widening. The setnewnarrowpats() method | ||||
# will currently always be there when using the core+narrowhg server, but | ||||
# other servers may include a changespec part even when not widening (e.g. | ||||
# because we're deepening a shallow repo). | ||||
|
r43385 | if util.safehasattr(repo, 'setnewnarrowpats'): | ||
|
r36096 | repo.setnewnarrowpats() | ||
|
r43346 | |||
|
r36096 | def handlechangegroup_widen(op, inpart): | ||
"""Changegroup exchange handler which restores temporarily-stripped nodes""" | ||||
# We saved a bundle with stripped node data we must now restore. | ||||
# This approach is based on mercurial/repair.py@6ee26a53c111. | ||||
repo = op.repo | ||||
ui = op.ui | ||||
chgrpfile = op._widen_bundle | ||||
del op._widen_bundle | ||||
vfs = repo.vfs | ||||
|
r43347 | ui.note(_(b"adding branch\n")) | ||
f = vfs.open(chgrpfile, b"rb") | ||||
|
r36096 | try: | ||
gen = exchange.readbundle(ui, f, chgrpfile, vfs) | ||||
r43165 | # silence internal shuffling chatter | |||
|
r43347 | override = {(b'ui', b'quiet'): True} | ||
r43165 | if ui.verbose: | |||
override = {} | ||||
with ui.configoverride(override): | ||||
if isinstance(gen, bundle2.unbundle20): | ||||
|
r43347 | with repo.transaction(b'strip') as tr: | ||
r43165 | bundle2.processbundle(repo, gen, lambda: tr) | |||
else: | ||||
|
r43347 | gen.apply( | ||
repo, b'strip', b'bundle:' + vfs.join(chgrpfile), True | ||||
) | ||||
|
r36096 | finally: | ||
f.close() | ||||
# remove undo files | ||||
for undovfs, undofile in repo.undofiles(): | ||||
try: | ||||
undovfs.unlink(undofile) | ||||
except OSError as e: | ||||
if e.errno != errno.ENOENT: | ||||
|
r43346 | ui.warn( | ||
|
r43347 | _(b'error removing %s: %s\n') | ||
|
r43346 | % (undovfs.join(undofile), stringutil.forcebytestr(e)) | ||
) | ||||
|
r36096 | |||
# Remove partial backup only if there were no exceptions | ||||
|
r38548 | op._widen_uninterr.__exit__(None, None, None) | ||
|
r36096 | vfs.unlink(chgrpfile) | ||
|
r43346 | |||
|
r36096 | def setup(): | ||
"""Enable narrow repo support in bundle2-related extension points.""" | ||||
|
r37631 | getbundleargs = wireprototypes.GETBUNDLE_ARGUMENTS | ||
|
r43347 | getbundleargs[b'narrow'] = b'boolean' | ||
getbundleargs[b'depth'] = b'plain' | ||||
getbundleargs[b'oldincludepats'] = b'csv' | ||||
getbundleargs[b'oldexcludepats'] = b'csv' | ||||
getbundleargs[b'known'] = b'csv' | ||||
|
r36096 | |||
# Extend changegroup serving to handle requests from narrow clients. | ||||
|
r43347 | origcgfn = exchange.getbundle2partsmapping[b'changegroup'] | ||
|
r43346 | |||
|
r36096 | def wrappedcgfn(*args, **kwargs): | ||
repo = args[1] | ||||
|
r36104 | if repo.ui.has_section(_NARROWACL_SECTION): | ||
|
r38843 | kwargs = exchange.applynarrowacl(repo, kwargs) | ||
|
r43906 | if kwargs.get('narrow', False) and repo.ui.configbool( | ||
|
r43347 | b'experimental', b'narrowservebrokenellipses' | ||
|
r43346 | ): | ||
|
r36096 | getbundlechangegrouppart_narrow(*args, **kwargs) | ||
else: | ||||
origcgfn(*args, **kwargs) | ||||
|
r43346 | |||
|
r43347 | exchange.getbundle2partsmapping[b'changegroup'] = wrappedcgfn | ||
|
r36096 | |||
# Extend changegroup receiver so client can fixup after widen requests. | ||||
|
r43347 | origcghandler = bundle2.parthandlermapping[b'changegroup'] | ||
|
r43346 | |||
|
r36096 | def wrappedcghandler(op, inpart): | ||
origcghandler(op, inpart) | ||||
|
r43385 | if util.safehasattr(op, '_widen_bundle'): | ||
|
r36096 | handlechangegroup_widen(op, inpart) | ||
|
r43385 | if util.safehasattr(op, '_bookmarksbackup'): | ||
|
r43346 | localrepo.localrepository._bookmarks.set( | ||
op.repo, op._bookmarksbackup | ||||
) | ||||
|
r40903 | del op._bookmarksbackup | ||
|
r36096 | wrappedcghandler.params = origcghandler.params | ||
|
r43347 | bundle2.parthandlermapping[b'changegroup'] = wrappedcghandler | ||