##// END OF EJS Templates
bdiff: drop duplicate definition of splitnewlines()...
bdiff: drop duplicate definition of splitnewlines() It was added in 29dd37a418aa (bdiff: write a native version of splitnewlines, 2018-01-25). Differential Revision: https://phab.mercurial-scm.org/D5618

File last commit:

r41272:44a51c1c default
r41305:8b7973d4 default
Show More
narrowbundle2.py
280 lines | 11.0 KiB | text/x-python | PythonLexer
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 # narrowbundle2.py - bundle2 extensions for narrow repository support
#
# Copyright 2017 Google, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import
import errno
import struct
from mercurial.i18n import _
from mercurial.node import (
bin,
nullid,
)
from mercurial import (
bundle2,
changegroup,
error,
exchange,
Martin von Zweigbergk
narrow: keep bookmarks temporarily stripped for as long as commits are...
r40903 localrepo,
Gregory Szorc
narrowspec: move module into core...
r36178 narrowspec,
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 repair,
Martin von Zweigbergk
narrow: move requirement constant from changegroup to repository...
r38871 repository,
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 util,
Gregory Szorc
wireproto: move gboptsmap to wireprototypes and rename (API)...
r37631 wireprototypes,
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 )
Yuya Nishihara
stringutil: bulk-replace call sites to point to new module...
r37102 from mercurial.utils import (
stringutil,
)
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096
Augie Fackler
narrowbundle2: mark most constants as module-private...
r36104 _NARROWACL_SECTION = 'narrowhgacl'
Pulkit Goyal
narrow: drop the bundle2 capability since we have server capabilities (BC)...
r40786 _CHANGESPECPART = 'narrow:changespec'
_SPECPART = 'narrow:spec'
Augie Fackler
narrowbundle2: mark most constants as module-private...
r36104 _SPECPART_INCLUDE = 'include'
_SPECPART_EXCLUDE = 'exclude'
_KILLNODESIGNAL = 'KILL'
_DONESIGNAL = 'DONE'
_ELIDEDCSHEADER = '>20s20s20sl' # cset id, p1, p2, len(text)
_ELIDEDMFHEADER = '>20s20s20s20sl' # manifest id, p1, p2, link id, len(text)
_CSHEADERSIZE = struct.calcsize(_ELIDEDCSHEADER)
_MFHEADERSIZE = struct.calcsize(_ELIDEDMFHEADER)
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096
# Serve a changegroup for a client with a narrow clone.
def getbundlechangegrouppart_narrow(bundler, repo, source,
bundlecaps=None, b2caps=None, heads=None,
common=None, **kwargs):
Gregory Szorc
exchange: move simple narrow changegroup generation from extension...
r38844 assert repo.ui.configbool('experimental', 'narrowservebrokenellipses')
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 cgversions = b2caps.get('changegroup')
if cgversions: # 3.1 and 3.2 ship with an empty value
cgversions = [v for v in cgversions
if v in changegroup.supportedoutgoingversions(repo)]
if not cgversions:
raise ValueError(_('no common changegroup version'))
Augie Fackler
narrowbundle2: drop legacy getcgkwargs variable...
r36374 version = max(cgversions)
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 else:
raise ValueError(_("server does not advertise changegroup version,"
" can't negotiate support for ellipsis nodes"))
Augie Fackler
narrowbundle2: use native string to get kwargs from dict...
r36375 include = sorted(filter(bool, kwargs.get(r'includepats', [])))
exclude = sorted(filter(bool, kwargs.get(r'excludepats', [])))
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 newmatch = narrowspec.match(repo.root, include=include, exclude=exclude)
Augie Fackler
narrowbundle2: more kwargs native string fixes...
r36377 depth = kwargs.get(r'depth', None)
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 if depth is not None:
depth = int(depth)
if depth < 1:
raise error.Abort(_('depth must be positive, got %d') % depth)
heads = set(heads or repo.heads())
common = set(common or [nullid])
Augie Fackler
narrowbundle2: more kwargs native string fixes...
r36377 oldinclude = sorted(filter(bool, kwargs.get(r'oldincludepats', [])))
oldexclude = sorted(filter(bool, kwargs.get(r'oldexcludepats', [])))
known = {bin(n) for n in kwargs.get(r'known', [])}
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 if known and (oldinclude != include or oldexclude != exclude):
# Steps:
# 1. Send kill for "$known & ::common"
#
# 2. Send changegroup for ::common
#
# 3. Proceed.
#
# In the future, we can send kills for only the specific
# nodes we know should go away or change shape, and then
# send a data stream that tells the client something like this:
#
# a) apply this changegroup
# b) apply nodes XXX, YYY, ZZZ that you already have
# c) goto a
#
# until they've built up the full new state.
# Convert to revnums and intersect with "common". The client should
# have made it a subset of "common" already, but let's be safe.
known = set(repo.revs("%ln & ::%ln", known, common))
# TODO: we could send only roots() of this set, and the
# list of nodes in common, and the client could work out
# what to strip, instead of us explicitly sending every
# single node.
deadrevs = known
def genkills():
for r in deadrevs:
Augie Fackler
narrowbundle2: mark most constants as module-private...
r36104 yield _KILLNODESIGNAL
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 yield repo.changelog.node(r)
Augie Fackler
narrowbundle2: mark most constants as module-private...
r36104 yield _DONESIGNAL
bundler.newpart(_CHANGESPECPART, data=genkills())
Gregory Szorc
exchange: move _computeellipsis() from narrow...
r38827 newvisit, newfull, newellipsis = exchange._computeellipsis(
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 repo, set(), common, known, newmatch)
if newvisit:
Gregory Szorc
changegroup: inline _packellipsischangegroup...
r38946 packer = changegroup.getbundler(version, repo,
Martin von Zweigbergk
narrow: when widening, don't include manifests the client already has...
r40380 matcher=newmatch,
Gregory Szorc
changegroup: inline _packellipsischangegroup...
r38946 ellipses=True,
shallow=depth is not None,
ellipsisroots=newellipsis,
fullnodes=newfull)
Pulkit Goyal
narrow: pass 'narrow_widen' as source while generating changegroup...
r39998 cgdata = packer.generate(common, newvisit, False, 'narrow_widen')
Gregory Szorc
changegroup: inline _packellipsischangegroup...
r38946
part = bundler.newpart('changegroup', data=cgdata)
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 part.addparam('version', version)
if 'treemanifest' in repo.requirements:
part.addparam('treemanifest', '1')
Gregory Szorc
exchange: move _computeellipsis() from narrow...
r38827 visitnodes, relevant_nodes, ellipsisroots = exchange._computeellipsis(
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 repo, common, heads, set(), newmatch, depth=depth)
repo.ui.debug('Found %d relevant revs\n' % len(relevant_nodes))
if visitnodes:
Gregory Szorc
changegroup: inline _packellipsischangegroup...
r38946 packer = changegroup.getbundler(version, repo,
Martin von Zweigbergk
narrow: when widening, don't include manifests the client already has...
r40380 matcher=newmatch,
Gregory Szorc
changegroup: inline _packellipsischangegroup...
r38946 ellipses=True,
shallow=depth is not None,
ellipsisroots=ellipsisroots,
fullnodes=relevant_nodes)
Pulkit Goyal
narrow: pass 'narrow_widen' as source while generating changegroup...
r39998 cgdata = packer.generate(common, visitnodes, False, 'narrow_widen')
Gregory Szorc
changegroup: inline _packellipsischangegroup...
r38946
part = bundler.newpart('changegroup', data=cgdata)
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 part.addparam('version', version)
if 'treemanifest' in repo.requirements:
part.addparam('treemanifest', '1')
Augie Fackler
narrowbundle2: mark most constants as module-private...
r36104 @bundle2.parthandler(_SPECPART, (_SPECPART_INCLUDE, _SPECPART_EXCLUDE))
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 def _handlechangespec_2(op, inpart):
Augie Fackler
narrowbundle2: mark most constants as module-private...
r36104 includepats = set(inpart.params.get(_SPECPART_INCLUDE, '').splitlines())
excludepats = set(inpart.params.get(_SPECPART_EXCLUDE, '').splitlines())
Gregory Szorc
narrow: validate patterns on incoming bundle2 part...
r39576 narrowspec.validatepatterns(includepats)
narrowspec.validatepatterns(excludepats)
Martin von Zweigbergk
narrow: move requirement constant from changegroup to repository...
r38871 if not repository.NARROW_REQUIREMENT in op.repo.requirements:
op.repo.requirements.add(repository.NARROW_REQUIREMENT)
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 op.repo._writerequirements()
Martin von Zweigbergk
narrow: reduce depedence on narrowspec.save()...
r36487 op.repo.setnarrowpats(includepats, excludepats)
Martin von Zweigbergk
narrow: move copytonarrowspec() out of setnarrowpats()...
r41272 narrowspec.copytoworkingcopy(op.repo)
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096
Augie Fackler
narrowbundle2: mark most constants as module-private...
r36104 @bundle2.parthandler(_CHANGESPECPART)
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 def _handlechangespec(op, inpart):
repo = op.repo
cl = repo.changelog
# changesets which need to be stripped entirely. either they're no longer
# needed in the new narrow spec, or the server is sending a replacement
# in the changegroup part.
clkills = set()
# A changespec part contains all the updates to ellipsis nodes
# that will happen as a result of widening or narrowing a
# repo. All the changes that this block encounters are ellipsis
# nodes or flags to kill an existing ellipsis.
chunksignal = changegroup.readexactly(inpart, 4)
Augie Fackler
narrowbundle2: mark most constants as module-private...
r36104 while chunksignal != _DONESIGNAL:
if chunksignal == _KILLNODESIGNAL:
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 # a node used to be an ellipsis but isn't anymore
ck = changegroup.readexactly(inpart, 20)
if cl.hasnode(ck):
clkills.add(ck)
else:
raise error.Abort(
_('unexpected changespec node chunk type: %s') % chunksignal)
chunksignal = changegroup.readexactly(inpart, 4)
if clkills:
# preserve bookmarks that repair.strip() would otherwise strip
Martin von Zweigbergk
narrow: keep bookmarks temporarily stripped for as long as commits are...
r40903 op._bookmarksbackup = repo._bookmarks
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 class dummybmstore(dict):
def applychanges(self, repo, tr, changes):
pass
Martin von Zweigbergk
narrow: keep bookmarks temporarily stripped for as long as commits are...
r40903 localrepo.localrepository._bookmarks.set(repo, dummybmstore())
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 chgrpfile = repair.strip(op.ui, repo, list(clkills), backup=True,
topic='widen')
if chgrpfile:
Kyle Lippincott
procutil: correct spelling of uninterruptable -> uninterruptible...
r41106 op._widen_uninterr = repo.ui.uninterruptible()
Augie Fackler
narrowbundle2: when we handle a widen, mark the operation as unsafe...
r38548 op._widen_uninterr.__enter__()
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 # presence of _widen_bundle attribute activates widen handler later
op._widen_bundle = chgrpfile
# Set the new narrowspec if we're widening. The setnewnarrowpats() method
# will currently always be there when using the core+narrowhg server, but
# other servers may include a changespec part even when not widening (e.g.
# because we're deepening a shallow repo).
if util.safehasattr(repo, 'setnewnarrowpats'):
repo.setnewnarrowpats()
def handlechangegroup_widen(op, inpart):
"""Changegroup exchange handler which restores temporarily-stripped nodes"""
# We saved a bundle with stripped node data we must now restore.
# This approach is based on mercurial/repair.py@6ee26a53c111.
repo = op.repo
ui = op.ui
chgrpfile = op._widen_bundle
del op._widen_bundle
vfs = repo.vfs
ui.note(_("adding branch\n"))
f = vfs.open(chgrpfile, "rb")
try:
gen = exchange.readbundle(ui, f, chgrpfile, vfs)
if not ui.verbose:
# silence internal shuffling chatter
ui.pushbuffer()
if isinstance(gen, bundle2.unbundle20):
with repo.transaction('strip') as tr:
bundle2.processbundle(repo, gen, lambda: tr)
else:
gen.apply(repo, 'strip', 'bundle:' + vfs.join(chgrpfile), True)
if not ui.verbose:
ui.popbuffer()
finally:
f.close()
# remove undo files
for undovfs, undofile in repo.undofiles():
try:
undovfs.unlink(undofile)
except OSError as e:
if e.errno != errno.ENOENT:
ui.warn(_('error removing %s: %s\n') %
Yuya Nishihara
stringutil: bulk-replace call sites to point to new module...
r37102 (undovfs.join(undofile), stringutil.forcebytestr(e)))
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096
# Remove partial backup only if there were no exceptions
Augie Fackler
narrowbundle2: when we handle a widen, mark the operation as unsafe...
r38548 op._widen_uninterr.__exit__(None, None, None)
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 vfs.unlink(chgrpfile)
def setup():
"""Enable narrow repo support in bundle2-related extension points."""
Gregory Szorc
wireproto: move gboptsmap to wireprototypes and rename (API)...
r37631 getbundleargs = wireprototypes.GETBUNDLE_ARGUMENTS
getbundleargs['narrow'] = 'boolean'
getbundleargs['depth'] = 'plain'
getbundleargs['oldincludepats'] = 'csv'
getbundleargs['oldexcludepats'] = 'csv'
getbundleargs['known'] = 'csv'
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096
# Extend changegroup serving to handle requests from narrow clients.
origcgfn = exchange.getbundle2partsmapping['changegroup']
def wrappedcgfn(*args, **kwargs):
repo = args[1]
Augie Fackler
narrowbundle2: mark most constants as module-private...
r36104 if repo.ui.has_section(_NARROWACL_SECTION):
Gregory Szorc
exchange: make narrow ACL presence imply narrow=True...
r38843 kwargs = exchange.applynarrowacl(repo, kwargs)
Gregory Szorc
exchange: move simple narrow changegroup generation from extension...
r38844 if (kwargs.get(r'narrow', False) and
repo.ui.configbool('experimental', 'narrowservebrokenellipses')):
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 getbundlechangegrouppart_narrow(*args, **kwargs)
else:
origcgfn(*args, **kwargs)
exchange.getbundle2partsmapping['changegroup'] = wrappedcgfn
# Extend changegroup receiver so client can fixup after widen requests.
origcghandler = bundle2.parthandlermapping['changegroup']
def wrappedcghandler(op, inpart):
origcghandler(op, inpart)
if util.safehasattr(op, '_widen_bundle'):
handlechangegroup_widen(op, inpart)
Martin von Zweigbergk
narrow: keep bookmarks temporarily stripped for as long as commits are...
r40903 if util.safehasattr(op, '_bookmarksbackup'):
localrepo.localrepository._bookmarks.set(op.repo,
op._bookmarksbackup)
del op._bookmarksbackup
Augie Fackler
narrow: import experimental extension from narrowhg revision cb51d673e9c5...
r36096 wrappedcghandler.params = origcghandler.params
bundle2.parthandlermapping['changegroup'] = wrappedcghandler