exchange.py
2006 lines
| 74.0 KiB
| text/x-python
|
PythonLexer
/ mercurial / exchange.py
|
r21024 | # exchange.py - utility to exchange data between repos. | ||
|
r20345 | # | ||
# Copyright 2005-2007 Matt Mackall <mpm@selenic.com> | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
|
r27523 | from __future__ import absolute_import | ||
import errno | ||||
|
r29341 | import hashlib | ||
|
r27523 | |||
from .i18n import _ | ||||
from .node import ( | ||||
hex, | ||||
nullid, | ||||
) | ||||
from . import ( | ||||
base85, | ||||
bookmarks as bookmod, | ||||
bundle2, | ||||
changegroup, | ||||
discovery, | ||||
error, | ||||
lock as lockmod, | ||||
obsolete, | ||||
phases, | ||||
pushkey, | ||||
scmutil, | ||||
sslutil, | ||||
streamclone, | ||||
tags, | ||||
url as urlmod, | ||||
util, | ||||
) | ||||
|
r20345 | |||
|
r28883 | urlerr = util.urlerr | ||
urlreq = util.urlreq | ||||
|
r26640 | # Maps bundle version human names to changegroup versions. | ||
_bundlespeccgversions = {'v1': '01', | ||||
'v2': '02', | ||||
|
r26756 | 'packed1': 's1', | ||
|
r26640 | 'bundle2': '02', #legacy | ||
} | ||||
|
r26646 | def parsebundlespec(repo, spec, strict=True, externalnames=False): | ||
|
r26640 | """Parse a bundle string specification into parts. | ||
Bundle specifications denote a well-defined bundle/exchange format. | ||||
The content of a given specification should not change over time in | ||||
order to ensure that bundles produced by a newer version of Mercurial are | ||||
readable from an older version. | ||||
The string currently has the form: | ||||
|
r26639 | |||
|
r26759 | <compression>-<type>[;<parameter0>[;<parameter1>]] | ||
|
r26640 | |||
Where <compression> is one of the supported compression formats | ||||
|
r26759 | and <type> is (currently) a version string. A ";" can follow the type and | ||
|
r30332 | all text afterwards is interpreted as URI encoded, ";" delimited key=value | ||
|
r26759 | pairs. | ||
|
r26639 | |||
|
r26640 | If ``strict`` is True (the default) <compression> is required. Otherwise, | ||
it is optional. | ||||
|
r26639 | |||
|
r26646 | If ``externalnames`` is False (the default), the human-centric names will | ||
be converted to their internal representation. | ||||
|
r26759 | Returns a 3-tuple of (compression, version, parameters). Compression will | ||
be ``None`` if not in strict mode and a compression isn't defined. | ||||
|
r26639 | |||
|
r26640 | An ``InvalidBundleSpecification`` is raised when the specification is | ||
not syntactically well formed. | ||||
An ``UnsupportedBundleSpecification`` is raised when the compression or | ||||
bundle type/version is not recognized. | ||||
|
r26639 | |||
|
r26640 | Note: this function will likely eventually return a more complex data | ||
structure, including bundle2 part information. | ||||
|
r26639 | """ | ||
|
r26759 | def parseparams(s): | ||
if ';' not in s: | ||||
return s, {} | ||||
params = {} | ||||
version, paramstr = s.split(';', 1) | ||||
for p in paramstr.split(';'): | ||||
if '=' not in p: | ||||
raise error.InvalidBundleSpecification( | ||||
_('invalid bundle specification: ' | ||||
'missing "=" in parameter: %s') % p) | ||||
key, value = p.split('=', 1) | ||||
|
r28883 | key = urlreq.unquote(key) | ||
value = urlreq.unquote(value) | ||||
|
r26759 | params[key] = value | ||
return version, params | ||||
|
r26640 | if strict and '-' not in spec: | ||
raise error.InvalidBundleSpecification( | ||||
_('invalid bundle specification; ' | ||||
'must be prefixed with compression: %s') % spec) | ||||
|
r26639 | |||
if '-' in spec: | ||||
|
r26640 | compression, version = spec.split('-', 1) | ||
|
r26639 | |||
|
r30440 | if compression not in util.compengines.supportedbundlenames: | ||
|
r26640 | raise error.UnsupportedBundleSpecification( | ||
_('%s compression is not supported') % compression) | ||||
|
r26759 | version, params = parseparams(version) | ||
|
r26640 | if version not in _bundlespeccgversions: | ||
raise error.UnsupportedBundleSpecification( | ||||
_('%s is not a recognized bundle version') % version) | ||||
|
r26639 | else: | ||
|
r26640 | # Value could be just the compression or just the version, in which | ||
# case some defaults are assumed (but only when not in strict mode). | ||||
assert not strict | ||||
|
r26639 | |||
|
r26759 | spec, params = parseparams(spec) | ||
|
r30440 | if spec in util.compengines.supportedbundlenames: | ||
|
r26640 | compression = spec | ||
version = 'v1' | ||||
if 'generaldelta' in repo.requirements: | ||||
version = 'v2' | ||||
elif spec in _bundlespeccgversions: | ||||
|
r26756 | if spec == 'packed1': | ||
compression = 'none' | ||||
else: | ||||
compression = 'bzip2' | ||||
|
r26640 | version = spec | ||
else: | ||||
raise error.UnsupportedBundleSpecification( | ||||
_('%s is not a recognized bundle specification') % spec) | ||||
|
r26639 | |||
|
r26760 | # The specification for packed1 can optionally declare the data formats | ||
# required to apply it. If we see this metadata, compare against what the | ||||
# repo supports and error if the bundle isn't compatible. | ||||
if version == 'packed1' and 'requirements' in params: | ||||
requirements = set(params['requirements'].split(',')) | ||||
missingreqs = requirements - repo.supportedformats | ||||
if missingreqs: | ||||
raise error.UnsupportedBundleSpecification( | ||||
_('missing support for repository features: %s') % | ||||
', '.join(sorted(missingreqs))) | ||||
|
r26646 | if not externalnames: | ||
|
r30440 | engine = util.compengines.forbundlename(compression) | ||
compression = engine.bundletype()[1] | ||||
|
r26646 | version = _bundlespeccgversions[version] | ||
|
r26759 | return compression, version, params | ||
|
r26639 | |||
|
r21064 | def readbundle(ui, fh, fname, vfs=None): | ||
|
r21065 | header = changegroup.readexactly(fh, 4) | ||
|
r21063 | |||
|
r21065 | alg = None | ||
|
r21063 | if not fname: | ||
fname = "stream" | ||||
if not header.startswith('HG') and header.startswith('\0'): | ||||
fh = changegroup.headerlessfixup(fh, header) | ||||
|
r21065 | header = "HG10" | ||
alg = 'UN' | ||||
|
r21063 | elif vfs: | ||
fname = vfs.join(fname) | ||||
|
r21065 | magic, version = header[0:2], header[2:4] | ||
|
r21063 | |||
if magic != 'HG': | ||||
|
r26587 | raise error.Abort(_('%s: not a Mercurial bundle') % fname) | ||
|
r21065 | if version == '10': | ||
if alg is None: | ||||
alg = changegroup.readexactly(fh, 2) | ||||
|
r22390 | return changegroup.cg1unpacker(fh, alg) | ||
|
r24649 | elif version.startswith('2'): | ||
|
r25640 | return bundle2.getunbundler(ui, fh, magicstring=magic + version) | ||
|
r26756 | elif version == 'S1': | ||
return streamclone.streamcloneapplier(fh) | ||||
|
r21065 | else: | ||
|
r26587 | raise error.Abort(_('%s: unknown bundle version %s') % (fname, version)) | ||
|
r21063 | |||
|
r27883 | def getbundlespec(ui, fh): | ||
"""Infer the bundlespec from a bundle file handle. | ||||
The input file handle is seeked and the original seek position is not | ||||
restored. | ||||
""" | ||||
def speccompression(alg): | ||||
|
r30440 | try: | ||
return util.compengines.forbundletype(alg).bundletype()[0] | ||||
except KeyError: | ||||
return None | ||||
|
r27883 | |||
b = readbundle(ui, fh, None) | ||||
if isinstance(b, changegroup.cg1unpacker): | ||||
alg = b._type | ||||
if alg == '_truncatedBZ': | ||||
alg = 'BZ' | ||||
comp = speccompression(alg) | ||||
if not comp: | ||||
raise error.Abort(_('unknown compression algorithm: %s') % alg) | ||||
return '%s-v1' % comp | ||||
elif isinstance(b, bundle2.unbundle20): | ||||
if 'Compression' in b.params: | ||||
comp = speccompression(b.params['Compression']) | ||||
if not comp: | ||||
raise error.Abort(_('unknown compression algorithm: %s') % comp) | ||||
else: | ||||
comp = 'none' | ||||
version = None | ||||
for part in b.iterparts(): | ||||
if part.type == 'changegroup': | ||||
version = part.params['version'] | ||||
if version in ('01', '02'): | ||||
version = 'v2' | ||||
else: | ||||
raise error.Abort(_('changegroup version %s does not have ' | ||||
'a known bundlespec') % version, | ||||
hint=_('try upgrading your Mercurial ' | ||||
'client')) | ||||
if not version: | ||||
raise error.Abort(_('could not identify changegroup version in ' | ||||
'bundle')) | ||||
return '%s-%s' % (comp, version) | ||||
elif isinstance(b, streamclone.streamcloneapplier): | ||||
requirements = streamclone.readbundle1header(fh)[2] | ||||
params = 'requirements=%s' % ','.join(sorted(requirements)) | ||||
|
r28883 | return 'none-packed1;%s' % urlreq.quote(params) | ||
|
r27883 | else: | ||
raise error.Abort(_('unknown bundle type: %s') % b) | ||||
|
r22346 | def buildobsmarkerspart(bundler, markers): | ||
"""add an obsmarker part to the bundler with <markers> | ||||
No part is created if markers is empty. | ||||
Raises ValueError if the bundler doesn't support any known obsmarker format. | ||||
""" | ||||
if markers: | ||||
remoteversions = bundle2.obsmarkersversion(bundler.capabilities) | ||||
version = obsolete.commonversion(remoteversions) | ||||
if version is None: | ||||
|
r26779 | raise ValueError('bundler does not support common obsmarker format') | ||
|
r22346 | stream = obsolete.encodemarkers(markers, True, version=version) | ||
|
r24686 | return bundler.newpart('obsmarkers', data=stream) | ||
|
r22346 | return None | ||
|
r20346 | |||
|
r29808 | def _computeoutgoing(repo, heads, common): | ||
"""Computes which revs are outgoing given a set of common | ||||
and a set of heads. | ||||
This is a separate function so extensions can have access to | ||||
the logic. | ||||
Returns a discovery.outgoing object. | ||||
""" | ||||
cl = repo.changelog | ||||
if common: | ||||
hasnode = cl.hasnode | ||||
common = [n for n in common if hasnode(n)] | ||||
else: | ||||
common = [nullid] | ||||
if not heads: | ||||
heads = cl.heads() | ||||
return discovery.outgoing(repo, common, heads) | ||||
|
r29682 | def _forcebundle1(op): | ||
"""return true if a pull/push must use bundle1 | ||||
|
r24650 | |||
|
r29683 | This function is used to allow testing of the older bundle version""" | ||
ui = op.repo.ui | ||||
forcebundle1 = False | ||||
|
r30332 | # The goal is this config is to allow developer to choose the bundle | ||
|
r29683 | # version used during exchanged. This is especially handy during test. | ||
# Value is a list of bundle version to be picked from, highest version | ||||
# should be used. | ||||
# | ||||
# developer config: devel.legacy.exchange | ||||
exchange = ui.configlist('devel', 'legacy.exchange') | ||||
|
r29689 | forcebundle1 = 'bundle2' not in exchange and 'bundle1' in exchange | ||
|
r29683 | return forcebundle1 or not op.remote.capable('bundle2') | ||
|
r24650 | |||
|
r20346 | class pushoperation(object): | ||
"""A object that represent a single push operation | ||||
|
r28456 | Its purpose is to carry push related state and very common operations. | ||
|
r20346 | |||
|
r28456 | A new pushoperation should be created at the beginning of each push and | ||
discarded afterward. | ||||
|
r20346 | """ | ||
|
r22623 | def __init__(self, repo, remote, force=False, revs=None, newbranch=False, | ||
bookmarks=()): | ||||
|
r20346 | # repo we push from | ||
self.repo = repo | ||||
|
r20347 | self.ui = repo.ui | ||
|
r20348 | # repo we push to | ||
self.remote = remote | ||||
|
r20349 | # force option provided | ||
self.force = force | ||||
|
r20350 | # revs to be pushed (None is "all") | ||
self.revs = revs | ||||
|
r22623 | # bookmark explicitly pushed | ||
self.bookmarks = bookmarks | ||||
|
r20351 | # allow push of new branch | ||
self.newbranch = newbranch | ||||
|
r20436 | # did a local lock get acquired? | ||
self.locallocked = None | ||||
|
r21901 | # step already performed | ||
# (used to check what steps have been already performed through bundle2) | ||||
self.stepsdone = set() | ||||
|
r22615 | # Integer version of the changegroup push result | ||
|
r20439 | # - None means nothing to push | ||
# - 0 means HTTP error | ||||
# - 1 means we pushed and remote head count is unchanged *or* | ||||
# we have outgoing changesets but refused to push | ||||
# - other values as described by addchangegroup() | ||||
|
r22615 | self.cgresult = None | ||
|
r22624 | # Boolean value for the bookmark push | ||
self.bkresult = None | ||||
|
r21024 | # discover.outgoing object (contains common and outgoing data) | ||
|
r20440 | self.outgoing = None | ||
|
r20462 | # all remote heads before the push | ||
self.remoteheads = None | ||||
|
r20464 | # testable as a boolean indicating if any nodes are missing locally. | ||
self.incoming = None | ||||
|
r22019 | # phases changes that must be pushed along side the changesets | ||
self.outdatedphases = None | ||||
# phases changes that must be pushed if changeset push fails | ||||
self.fallbackoutdatedphases = None | ||||
|
r22034 | # outgoing obsmarkers | ||
|
r22035 | self.outobsmarkers = set() | ||
|
r22239 | # outgoing bookmarks | ||
self.outbookmarks = [] | ||||
|
r23437 | # transaction manager | ||
self.trmanager = None | ||||
|
r25485 | # map { pushkey partid -> callback handling failure} | ||
# used to handle exception from mandatory pushkey part failure | ||||
self.pkfailcb = {} | ||||
|
r20346 | |||
|
r22014 | @util.propertycache | ||
def futureheads(self): | ||||
"""future remote heads if the changeset push succeeds""" | ||||
return self.outgoing.missingheads | ||||
|
r22015 | @util.propertycache | ||
def fallbackheads(self): | ||||
"""future remote heads if the changeset push fails""" | ||||
if self.revs is None: | ||||
# not target to push, all common are relevant | ||||
return self.outgoing.commonheads | ||||
unfi = self.repo.unfiltered() | ||||
# I want cheads = heads(::missingheads and ::commonheads) | ||||
# (missingheads is revs with secret changeset filtered out) | ||||
# | ||||
# This can be expressed as: | ||||
# cheads = ( (missingheads and ::commonheads) | ||||
# + (commonheads and ::missingheads))" | ||||
# ) | ||||
# | ||||
# while trying to push we already computed the following: | ||||
# common = (::commonheads) | ||||
# missing = ((commonheads::missingheads) - commonheads) | ||||
# | ||||
# We can pick: | ||||
# * missingheads part of common (::commonheads) | ||||
|
r26184 | common = self.outgoing.common | ||
|
r22015 | nm = self.repo.changelog.nodemap | ||
cheads = [node for node in self.revs if nm[node] in common] | ||||
# and | ||||
# * commonheads parents on missing | ||||
revset = unfi.set('%ln and parents(roots(%ln))', | ||||
self.outgoing.commonheads, | ||||
self.outgoing.missing) | ||||
cheads.extend(c.node() for c in revset) | ||||
return cheads | ||||
|
r22016 | @property | ||
def commonheads(self): | ||||
"""set of all common heads after changeset bundle push""" | ||||
|
r22615 | if self.cgresult: | ||
|
r22016 | return self.futureheads | ||
else: | ||||
return self.fallbackheads | ||||
|
r22015 | |||
|
r22650 | # mapping of message used when pushing bookmark | ||
bookmsgmap = {'update': (_("updating bookmark %s\n"), | ||||
_('updating bookmark %s failed!\n')), | ||||
'export': (_("exporting bookmark %s\n"), | ||||
_('exporting bookmark %s failed!\n')), | ||||
'delete': (_("deleting remote bookmark %s\n"), | ||||
_('deleting remote bookmark %s failed!\n')), | ||||
} | ||||
|
r26729 | def push(repo, remote, force=False, revs=None, newbranch=False, bookmarks=(), | ||
opargs=None): | ||||
|
r20345 | '''Push outgoing changesets (limited by revs) from a local | ||
repository to remote. Return an integer: | ||||
- None means nothing to push | ||||
- 0 means HTTP error | ||||
- 1 means we pushed and remote head count is unchanged *or* | ||||
we have outgoing changesets but refused to push | ||||
- other values as described by addchangegroup() | ||||
''' | ||||
|
r26729 | if opargs is None: | ||
opargs = {} | ||||
pushop = pushoperation(repo, remote, force, revs, newbranch, bookmarks, | ||||
**opargs) | ||||
|
r20348 | if pushop.remote.local(): | ||
missing = (set(pushop.repo.requirements) | ||||
- pushop.remote.local().supported) | ||||
|
r20345 | if missing: | ||
msg = _("required features are not" | ||||
" supported in the destination:" | ||||
" %s") % (', '.join(sorted(missing))) | ||||
|
r26587 | raise error.Abort(msg) | ||
|
r20345 | |||
# there are two ways to push to remote repo: | ||||
# | ||||
# addchangegroup assumes local user can lock remote | ||||
# repo (local filesystem, old ssh servers). | ||||
# | ||||
# unbundle assumes local user cannot lock remote repo (new ssh | ||||
# servers, http servers). | ||||
|
r20348 | if not pushop.remote.canpush(): | ||
|
r26587 | raise error.Abort(_("destination does not support push")) | ||
|
r20345 | # get local lock as we might write phase data | ||
|
r24754 | localwlock = locallock = None | ||
|
r20345 | try: | ||
|
r24754 | # bundle2 push may receive a reply bundle touching bookmarks or other | ||
# things requiring the wlock. Take it now to ensure proper ordering. | ||||
maypushback = pushop.ui.configbool('experimental', 'bundle2.pushback') | ||||
|
r29682 | if (not _forcebundle1(pushop)) and maypushback: | ||
|
r24754 | localwlock = pushop.repo.wlock() | ||
|
r20346 | locallock = pushop.repo.lock() | ||
|
r20436 | pushop.locallocked = True | ||
|
r25660 | except IOError as err: | ||
|
r20436 | pushop.locallocked = False | ||
|
r20345 | if err.errno != errno.EACCES: | ||
raise | ||||
# source repo cannot be locked. | ||||
# We do not abort the push, but just disable the local phase | ||||
# synchronisation. | ||||
msg = 'cannot lock source repository: %s\n' % err | ||||
|
r20347 | pushop.ui.debug(msg) | ||
|
r20345 | try: | ||
|
r23437 | if pushop.locallocked: | ||
|
r26672 | pushop.trmanager = transactionmanager(pushop.repo, | ||
|
r23437 | 'push-response', | ||
pushop.remote.url()) | ||||
|
r20924 | pushop.repo.checkpush(pushop) | ||
|
r20345 | lock = None | ||
|
r20348 | unbundle = pushop.remote.capable('unbundle') | ||
|
r20345 | if not unbundle: | ||
|
r20348 | lock = pushop.remote.lock() | ||
|
r20345 | try: | ||
|
r20466 | _pushdiscovery(pushop) | ||
|
r29682 | if not _forcebundle1(pushop): | ||
|
r21903 | _pushbundle2(pushop) | ||
_pushchangeset(pushop) | ||||
|
r20441 | _pushsyncphase(pushop) | ||
|
r20433 | _pushobsolete(pushop) | ||
|
r22224 | _pushbookmark(pushop) | ||
|
r20345 | finally: | ||
if lock is not None: | ||||
lock.release() | ||||
|
r23437 | if pushop.trmanager: | ||
pushop.trmanager.close() | ||||
|
r20345 | finally: | ||
|
r23437 | if pushop.trmanager: | ||
pushop.trmanager.release() | ||||
|
r20345 | if locallock is not None: | ||
locallock.release() | ||||
|
r24754 | if localwlock is not None: | ||
localwlock.release() | ||||
|
r20345 | |||
|
r22616 | return pushop | ||
|
r20352 | |||
|
r22018 | # list of steps to perform discovery before push | ||
pushdiscoveryorder = [] | ||||
# Mapping between step name and function | ||||
# | ||||
# This exists to help extensions wrap steps if necessary | ||||
pushdiscoverymapping = {} | ||||
def pushdiscovery(stepname): | ||||
"""decorator for function performing discovery before push | ||||
The function is added to the step -> function mapping and appended to the | ||||
list of steps. Beware that decorated function will be added in order (this | ||||
may matter). | ||||
You can only use this decorator for a new step, if you want to wrap a step | ||||
from an extension, change the pushdiscovery dictionary directly.""" | ||||
def dec(func): | ||||
assert stepname not in pushdiscoverymapping | ||||
pushdiscoverymapping[stepname] = func | ||||
pushdiscoveryorder.append(stepname) | ||||
return func | ||||
return dec | ||||
|
r20466 | def _pushdiscovery(pushop): | ||
|
r22018 | """Run all discovery steps""" | ||
for stepname in pushdiscoveryorder: | ||||
step = pushdiscoverymapping[stepname] | ||||
step(pushop) | ||||
@pushdiscovery('changeset') | ||||
def _pushdiscoverychangeset(pushop): | ||||
"""discover the changeset that need to be pushed""" | ||||
|
r20466 | fci = discovery.findcommonincoming | ||
|
r23848 | commoninc = fci(pushop.repo, pushop.remote, force=pushop.force) | ||
|
r20466 | common, inc, remoteheads = commoninc | ||
fco = discovery.findcommonoutgoing | ||||
|
r23848 | outgoing = fco(pushop.repo, pushop.remote, onlyheads=pushop.revs, | ||
|
r20466 | commoninc=commoninc, force=pushop.force) | ||
pushop.outgoing = outgoing | ||||
pushop.remoteheads = remoteheads | ||||
pushop.incoming = inc | ||||
|
r22019 | @pushdiscovery('phase') | ||
def _pushdiscoveryphase(pushop): | ||||
"""discover the phase that needs to be pushed | ||||
(computed for both success and failure case for changesets push)""" | ||||
outgoing = pushop.outgoing | ||||
unfi = pushop.repo.unfiltered() | ||||
remotephases = pushop.remote.listkeys('phases') | ||||
publishing = remotephases.get('publishing', False) | ||||
|
r25337 | if (pushop.ui.configbool('ui', '_usedassubrepo', False) | ||
and remotephases # server supports phases | ||||
and not pushop.outgoing.missing # no changesets to be pushed | ||||
and publishing): | ||||
# When: | ||||
# - this is a subrepo push | ||||
# - and remote support phase | ||||
# - and no changeset are to be pushed | ||||
# - and remote is publishing | ||||
# We may be in issue 3871 case! | ||||
# We drop the possible phase synchronisation done by | ||||
# courtesy to publish changesets possibly locally draft | ||||
# on the remote. | ||||
remotephases = {'publishing': 'True'} | ||||
|
r22019 | ana = phases.analyzeremotephases(pushop.repo, | ||
pushop.fallbackheads, | ||||
remotephases) | ||||
pheads, droots = ana | ||||
extracond = '' | ||||
if not publishing: | ||||
extracond = ' and public()' | ||||
revset = 'heads((%%ln::%%ln) %s)' % extracond | ||||
# Get the list of all revs draft on remote by public here. | ||||
# XXX Beware that revset break if droots is not strictly | ||||
# XXX root we may want to ensure it is but it is costly | ||||
fallback = list(unfi.set(revset, droots, pushop.fallbackheads)) | ||||
if not outgoing.missing: | ||||
future = fallback | ||||
else: | ||||
# adds changeset we are going to push as draft | ||||
# | ||||
|
r23139 | # should not be necessary for publishing server, but because of an | ||
|
r22019 | # issue fixed in xxxxx we have to do it anyway. | ||
fdroots = list(unfi.set('roots(%ln + %ln::)', | ||||
outgoing.missing, droots)) | ||||
fdroots = [f.node() for f in fdroots] | ||||
future = list(unfi.set(revset, fdroots, pushop.futureheads)) | ||||
pushop.outdatedphases = future | ||||
pushop.fallbackoutdatedphases = fallback | ||||
|
r22035 | @pushdiscovery('obsmarker') | ||
def _pushdiscoveryobsmarkers(pushop): | ||||
|
r22953 | if (obsolete.isenabled(pushop.repo, obsolete.exchangeopt) | ||
|
r22269 | and pushop.repo.obsstore | ||
and 'obsolete' in pushop.remote.listkeys('namespaces')): | ||||
|
r22350 | repo = pushop.repo | ||
# very naive computation, that can be quite expensive on big repo. | ||||
# However: evolution is currently slow on them anyway. | ||||
nodes = (c.node() for c in repo.set('::%ln', pushop.futureheads)) | ||||
pushop.outobsmarkers = pushop.repo.obsstore.relevantmarkers(nodes) | ||||
|
r22035 | |||
|
r22239 | @pushdiscovery('bookmarks') | ||
def _pushdiscoverybookmarks(pushop): | ||||
ui = pushop.ui | ||||
repo = pushop.repo.unfiltered() | ||||
remote = pushop.remote | ||||
ui.debug("checking for updated bookmarks\n") | ||||
ancestors = () | ||||
if pushop.revs: | ||||
revnums = map(repo.changelog.rev, pushop.revs) | ||||
ancestors = repo.changelog.ancestors(revnums, inclusive=True) | ||||
remotebookmark = remote.listkeys('bookmarks') | ||||
|
r28182 | explicit = set([repo._bookmarks.expandname(bookmark) | ||
for bookmark in pushop.bookmarks]) | ||||
|
r22651 | |||
|
r30583 | remotebookmark = bookmod.unhexlifybookmarks(remotebookmark) | ||
comp = bookmod.comparebookmarks(repo, repo._bookmarks, remotebookmark) | ||||
def safehex(x): | ||||
if x is None: | ||||
return x | ||||
return hex(x) | ||||
def hexifycompbookmarks(bookmarks): | ||||
for b, scid, dcid in bookmarks: | ||||
yield b, safehex(scid), safehex(dcid) | ||||
comp = [hexifycompbookmarks(marks) for marks in comp] | ||||
|
r23081 | addsrc, adddst, advsrc, advdst, diverge, differ, invalid, same = comp | ||
|
r30583 | |||
|
r22239 | for b, scid, dcid in advsrc: | ||
|
r22651 | if b in explicit: | ||
explicit.remove(b) | ||||
|
r22239 | if not ancestors or repo[scid].rev() in ancestors: | ||
pushop.outbookmarks.append((b, dcid, scid)) | ||||
|
r22651 | # search added bookmark | ||
for b, scid, dcid in addsrc: | ||||
if b in explicit: | ||||
explicit.remove(b) | ||||
pushop.outbookmarks.append((b, '', scid)) | ||||
# search for overwritten bookmark | ||||
|
r30583 | for b, scid, dcid in list(advdst) + list(diverge) + list(differ): | ||
|
r22651 | if b in explicit: | ||
explicit.remove(b) | ||||
pushop.outbookmarks.append((b, dcid, scid)) | ||||
# search for bookmark to delete | ||||
for b, scid, dcid in adddst: | ||||
if b in explicit: | ||||
explicit.remove(b) | ||||
# treat as "deleted locally" | ||||
pushop.outbookmarks.append((b, dcid, '')) | ||||
|
r23082 | # identical bookmarks shouldn't get reported | ||
for b, scid, dcid in same: | ||||
if b in explicit: | ||||
explicit.remove(b) | ||||
|
r22651 | |||
if explicit: | ||||
explicit = sorted(explicit) | ||||
# we should probably list all of them | ||||
ui.warn(_('bookmark %s does not exist on the local ' | ||||
'or remote repository!\n') % explicit[0]) | ||||
pushop.bkresult = 2 | ||||
pushop.outbookmarks.sort() | ||||
|
r22239 | |||
|
r20465 | def _pushcheckoutgoing(pushop): | ||
outgoing = pushop.outgoing | ||||
unfi = pushop.repo.unfiltered() | ||||
if not outgoing.missing: | ||||
# nothing to push | ||||
scmutil.nochangesfound(unfi.ui, unfi, outgoing.excluded) | ||||
return False | ||||
# something to push | ||||
if not pushop.force: | ||||
# if repo.obsstore == False --> no obsolete | ||||
# then, save the iteration | ||||
if unfi.obsstore: | ||||
# this message are here for 80 char limit reason | ||||
mso = _("push includes obsolete changeset: %s!") | ||||
|
r22628 | mst = {"unstable": _("push includes unstable changeset: %s!"), | ||
"bumped": _("push includes bumped changeset: %s!"), | ||||
"divergent": _("push includes divergent changeset: %s!")} | ||||
|
r20465 | # If we are to push if there is at least one | ||
# obsolete or unstable changeset in missing, at | ||||
# least one of the missinghead will be obsolete or | ||||
# unstable. So checking heads only is ok | ||||
for node in outgoing.missingheads: | ||||
ctx = unfi[node] | ||||
if ctx.obsolete(): | ||||
|
r26587 | raise error.Abort(mso % ctx) | ||
|
r20465 | elif ctx.troubled(): | ||
|
r26587 | raise error.Abort(mst[ctx.troubles()[0]] % ctx) | ||
|
r25836 | |||
|
r26935 | discovery.checkheads(pushop) | ||
|
r20465 | return True | ||
|
r22017 | # List of names of steps to perform for an outgoing bundle2, order matters. | ||
b2partsgenorder = [] | ||||
# Mapping between step name and function | ||||
# | ||||
# This exists to help extensions wrap steps if necessary | ||||
b2partsgenmapping = {} | ||||
|
r24731 | def b2partsgenerator(stepname, idx=None): | ||
|
r22017 | """decorator for function generating bundle2 part | ||
The function is added to the step -> function mapping and appended to the | ||||
list of steps. Beware that decorated functions will be added in order | ||||
(this may matter). | ||||
You can only use this decorator for new steps, if you want to wrap a step | ||||
from an extension, attack the b2partsgenmapping dictionary directly.""" | ||||
def dec(func): | ||||
assert stepname not in b2partsgenmapping | ||||
b2partsgenmapping[stepname] = func | ||||
|
r24731 | if idx is None: | ||
b2partsgenorder.append(stepname) | ||||
else: | ||||
b2partsgenorder.insert(idx, stepname) | ||||
|
r22017 | return func | ||
return dec | ||||
|
r26428 | def _pushb2ctxcheckheads(pushop, bundler): | ||
"""Generate race condition checking parts | ||||
|
r26781 | Exists as an independent function to aid extensions | ||
|
r26428 | """ | ||
if not pushop.force: | ||||
bundler.newpart('check:heads', data=iter(pushop.remoteheads)) | ||||
|
r22017 | @b2partsgenerator('changeset') | ||
|
r21899 | def _pushb2ctx(pushop, bundler): | ||
"""handle changegroup push through bundle2 | ||||
|
r22615 | addchangegroup result is stored in the ``pushop.cgresult`` attribute. | ||
|
r21899 | """ | ||
|
r21902 | if 'changesets' in pushop.stepsdone: | ||
return | ||||
pushop.stepsdone.add('changesets') | ||||
|
r21899 | # Send known heads to the server for race detection. | ||
|
r21903 | if not _pushcheckoutgoing(pushop): | ||
return | ||||
|
r28876 | pushop.repo.prepushoutgoinghooks(pushop) | ||
|
r26428 | |||
_pushb2ctxcheckheads(pushop, bundler) | ||||
|
r23180 | b2caps = bundle2.bundle2caps(pushop.remote) | ||
|
r28668 | version = '01' | ||
|
r24686 | cgversions = b2caps.get('changegroup') | ||
|
r28668 | if cgversions: # 3.1 and 3.2 ship with an empty value | ||
|
r27751 | cgversions = [v for v in cgversions | ||
|
r27953 | if v in changegroup.supportedoutgoingversions( | ||
pushop.repo)] | ||||
|
r23180 | if not cgversions: | ||
raise ValueError(_('no common changegroup version')) | ||||
version = max(cgversions) | ||||
|
r28668 | cg = changegroup.getlocalchangegroupraw(pushop.repo, 'push', | ||
pushop.outgoing, | ||||
version=version) | ||||
|
r24686 | cgpart = bundler.newpart('changegroup', data=cg) | ||
|
r28668 | if cgversions: | ||
|
r23180 | cgpart.addparam('version', version) | ||
|
r27938 | if 'treemanifest' in pushop.repo.requirements: | ||
cgpart.addparam('treemanifest', '1') | ||||
|
r21899 | def handlereply(op): | ||
|
r23139 | """extract addchangegroup returns from server reply""" | ||
|
r21899 | cgreplies = op.records.getreplies(cgpart.id) | ||
assert len(cgreplies['changegroup']) == 1 | ||||
|
r22615 | pushop.cgresult = cgreplies['changegroup'][0]['return'] | ||
|
r21899 | return handlereply | ||
|
r22020 | @b2partsgenerator('phase') | ||
def _pushb2phases(pushop, bundler): | ||||
"""handle phase push through bundle2""" | ||||
if 'phases' in pushop.stepsdone: | ||||
return | ||||
b2caps = bundle2.bundle2caps(pushop.remote) | ||||
|
r24686 | if not 'pushkey' in b2caps: | ||
|
r22020 | return | ||
pushop.stepsdone.add('phases') | ||||
part2node = [] | ||||
|
r25502 | |||
def handlefailure(pushop, exc): | ||||
targetid = int(exc.partid) | ||||
for partid, node in part2node: | ||||
if partid == targetid: | ||||
raise error.Abort(_('updating %s to public failed') % node) | ||||
|
r22020 | enc = pushkey.encode | ||
for newremotehead in pushop.outdatedphases: | ||||
|
r25502 | part = bundler.newpart('pushkey') | ||
|
r22020 | part.addparam('namespace', enc('phases')) | ||
part.addparam('key', enc(newremotehead.hex())) | ||||
part.addparam('old', enc(str(phases.draft))) | ||||
part.addparam('new', enc(str(phases.public))) | ||||
part2node.append((part.id, newremotehead)) | ||||
|
r25502 | pushop.pkfailcb[part.id] = handlefailure | ||
|
r22020 | def handlereply(op): | ||
for partid, node in part2node: | ||||
partrep = op.records.getreplies(partid) | ||||
results = partrep['pushkey'] | ||||
assert len(results) <= 1 | ||||
msg = None | ||||
if not results: | ||||
msg = _('server ignored update of %s to public!\n') % node | ||||
elif not int(results[0]['return']): | ||||
msg = _('updating %s to public failed!\n') % node | ||||
if msg is not None: | ||||
pushop.ui.warn(msg) | ||||
return handlereply | ||||
|
r21904 | |||
|
r22347 | @b2partsgenerator('obsmarkers') | ||
def _pushb2obsmarkers(pushop, bundler): | ||||
if 'obsmarkers' in pushop.stepsdone: | ||||
return | ||||
remoteversions = bundle2.obsmarkersversion(bundler.capabilities) | ||||
if obsolete.commonversion(remoteversions) is None: | ||||
return | ||||
pushop.stepsdone.add('obsmarkers') | ||||
if pushop.outobsmarkers: | ||||
|
r25118 | markers = sorted(pushop.outobsmarkers) | ||
buildobsmarkerspart(bundler, markers) | ||||
|
r22347 | |||
|
r22242 | @b2partsgenerator('bookmarks') | ||
def _pushb2bookmarks(pushop, bundler): | ||||
|
r25895 | """handle bookmark push through bundle2""" | ||
|
r22242 | if 'bookmarks' in pushop.stepsdone: | ||
return | ||||
b2caps = bundle2.bundle2caps(pushop.remote) | ||||
|
r24686 | if 'pushkey' not in b2caps: | ||
|
r22242 | return | ||
pushop.stepsdone.add('bookmarks') | ||||
part2book = [] | ||||
enc = pushkey.encode | ||||
|
r25501 | |||
def handlefailure(pushop, exc): | ||||
targetid = int(exc.partid) | ||||
for partid, book, action in part2book: | ||||
if partid == targetid: | ||||
raise error.Abort(bookmsgmap[action][1].rstrip() % book) | ||||
# we should not be called for part we did not generated | ||||
assert False | ||||
|
r22242 | for book, old, new in pushop.outbookmarks: | ||
|
r25501 | part = bundler.newpart('pushkey') | ||
|
r22242 | part.addparam('namespace', enc('bookmarks')) | ||
part.addparam('key', enc(book)) | ||||
part.addparam('old', enc(old)) | ||||
part.addparam('new', enc(new)) | ||||
|
r22650 | action = 'update' | ||
if not old: | ||||
action = 'export' | ||||
elif not new: | ||||
action = 'delete' | ||||
part2book.append((part.id, book, action)) | ||||
|
r25501 | pushop.pkfailcb[part.id] = handlefailure | ||
|
r22650 | |||
|
r22242 | def handlereply(op): | ||
|
r22650 | ui = pushop.ui | ||
for partid, book, action in part2book: | ||||
|
r22242 | partrep = op.records.getreplies(partid) | ||
results = partrep['pushkey'] | ||||
assert len(results) <= 1 | ||||
if not results: | ||||
pushop.ui.warn(_('server ignored bookmark %s update\n') % book) | ||||
else: | ||||
ret = int(results[0]['return']) | ||||
if ret: | ||||
|
r22650 | ui.status(bookmsgmap[action][0] % book) | ||
|
r22242 | else: | ||
|
r22650 | ui.warn(bookmsgmap[action][1] % book) | ||
|
r22649 | if pushop.bkresult is not None: | ||
pushop.bkresult = 1 | ||||
|
r22242 | return handlereply | ||
|
r21061 | def _pushbundle2(pushop): | ||
"""push data to the remote using bundle2 | ||||
The only currently supported type of data is changegroup but this will | ||||
evolve in the future.""" | ||||
|
r21644 | bundler = bundle2.bundle20(pushop.ui, bundle2.bundle2caps(pushop.remote)) | ||
|
r23439 | pushback = (pushop.trmanager | ||
and pushop.ui.configbool('experimental', 'bundle2.pushback')) | ||||
|
r21142 | # create reply capability | ||
|
r23439 | capsblob = bundle2.encodecaps(bundle2.getrepocaps(pushop.repo, | ||
allowpushback=pushback)) | ||||
|
r24686 | bundler.newpart('replycaps', data=capsblob) | ||
|
r21904 | replyhandlers = [] | ||
|
r22017 | for partgenname in b2partsgenorder: | ||
partgen = b2partsgenmapping[partgenname] | ||||
|
r21904 | ret = partgen(pushop, bundler) | ||
|
r21941 | if callable(ret): | ||
replyhandlers.append(ret) | ||||
|
r21904 | # do not push if nothing to push | ||
|
r21903 | if bundler.nbparts <= 1: | ||
return | ||||
|
r21061 | stream = util.chunkbuffer(bundler.getchunks()) | ||
|
r21182 | try: | ||
|
r25485 | try: | ||
|
r29704 | reply = pushop.remote.unbundle( | ||
stream, ['force'], pushop.remote.url()) | ||||
|
r25660 | except error.BundleValueError as exc: | ||
|
r29389 | raise error.Abort(_('missing support for %s') % exc) | ||
|
r25485 | try: | ||
trgetter = None | ||||
if pushback: | ||||
trgetter = pushop.trmanager.transaction | ||||
op = bundle2.processbundle(pushop.repo, reply, trgetter) | ||||
|
r25660 | except error.BundleValueError as exc: | ||
|
r29389 | raise error.Abort(_('missing support for %s') % exc) | ||
|
r26829 | except bundle2.AbortFromPart as exc: | ||
pushop.ui.status(_('remote: %s\n') % exc) | ||||
raise error.Abort(_('push failed on remote'), hint=exc.hint) | ||||
|
r25660 | except error.PushkeyFailed as exc: | ||
|
r25485 | partid = int(exc.partid) | ||
if partid not in pushop.pkfailcb: | ||||
raise | ||||
pushop.pkfailcb[partid](pushop, exc) | ||||
|
r21904 | for rephand in replyhandlers: | ||
rephand(op) | ||||
|
r21061 | |||
|
r20463 | def _pushchangeset(pushop): | ||
"""Make the actual push of changeset bundle to remote repo""" | ||||
|
r21902 | if 'changesets' in pushop.stepsdone: | ||
return | ||||
pushop.stepsdone.add('changesets') | ||||
|
r21903 | if not _pushcheckoutgoing(pushop): | ||
return | ||||
|
r28876 | pushop.repo.prepushoutgoinghooks(pushop) | ||
|
r20463 | outgoing = pushop.outgoing | ||
unbundle = pushop.remote.capable('unbundle') | ||||
# TODO: get bundlecaps from remote | ||||
bundlecaps = None | ||||
# create a changegroup from local | ||||
if pushop.revs is None and not (outgoing.excluded | ||||
or pushop.repo.changelog.filteredrevs): | ||||
# push everything, | ||||
# use the fast path, no race possible on push | ||||
|
r22390 | bundler = changegroup.cg1packer(pushop.repo, bundlecaps) | ||
|
r20925 | cg = changegroup.getsubset(pushop.repo, | ||
outgoing, | ||||
bundler, | ||||
'push', | ||||
fastpath=True) | ||||
|
r20463 | else: | ||
|
r22390 | cg = changegroup.getlocalchangegroup(pushop.repo, 'push', outgoing, | ||
|
r20928 | bundlecaps) | ||
|
r20463 | |||
# apply changegroup to remote | ||||
if unbundle: | ||||
# local repo finds heads on server, finds out what | ||||
# revs it must push. once revs transferred, if server | ||||
# finds it has different heads (someone else won | ||||
# commit/push race), server aborts. | ||||
if pushop.force: | ||||
remoteheads = ['force'] | ||||
else: | ||||
remoteheads = pushop.remoteheads | ||||
# ssh: return remote's addchangegroup() | ||||
# http: return remote's addchangegroup() or 0 for error | ||||
|
r22615 | pushop.cgresult = pushop.remote.unbundle(cg, remoteheads, | ||
|
r21761 | pushop.repo.url()) | ||
|
r20463 | else: | ||
# we return an integer indicating remote head count | ||||
# change | ||||
|
r22615 | pushop.cgresult = pushop.remote.addchangegroup(cg, 'push', | ||
pushop.repo.url()) | ||||
|
r20463 | |||
|
r20441 | def _pushsyncphase(pushop): | ||
|
r21024 | """synchronise phase information locally and remotely""" | ||
|
r20468 | cheads = pushop.commonheads | ||
|
r20441 | # even when we don't push, exchanging phase data is useful | ||
remotephases = pushop.remote.listkeys('phases') | ||||
if (pushop.ui.configbool('ui', '_usedassubrepo', False) | ||||
and remotephases # server supports phases | ||||
|
r22615 | and pushop.cgresult is None # nothing was pushed | ||
|
r20441 | and remotephases.get('publishing', False)): | ||
# When: | ||||
# - this is a subrepo push | ||||
# - and remote support phase | ||||
# - and no changeset was pushed | ||||
# - and remote is publishing | ||||
# We may be in issue 3871 case! | ||||
# We drop the possible phase synchronisation done by | ||||
# courtesy to publish changesets possibly locally draft | ||||
# on the remote. | ||||
remotephases = {'publishing': 'True'} | ||||
|
r21012 | if not remotephases: # old server or public only reply from non-publishing | ||
|
r20441 | _localphasemove(pushop, cheads) | ||
# don't push any phase data as there is nothing to push | ||||
else: | ||||
ana = phases.analyzeremotephases(pushop.repo, cheads, | ||||
remotephases) | ||||
pheads, droots = ana | ||||
### Apply remote phase on local | ||||
if remotephases.get('publishing', False): | ||||
_localphasemove(pushop, cheads) | ||||
else: # publish = False | ||||
_localphasemove(pushop, pheads) | ||||
_localphasemove(pushop, cheads, phases.draft) | ||||
### Apply local phase on remote | ||||
|
r22615 | if pushop.cgresult: | ||
|
r22020 | if 'phases' in pushop.stepsdone: | ||
# phases already pushed though bundle2 | ||||
return | ||||
|
r22019 | outdated = pushop.outdatedphases | ||
else: | ||||
outdated = pushop.fallbackoutdatedphases | ||||
|
r22020 | pushop.stepsdone.add('phases') | ||
|
r22019 | # filter heads already turned public by the push | ||
outdated = [c for c in outdated if c.node() not in pheads] | ||||
|
r23376 | # fallback to independent pushkey command | ||
for newremotehead in outdated: | ||||
r = pushop.remote.pushkey('phases', | ||||
newremotehead.hex(), | ||||
str(phases.draft), | ||||
str(phases.public)) | ||||
if not r: | ||||
pushop.ui.warn(_('updating %s to public failed!\n') | ||||
% newremotehead) | ||||
|
r20441 | |||
|
r20438 | def _localphasemove(pushop, nodes, phase=phases.public): | ||
"""move <nodes> to <phase> in the local source repo""" | ||||
|
r23437 | if pushop.trmanager: | ||
phases.advanceboundary(pushop.repo, | ||||
pushop.trmanager.transaction(), | ||||
phase, | ||||
nodes) | ||||
|
r20438 | else: | ||
# repo is not locked, do not change any phases! | ||||
# Informs the user that phases should have been moved when | ||||
# applicable. | ||||
actualmoves = [n for n in nodes if phase < pushop.repo[n].phase()] | ||||
phasestr = phases.phasenames[phase] | ||||
if actualmoves: | ||||
pushop.ui.status(_('cannot lock source repo, skipping ' | ||||
'local %s phase update\n') % phasestr) | ||||
|
r20433 | def _pushobsolete(pushop): | ||
|
r20434 | """utility function to push obsolete markers to a remote""" | ||
|
r22036 | if 'obsmarkers' in pushop.stepsdone: | ||
return | ||||
|
r20433 | repo = pushop.repo | ||
remote = pushop.remote | ||||
|
r22036 | pushop.stepsdone.add('obsmarkers') | ||
|
r22350 | if pushop.outobsmarkers: | ||
|
r25559 | pushop.ui.debug('try to push obsolete markers to remote\n') | ||
|
r20432 | rslts = [] | ||
|
r25118 | remotedata = obsolete._pushkeyescape(sorted(pushop.outobsmarkers)) | ||
|
r20432 | for key in sorted(remotedata, reverse=True): | ||
# reverse sort to ensure we end with dump0 | ||||
data = remotedata[key] | ||||
rslts.append(remote.pushkey('obsolete', key, '', data)) | ||||
if [r for r in rslts if not r]: | ||||
msg = _('failed to push some obsolete markers!\n') | ||||
repo.ui.warn(msg) | ||||
|
r20431 | def _pushbookmark(pushop): | ||
|
r20352 | """Update bookmark position on remote""" | ||
|
r22615 | if pushop.cgresult == 0 or 'bookmarks' in pushop.stepsdone: | ||
|
r22228 | return | ||
|
r22240 | pushop.stepsdone.add('bookmarks') | ||
|
r20431 | ui = pushop.ui | ||
remote = pushop.remote | ||||
|
r22650 | |||
|
r22239 | for b, old, new in pushop.outbookmarks: | ||
|
r22650 | action = 'update' | ||
if not old: | ||||
action = 'export' | ||||
elif not new: | ||||
action = 'delete' | ||||
|
r22239 | if remote.pushkey('bookmarks', b, old, new): | ||
|
r22650 | ui.status(bookmsgmap[action][0] % b) | ||
|
r20352 | else: | ||
|
r22650 | ui.warn(bookmsgmap[action][1] % b) | ||
# discovery can have set the value form invalid entry | ||||
if pushop.bkresult is not None: | ||||
pushop.bkresult = 1 | ||||
|
r20469 | |||
|
r20472 | class pulloperation(object): | ||
"""A object that represent a single pull operation | ||||
|
r23219 | It purpose is to carry pull related state and very common operation. | ||
|
r20472 | |||
|
r21024 | A new should be created at the beginning of each pull and discarded | ||
|
r20472 | afterward. | ||
""" | ||||
|
r25446 | def __init__(self, repo, remote, heads=None, force=False, bookmarks=(), | ||
|
r26448 | remotebookmarks=None, streamclonerequested=None): | ||
|
r20596 | # repo we pull into | ||
|
r20472 | self.repo = repo | ||
|
r20596 | # repo we pull from | ||
|
r20473 | self.remote = remote | ||
|
r20474 | # revision we try to pull (None is "all") | ||
self.heads = heads | ||||
|
r22654 | # bookmark pulled explicitly | ||
|
r29376 | self.explicitbookmarks = [repo._bookmarks.expandname(bookmark) | ||
for bookmark in bookmarks] | ||||
|
r20475 | # do we force pull? | ||
self.force = force | ||||
|
r26448 | # whether a streaming clone was requested | ||
self.streamclonerequested = streamclonerequested | ||||
|
r23436 | # transaction manager | ||
self.trmanager = None | ||||
|
r20487 | # set of common changeset between local and remote before pull | ||
self.common = None | ||||
# set of pulled head | ||||
self.rheads = None | ||||
|
r21024 | # list of missing changeset to fetch remotely | ||
|
r20488 | self.fetch = None | ||
|
r22654 | # remote bookmarks data | ||
|
r25446 | self.remotebookmarks = remotebookmarks | ||
|
r21024 | # result of changegroup pulling (used as return code by pull) | ||
|
r20898 | self.cgresult = None | ||
|
r22937 | # list of step already done | ||
self.stepsdone = set() | ||||
|
r26689 | # Whether we attempted a clone from pre-generated bundles. | ||
self.clonebundleattempted = False | ||||
|
r20487 | |||
@util.propertycache | ||||
def pulledsubset(self): | ||||
"""heads of the set of changeset target by the pull""" | ||||
# compute target subset | ||||
if self.heads is None: | ||||
# We pulled every thing possible | ||||
# sync on everything common | ||||
|
r20878 | c = set(self.common) | ||
ret = list(self.common) | ||||
for n in self.rheads: | ||||
if n not in c: | ||||
ret.append(n) | ||||
return ret | ||||
|
r20487 | else: | ||
# We pulled a specific subset | ||||
# sync on this subset | ||||
return self.heads | ||||
|
r20477 | |||
|
r26464 | @util.propertycache | ||
|
r26465 | def canusebundle2(self): | ||
|
r29682 | return not _forcebundle1(self) | ||
|
r26465 | |||
@util.propertycache | ||||
|
r26464 | def remotebundle2caps(self): | ||
return bundle2.bundle2caps(self.remote) | ||||
|
r20477 | def gettransaction(self): | ||
|
r23436 | # deprecated; talk to trmanager directly | ||
return self.trmanager.transaction() | ||||
class transactionmanager(object): | ||||
|
r23543 | """An object to manage the life cycle of a transaction | ||
|
r23436 | |||
It creates the transaction on demand and calls the appropriate hooks when | ||||
closing the transaction.""" | ||||
def __init__(self, repo, source, url): | ||||
self.repo = repo | ||||
self.source = source | ||||
self.url = url | ||||
self._tr = None | ||||
def transaction(self): | ||||
"""Return an open transaction object, constructing if necessary""" | ||||
if not self._tr: | ||||
trname = '%s\n%s' % (self.source, util.hidepassword(self.url)) | ||||
self._tr = self.repo.transaction(trname) | ||||
self._tr.hookargs['source'] = self.source | ||||
self._tr.hookargs['url'] = self.url | ||||
|
r20477 | return self._tr | ||
|
r23436 | def close(self): | ||
|
r20477 | """close transaction if created""" | ||
if self._tr is not None: | ||||
|
r23222 | self._tr.close() | ||
|
r20477 | |||
|
r23436 | def release(self): | ||
|
r20477 | """release transaction if created""" | ||
if self._tr is not None: | ||||
self._tr.release() | ||||
|
r20469 | |||
|
r26448 | def pull(repo, remote, heads=None, force=False, bookmarks=(), opargs=None, | ||
streamclonerequested=None): | ||||
|
r26440 | """Fetch repository data from a remote. | ||
This is the main function used to retrieve data from a remote repository. | ||||
``repo`` is the local repository to clone into. | ||||
``remote`` is a peer instance. | ||||
``heads`` is an iterable of revisions we want to pull. ``None`` (the | ||||
default) means to pull everything from the remote. | ||||
``bookmarks`` is an iterable of bookmarks requesting to be pulled. By | ||||
default, all remote bookmarks are pulled. | ||||
``opargs`` are additional keyword arguments to pass to ``pulloperation`` | ||||
initialization. | ||||
|
r26448 | ``streamclonerequested`` is a boolean indicating whether a "streaming | ||
clone" is requested. A "streaming clone" is essentially a raw file copy | ||||
of revlogs from the server. This only works when the local repository is | ||||
empty. The default value of ``None`` means to respect the server | ||||
configuration for preferring stream clones. | ||||
|
r26440 | |||
Returns the ``pulloperation`` created for this pull. | ||||
""" | ||||
|
r25445 | if opargs is None: | ||
opargs = {} | ||||
pullop = pulloperation(repo, remote, heads, force, bookmarks=bookmarks, | ||||
|
r26448 | streamclonerequested=streamclonerequested, **opargs) | ||
|
r20473 | if pullop.remote.local(): | ||
missing = set(pullop.remote.requirements) - pullop.repo.supported | ||||
|
r20469 | if missing: | ||
msg = _("required features are not" | ||||
" supported in the destination:" | ||||
" %s") % (', '.join(sorted(missing))) | ||||
|
r26587 | raise error.Abort(msg) | ||
|
r20469 | |||
|
r30068 | wlock = lock = None | ||
|
r20469 | try: | ||
|
r30068 | wlock = pullop.repo.wlock() | ||
lock = pullop.repo.lock() | ||||
|
r23436 | pullop.trmanager = transactionmanager(repo, 'pull', remote.url()) | ||
|
r26462 | streamclone.maybeperformlegacystreamclone(pullop) | ||
|
r26623 | # This should ideally be in _pullbundle2(). However, it needs to run | ||
# before discovery to avoid extra work. | ||||
_maybeapplyclonebundle(pullop) | ||||
|
r20900 | _pulldiscovery(pullop) | ||
|
r26465 | if pullop.canusebundle2: | ||
|
r20955 | _pullbundle2(pullop) | ||
|
r22653 | _pullchangeset(pullop) | ||
_pullphase(pullop) | ||||
|
r22655 | _pullbookmarks(pullop) | ||
|
r22653 | _pullobsolete(pullop) | ||
|
r23436 | pullop.trmanager.close() | ||
|
r20469 | finally: | ||
|
r30068 | lockmod.release(pullop.trmanager, lock, wlock) | ||
|
r20469 | |||
|
r22693 | return pullop | ||
|
r20476 | |||
|
r22936 | # list of steps to perform discovery before pull | ||
pulldiscoveryorder = [] | ||||
# Mapping between step name and function | ||||
# | ||||
# This exists to help extensions wrap steps if necessary | ||||
pulldiscoverymapping = {} | ||||
def pulldiscovery(stepname): | ||||
"""decorator for function performing discovery before pull | ||||
The function is added to the step -> function mapping and appended to the | ||||
list of steps. Beware that decorated function will be added in order (this | ||||
may matter). | ||||
You can only use this decorator for a new step, if you want to wrap a step | ||||
from an extension, change the pulldiscovery dictionary directly.""" | ||||
def dec(func): | ||||
assert stepname not in pulldiscoverymapping | ||||
pulldiscoverymapping[stepname] = func | ||||
pulldiscoveryorder.append(stepname) | ||||
return func | ||||
return dec | ||||
|
r20900 | def _pulldiscovery(pullop): | ||
|
r22936 | """Run all discovery steps""" | ||
for stepname in pulldiscoveryorder: | ||||
step = pulldiscoverymapping[stepname] | ||||
step(pullop) | ||||
|
r25369 | @pulldiscovery('b1:bookmarks') | ||
def _pullbookmarkbundle1(pullop): | ||||
"""fetch bookmark data in bundle1 case | ||||
If not using bundle2, we have to fetch bookmarks before changeset | ||||
discovery to reduce the chance and impact of race conditions.""" | ||||
|
r25443 | if pullop.remotebookmarks is not None: | ||
return | ||||
|
r26465 | if pullop.canusebundle2 and 'listkeys' in pullop.remotebundle2caps: | ||
|
r25479 | # all known bundle2 servers now support listkeys, but lets be nice with | ||
# new implementation. | ||||
return | ||||
pullop.remotebookmarks = pullop.remote.listkeys('bookmarks') | ||||
|
r25369 | |||
|
r22936 | @pulldiscovery('changegroup') | ||
def _pulldiscoverychangegroup(pullop): | ||||
|
r20900 | """discovery phase for the pull | ||
Current handle changeset discovery only, will change handle all discovery | ||||
at some point.""" | ||||
|
r23848 | tmp = discovery.findcommonincoming(pullop.repo, | ||
|
r20900 | pullop.remote, | ||
heads=pullop.heads, | ||||
force=pullop.force) | ||||
|
r23848 | common, fetch, rheads = tmp | ||
nm = pullop.repo.unfiltered().changelog.nodemap | ||||
if fetch and rheads: | ||||
# If a remote heads in filtered locally, lets drop it from the unknown | ||||
# remote heads and put in back in common. | ||||
# | ||||
# This is a hackish solution to catch most of "common but locally | ||||
# hidden situation". We do not performs discovery on unfiltered | ||||
# repository because it end up doing a pathological amount of round | ||||
# trip for w huge amount of changeset we do not care about. | ||||
# | ||||
# If a set of such "common but filtered" changeset exist on the server | ||||
# but are not including a remote heads, we'll not be able to detect it, | ||||
scommon = set(common) | ||||
filteredrheads = [] | ||||
for n in rheads: | ||||
|
r23975 | if n in nm: | ||
if n not in scommon: | ||||
common.append(n) | ||||
|
r23848 | else: | ||
filteredrheads.append(n) | ||||
if not filteredrheads: | ||||
fetch = [] | ||||
rheads = filteredrheads | ||||
pullop.common = common | ||||
pullop.fetch = fetch | ||||
pullop.rheads = rheads | ||||
|
r20900 | |||
|
r20955 | def _pullbundle2(pullop): | ||
"""pull data using bundle2 | ||||
For now, the only supported data are changegroup.""" | ||||
|
r21645 | kwargs = {'bundlecaps': caps20to10(pullop.repo)} | ||
|
r26471 | |||
streaming, streamreqs = streamclone.canperformstreamclone(pullop) | ||||
|
r20955 | # pulling changegroup | ||
|
r22937 | pullop.stepsdone.add('changegroup') | ||
|
r21259 | |||
kwargs['common'] = pullop.common | ||||
kwargs['heads'] = pullop.heads or pullop.rheads | ||||
|
r22352 | kwargs['cg'] = pullop.fetch | ||
|
r26464 | if 'listkeys' in pullop.remotebundle2caps: | ||
|
r29064 | kwargs['listkeys'] = ['phases'] | ||
|
r25444 | if pullop.remotebookmarks is None: | ||
# make sure to always includes bookmark data when migrating | ||||
# `hg incoming --bundle` to using this function. | ||||
kwargs['listkeys'].append('bookmarks') | ||||
|
r26690 | |||
# If this is a full pull / clone and the server supports the clone bundles | ||||
# feature, tell the server whether we attempted a clone bundle. The | ||||
# presence of this flag indicates the client supports clone bundles. This | ||||
# will enable the server to treat clients that support clone bundles | ||||
# differently from those that don't. | ||||
if (pullop.remote.capable('clonebundles') | ||||
and pullop.heads is None and list(pullop.common) == [nullid]): | ||||
kwargs['cbattempted'] = pullop.clonebundleattempted | ||||
|
r26471 | if streaming: | ||
pullop.repo.ui.status(_('streaming all changes\n')) | ||||
elif not pullop.fetch: | ||||
|
r21258 | pullop.repo.ui.status(_("no changes found\n")) | ||
pullop.cgresult = 0 | ||||
|
r20955 | else: | ||
if pullop.heads is None and list(pullop.common) == [nullid]: | ||||
pullop.repo.ui.status(_("requesting all changes\n")) | ||||
|
r22953 | if obsolete.isenabled(pullop.repo, obsolete.exchangeopt): | ||
|
r26464 | remoteversions = bundle2.obsmarkersversion(pullop.remotebundle2caps) | ||
|
r22354 | if obsolete.commonversion(remoteversions) is not None: | ||
kwargs['obsmarkers'] = True | ||||
|
r22937 | pullop.stepsdone.add('obsmarkers') | ||
|
r21159 | _pullbundle2extraprepare(pullop, kwargs) | ||
|
r20955 | bundle = pullop.remote.getbundle('pull', **kwargs) | ||
try: | ||||
op = bundle2.processbundle(pullop.repo, bundle, pullop.gettransaction) | ||||
|
r25660 | except error.BundleValueError as exc: | ||
|
r29389 | raise error.Abort(_('missing support for %s') % exc) | ||
|
r21259 | |||
if pullop.fetch: | ||||
|
r23890 | results = [cg['return'] for cg in op.records['changegroup']] | ||
pullop.cgresult = changegroup.combineresults(results) | ||||
|
r20955 | |||
|
r21658 | # processing phases change | ||
for namespace, value in op.records['listkeys']: | ||||
if namespace == 'phases': | ||||
_pullapplyphases(pullop, value) | ||||
|
r22656 | # processing bookmark update | ||
for namespace, value in op.records['listkeys']: | ||||
if namespace == 'bookmarks': | ||||
pullop.remotebookmarks = value | ||||
|
r25444 | |||
# bookmark data were either already there or pulled in the bundle | ||||
if pullop.remotebookmarks is not None: | ||||
_pullbookmarks(pullop) | ||||
|
r22656 | |||
|
r21159 | def _pullbundle2extraprepare(pullop, kwargs): | ||
"""hook function so that extensions can extend the getbundle call""" | ||||
pass | ||||
|
r20489 | def _pullchangeset(pullop): | ||
"""pull changeset from unbundle into the local repo""" | ||||
# We delay the open of the transaction as late as possible so we | ||||
# don't open transaction for nothing or you break future useful | ||||
# rollback call | ||||
|
r22937 | if 'changegroup' in pullop.stepsdone: | ||
|
r22653 | return | ||
|
r22937 | pullop.stepsdone.add('changegroup') | ||
|
r20899 | if not pullop.fetch: | ||
|
r23217 | pullop.repo.ui.status(_("no changes found\n")) | ||
pullop.cgresult = 0 | ||||
return | ||||
|
r20489 | pullop.gettransaction() | ||
if pullop.heads is None and list(pullop.common) == [nullid]: | ||||
pullop.repo.ui.status(_("requesting all changes\n")) | ||||
elif pullop.heads is None and pullop.remote.capable('changegroupsubset'): | ||||
# issue1320, avoid a race if remote changed after discovery | ||||
pullop.heads = pullop.rheads | ||||
if pullop.remote.capable('getbundle'): | ||||
# TODO: get bundlecaps from remote | ||||
cg = pullop.remote.getbundle('pull', common=pullop.common, | ||||
heads=pullop.heads or pullop.rheads) | ||||
elif pullop.heads is None: | ||||
cg = pullop.remote.changegroup(pullop.fetch, 'pull') | ||||
elif not pullop.remote.capable('changegroupsubset'): | ||||
|
r26587 | raise error.Abort(_("partial pull cannot be done because " | ||
|
r21554 | "other repository doesn't support " | ||
"changegroupsubset.")) | ||||
|
r20489 | else: | ||
cg = pullop.remote.changegroupsubset(pullop.fetch, pullop.heads, 'pull') | ||||
|
r26700 | pullop.cgresult = cg.apply(pullop.repo, 'pull', pullop.remote.url()) | ||
|
r20489 | |||
|
r20486 | def _pullphase(pullop): | ||
# Get remote phases data from remote | ||||
|
r22937 | if 'phases' in pullop.stepsdone: | ||
|
r22653 | return | ||
|
r21654 | remotephases = pullop.remote.listkeys('phases') | ||
_pullapplyphases(pullop, remotephases) | ||||
def _pullapplyphases(pullop, remotephases): | ||||
"""apply phase movement from observed remote state""" | ||||
|
r22937 | if 'phases' in pullop.stepsdone: | ||
return | ||||
pullop.stepsdone.add('phases') | ||||
|
r20486 | publishing = bool(remotephases.get('publishing', False)) | ||
if remotephases and not publishing: | ||||
|
r30332 | # remote is new and non-publishing | ||
|
r20486 | pheads, _dr = phases.analyzeremotephases(pullop.repo, | ||
pullop.pulledsubset, | ||||
remotephases) | ||||
|
r22068 | dheads = pullop.pulledsubset | ||
|
r20486 | else: | ||
# Remote is old or publishing all common changesets | ||||
# should be seen as public | ||||
|
r22068 | pheads = pullop.pulledsubset | ||
dheads = [] | ||||
unfi = pullop.repo.unfiltered() | ||||
phase = unfi._phasecache.phase | ||||
rev = unfi.changelog.nodemap.get | ||||
public = phases.public | ||||
draft = phases.draft | ||||
# exclude changesets already public locally and update the others | ||||
pheads = [pn for pn in pheads if phase(unfi, rev(pn)) > public] | ||||
if pheads: | ||||
|
r22069 | tr = pullop.gettransaction() | ||
phases.advanceboundary(pullop.repo, tr, public, pheads) | ||||
|
r22068 | |||
# exclude changesets already draft locally and update the others | ||||
dheads = [pn for pn in dheads if phase(unfi, rev(pn)) > draft] | ||||
if dheads: | ||||
|
r22069 | tr = pullop.gettransaction() | ||
phases.advanceboundary(pullop.repo, tr, draft, dheads) | ||||
|
r20486 | |||
|
r22654 | def _pullbookmarks(pullop): | ||
"""process the remote bookmark information to update the local one""" | ||||
|
r22937 | if 'bookmarks' in pullop.stepsdone: | ||
|
r22654 | return | ||
|
r22937 | pullop.stepsdone.add('bookmarks') | ||
|
r22654 | repo = pullop.repo | ||
remotebookmarks = pullop.remotebookmarks | ||||
|
r30583 | remotebookmarks = bookmod.unhexlifybookmarks(remotebookmarks) | ||
|
r22654 | bookmod.updatefromremote(repo.ui, repo, remotebookmarks, | ||
|
r22658 | pullop.remote.url(), | ||
|
r22666 | pullop.gettransaction, | ||
|
r22658 | explicit=pullop.explicitbookmarks) | ||
|
r22654 | |||
|
r20478 | def _pullobsolete(pullop): | ||
|
r20476 | """utility function to pull obsolete markers from a remote | ||
The `gettransaction` is function that return the pull transaction, creating | ||||
one if necessary. We return the transaction to inform the calling code that | ||||
a new transaction have been created (when applicable). | ||||
Exists mostly to allow overriding for experimentation purpose""" | ||||
|
r22937 | if 'obsmarkers' in pullop.stepsdone: | ||
|
r22653 | return | ||
|
r22937 | pullop.stepsdone.add('obsmarkers') | ||
|
r20476 | tr = None | ||
|
r22953 | if obsolete.isenabled(pullop.repo, obsolete.exchangeopt): | ||
|
r20478 | pullop.repo.ui.debug('fetching remote obsolete markers\n') | ||
remoteobs = pullop.remote.listkeys('obsolete') | ||||
|
r20476 | if 'dump0' in remoteobs: | ||
|
r20478 | tr = pullop.gettransaction() | ||
|
r27558 | markers = [] | ||
|
r20476 | for key in sorted(remoteobs, reverse=True): | ||
if key.startswith('dump'): | ||||
data = base85.b85decode(remoteobs[key]) | ||||
|
r27558 | version, newmarks = obsolete._readmarkers(data) | ||
markers += newmarks | ||||
if markers: | ||||
pullop.repo.obsstore.add(tr, markers) | ||||
|
r20478 | pullop.repo.invalidatevolatilesets() | ||
|
r20476 | return tr | ||
|
r21645 | def caps20to10(repo): | ||
"""return a set with appropriate options to use bundle20 during getbundle""" | ||||
|
r24686 | caps = set(['HG20']) | ||
|
r22342 | capsblob = bundle2.encodecaps(bundle2.getrepocaps(repo)) | ||
|
r28883 | caps.add('bundle2=' + urlreq.quote(capsblob)) | ||
|
r21645 | return caps | ||
|
r22542 | # List of names of steps to perform for a bundle2 for getbundle, order matters. | ||
getbundle2partsorder = [] | ||||
# Mapping between step name and function | ||||
# | ||||
# This exists to help extensions wrap steps if necessary | ||||
getbundle2partsmapping = {} | ||||
|
r24732 | def getbundle2partsgenerator(stepname, idx=None): | ||
|
r22542 | """decorator for function generating bundle2 part for getbundle | ||
The function is added to the step -> function mapping and appended to the | ||||
list of steps. Beware that decorated functions will be added in order | ||||
(this may matter). | ||||
You can only use this decorator for new steps, if you want to wrap a step | ||||
from an extension, attack the getbundle2partsmapping dictionary directly.""" | ||||
def dec(func): | ||||
assert stepname not in getbundle2partsmapping | ||||
getbundle2partsmapping[stepname] = func | ||||
|
r24732 | if idx is None: | ||
getbundle2partsorder.append(stepname) | ||||
else: | ||||
getbundle2partsorder.insert(idx, stepname) | ||||
|
r22542 | return func | ||
return dec | ||||
|
r27244 | def bundle2requested(bundlecaps): | ||
if bundlecaps is not None: | ||||
return any(cap.startswith('HG2') for cap in bundlecaps) | ||||
return False | ||||
|
r30187 | def getbundlechunks(repo, source, heads=None, common=None, bundlecaps=None, | ||
**kwargs): | ||||
"""Return chunks constituting a bundle's raw data. | ||||
|
r20954 | |||
|
r24686 | Could be a bundle HG10 or a bundle HG20 depending on bundlecaps | ||
|
r30187 | passed. | ||
|
r20954 | |||
|
r30187 | Returns an iterator over raw chunks (of varying sizes). | ||
|
r20954 | """ | ||
|
r27244 | usebundle2 = bundle2requested(bundlecaps) | ||
|
r22542 | # bundle10 case | ||
|
r24649 | if not usebundle2: | ||
|
r22542 | if bundlecaps and not kwargs.get('cg', True): | ||
raise ValueError(_('request for bundle10 must include changegroup')) | ||||
|
r21656 | if kwargs: | ||
raise ValueError(_('unsupported getbundle arguments: %s') | ||||
% ', '.join(sorted(kwargs.keys()))) | ||||
|
r29808 | outgoing = _computeoutgoing(repo, heads, common) | ||
|
r30187 | bundler = changegroup.getbundler('01', repo, bundlecaps) | ||
return changegroup.getsubsetraw(repo, outgoing, bundler, source) | ||||
|
r22542 | |||
# bundle20 case | ||||
|
r21143 | b2caps = {} | ||
for bcaps in bundlecaps: | ||||
if bcaps.startswith('bundle2='): | ||||
|
r28883 | blob = urlreq.unquote(bcaps[len('bundle2='):]) | ||
|
r21143 | b2caps.update(bundle2.decodecaps(blob)) | ||
bundler = bundle2.bundle20(repo.ui, b2caps) | ||||
|
r22542 | |||
|
r23218 | kwargs['heads'] = heads | ||
kwargs['common'] = common | ||||
|
r22542 | for name in getbundle2partsorder: | ||
func = getbundle2partsmapping[name] | ||||
|
r22543 | func(bundler, repo, source, bundlecaps=bundlecaps, b2caps=b2caps, | ||
**kwargs) | ||||
|
r22542 | |||
|
r30187 | return bundler.getchunks() | ||
|
r22542 | |||
@getbundle2partsgenerator('changegroup') | ||||
|
r22543 | def _getbundlechangegrouppart(bundler, repo, source, bundlecaps=None, | ||
b2caps=None, heads=None, common=None, **kwargs): | ||||
|
r22542 | """add a changegroup part to the requested bundle""" | ||
cg = None | ||||
if kwargs.get('cg', True): | ||||
# build changegroup bundle here. | ||||
|
r28667 | version = '01' | ||
|
r24686 | cgversions = b2caps.get('changegroup') | ||
|
r25503 | if cgversions: # 3.1 and 3.2 ship with an empty value | ||
|
r27751 | cgversions = [v for v in cgversions | ||
|
r27953 | if v in changegroup.supportedoutgoingversions(repo)] | ||
|
r23179 | if not cgversions: | ||
raise ValueError(_('no common changegroup version')) | ||||
|
r28667 | version = max(cgversions) | ||
|
r29808 | outgoing = _computeoutgoing(repo, heads, common) | ||
|
r25504 | cg = changegroup.getlocalchangegroupraw(repo, source, outgoing, | ||
bundlecaps=bundlecaps, | ||||
|
r28667 | version=version) | ||
|
r22542 | |||
|
r21259 | if cg: | ||
|
r24686 | part = bundler.newpart('changegroup', data=cg) | ||
|
r28667 | if cgversions: | ||
|
r23179 | part.addparam('version', version) | ||
|
r25516 | part.addparam('nbchanges', str(len(outgoing.missing)), mandatory=False) | ||
|
r27734 | if 'treemanifest' in repo.requirements: | ||
part.addparam('treemanifest', '1') | ||||
|
r22542 | |||
@getbundle2partsgenerator('listkeys') | ||||
|
r22543 | def _getbundlelistkeysparts(bundler, repo, source, bundlecaps=None, | ||
b2caps=None, **kwargs): | ||||
|
r22542 | """add parts containing listkeys namespaces to the requested bundle""" | ||
|
r21657 | listkeys = kwargs.get('listkeys', ()) | ||
for namespace in listkeys: | ||||
|
r24686 | part = bundler.newpart('listkeys') | ||
|
r21657 | part.addparam('namespace', namespace) | ||
keys = repo.listkeys(namespace).items() | ||||
part.data = pushkey.encodekeys(keys) | ||||
|
r20967 | |||
|
r22542 | @getbundle2partsgenerator('obsmarkers') | ||
|
r22543 | def _getbundleobsmarkerpart(bundler, repo, source, bundlecaps=None, | ||
b2caps=None, heads=None, **kwargs): | ||||
|
r22541 | """add an obsolescence markers part to the requested bundle""" | ||
|
r22353 | if kwargs.get('obsmarkers', False): | ||
if heads is None: | ||||
heads = repo.heads() | ||||
subset = [c.node() for c in repo.set('::%ln', heads)] | ||||
markers = repo.obsstore.relevantmarkers(subset) | ||||
|
r25118 | markers = sorted(markers) | ||
|
r22353 | buildobsmarkerspart(bundler, markers) | ||
|
r25402 | @getbundle2partsgenerator('hgtagsfnodes') | ||
def _getbundletagsfnodes(bundler, repo, source, bundlecaps=None, | ||||
b2caps=None, heads=None, common=None, | ||||
**kwargs): | ||||
"""Transfer the .hgtags filenodes mapping. | ||||
Only values for heads in this bundle will be transferred. | ||||
The part data consists of pairs of 20 byte changeset node and .hgtags | ||||
filenodes raw values. | ||||
""" | ||||
# Don't send unless: | ||||
# - changeset are being exchanged, | ||||
# - the client supports it. | ||||
if not (kwargs.get('cg', True) and 'hgtagsfnodes' in b2caps): | ||||
return | ||||
|
r29808 | outgoing = _computeoutgoing(repo, heads, common) | ||
|
r25402 | |||
if not outgoing.missingheads: | ||||
return | ||||
cache = tags.hgtagsfnodescache(repo.unfiltered()) | ||||
chunks = [] | ||||
# .hgtags fnodes are only relevant for head changesets. While we could | ||||
# transfer values for all known nodes, there will likely be little to | ||||
# no benefit. | ||||
# | ||||
# We don't bother using a generator to produce output data because | ||||
# a) we only have 40 bytes per head and even esoteric numbers of heads | ||||
# consume little memory (1M heads is 40MB) b) we don't want to send the | ||||
# part if we don't have entries and knowing if we have entries requires | ||||
# cache lookups. | ||||
for node in outgoing.missingheads: | ||||
# Don't compute missing, as this may slow down serving. | ||||
fnode = cache.getfnode(node, computemissing=False) | ||||
if fnode is not None: | ||||
chunks.extend([node, fnode]) | ||||
if chunks: | ||||
bundler.newpart('hgtagsfnodes', data=''.join(chunks)) | ||||
|
r30483 | def _getbookmarks(repo, **kwargs): | ||
"""Returns bookmark to node mapping. | ||||
This function is primarily used to generate `bookmarks` bundle2 part. | ||||
It is a separate function in order to make it easy to wrap it | ||||
in extensions. Passing `kwargs` to the function makes it easy to | ||||
add new parameters in extensions. | ||||
""" | ||||
return dict(bookmod.listbinbookmarks(repo)) | ||||
|
r20967 | def check_heads(repo, their_heads, context): | ||
"""check if the heads of a repo have been modified | ||||
Used by peer for unbundling. | ||||
""" | ||||
heads = repo.heads() | ||||
|
r29341 | heads_hash = hashlib.sha1(''.join(sorted(heads))).digest() | ||
|
r20967 | if not (their_heads == ['force'] or their_heads == heads or | ||
their_heads == ['hashed', heads_hash]): | ||||
# someone else committed/pushed/unbundled while we | ||||
# were transferring data | ||||
|
r21184 | raise error.PushRaced('repository changed while %s - ' | ||
'please try again' % context) | ||||
|
r20968 | |||
def unbundle(repo, cg, heads, source, url): | ||||
"""Apply a bundle to a repo. | ||||
this function makes sure the repo is locked during the application and have | ||||
|
r21024 | mechanism to check that no push race occurred between the creation of the | ||
|
r20968 | bundle and its application. | ||
If the push was raced as PushRaced exception is raised.""" | ||||
r = 0 | ||||
|
r21061 | # need a transaction when processing a bundle2 stream | ||
|
r26566 | # [wlock, lock, tr] - needs to be an array so nested functions can modify it | ||
lockandtr = [None, None, None] | ||||
|
r24847 | recordout = None | ||
|
r24878 | # quick fix for output mismatch with bundle2 in 3.4 | ||
captureoutput = repo.ui.configbool('experimental', 'bundle2-output-capture', | ||||
False) | ||||
|
r25423 | if url.startswith('remote:http:') or url.startswith('remote:https:'): | ||
|
r24878 | captureoutput = True | ||
|
r20968 | try: | ||
check_heads(repo, heads, 'uploading changes') | ||||
# push can proceed | ||||
|
r21061 | if util.safehasattr(cg, 'params'): | ||
|
r24795 | r = None | ||
|
r21187 | try: | ||
|
r26566 | def gettransaction(): | ||
if not lockandtr[2]: | ||||
lockandtr[0] = repo.wlock() | ||||
lockandtr[1] = repo.lock() | ||||
lockandtr[2] = repo.transaction(source) | ||||
lockandtr[2].hookargs['source'] = source | ||||
lockandtr[2].hookargs['url'] = url | ||||
lockandtr[2].hookargs['bundle2'] = '1' | ||||
return lockandtr[2] | ||||
# Do greedy locking by default until we're satisfied with lazy | ||||
# locking. | ||||
if not repo.ui.configbool('experimental', 'bundle2lazylocking'): | ||||
gettransaction() | ||||
op = bundle2.bundleoperation(repo, gettransaction, | ||||
|
r24878 | captureoutput=captureoutput) | ||
|
r24851 | try: | ||
|
r25896 | op = bundle2.processbundle(repo, cg, op=op) | ||
|
r24851 | finally: | ||
r = op.reply | ||||
|
r24878 | if captureoutput and r is not None: | ||
|
r24851 | repo.ui.pushbuffer(error=True, subproc=True) | ||
def recordout(output): | ||||
r.newpart('output', data=output, mandatory=False) | ||||
|
r26566 | if lockandtr[2] is not None: | ||
lockandtr[2].close() | ||||
|
r25660 | except BaseException as exc: | ||
|
r21187 | exc.duringunbundle2 = True | ||
|
r24878 | if captureoutput and r is not None: | ||
|
r24847 | parts = exc._bundle2salvagedoutput = r.salvageoutput() | ||
def recordout(output): | ||||
part = bundle2.bundlepart('output', data=output, | ||||
mandatory=False) | ||||
parts.append(part) | ||||
|
r21187 | raise | ||
|
r21061 | else: | ||
|
r26566 | lockandtr[1] = repo.lock() | ||
|
r26700 | r = cg.apply(repo, source, url) | ||
|
r20968 | finally: | ||
|
r26566 | lockmod.release(lockandtr[2], lockandtr[1], lockandtr[0]) | ||
|
r24847 | if recordout is not None: | ||
recordout(repo.ui.popbuffer()) | ||||
|
r20968 | return r | ||
|
r26623 | |||
def _maybeapplyclonebundle(pullop): | ||||
"""Apply a clone bundle from a remote, if possible.""" | ||||
repo = pullop.repo | ||||
remote = pullop.remote | ||||
|
r27738 | if not repo.ui.configbool('ui', 'clonebundles', True): | ||
|
r26623 | return | ||
|
r26855 | # Only run if local repo is empty. | ||
if len(repo): | ||||
return | ||||
|
r26623 | if pullop.heads: | ||
return | ||||
if not remote.capable('clonebundles'): | ||||
return | ||||
res = remote._call('clonebundles') | ||||
|
r26689 | |||
# If we call the wire protocol command, that's good enough to record the | ||||
# attempt. | ||||
pullop.clonebundleattempted = True | ||||
|
r26647 | entries = parseclonebundlesmanifest(repo, res) | ||
|
r26623 | if not entries: | ||
repo.ui.note(_('no clone bundles available on remote; ' | ||||
'falling back to regular clone\n')) | ||||
return | ||||
|
r26644 | entries = filterclonebundleentries(repo, entries) | ||
if not entries: | ||||
# There is a thundering herd concern here. However, if a server | ||||
# operator doesn't advertise bundles appropriate for its clients, | ||||
# they deserve what's coming. Furthermore, from a client's | ||||
# perspective, no automatic fallback would mean not being able to | ||||
# clone! | ||||
repo.ui.warn(_('no compatible clone bundles available on server; ' | ||||
'falling back to regular clone\n')) | ||||
repo.ui.warn(_('(you may want to report this to the server ' | ||||
'operator)\n')) | ||||
return | ||||
|
r26648 | entries = sortclonebundleentries(repo.ui, entries) | ||
|
r26644 | |||
|
r26623 | url = entries[0]['URL'] | ||
repo.ui.status(_('applying clone bundle from %s\n') % url) | ||||
if trypullbundlefromurl(repo.ui, repo, url): | ||||
repo.ui.status(_('finished applying clone bundle\n')) | ||||
# Bundle failed. | ||||
# | ||||
# We abort by default to avoid the thundering herd of | ||||
# clients flooding a server that was expecting expensive | ||||
# clone load to be offloaded. | ||||
elif repo.ui.configbool('ui', 'clonebundlefallback', False): | ||||
repo.ui.warn(_('falling back to normal clone\n')) | ||||
else: | ||||
raise error.Abort(_('error applying bundle'), | ||||
|
r26688 | hint=_('if this error persists, consider contacting ' | ||
'the server operator or disable clone ' | ||||
'bundles via ' | ||||
|
r27738 | '"--config ui.clonebundles=false"')) | ||
|
r26623 | |||
|
r26647 | def parseclonebundlesmanifest(repo, s): | ||
|
r26623 | """Parses the raw text of a clone bundles manifest. | ||
Returns a list of dicts. The dicts have a ``URL`` key corresponding | ||||
to the URL and other keys are the attributes for the entry. | ||||
""" | ||||
m = [] | ||||
for line in s.splitlines(): | ||||
fields = line.split() | ||||
if not fields: | ||||
continue | ||||
attrs = {'URL': fields[0]} | ||||
for rawattr in fields[1:]: | ||||
key, value = rawattr.split('=', 1) | ||||
|
r28883 | key = urlreq.unquote(key) | ||
value = urlreq.unquote(value) | ||||
|
r26647 | attrs[key] = value | ||
# Parse BUNDLESPEC into components. This makes client-side | ||||
# preferences easier to specify since you can prefer a single | ||||
# component of the BUNDLESPEC. | ||||
if key == 'BUNDLESPEC': | ||||
try: | ||||
|
r26759 | comp, version, params = parsebundlespec(repo, value, | ||
externalnames=True) | ||||
|
r26647 | attrs['COMPRESSION'] = comp | ||
attrs['VERSION'] = version | ||||
except error.InvalidBundleSpecification: | ||||
pass | ||||
except error.UnsupportedBundleSpecification: | ||||
pass | ||||
|
r26623 | |||
m.append(attrs) | ||||
return m | ||||
|
r26644 | def filterclonebundleentries(repo, entries): | ||
|
r26687 | """Remove incompatible clone bundle manifest entries. | ||
Accepts a list of entries parsed with ``parseclonebundlesmanifest`` | ||||
and returns a new list consisting of only the entries that this client | ||||
should be able to apply. | ||||
There is no guarantee we'll be able to apply all returned entries because | ||||
the metadata we use to filter on may be missing or wrong. | ||||
""" | ||||
|
r26644 | newentries = [] | ||
for entry in entries: | ||||
spec = entry.get('BUNDLESPEC') | ||||
if spec: | ||||
try: | ||||
parsebundlespec(repo, spec, strict=True) | ||||
except error.InvalidBundleSpecification as e: | ||||
repo.ui.debug(str(e) + '\n') | ||||
continue | ||||
except error.UnsupportedBundleSpecification as e: | ||||
repo.ui.debug('filtering %s because unsupported bundle ' | ||||
'spec: %s\n' % (entry['URL'], str(e))) | ||||
continue | ||||
|
r26645 | if 'REQUIRESNI' in entry and not sslutil.hassni: | ||
repo.ui.debug('filtering %s because SNI not supported\n' % | ||||
entry['URL']) | ||||
continue | ||||
|
r26644 | newentries.append(entry) | ||
return newentries | ||||
|
r30685 | class clonebundleentry(object): | ||
"""Represents an item in a clone bundles manifest. | ||||
This rich class is needed to support sorting since sorted() in Python 3 | ||||
doesn't support ``cmp`` and our comparison is complex enough that ``key=`` | ||||
won't work. | ||||
""" | ||||
|
r26648 | |||
|
r30685 | def __init__(self, value, prefers): | ||
self.value = value | ||||
self.prefers = prefers | ||||
|
r26648 | |||
|
r30685 | def _cmp(self, other): | ||
for prefkey, prefvalue in self.prefers: | ||||
avalue = self.value.get(prefkey) | ||||
bvalue = other.value.get(prefkey) | ||||
|
r26648 | |||
# Special case for b missing attribute and a matches exactly. | ||||
if avalue is not None and bvalue is None and avalue == prefvalue: | ||||
return -1 | ||||
# Special case for a missing attribute and b matches exactly. | ||||
if bvalue is not None and avalue is None and bvalue == prefvalue: | ||||
return 1 | ||||
# We can't compare unless attribute present on both. | ||||
if avalue is None or bvalue is None: | ||||
continue | ||||
# Same values should fall back to next attribute. | ||||
if avalue == bvalue: | ||||
continue | ||||
# Exact matches come first. | ||||
if avalue == prefvalue: | ||||
return -1 | ||||
if bvalue == prefvalue: | ||||
return 1 | ||||
# Fall back to next attribute. | ||||
continue | ||||
# If we got here we couldn't sort by attributes and prefers. Fall | ||||
# back to index order. | ||||
return 0 | ||||
|
r30685 | def __lt__(self, other): | ||
return self._cmp(other) < 0 | ||||
def __gt__(self, other): | ||||
return self._cmp(other) > 0 | ||||
def __eq__(self, other): | ||||
return self._cmp(other) == 0 | ||||
def __le__(self, other): | ||||
return self._cmp(other) <= 0 | ||||
def __ge__(self, other): | ||||
return self._cmp(other) >= 0 | ||||
def __ne__(self, other): | ||||
return self._cmp(other) != 0 | ||||
def sortclonebundleentries(ui, entries): | ||||
prefers = ui.configlist('ui', 'clonebundleprefers', default=[]) | ||||
if not prefers: | ||||
return list(entries) | ||||
prefers = [p.split('=', 1) for p in prefers] | ||||
items = sorted(clonebundleentry(v, prefers) for v in entries) | ||||
return [i.value for i in items] | ||||
|
r26648 | |||
|
r26623 | def trypullbundlefromurl(ui, repo, url): | ||
"""Attempt to apply a bundle from a URL.""" | ||||
lock = repo.lock() | ||||
try: | ||||
tr = repo.transaction('bundleurl') | ||||
try: | ||||
try: | ||||
fh = urlmod.open(ui, url) | ||||
cg = readbundle(ui, fh, 'stream') | ||||
|
r26643 | |||
if isinstance(cg, bundle2.unbundle20): | ||||
bundle2.processbundle(repo, cg, lambda: tr) | ||||
|
r26761 | elif isinstance(cg, streamclone.streamcloneapplier): | ||
cg.apply(repo) | ||||
|
r26643 | else: | ||
|
r26700 | cg.apply(repo, 'clonebundles', url) | ||
|
r26623 | tr.close() | ||
return True | ||||
|
r28883 | except urlerr.httperror as e: | ||
|
r26623 | ui.warn(_('HTTP error fetching bundle: %s\n') % str(e)) | ||
|
r28883 | except urlerr.urlerror as e: | ||
|
r26732 | ui.warn(_('error fetching bundle: %s\n') % e.reason[1]) | ||
|
r26623 | |||
return False | ||||
finally: | ||||
tr.release() | ||||
finally: | ||||
lock.release() | ||||