__init__.py
1344 lines
| 44.6 KiB
| text/x-python
|
PythonLexer
|
r37204 | # Infinite push | ||
# | ||||
# Copyright 2016 Facebook, Inc. | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
""" store some pushes in a remote blob store on the server (EXPERIMENTAL) | ||||
|
r43381 | IMPORTANT: if you use this extension, please contact | ||
mercurial-devel@mercurial-scm.org ASAP. This extension is believed to | ||||
be unused and barring learning of users of this functionality, we will | ||||
delete this code at the end of 2020. | ||||
|
r37204 | [infinitepush] | ||
# Server-side and client-side option. Pattern of the infinitepush bookmark | ||||
branchpattern = PATTERN | ||||
# Server or client | ||||
server = False | ||||
# Server-side option. Possible values: 'disk' or 'sql'. Fails if not set | ||||
indextype = disk | ||||
# Server-side option. Used only if indextype=sql. | ||||
# Format: 'IP:PORT:DB_NAME:USER:PASSWORD' | ||||
sqlhost = IP:PORT:DB_NAME:USER:PASSWORD | ||||
# Server-side option. Used only if indextype=disk. | ||||
# Filesystem path to the index store | ||||
indexpath = PATH | ||||
# Server-side option. Possible values: 'disk' or 'external' | ||||
# Fails if not set | ||||
storetype = disk | ||||
# Server-side option. | ||||
# Path to the binary that will save bundle to the bundlestore | ||||
# Formatted cmd line will be passed to it (see `put_args`) | ||||
put_binary = put | ||||
# Serser-side option. Used only if storetype=external. | ||||
# Format cmd-line string for put binary. Placeholder: {filename} | ||||
put_args = {filename} | ||||
# Server-side option. | ||||
# Path to the binary that get bundle from the bundlestore. | ||||
# Formatted cmd line will be passed to it (see `get_args`) | ||||
get_binary = get | ||||
# Serser-side option. Used only if storetype=external. | ||||
# Format cmd-line string for get binary. Placeholders: {filename} {handle} | ||||
get_args = {filename} {handle} | ||||
# Server-side option | ||||
logfile = FIlE | ||||
# Server-side option | ||||
loglevel = DEBUG | ||||
# Server-side option. Used only if indextype=sql. | ||||
# Sets mysql wait_timeout option. | ||||
waittimeout = 300 | ||||
# Server-side option. Used only if indextype=sql. | ||||
# Sets mysql innodb_lock_wait_timeout option. | ||||
locktimeout = 120 | ||||
# Server-side option. Used only if indextype=sql. | ||||
# Name of the repository | ||||
reponame = '' | ||||
# Client-side option. Used by --list-remote option. List of remote scratch | ||||
# patterns to list if no patterns are specified. | ||||
defaultremotepatterns = ['*'] | ||||
# Instructs infinitepush to forward all received bundle2 parts to the | ||||
# bundle for storage. Defaults to False. | ||||
storeallparts = True | ||||
|
r37223 | # routes each incoming push to the bundlestore. defaults to False | ||
pushtobundlestore = True | ||||
|
r37204 | [remotenames] | ||
# Client-side option | ||||
# This option should be set only if remotenames extension is enabled. | ||||
# Whether remote bookmarks are tracked by remotenames extension. | ||||
bookmarks = True | ||||
""" | ||||
from __future__ import absolute_import | ||||
import collections | ||||
import contextlib | ||||
import errno | ||||
import functools | ||||
import logging | ||||
import os | ||||
import random | ||||
import re | ||||
import socket | ||||
import subprocess | ||||
import time | ||||
from mercurial.node import ( | ||||
bin, | ||||
hex, | ||||
) | ||||
from mercurial.i18n import _ | ||||
|
r43359 | from mercurial.pycompat import ( | ||
getattr, | ||||
open, | ||||
) | ||||
|
r43355 | |||
|
r37221 | from mercurial.utils import ( | ||
procutil, | ||||
stringutil, | ||||
) | ||||
|
r37204 | from mercurial import ( | ||
bundle2, | ||||
changegroup, | ||||
commands, | ||||
discovery, | ||||
encoding, | ||||
error, | ||||
exchange, | ||||
extensions, | ||||
hg, | ||||
localrepo, | ||||
phases, | ||||
pushkey, | ||||
|
r37595 | pycompat, | ||
|
r37204 | registrar, | ||
util, | ||||
|
r37630 | wireprototypes, | ||
|
r37632 | wireprotov1peer, | ||
|
r37803 | wireprotov1server, | ||
|
r37204 | ) | ||
from . import ( | ||||
bundleparts, | ||||
common, | ||||
) | ||||
# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for | ||||
# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should | ||||
# be specifying the version(s) of Mercurial they are tested with, or | ||||
# leave the attribute unspecified. | ||||
|
r43347 | testedwith = b'ships-with-hg-core' | ||
|
r37204 | |||
configtable = {} | ||||
configitem = registrar.configitem(configtable) | ||||
|
r43346 | configitem( | ||
|
r43347 | b'infinitepush', b'server', default=False, | ||
|
r37204 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'infinitepush', b'storetype', default=b'', | ||
|
r37204 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'infinitepush', b'indextype', default=b'', | ||
|
r37204 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'infinitepush', b'indexpath', default=b'', | ||
|
r37204 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'infinitepush', b'storeallparts', default=False, | ||
|
r37204 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'infinitepush', b'reponame', default=b'', | ||
|
r37204 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'scratchbranch', b'storepath', default=b'', | ||
|
r37204 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'infinitepush', b'branchpattern', default=b'', | ||
|
r37204 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'infinitepush', b'pushtobundlestore', default=False, | ||
|
r37223 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'experimental', b'server-bundlestore-bookmark', default=b'', | ||
|
r37204 | ) | ||
|
r43346 | configitem( | ||
|
r43347 | b'experimental', b'infinitepush-scratchpush', default=False, | ||
|
r37204 | ) | ||
|
r43347 | experimental = b'experimental' | ||
configbookmark = b'server-bundlestore-bookmark' | ||||
configscratchpush = b'infinitepush-scratchpush' | ||||
|
r37204 | |||
scratchbranchparttype = bundleparts.scratchbranchparttype | ||||
|
r37205 | revsetpredicate = registrar.revsetpredicate() | ||
templatekeyword = registrar.templatekeyword() | ||||
|
r37204 | _scratchbranchmatcher = lambda x: False | ||
_maybehash = re.compile(r'^[a-f0-9]+$').search | ||||
|
r43346 | |||
|
r37204 | def _buildexternalbundlestore(ui): | ||
|
r43347 | put_args = ui.configlist(b'infinitepush', b'put_args', []) | ||
put_binary = ui.config(b'infinitepush', b'put_binary') | ||||
|
r37204 | if not put_binary: | ||
|
r43347 | raise error.Abort(b'put binary is not specified') | ||
get_args = ui.configlist(b'infinitepush', b'get_args', []) | ||||
get_binary = ui.config(b'infinitepush', b'get_binary') | ||||
|
r37204 | if not get_binary: | ||
|
r43347 | raise error.Abort(b'get binary is not specified') | ||
|
r37204 | from . import store | ||
|
r43346 | |||
|
r37204 | return store.externalbundlestore(put_binary, put_args, get_binary, get_args) | ||
|
r43346 | |||
|
r37204 | def _buildsqlindex(ui): | ||
|
r43347 | sqlhost = ui.config(b'infinitepush', b'sqlhost') | ||
|
r37204 | if not sqlhost: | ||
|
r43347 | raise error.Abort(_(b'please set infinitepush.sqlhost')) | ||
host, port, db, user, password = sqlhost.split(b':') | ||||
reponame = ui.config(b'infinitepush', b'reponame') | ||||
|
r37204 | if not reponame: | ||
|
r43347 | raise error.Abort(_(b'please set infinitepush.reponame')) | ||
|
r37204 | |||
|
r43347 | logfile = ui.config(b'infinitepush', b'logfile', b'') | ||
waittimeout = ui.configint(b'infinitepush', b'waittimeout', 300) | ||||
locktimeout = ui.configint(b'infinitepush', b'locktimeout', 120) | ||||
|
r37204 | from . import sqlindexapi | ||
|
r43346 | |||
|
r37204 | return sqlindexapi.sqlindexapi( | ||
|
r43346 | reponame, | ||
host, | ||||
port, | ||||
db, | ||||
user, | ||||
password, | ||||
logfile, | ||||
_getloglevel(ui), | ||||
waittimeout=waittimeout, | ||||
locktimeout=locktimeout, | ||||
) | ||||
|
r37204 | |||
def _getloglevel(ui): | ||||
|
r43347 | loglevel = ui.config(b'infinitepush', b'loglevel', b'DEBUG') | ||
|
r37204 | numeric_loglevel = getattr(logging, loglevel.upper(), None) | ||
if not isinstance(numeric_loglevel, int): | ||||
|
r43347 | raise error.Abort(_(b'invalid log level %s') % loglevel) | ||
|
r37204 | return numeric_loglevel | ||
|
r43346 | |||
|
r37204 | def _tryhoist(ui, remotebookmark): | ||
'''returns a bookmarks with hoisted part removed | ||||
Remotenames extension has a 'hoist' config that allows to use remote | ||||
bookmarks without specifying remote path. For example, 'hg update master' | ||||
works as well as 'hg update remote/master'. We want to allow the same in | ||||
infinitepush. | ||||
''' | ||||
if common.isremotebooksenabled(ui): | ||||
|
r43347 | hoist = ui.config(b'remotenames', b'hoistedpeer') + b'/' | ||
|
r37204 | if remotebookmark.startswith(hoist): | ||
|
r43346 | return remotebookmark[len(hoist) :] | ||
|
r37204 | return remotebookmark | ||
|
r43346 | |||
|
r37204 | class bundlestore(object): | ||
def __init__(self, repo): | ||||
self._repo = repo | ||||
|
r43347 | storetype = self._repo.ui.config(b'infinitepush', b'storetype') | ||
if storetype == b'disk': | ||||
|
r37204 | from . import store | ||
|
r43346 | |||
|
r37204 | self.store = store.filebundlestore(self._repo.ui, self._repo) | ||
|
r43347 | elif storetype == b'external': | ||
|
r37204 | self.store = _buildexternalbundlestore(self._repo.ui) | ||
else: | ||||
raise error.Abort( | ||||
|
r43347 | _(b'unknown infinitepush store type specified %s') % storetype | ||
|
r43346 | ) | ||
|
r37204 | |||
|
r43347 | indextype = self._repo.ui.config(b'infinitepush', b'indextype') | ||
if indextype == b'disk': | ||||
|
r37204 | from . import fileindexapi | ||
|
r43346 | |||
|
r37204 | self.index = fileindexapi.fileindexapi(self._repo) | ||
|
r43347 | elif indextype == b'sql': | ||
|
r37204 | self.index = _buildsqlindex(self._repo.ui) | ||
else: | ||||
raise error.Abort( | ||||
|
r43347 | _(b'unknown infinitepush index type specified %s') % indextype | ||
|
r43346 | ) | ||
|
r37204 | |||
def _isserver(ui): | ||||
|
r43347 | return ui.configbool(b'infinitepush', b'server') | ||
|
r37204 | |||
|
r43346 | |||
|
r37204 | def reposetup(ui, repo): | ||
if _isserver(ui) and repo.local(): | ||||
repo.bundlestore = bundlestore(repo) | ||||
|
r43346 | |||
|
r37204 | def extsetup(ui): | ||
commonsetup(ui) | ||||
if _isserver(ui): | ||||
serverextsetup(ui) | ||||
else: | ||||
clientextsetup(ui) | ||||
|
r43346 | |||
|
r37204 | def commonsetup(ui): | ||
|
r43347 | wireprotov1server.commands[b'listkeyspatterns'] = ( | ||
|
r43346 | wireprotolistkeyspatterns, | ||
|
r43347 | b'namespace patterns', | ||
|
r43346 | ) | ||
|
r43347 | scratchbranchpat = ui.config(b'infinitepush', b'branchpattern') | ||
|
r37204 | if scratchbranchpat: | ||
global _scratchbranchmatcher | ||||
|
r43346 | kind, pat, _scratchbranchmatcher = stringutil.stringmatcher( | ||
scratchbranchpat | ||||
) | ||||
|
r37204 | |||
def serverextsetup(ui): | ||||
|
r43347 | origpushkeyhandler = bundle2.parthandlermapping[b'pushkey'] | ||
|
r37204 | |||
def newpushkeyhandler(*args, **kwargs): | ||||
bundle2pushkey(origpushkeyhandler, *args, **kwargs) | ||||
|
r43346 | |||
|
r37204 | newpushkeyhandler.params = origpushkeyhandler.params | ||
|
r43347 | bundle2.parthandlermapping[b'pushkey'] = newpushkeyhandler | ||
|
r37204 | |||
|
r43347 | orighandlephasehandler = bundle2.parthandlermapping[b'phase-heads'] | ||
|
r41925 | newphaseheadshandler = lambda *args, **kwargs: bundle2handlephases( | ||
|
r43346 | orighandlephasehandler, *args, **kwargs | ||
) | ||||
|
r37204 | newphaseheadshandler.params = orighandlephasehandler.params | ||
|
r43347 | bundle2.parthandlermapping[b'phase-heads'] = newphaseheadshandler | ||
|
r37204 | |||
|
r43346 | extensions.wrapfunction( | ||
|
r43347 | localrepo.localrepository, b'listkeys', localrepolistkeys | ||
|
r43346 | ) | ||
|
r43347 | wireprotov1server.commands[b'lookup'] = ( | ||
_lookupwrap(wireprotov1server.commands[b'lookup'][0]), | ||||
b'key', | ||||
|
r43346 | ) | ||
|
r43347 | extensions.wrapfunction(exchange, b'getbundlechunks', getbundlechunks) | ||
|
r37204 | |||
|
r43347 | extensions.wrapfunction(bundle2, b'processparts', processparts) | ||
|
r37204 | |||
|
r43346 | |||
|
r37204 | def clientextsetup(ui): | ||
|
r43347 | entry = extensions.wrapcommand(commands.table, b'push', _push) | ||
|
r37204 | |||
entry[1].append( | ||||
|
r43346 | ( | ||
|
r43347 | b'', | ||
b'bundle-store', | ||||
|
r43346 | None, | ||
|
r43347 | _(b'force push to go to bundle store (EXPERIMENTAL)'), | ||
|
r43346 | ) | ||
) | ||||
|
r37204 | |||
|
r43347 | extensions.wrapcommand(commands.table, b'pull', _pull) | ||
|
r37204 | |||
|
r43347 | extensions.wrapfunction(discovery, b'checkheads', _checkheads) | ||
|
r37204 | |||
|
r37632 | wireprotov1peer.wirepeer.listkeyspatterns = listkeyspatterns | ||
|
r37204 | |||
partorder = exchange.b2partsgenorder | ||||
|
r43347 | index = partorder.index(b'changeset') | ||
|
r37204 | partorder.insert( | ||
|
r43346 | index, partorder.pop(partorder.index(scratchbranchparttype)) | ||
) | ||||
|
r37204 | |||
def _checkheads(orig, pushop): | ||||
if pushop.ui.configbool(experimental, configscratchpush, False): | ||||
return | ||||
return orig(pushop) | ||||
|
r43346 | |||
|
r37204 | def wireprotolistkeyspatterns(repo, proto, namespace, patterns): | ||
|
r37630 | patterns = wireprototypes.decodelist(patterns) | ||
|
r43375 | d = pycompat.iteritems(repo.listkeys(encoding.tolocal(namespace), patterns)) | ||
|
r37204 | return pushkey.encodekeys(d) | ||
|
r43346 | |||
|
r37204 | def localrepolistkeys(orig, self, namespace, patterns=None): | ||
|
r43347 | if namespace == b'bookmarks' and patterns: | ||
|
r37204 | index = self.bundlestore.index | ||
results = {} | ||||
bookmarks = orig(self, namespace) | ||||
for pattern in patterns: | ||||
results.update(index.getbookmarks(pattern)) | ||||
|
r43347 | if pattern.endswith(b'*'): | ||
pattern = b're:^' + pattern[:-1] + b'.*' | ||||
|
r37221 | kind, pat, matcher = stringutil.stringmatcher(pattern) | ||
|
r43375 | for bookmark, node in pycompat.iteritems(bookmarks): | ||
|
r37204 | if matcher(bookmark): | ||
results[bookmark] = node | ||||
return results | ||||
else: | ||||
return orig(self, namespace) | ||||
|
r43346 | |||
|
r37633 | @wireprotov1peer.batchable | ||
|
r37204 | def listkeyspatterns(self, namespace, patterns): | ||
|
r43347 | if not self.capable(b'pushkey'): | ||
|
r37204 | yield {}, None | ||
|
r37633 | f = wireprotov1peer.future() | ||
|
r43347 | self.ui.debug(b'preparing listkeys for "%s"\n' % namespace) | ||
|
r37204 | yield { | ||
|
r43347 | b'namespace': encoding.fromlocal(namespace), | ||
b'patterns': wireprototypes.encodelist(patterns), | ||||
|
r37204 | }, f | ||
d = f.value | ||||
|
r43347 | self.ui.debug( | ||
b'received listkey for "%s": %i bytes\n' % (namespace, len(d)) | ||||
) | ||||
|
r37204 | yield pushkey.decodekeys(d) | ||
|
r43346 | |||
|
r37204 | def _readbundlerevs(bundlerepo): | ||
|
r43347 | return list(bundlerepo.revs(b'bundle()')) | ||
|
r37204 | |||
|
r43346 | |||
|
r37204 | def _includefilelogstobundle(bundlecaps, bundlerepo, bundlerevs, ui): | ||
'''Tells remotefilelog to include all changed files to the changegroup | ||||
By default remotefilelog doesn't include file content to the changegroup. | ||||
But we need to include it if we are fetching from bundlestore. | ||||
''' | ||||
changedfiles = set() | ||||
cl = bundlerepo.changelog | ||||
for r in bundlerevs: | ||||
# [3] means changed files | ||||
changedfiles.update(cl.read(r)[3]) | ||||
if not changedfiles: | ||||
return bundlecaps | ||||
|
r43347 | changedfiles = b'\0'.join(changedfiles) | ||
|
r37204 | newcaps = [] | ||
appended = False | ||||
|
r43346 | for cap in bundlecaps or []: | ||
|
r43347 | if cap.startswith(b'excludepattern='): | ||
newcaps.append(b'\0'.join((cap, changedfiles))) | ||||
|
r37204 | appended = True | ||
else: | ||||
newcaps.append(cap) | ||||
if not appended: | ||||
# Not found excludepattern cap. Just append it | ||||
|
r43347 | newcaps.append(b'excludepattern=' + changedfiles) | ||
|
r37204 | |||
return newcaps | ||||
|
r43346 | |||
|
r37204 | def _rebundle(bundlerepo, bundleroots, unknownhead): | ||
''' | ||||
Bundle may include more revision then user requested. For example, | ||||
if user asks for revision but bundle also consists its descendants. | ||||
This function will filter out all revision that user is not requested. | ||||
''' | ||||
parts = [] | ||||
|
r43347 | version = b'02' | ||
|
r43346 | outgoing = discovery.outgoing( | ||
bundlerepo, commonheads=bundleroots, missingheads=[unknownhead] | ||||
) | ||||
|
r43347 | cgstream = changegroup.makestream(bundlerepo, outgoing, version, b'pull') | ||
|
r37204 | cgstream = util.chunkbuffer(cgstream).read() | ||
|
r43347 | cgpart = bundle2.bundlepart(b'changegroup', data=cgstream) | ||
cgpart.addparam(b'version', version) | ||||
|
r37204 | parts.append(cgpart) | ||
return parts | ||||
|
r43346 | |||
|
r37204 | def _getbundleroots(oldrepo, bundlerepo, bundlerevs): | ||
cl = bundlerepo.changelog | ||||
bundleroots = [] | ||||
for rev in bundlerevs: | ||||
node = cl.node(rev) | ||||
parents = cl.parents(node) | ||||
for parent in parents: | ||||
# include all revs that exist in the main repo | ||||
# to make sure that bundle may apply client-side | ||||
if parent in oldrepo: | ||||
bundleroots.append(parent) | ||||
return bundleroots | ||||
|
r43346 | |||
|
r37204 | def _needsrebundling(head, bundlerepo): | ||
|
r43347 | bundleheads = list(bundlerepo.revs(b'heads(bundle())')) | ||
|
r43346 | return not ( | ||
len(bundleheads) == 1 and bundlerepo[bundleheads[0]].node() == head | ||||
) | ||||
|
r37204 | |||
def _generateoutputparts(head, bundlerepo, bundleroots, bundlefile): | ||||
'''generates bundle that will be send to the user | ||||
returns tuple with raw bundle string and bundle type | ||||
''' | ||||
parts = [] | ||||
if not _needsrebundling(head, bundlerepo): | ||||
|
r43347 | with util.posixfile(bundlefile, b"rb") as f: | ||
|
r37204 | unbundler = exchange.readbundle(bundlerepo.ui, f, bundlefile) | ||
if isinstance(unbundler, changegroup.cg1unpacker): | ||||
|
r43346 | part = bundle2.bundlepart( | ||
|
r43347 | b'changegroup', data=unbundler._stream.read() | ||
|
r43346 | ) | ||
|
r43347 | part.addparam(b'version', b'01') | ||
|
r37204 | parts.append(part) | ||
elif isinstance(unbundler, bundle2.unbundle20): | ||||
haschangegroup = False | ||||
for part in unbundler.iterparts(): | ||||
|
r43347 | if part.type == b'changegroup': | ||
|
r37204 | haschangegroup = True | ||
newpart = bundle2.bundlepart(part.type, data=part.read()) | ||||
|
r43375 | for key, value in pycompat.iteritems(part.params): | ||
|
r37204 | newpart.addparam(key, value) | ||
parts.append(newpart) | ||||
if not haschangegroup: | ||||
raise error.Abort( | ||||
|
r43347 | b'unexpected bundle without changegroup part, ' | ||
+ b'head: %s' % hex(head), | ||||
hint=b'report to administrator', | ||||
|
r43346 | ) | ||
|
r37204 | else: | ||
|
r43347 | raise error.Abort(b'unknown bundle type') | ||
|
r37204 | else: | ||
parts = _rebundle(bundlerepo, bundleroots, head) | ||||
return parts | ||||
|
r43346 | |||
|
r37204 | def getbundlechunks(orig, repo, source, heads=None, bundlecaps=None, **kwargs): | ||
heads = heads or [] | ||||
# newheads are parents of roots of scratch bundles that were requested | ||||
newphases = {} | ||||
scratchbundles = [] | ||||
newheads = [] | ||||
scratchheads = [] | ||||
nodestobundle = {} | ||||
allbundlestocleanup = [] | ||||
try: | ||||
for head in heads: | ||||
if head not in repo.changelog.nodemap: | ||||
if head not in nodestobundle: | ||||
newbundlefile = common.downloadbundle(repo, head) | ||||
|
r43347 | bundlepath = b"bundle:%s+%s" % (repo.root, newbundlefile) | ||
|
r37204 | bundlerepo = hg.repository(repo.ui, bundlepath) | ||
allbundlestocleanup.append((bundlerepo, newbundlefile)) | ||||
bundlerevs = set(_readbundlerevs(bundlerepo)) | ||||
bundlecaps = _includefilelogstobundle( | ||||
|
r43346 | bundlecaps, bundlerepo, bundlerevs, repo.ui | ||
) | ||||
|
r37204 | cl = bundlerepo.changelog | ||
bundleroots = _getbundleroots(repo, bundlerepo, bundlerevs) | ||||
for rev in bundlerevs: | ||||
node = cl.node(rev) | ||||
newphases[hex(node)] = str(phases.draft) | ||||
|
r43346 | nodestobundle[node] = ( | ||
bundlerepo, | ||||
bundleroots, | ||||
newbundlefile, | ||||
) | ||||
|
r37204 | |||
scratchbundles.append( | ||||
|
r43346 | _generateoutputparts(head, *nodestobundle[head]) | ||
) | ||||
|
r37204 | newheads.extend(bundleroots) | ||
scratchheads.append(head) | ||||
finally: | ||||
for bundlerepo, bundlefile in allbundlestocleanup: | ||||
bundlerepo.close() | ||||
try: | ||||
os.unlink(bundlefile) | ||||
except (IOError, OSError): | ||||
# if we can't cleanup the file then just ignore the error, | ||||
# no need to fail | ||||
pass | ||||
pullfrombundlestore = bool(scratchbundles) | ||||
wrappedchangegrouppart = False | ||||
wrappedlistkeys = False | ||||
|
r43347 | oldchangegrouppart = exchange.getbundle2partsmapping[b'changegroup'] | ||
|
r37204 | try: | ||
|
r43346 | |||
|
r37204 | def _changegrouppart(bundler, *args, **kwargs): | ||
# Order is important here. First add non-scratch part | ||||
# and only then add parts with scratch bundles because | ||||
# non-scratch part contains parents of roots of scratch bundles. | ||||
result = oldchangegrouppart(bundler, *args, **kwargs) | ||||
for bundle in scratchbundles: | ||||
for part in bundle: | ||||
bundler.addpart(part) | ||||
return result | ||||
|
r43347 | exchange.getbundle2partsmapping[b'changegroup'] = _changegrouppart | ||
|
r37204 | wrappedchangegrouppart = True | ||
def _listkeys(orig, self, namespace): | ||||
origvalues = orig(self, namespace) | ||||
|
r43347 | if namespace == b'phases' and pullfrombundlestore: | ||
if origvalues.get(b'publishing') == b'True': | ||||
|
r37204 | # Make repo non-publishing to preserve draft phase | ||
|
r43347 | del origvalues[b'publishing'] | ||
|
r37204 | origvalues.update(newphases) | ||
return origvalues | ||||
|
r43346 | extensions.wrapfunction( | ||
|
r43347 | localrepo.localrepository, b'listkeys', _listkeys | ||
|
r43346 | ) | ||
|
r37204 | wrappedlistkeys = True | ||
heads = list((set(newheads) | set(heads)) - set(scratchheads)) | ||||
|
r43346 | result = orig( | ||
repo, source, heads=heads, bundlecaps=bundlecaps, **kwargs | ||||
) | ||||
|
r37204 | finally: | ||
if wrappedchangegrouppart: | ||||
|
r43347 | exchange.getbundle2partsmapping[b'changegroup'] = oldchangegrouppart | ||
|
r37204 | if wrappedlistkeys: | ||
|
r43346 | extensions.unwrapfunction( | ||
|
r43347 | localrepo.localrepository, b'listkeys', _listkeys | ||
|
r43346 | ) | ||
|
r37204 | return result | ||
|
r43346 | |||
|
r37204 | def _lookupwrap(orig): | ||
def _lookup(repo, proto, key): | ||||
localkey = encoding.tolocal(key) | ||||
if isinstance(localkey, str) and _scratchbranchmatcher(localkey): | ||||
scratchnode = repo.bundlestore.index.getnode(localkey) | ||||
if scratchnode: | ||||
|
r43347 | return b"%d %s\n" % (1, scratchnode) | ||
|
r37204 | else: | ||
|
r43347 | return b"%d %s\n" % ( | ||
0, | ||||
b'scratch branch %s not found' % localkey, | ||||
) | ||||
|
r37204 | else: | ||
try: | ||||
r = hex(repo.lookup(localkey)) | ||||
|
r43347 | return b"%d %s\n" % (1, r) | ||
|
r37204 | except Exception as inst: | ||
if repo.bundlestore.index.getbundle(localkey): | ||||
|
r43347 | return b"%d %s\n" % (1, localkey) | ||
|
r37204 | else: | ||
|
r38094 | r = stringutil.forcebytestr(inst) | ||
|
r43347 | return b"%d %s\n" % (0, r) | ||
|
r43346 | |||
|
r37204 | return _lookup | ||
|
r43346 | |||
|
r43347 | def _pull(orig, ui, repo, source=b"default", **opts): | ||
|
r37595 | opts = pycompat.byteskwargs(opts) | ||
|
r37204 | # Copy paste from `pull` command | ||
|
r43347 | source, branches = hg.parseurl(ui.expandpath(source), opts.get(b'branch')) | ||
|
r37204 | |||
scratchbookmarks = {} | ||||
unfi = repo.unfiltered() | ||||
unknownnodes = [] | ||||
|
r43347 | for rev in opts.get(b'rev', []): | ||
|
r37204 | if rev not in unfi: | ||
unknownnodes.append(rev) | ||||
|
r43347 | if opts.get(b'bookmark'): | ||
|
r37204 | bookmarks = [] | ||
|
r43347 | revs = opts.get(b'rev') or [] | ||
for bookmark in opts.get(b'bookmark'): | ||||
|
r37204 | if _scratchbranchmatcher(bookmark): | ||
# rev is not known yet | ||||
# it will be fetched with listkeyspatterns next | ||||
|
r43347 | scratchbookmarks[bookmark] = b'REVTOFETCH' | ||
|
r37204 | else: | ||
bookmarks.append(bookmark) | ||||
if scratchbookmarks: | ||||
other = hg.peer(repo, opts, source) | ||||
fetchedbookmarks = other.listkeyspatterns( | ||||
|
r43347 | b'bookmarks', patterns=scratchbookmarks | ||
|
r43346 | ) | ||
|
r37204 | for bookmark in scratchbookmarks: | ||
if bookmark not in fetchedbookmarks: | ||||
|
r43346 | raise error.Abort( | ||
|
r43347 | b'remote bookmark %s not found!' % bookmark | ||
|
r43346 | ) | ||
|
r37204 | scratchbookmarks[bookmark] = fetchedbookmarks[bookmark] | ||
revs.append(fetchedbookmarks[bookmark]) | ||||
|
r43347 | opts[b'bookmark'] = bookmarks | ||
opts[b'rev'] = revs | ||||
|
r37204 | |||
if scratchbookmarks or unknownnodes: | ||||
# Set anyincoming to True | ||||
|
r43346 | extensions.wrapfunction( | ||
|
r43347 | discovery, b'findcommonincoming', _findcommonincoming | ||
|
r43346 | ) | ||
|
r37204 | try: | ||
# Remote scratch bookmarks will be deleted because remotenames doesn't | ||||
# know about them. Let's save it before pull and restore after | ||||
remotescratchbookmarks = _readscratchremotebookmarks(ui, repo, source) | ||||
|
r37595 | result = orig(ui, repo, source, **pycompat.strkwargs(opts)) | ||
|
r37204 | # TODO(stash): race condition is possible | ||
# if scratch bookmarks was updated right after orig. | ||||
# But that's unlikely and shouldn't be harmful. | ||||
if common.isremotebooksenabled(ui): | ||||
remotescratchbookmarks.update(scratchbookmarks) | ||||
_saveremotebookmarks(repo, remotescratchbookmarks, source) | ||||
else: | ||||
_savelocalbookmarks(repo, scratchbookmarks) | ||||
return result | ||||
finally: | ||||
if scratchbookmarks: | ||||
|
r43347 | extensions.unwrapfunction(discovery, b'findcommonincoming') | ||
|
r37204 | |||
|
r43346 | |||
|
r37204 | def _readscratchremotebookmarks(ui, repo, other): | ||
if common.isremotebooksenabled(ui): | ||||
|
r43347 | remotenamesext = extensions.find(b'remotenames') | ||
|
r37204 | remotepath = remotenamesext.activepath(repo.ui, other) | ||
result = {} | ||||
# Let's refresh remotenames to make sure we have it up to date | ||||
# Seems that `repo.names['remotebookmarks']` may return stale bookmarks | ||||
# and it results in deleting scratch bookmarks. Our best guess how to | ||||
# fix it is to use `clearnames()` | ||||
repo._remotenames.clearnames() | ||||
|
r43347 | for remotebookmark in repo.names[b'remotebookmarks'].listnames(repo): | ||
|
r37204 | path, bookname = remotenamesext.splitremotename(remotebookmark) | ||
if path == remotepath and _scratchbranchmatcher(bookname): | ||||
|
r43347 | nodes = repo.names[b'remotebookmarks'].nodes( | ||
|
r43346 | repo, remotebookmark | ||
) | ||||
|
r37204 | if nodes: | ||
result[bookname] = hex(nodes[0]) | ||||
return result | ||||
else: | ||||
return {} | ||||
|
r43346 | |||
|
r37204 | def _saveremotebookmarks(repo, newbookmarks, remote): | ||
|
r43347 | remotenamesext = extensions.find(b'remotenames') | ||
|
r37204 | remotepath = remotenamesext.activepath(repo.ui, remote) | ||
branches = collections.defaultdict(list) | ||||
bookmarks = {} | ||||
remotenames = remotenamesext.readremotenames(repo) | ||||
for hexnode, nametype, remote, rname in remotenames: | ||||
if remote != remotepath: | ||||
continue | ||||
|
r43347 | if nametype == b'bookmarks': | ||
|
r37204 | if rname in newbookmarks: | ||
# It's possible if we have a normal bookmark that matches | ||||
# scratch branch pattern. In this case just use the current | ||||
# bookmark node | ||||
del newbookmarks[rname] | ||||
bookmarks[rname] = hexnode | ||||
|
r43347 | elif nametype == b'branches': | ||
|
r37204 | # saveremotenames expects 20 byte binary nodes for branches | ||
branches[rname].append(bin(hexnode)) | ||||
|
r43375 | for bookmark, hexnode in pycompat.iteritems(newbookmarks): | ||
|
r37204 | bookmarks[bookmark] = hexnode | ||
remotenamesext.saveremotenames(repo, remotepath, branches, bookmarks) | ||||
|
r43346 | |||
|
r37204 | def _savelocalbookmarks(repo, bookmarks): | ||
if not bookmarks: | ||||
return | ||||
|
r43347 | with repo.wlock(), repo.lock(), repo.transaction(b'bookmark') as tr: | ||
|
r37204 | changes = [] | ||
|
r43375 | for scratchbook, node in pycompat.iteritems(bookmarks): | ||
|
r37204 | changectx = repo[node] | ||
changes.append((scratchbook, changectx.node())) | ||||
repo._bookmarks.applychanges(repo, tr, changes) | ||||
|
r43346 | |||
|
r37204 | def _findcommonincoming(orig, *args, **kwargs): | ||
common, inc, remoteheads = orig(*args, **kwargs) | ||||
return common, True, remoteheads | ||||
|
r43346 | |||
|
r37204 | def _push(orig, ui, repo, dest=None, *args, **opts): | ||
|
r40288 | opts = pycompat.byteskwargs(opts) | ||
|
r43347 | bookmark = opts.get(b'bookmark') | ||
|
r37216 | # we only support pushing one infinitepush bookmark at once | ||
if len(bookmark) == 1: | ||||
bookmark = bookmark[0] | ||||
else: | ||||
|
r43347 | bookmark = b'' | ||
|
r37204 | |||
oldphasemove = None | ||||
|
r37215 | overrides = {(experimental, configbookmark): bookmark} | ||
|
r37204 | |||
|
r43347 | with ui.configoverride(overrides, b'infinitepush'): | ||
scratchpush = opts.get(b'bundle_store') | ||||
|
r37204 | if _scratchbranchmatcher(bookmark): | ||
scratchpush = True | ||||
# bundle2 can be sent back after push (for example, bundle2 | ||||
# containing `pushkey` part to update bookmarks) | ||||
|
r43347 | ui.setconfig(experimental, b'bundle2.pushback', True) | ||
|
r37204 | |||
if scratchpush: | ||||
|
r37216 | # this is an infinitepush, we don't want the bookmark to be applied | ||
# rather that should be stored in the bundlestore | ||||
|
r43347 | opts[b'bookmark'] = [] | ||
|
r37204 | ui.setconfig(experimental, configscratchpush, True) | ||
|
r43346 | oldphasemove = extensions.wrapfunction( | ||
|
r43347 | exchange, b'_localphasemove', _phasemove | ||
|
r43346 | ) | ||
|
r37204 | # Copy-paste from `push` command | ||
|
r43347 | path = ui.paths.getpath(dest, default=(b'default-push', b'default')) | ||
|
r37204 | if not path: | ||
|
r43346 | raise error.Abort( | ||
|
r43347 | _(b'default repository not configured!'), | ||
hint=_(b"see 'hg help config.paths'"), | ||||
|
r43346 | ) | ||
|
r37204 | destpath = path.pushloc or path.loc | ||
# Remote scratch bookmarks will be deleted because remotenames doesn't | ||||
# know about them. Let's save it before push and restore after | ||||
remotescratchbookmarks = _readscratchremotebookmarks(ui, repo, destpath) | ||||
|
r40288 | result = orig(ui, repo, dest, *args, **pycompat.strkwargs(opts)) | ||
|
r37204 | if common.isremotebooksenabled(ui): | ||
if bookmark and scratchpush: | ||||
other = hg.peer(repo, opts, destpath) | ||||
|
r43346 | fetchedbookmarks = other.listkeyspatterns( | ||
|
r43347 | b'bookmarks', patterns=[bookmark] | ||
|
r43346 | ) | ||
|
r37204 | remotescratchbookmarks.update(fetchedbookmarks) | ||
_saveremotebookmarks(repo, remotescratchbookmarks, destpath) | ||||
if oldphasemove: | ||||
exchange._localphasemove = oldphasemove | ||||
return result | ||||
|
r43346 | |||
|
r37204 | def _deleteinfinitepushbookmarks(ui, repo, path, names): | ||
"""Prune remote names by removing the bookmarks we don't want anymore, | ||||
then writing the result back to disk | ||||
""" | ||||
|
r43347 | remotenamesext = extensions.find(b'remotenames') | ||
|
r37204 | |||
# remotename format is: | ||||
# (node, nametype ("branches" or "bookmarks"), remote, name) | ||||
nametype_idx = 1 | ||||
remote_idx = 2 | ||||
name_idx = 3 | ||||
|
r43346 | remotenames = [ | ||
remotename | ||||
for remotename in remotenamesext.readremotenames(repo) | ||||
if remotename[remote_idx] == path | ||||
] | ||||
remote_bm_names = [ | ||||
remotename[name_idx] | ||||
for remotename in remotenames | ||||
|
r43347 | if remotename[nametype_idx] == b"bookmarks" | ||
|
r43346 | ] | ||
|
r37204 | |||
for name in names: | ||||
if name not in remote_bm_names: | ||||
|
r43346 | raise error.Abort( | ||
_( | ||||
|
r43347 | b"infinitepush bookmark '{}' does not exist " | ||
b"in path '{}'" | ||||
|
r43346 | ).format(name, path) | ||
) | ||||
|
r37204 | |||
bookmarks = {} | ||||
branches = collections.defaultdict(list) | ||||
for node, nametype, remote, name in remotenames: | ||||
|
r43347 | if nametype == b"bookmarks" and name not in names: | ||
|
r37204 | bookmarks[name] = node | ||
|
r43347 | elif nametype == b"branches": | ||
|
r37204 | # saveremotenames wants binary nodes for branches | ||
branches[name].append(bin(node)) | ||||
remotenamesext.saveremotenames(repo, path, branches, bookmarks) | ||||
|
r43346 | |||
|
r37204 | def _phasemove(orig, pushop, nodes, phase=phases.public): | ||
"""prevent commits from being marked public | ||||
Since these are going to a scratch branch, they aren't really being | ||||
published.""" | ||||
if phase != phases.public: | ||||
orig(pushop, nodes, phase) | ||||
|
r43346 | |||
|
r37204 | @exchange.b2partsgenerator(scratchbranchparttype) | ||
def partgen(pushop, bundler): | ||||
bookmark = pushop.ui.config(experimental, configbookmark) | ||||
scratchpush = pushop.ui.configbool(experimental, configscratchpush) | ||||
|
r43347 | if b'changesets' in pushop.stepsdone or not scratchpush: | ||
|
r37204 | return | ||
if scratchbranchparttype not in bundle2.bundle2caps(pushop.remote): | ||||
return | ||||
|
r43347 | pushop.stepsdone.add(b'changesets') | ||
|
r37204 | if not pushop.outgoing.missing: | ||
|
r43347 | pushop.ui.status(_(b'no changes found\n')) | ||
|
r37204 | pushop.cgresult = 0 | ||
return | ||||
# This parameter tells the server that the following bundle is an | ||||
# infinitepush. This let's it switch the part processing to our infinitepush | ||||
# code path. | ||||
|
r43347 | bundler.addparam(b"infinitepush", b"True") | ||
|
r37204 | |||
|
r43346 | scratchparts = bundleparts.getscratchbranchparts( | ||
pushop.repo, pushop.remote, pushop.outgoing, pushop.ui, bookmark | ||||
) | ||||
|
r37204 | |||
for scratchpart in scratchparts: | ||||
bundler.addpart(scratchpart) | ||||
def handlereply(op): | ||||
# server either succeeds or aborts; no code to read | ||||
pushop.cgresult = 1 | ||||
return handlereply | ||||
|
r43346 | |||
|
r37204 | bundle2.capabilities[bundleparts.scratchbranchparttype] = () | ||
|
r43346 | |||
|
r37204 | def _getrevs(bundle, oldnode, force, bookmark): | ||
|
r43347 | b'extracts and validates the revs to be imported' | ||
revs = [bundle[r] for r in bundle.revs(b'sort(bundle())')] | ||||
|
r37204 | |||
# new bookmark | ||||
if oldnode is None: | ||||
return revs | ||||
# Fast forward update | ||||
|
r43347 | if oldnode in bundle and list(bundle.set(b'bundle() & %s::', oldnode)): | ||
|
r37204 | return revs | ||
|
r37220 | return revs | ||
|
r37204 | |||
|
r43346 | |||
|
r37204 | @contextlib.contextmanager | ||
def logservicecall(logger, service, **kwargs): | ||||
start = time.time() | ||||
|
r43347 | logger(service, eventtype=b'start', **kwargs) | ||
|
r37204 | try: | ||
yield | ||||
|
r43346 | logger( | ||
service, | ||||
|
r43347 | eventtype=b'success', | ||
|
r43346 | elapsedms=(time.time() - start) * 1000, | ||
**kwargs | ||||
) | ||||
|
r37204 | except Exception as e: | ||
|
r43346 | logger( | ||
service, | ||||
|
r43347 | eventtype=b'failure', | ||
|
r43346 | elapsedms=(time.time() - start) * 1000, | ||
errormsg=str(e), | ||||
**kwargs | ||||
) | ||||
|
r37204 | raise | ||
|
r43346 | |||
|
r37204 | def _getorcreateinfinitepushlogger(op): | ||
|
r43347 | logger = op.records[b'infinitepushlogger'] | ||
|
r37204 | if not logger: | ||
ui = op.repo.ui | ||||
try: | ||||
|
r37221 | username = procutil.getuser() | ||
|
r37204 | except Exception: | ||
|
r43347 | username = b'unknown' | ||
|
r37204 | # Generate random request id to be able to find all logged entries | ||
# for the same request. Since requestid is pseudo-generated it may | ||||
# not be unique, but we assume that (hostname, username, requestid) | ||||
# is unique. | ||||
random.seed() | ||||
requestid = random.randint(0, 2000000000) | ||||
hostname = socket.gethostname() | ||||
|
r43346 | logger = functools.partial( | ||
ui.log, | ||||
|
r43347 | b'infinitepush', | ||
|
r43346 | user=username, | ||
requestid=requestid, | ||||
hostname=hostname, | ||||
|
r43347 | reponame=ui.config(b'infinitepush', b'reponame'), | ||
|
r43346 | ) | ||
|
r43347 | op.records.add(b'infinitepushlogger', logger) | ||
|
r37204 | else: | ||
logger = logger[0] | ||||
return logger | ||||
|
r43346 | |||
|
r37223 | def storetobundlestore(orig, repo, op, unbundler): | ||
"""stores the incoming bundle coming from push command to the bundlestore | ||||
instead of applying on the revlogs""" | ||||
|
r43347 | repo.ui.status(_(b"storing changesets on the bundlestore\n")) | ||
|
r37223 | bundler = bundle2.bundle20(repo.ui) | ||
# processing each part and storing it in bundler | ||||
with bundle2.partiterator(repo, op, unbundler) as parts: | ||||
for part in parts: | ||||
bundlepart = None | ||||
|
r43347 | if part.type == b'replycaps': | ||
|
r37223 | # This configures the current operation to allow reply parts. | ||
bundle2._processpart(op, part) | ||||
else: | ||||
bundlepart = bundle2.bundlepart(part.type, data=part.read()) | ||||
|
r43375 | for key, value in pycompat.iteritems(part.params): | ||
|
r37223 | bundlepart.addparam(key, value) | ||
# Certain parts require a response | ||||
|
r43347 | if part.type in (b'pushkey', b'changegroup'): | ||
|
r37223 | if op.reply is not None: | ||
|
r43347 | rpart = op.reply.newpart(b'reply:%s' % part.type) | ||
|
r43346 | rpart.addparam( | ||
|
r43347 | b'in-reply-to', b'%d' % part.id, mandatory=False | ||
|
r43346 | ) | ||
|
r43347 | rpart.addparam(b'return', b'1', mandatory=False) | ||
|
r37223 | |||
|
r43347 | op.records.add(part.type, {b'return': 1,}) | ||
|
r37223 | if bundlepart: | ||
bundler.addpart(bundlepart) | ||||
# storing the bundle in the bundlestore | ||||
buf = util.chunkbuffer(bundler.getchunks()) | ||||
|
r38182 | fd, bundlefile = pycompat.mkstemp() | ||
|
r37223 | try: | ||
try: | ||||
|
r37596 | fp = os.fdopen(fd, r'wb') | ||
|
r37223 | fp.write(buf.read()) | ||
finally: | ||||
fp.close() | ||||
storebundle(op, {}, bundlefile) | ||||
finally: | ||||
try: | ||||
os.unlink(bundlefile) | ||||
except Exception: | ||||
# we would rather see the original exception | ||||
pass | ||||
|
r43346 | |||
|
r37204 | def processparts(orig, repo, op, unbundler): | ||
|
r37222 | |||
# make sure we don't wrap processparts in case of `hg unbundle` | ||||
|
r43347 | if op.source == b'unbundle': | ||
|
r37256 | return orig(repo, op, unbundler) | ||
|
r37222 | |||
|
r37223 | # this server routes each push to bundle store | ||
|
r43347 | if repo.ui.configbool(b'infinitepush', b'pushtobundlestore'): | ||
|
r37223 | return storetobundlestore(orig, repo, op, unbundler) | ||
|
r43347 | if unbundler.params.get(b'infinitepush') != b'True': | ||
|
r37204 | return orig(repo, op, unbundler) | ||
|
r43347 | handleallparts = repo.ui.configbool(b'infinitepush', b'storeallparts') | ||
|
r37204 | |||
bundler = bundle2.bundle20(repo.ui) | ||||
cgparams = None | ||||
with bundle2.partiterator(repo, op, unbundler) as parts: | ||||
for part in parts: | ||||
bundlepart = None | ||||
|
r43347 | if part.type == b'replycaps': | ||
|
r37204 | # This configures the current operation to allow reply parts. | ||
bundle2._processpart(op, part) | ||||
elif part.type == bundleparts.scratchbranchparttype: | ||||
# Scratch branch parts need to be converted to normal | ||||
# changegroup parts, and the extra parameters stored for later | ||||
# when we upload to the store. Eventually those parameters will | ||||
# be put on the actual bundle instead of this part, then we can | ||||
# send a vanilla changegroup instead of the scratchbranch part. | ||||
|
r43347 | cgversion = part.params.get(b'cgversion', b'01') | ||
bundlepart = bundle2.bundlepart( | ||||
b'changegroup', data=part.read() | ||||
) | ||||
bundlepart.addparam(b'version', cgversion) | ||||
|
r37204 | cgparams = part.params | ||
# If we're not dumping all parts into the new bundle, we need to | ||||
# alert the future pushkey and phase-heads handler to skip | ||||
# the part. | ||||
if not handleallparts: | ||||
|
r43346 | op.records.add( | ||
|
r43347 | scratchbranchparttype + b'_skippushkey', True | ||
) | ||||
op.records.add( | ||||
scratchbranchparttype + b'_skipphaseheads', True | ||||
|
r43346 | ) | ||
|
r37204 | else: | ||
|
r37214 | if handleallparts: | ||
|
r37204 | # Ideally we would not process any parts, and instead just | ||
# forward them to the bundle for storage, but since this | ||||
# differs from previous behavior, we need to put it behind a | ||||
# config flag for incremental rollout. | ||||
bundlepart = bundle2.bundlepart(part.type, data=part.read()) | ||||
|
r43375 | for key, value in pycompat.iteritems(part.params): | ||
|
r37204 | bundlepart.addparam(key, value) | ||
# Certain parts require a response | ||||
|
r43347 | if part.type == b'pushkey': | ||
|
r37204 | if op.reply is not None: | ||
|
r43347 | rpart = op.reply.newpart(b'reply:pushkey') | ||
|
r43346 | rpart.addparam( | ||
|
r43347 | b'in-reply-to', str(part.id), mandatory=False | ||
|
r43346 | ) | ||
|
r43347 | rpart.addparam(b'return', b'1', mandatory=False) | ||
|
r37204 | else: | ||
bundle2._processpart(op, part) | ||||
if handleallparts: | ||||
|
r43347 | op.records.add(part.type, {b'return': 1,}) | ||
|
r37204 | if bundlepart: | ||
bundler.addpart(bundlepart) | ||||
# If commits were sent, store them | ||||
if cgparams: | ||||
buf = util.chunkbuffer(bundler.getchunks()) | ||||
|
r38182 | fd, bundlefile = pycompat.mkstemp() | ||
|
r37204 | try: | ||
try: | ||||
|
r37596 | fp = os.fdopen(fd, r'wb') | ||
|
r37204 | fp.write(buf.read()) | ||
finally: | ||||
fp.close() | ||||
storebundle(op, cgparams, bundlefile) | ||||
finally: | ||||
try: | ||||
os.unlink(bundlefile) | ||||
except Exception: | ||||
# we would rather see the original exception | ||||
pass | ||||
|
r43346 | |||
|
r37204 | def storebundle(op, params, bundlefile): | ||
log = _getorcreateinfinitepushlogger(op) | ||||
parthandlerstart = time.time() | ||||
|
r43347 | log(scratchbranchparttype, eventtype=b'start') | ||
|
r37204 | index = op.repo.bundlestore.index | ||
store = op.repo.bundlestore.store | ||||
|
r43347 | op.records.add(scratchbranchparttype + b'_skippushkey', True) | ||
|
r37204 | |||
bundle = None | ||||
try: # guards bundle | ||||
|
r43347 | bundlepath = b"bundle:%s+%s" % (op.repo.root, bundlefile) | ||
|
r37204 | bundle = hg.repository(op.repo.ui, bundlepath) | ||
|
r43347 | bookmark = params.get(b'bookmark') | ||
bookprevnode = params.get(b'bookprevnode', b'') | ||||
force = params.get(b'force') | ||||
|
r37204 | |||
if bookmark: | ||||
oldnode = index.getnode(bookmark) | ||||
else: | ||||
oldnode = None | ||||
|
r43347 | bundleheads = bundle.revs(b'heads(bundle())') | ||
|
r37204 | if bookmark and len(bundleheads) > 1: | ||
raise error.Abort( | ||||
|
r43347 | _(b'cannot push more than one head to a scratch branch') | ||
|
r43346 | ) | ||
|
r37204 | |||
revs = _getrevs(bundle, oldnode, force, bookmark) | ||||
# Notify the user of what is being pushed | ||||
|
r43347 | plural = b's' if len(revs) > 1 else b'' | ||
op.repo.ui.warn(_(b"pushing %d commit%s:\n") % (len(revs), plural)) | ||||
|
r37204 | maxoutput = 10 | ||
for i in range(0, min(len(revs), maxoutput)): | ||||
|
r43347 | firstline = bundle[revs[i]].description().split(b'\n')[0][:50] | ||
op.repo.ui.warn(b" %s %s\n" % (revs[i], firstline)) | ||||
|
r37204 | |||
if len(revs) > maxoutput + 1: | ||||
|
r43347 | op.repo.ui.warn(b" ...\n") | ||
firstline = bundle[revs[-1]].description().split(b'\n')[0][:50] | ||||
op.repo.ui.warn(b" %s %s\n" % (revs[-1], firstline)) | ||||
|
r37204 | |||
nodesctx = [bundle[rev] for rev in revs] | ||||
inindex = lambda rev: bool(index.getbundle(bundle[rev].hex())) | ||||
if bundleheads: | ||||
newheadscount = sum(not inindex(rev) for rev in bundleheads) | ||||
else: | ||||
newheadscount = 0 | ||||
# If there's a bookmark specified, there should be only one head, | ||||
# so we choose the last node, which will be that head. | ||||
# If a bug or malicious client allows there to be a bookmark | ||||
# with multiple heads, we will place the bookmark on the last head. | ||||
bookmarknode = nodesctx[-1].hex() if nodesctx else None | ||||
key = None | ||||
if newheadscount: | ||||
|
r43347 | with open(bundlefile, b'rb') as f: | ||
|
r37204 | bundledata = f.read() | ||
|
r43346 | with logservicecall( | ||
|
r43347 | log, b'bundlestore', bundlesize=len(bundledata) | ||
|
r43346 | ): | ||
|
r37204 | bundlesizelimit = 100 * 1024 * 1024 # 100 MB | ||
if len(bundledata) > bundlesizelimit: | ||||
|
r43346 | error_msg = ( | ||
|
r43347 | b'bundle is too big: %d bytes. ' | ||
+ b'max allowed size is 100 MB' | ||||
|
r43346 | ) | ||
|
r37204 | raise error.Abort(error_msg % (len(bundledata),)) | ||
key = store.write(bundledata) | ||||
|
r43347 | with logservicecall(log, b'index', newheadscount=newheadscount), index: | ||
|
r37204 | if key: | ||
index.addbundle(key, nodesctx) | ||||
if bookmark: | ||||
index.addbookmark(bookmark, bookmarknode) | ||||
|
r43346 | _maybeaddpushbackpart( | ||
op, bookmark, bookmarknode, bookprevnode, params | ||||
) | ||||
log( | ||||
scratchbranchparttype, | ||||
|
r43347 | eventtype=b'success', | ||
|
r43346 | elapsedms=(time.time() - parthandlerstart) * 1000, | ||
) | ||||
|
r37204 | |||
except Exception as e: | ||||
|
r43346 | log( | ||
scratchbranchparttype, | ||||
|
r43347 | eventtype=b'failure', | ||
|
r37204 | elapsedms=(time.time() - parthandlerstart) * 1000, | ||
|
r43346 | errormsg=str(e), | ||
) | ||||
|
r37204 | raise | ||
finally: | ||||
if bundle: | ||||
bundle.close() | ||||
|
r43346 | |||
@bundle2.parthandler( | ||||
scratchbranchparttype, | ||||
|
r43347 | ( | ||
b'bookmark', | ||||
b'bookprevnode', | ||||
b'force', | ||||
b'pushbackbookmarks', | ||||
b'cgversion', | ||||
), | ||||
|
r43346 | ) | ||
|
r37204 | def bundle2scratchbranch(op, part): | ||
'''unbundle a bundle2 part containing a changegroup to store''' | ||||
bundler = bundle2.bundle20(op.repo.ui) | ||||
|
r43347 | cgversion = part.params.get(b'cgversion', b'01') | ||
cgpart = bundle2.bundlepart(b'changegroup', data=part.read()) | ||||
cgpart.addparam(b'version', cgversion) | ||||
|
r37204 | bundler.addpart(cgpart) | ||
buf = util.chunkbuffer(bundler.getchunks()) | ||||
|
r38182 | fd, bundlefile = pycompat.mkstemp() | ||
|
r37204 | try: | ||
try: | ||||
|
r37596 | fp = os.fdopen(fd, r'wb') | ||
|
r37204 | fp.write(buf.read()) | ||
finally: | ||||
fp.close() | ||||
storebundle(op, part.params, bundlefile) | ||||
finally: | ||||
try: | ||||
os.unlink(bundlefile) | ||||
except OSError as e: | ||||
if e.errno != errno.ENOENT: | ||||
raise | ||||
return 1 | ||||
|
r43346 | |||
|
r37204 | def _maybeaddpushbackpart(op, bookmark, newnode, oldnode, params): | ||
|
r43347 | if params.get(b'pushbackbookmarks'): | ||
if op.reply and b'pushback' in op.reply.capabilities: | ||||
|
r37204 | params = { | ||
|
r43347 | b'namespace': b'bookmarks', | ||
b'key': bookmark, | ||||
b'new': newnode, | ||||
b'old': oldnode, | ||||
|
r37204 | } | ||
|
r43375 | op.reply.newpart( | ||
b'pushkey', mandatoryparams=pycompat.iteritems(params) | ||||
) | ||||
|
r37204 | |||
|
r43346 | |||
|
r37204 | def bundle2pushkey(orig, op, part): | ||
'''Wrapper of bundle2.handlepushkey() | ||||
The only goal is to skip calling the original function if flag is set. | ||||
It's set if infinitepush push is happening. | ||||
''' | ||||
|
r43347 | if op.records[scratchbranchparttype + b'_skippushkey']: | ||
|
r37204 | if op.reply is not None: | ||
|
r43347 | rpart = op.reply.newpart(b'reply:pushkey') | ||
rpart.addparam(b'in-reply-to', str(part.id), mandatory=False) | ||||
rpart.addparam(b'return', b'1', mandatory=False) | ||||
|
r37204 | return 1 | ||
return orig(op, part) | ||||
|
r43346 | |||
|
r37204 | def bundle2handlephases(orig, op, part): | ||
'''Wrapper of bundle2.handlephases() | ||||
The only goal is to skip calling the original function if flag is set. | ||||
It's set if infinitepush push is happening. | ||||
''' | ||||
|
r43347 | if op.records[scratchbranchparttype + b'_skipphaseheads']: | ||
|
r37204 | return | ||
return orig(op, part) | ||||
|
r43346 | |||
|
r37204 | def _asyncsavemetadata(root, nodes): | ||
'''starts a separate process that fills metadata for the nodes | ||||
This function creates a separate process and doesn't wait for it's | ||||
completion. This was done to avoid slowing down pushes | ||||
''' | ||||
maxnodes = 50 | ||||
if len(nodes) > maxnodes: | ||||
return | ||||
nodesargs = [] | ||||
for node in nodes: | ||||
|
r43347 | nodesargs.append(b'--node') | ||
|
r37204 | nodesargs.append(node) | ||
|
r43347 | with open(os.devnull, b'w+b') as devnull: | ||
|
r43346 | cmdline = [ | ||
util.hgexecutable(), | ||||
|
r43347 | b'debugfillinfinitepushmetadata', | ||
b'-R', | ||||
|
r43346 | root, | ||
] + nodesargs | ||||
|
r37204 | # Process will run in background. We don't care about the return code | ||
|
r43346 | subprocess.Popen( | ||
pycompat.rapply(procutil.tonativestr, cmdline), | ||||
close_fds=True, | ||||
shell=False, | ||||
stdin=devnull, | ||||
stdout=devnull, | ||||
stderr=devnull, | ||||
) | ||||