##// END OF EJS Templates
clonebundles: add support for inline (streaming) clonebundles...
clonebundles: add support for inline (streaming) clonebundles The idea behind inline clonebundles is to send them through the ssh or https connection to the Mercurial server. We've been using this specifically for streaming clonebundles, although it works for 'regular' clonebundles as well (but is less relevant, since pullbundles exist). We've had this enabled for around 9 months for a part of our users. A few benefits are: - no need to secure an external system, since everything goes through the same Mercurial server - easier scaling (in our case: no risk of inconsistencies between multiple mercurial-server mirrors and nginx clonebundles hosts) Remaining topics/questions right now: - The inline clonebundles don't work for https yet. This is because httppeer doesn't seem to support sending client capabilities. I didn't focus on that as my main goal was to get this working for ssh.

File last commit:

r51309:3a2df812 default
r51559:60f9602b default
Show More
unionrepo.py
328 lines | 9.9 KiB | text/x-python | PythonLexer
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 # unionrepo.py - repository class for viewing union of repository changesets
#
# Derived from bundlerepo.py
# Copyright 2006, 2007 Benoit Boissinot <bboissin@gmail.com>
# Copyright 2013 Unity Technologies, Mads Kiilerich <madski@unity3d.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
"""Repository class for "in-memory pull" of one local repository to another,
allowing operations like diff and log with revsets.
"""
Gregory Szorc
unionrepo: use absolute_import
r25988
from .i18n import _
Gregory Szorc
py3: manually import getattr where it is needed...
r43359 from .pycompat import getattr
Gregory Szorc
unionrepo: use absolute_import
r25988
from . import (
changelog,
cmdutil,
Matt Harbison
py3: rename pycompat.getcwd() to encoding.getcwd() (API)...
r39843 encoding,
Pierre-Yves David
error: get Abort from 'error' instead of 'util'...
r26587 error,
Gregory Szorc
unionrepo: use absolute_import
r25988 filelog,
localrepo,
manifest,
mdiff,
pathutil,
revlog,
util,
Pierre-Yves David
vfs: use 'vfs' module directly in 'mercurial.unionrepo'...
r31242 vfs as vfsmod,
Gregory Szorc
unionrepo: use absolute_import
r25988 )
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
revlog: add a "data compression mode" entry in the index tuple...
r48023 from .revlogutils import (
constants as revlog_constants,
)
Augie Fackler
formatting: blacken the codebase...
r43346
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 class unionrevlog(revlog.revlog):
revlog: use a "radix" to address revlog...
r47921 def __init__(self, opener, radix, revlog2, linkmapper):
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 # How it works:
# To retrieve a revision, we just need to know the node id so we can
# look it up in revlog2.
#
# To differentiate a rev in the second revlog from a rev in the revlog,
# we check revision against repotiprev.
Pierre-Yves David
vfs: use 'vfs' module directly in 'mercurial.unionrepo'...
r31242 opener = vfsmod.readonlyvfs(opener)
revlog: introduce an explicit tracking of what the revlog is about...
r47838 target = getattr(revlog2, 'target', None)
if target is None:
# a revlog wrapper, eg: the manifestlog that is not an actual revlog
target = revlog2._revlog.target
revlog: use a "radix" to address revlog...
r47921 revlog.revlog.__init__(self, opener, target=target, radix=radix)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 self.revlog2 = revlog2
n = len(self)
self.repotiprev = n - 1
Augie Fackler
formatting: blacken the codebase...
r43346 self.bundlerevs = set() # used by 'bundle()' revset expression
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 for rev2 in self.revlog2:
rev = self.revlog2.index[rev2]
# rev numbers - in revlog2, very different from self.rev
revlog: make the index always return the same tuple...
r47913 (
_start,
_csize,
rsize,
base,
linkrev,
p1rev,
p2rev,
node,
_sdo,
_sds,
revlog: add a "data compression mode" entry in the index tuple...
r48023 _dcm,
revlog: introduce a compression mode for sidedata in the revlog index...
r48030 _sdcm,
rank: add a "rank" value to the revlog-entry tuple...
r49330 rank,
revlog: make the index always return the same tuple...
r47913 ) = rev
Mike Edgar
changegroup: add flags field to cg3 delta header...
r27433 flags = _start & 0xFFFF
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
Augie Fackler
formatting: blacken the codebase...
r43346 if linkmapper is None: # link is to same revlog
assert linkrev == rev2 # we never link back
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 link = n
Augie Fackler
formatting: blacken the codebase...
r43346 else: # rev must be mapped from repo2 cl to unified cl by linkmapper
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 link = linkmapper(linkrev)
Augie Fackler
formatting: blacken the codebase...
r43346 if linkmapper is not None: # link is to same revlog
Pierre-Yves David
unionrepo: take delta base in account with building unified revlog...
r26230 base = linkmapper(base)
index: use `index.get_rev` in `unionrepo.unionrevlog`...
r43965 this_rev = self.index.get_rev(node)
if this_rev is not None:
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 # this happens for the common revlog revisions
index: use `index.get_rev` in `unionrepo.unionrevlog`...
r43965 self.bundlerevs.add(this_rev)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 continue
p1node = self.revlog2.node(p1rev)
p2node = self.revlog2.node(p2rev)
Joerg Sonnenberger
unionrepo: don't insert index tuples with None as int field...
r46419 # TODO: it's probably wrong to set compressed length to -1, but
Yuya Nishihara
unionrepo: fill in uncompressed length of revlog entry...
r38194 # I have no idea if csize is valid in the base revlog context.
Augie Fackler
formatting: blacken the codebase...
r43346 e = (
flags,
Joerg Sonnenberger
unionrepo: don't insert index tuples with None as int field...
r46419 -1,
Augie Fackler
formatting: blacken the codebase...
r43346 rsize,
base,
link,
self.rev(p1node),
self.rev(p2node),
node,
revlog: always "append" full size tuple...
r47914 0, # sidedata offset
0, # sidedata size
revlog: add a "data compression mode" entry in the index tuple...
r48023 revlog_constants.COMP_MODE_INLINE,
revlog: introduce a compression mode for sidedata in the revlog index...
r48030 revlog_constants.COMP_MODE_INLINE,
rank: add a "rank" value to the revlog-entry tuple...
r49330 rank,
Augie Fackler
formatting: blacken the codebase...
r43346 )
Martin von Zweigbergk
index: replace insert(-1, e) method by append(e) method...
r38886 self.index.append(e)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 self.bundlerevs.add(n)
n += 1
Matt Harbison
unionrepo: resync several methods to actually override superclass methods...
r50869 def _chunk(self, rev, df=None):
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if rev <= self.repotiprev:
return revlog.revlog._chunk(self, rev)
return self.revlog2._chunk(self.node(rev))
def revdiff(self, rev1, rev2):
"""return or calculate a delta between two revisions"""
if rev1 > self.repotiprev and rev2 > self.repotiprev:
return self.revlog2.revdiff(
self.revlog2.rev(self.node(rev1)),
Augie Fackler
formatting: blacken the codebase...
r43346 self.revlog2.rev(self.node(rev2)),
)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 elif rev1 <= self.repotiprev and rev2 <= self.repotiprev:
unionrepo: use normal inheritance scheme to call revdiff
r43095 return super(unionrevlog, self).revdiff(rev1, rev2)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
unionrepo: fix `revdiff` implementation to use `rawdata`...
r43094 return mdiff.textdiff(self.rawdata(rev1), self.rawdata(rev2))
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
unionrepo: use a lower level overide in unionrepo too...
r43092 def _revisiondata(self, nodeorrev, _df=None, raw=False):
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if isinstance(nodeorrev, int):
rev = nodeorrev
node = self.node(rev)
else:
node = nodeorrev
rev = self.rev(node)
if rev > self.repotiprev:
unionrepo: use a lower level overide in unionrepo too...
r43092 # work around manifestrevlog NOT being a revlog
revlog2 = getattr(self.revlog2, '_revlog', self.revlog2)
func = revlog2._revisiondata
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 else:
unionrepo: use a lower level overide in unionrepo too...
r43092 func = super(unionrevlog, self)._revisiondata
return func(node, _df=_df, raw=raw)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
Matt Harbison
unionrepo: resync several methods to actually override superclass methods...
r50869 def addrevision(
self,
text,
transaction,
link,
p1,
p2,
cachedelta=None,
node=None,
flags=revlog.REVIDX_DEFAULT_FLAGS,
deltacomputer=None,
sidedata=None,
):
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 raise NotImplementedError
Augie Fackler
formatting: blacken the codebase...
r43346
def addgroup(
self,
deltas,
linkmapper,
transaction,
Joerg Sonnenberger
revlog: decouple caching from addrevision callback for addgroup...
r47085 alwayscache=False,
Augie Fackler
formatting: blacken the codebase...
r43346 addrevisioncb=None,
Joerg Sonnenberger
revlog: extend addgroup() with callback for duplicates...
r46373 duplicaterevisioncb=None,
Matt Harbison
unionrepo: resync several methods to actually override superclass methods...
r50869 debug_info=None,
delta_base_reuse_policy=None,
Augie Fackler
formatting: blacken the codebase...
r43346 ):
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 raise NotImplementedError
Augie Fackler
formatting: blacken the codebase...
r43346
Joerg Sonnenberger
unionrepo: sync with repository API...
r42382 def strip(self, minlink, transaction):
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 raise NotImplementedError
Augie Fackler
formatting: blacken the codebase...
r43346
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 def checksize(self):
raise NotImplementedError
Augie Fackler
formatting: blacken the codebase...
r43346
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 class unionchangelog(unionrevlog, changelog.changelog):
def __init__(self, opener, opener2):
changelog.changelog.__init__(self, opener)
linkmapper = None
changelog2 = changelog.changelog(opener2)
revlog: use a "radix" to address revlog...
r47921 unionrevlog.__init__(self, opener, self.radix, changelog2, linkmapper)
Augie Fackler
formatting: blacken the codebase...
r43346
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
Durham Goode
manifest: add unionmanifestlog support...
r30374 class unionmanifest(unionrevlog, manifest.manifestrevlog):
Joerg Sonnenberger
node: introduce nodeconstants class...
r47538 def __init__(self, nodeconstants, opener, opener2, linkmapper):
manifest.manifestrevlog.__init__(self, nodeconstants, opener)
manifest2 = manifest.manifestrevlog(nodeconstants, opener2)
Augie Fackler
formatting: blacken the codebase...
r43346 unionrevlog.__init__(
revlog: use a "radix" to address revlog...
r47921 self, opener, self._revlog.radix, manifest2, linkmapper
Augie Fackler
formatting: blacken the codebase...
r43346 )
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
Gregory Szorc
filelog: wrap revlog instead of inheriting it (API)...
r37515 class unionfilelog(filelog.filelog):
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 def __init__(self, opener, path, opener2, linkmapper, repo):
filelog.filelog.__init__(self, opener, path)
filelog2 = filelog.filelog(opener2, path)
Augie Fackler
formatting: blacken the codebase...
r43346 self._revlog = unionrevlog(
revlog: use a "radix" to address revlog...
r47921 opener, self._revlog.radix, filelog2._revlog, linkmapper
Augie Fackler
formatting: blacken the codebase...
r43346 )
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 self._repo = repo
Gregory Szorc
filelog: wrap revlog instead of inheriting it (API)...
r37515 self.repotiprev = self._revlog.repotiprev
self.revlog2 = self._revlog.revlog2
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
Mike Edgar
revlog: add "iscensored()" to revlog public API...
r24118 def iscensored(self, rev):
"""Check if a revision is censored."""
if rev <= self.repotiprev:
return filelog.filelog.iscensored(self, rev)
Sean Farley
unionrepo: fix wrong rev being checked in iscensored (issue5024)
r27723 node = self.node(rev)
return self.revlog2.iscensored(self.revlog2.rev(node))
Mike Edgar
revlog: add "iscensored()" to revlog public API...
r24118
Augie Fackler
formatting: blacken the codebase...
r43346
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 class unionpeer(localrepo.localpeer):
def canpush(self):
return False
Augie Fackler
formatting: blacken the codebase...
r43346
Gregory Szorc
py3: use class X: instead of class X(object):...
r49801 class unionrepository:
Gregory Szorc
unionrepo: dynamically create repository type from base repository...
r39641 """Represents the union of data in 2 repositories.
Instances are not usable if constructed directly. Use ``instance()``
or ``makeunionrepository()`` to create a usable instance.
"""
Augie Fackler
formatting: blacken the codebase...
r43346
Gregory Szorc
unionrepo: dynamically create repository type from base repository...
r39641 def __init__(self, repo2, url):
self.repo2 = repo2
self._url = url
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 self.ui.setconfig(b'phases', b'publish', False, b'unionrepo')
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
@localrepo.unfilteredpropertycache
def changelog(self):
Angel Ezquerra
localrepo: remove all external users of localrepo.sopener...
r23878 return unionchangelog(self.svfs, self.repo2.svfs)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
Gregory Szorc
localrepo: pass root manifest into manifestlog.__init__...
r39799 @localrepo.unfilteredpropertycache
def manifestlog(self):
Augie Fackler
formatting: blacken the codebase...
r43346 rootstore = unionmanifest(
Joerg Sonnenberger
node: introduce nodeconstants class...
r47538 self.nodeconstants,
self.svfs,
self.repo2.svfs,
self.unfiltered()._clrev,
Augie Fackler
formatting: blacken the codebase...
r43346 )
return manifest.manifestlog(
self.svfs, self, rootstore, self.narrowmatch()
)
Gregory Szorc
localrepo: pass root manifest into manifestlog.__init__...
r39799
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 def _clrev(self, rev2):
"""map from repo2 changelog rev to temporary rev in self.changelog"""
node = self.repo2.changelog.node(rev2)
return self.changelog.rev(node)
def url(self):
return self._url
def file(self, f):
Augie Fackler
formatting: blacken the codebase...
r43346 return unionfilelog(
self.svfs, f, self.repo2.svfs, self.unfiltered()._clrev, self
)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
def close(self):
self.repo2.close()
def cancopy(self):
return False
Manuel Jacob
pull: add --remote-hidden option and pass it through peer creation...
r51309 def peer(self, path=None, remotehidden=False):
return unionpeer(self, path=None, remotehidden=remotehidden)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
def getcwd(self):
Augie Fackler
formatting: blacken the codebase...
r43346 return encoding.getcwd() # always outside the repo
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
Gregory Szorc
hg: allow extra arguments to be passed to repo creation (API)...
r39585 def instance(ui, path, create, intents=None, createopts=None):
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if create:
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 raise error.Abort(_(b'cannot create new union repository'))
parentpath = ui.config(b"bundle", b"mainreporoot")
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if not parentpath:
# try to find the correct path to the working directory repo
Matt Harbison
py3: rename pycompat.getcwd() to encoding.getcwd() (API)...
r39843 parentpath = cmdutil.findrepo(encoding.getcwd())
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if parentpath is None:
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 parentpath = b''
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if parentpath:
# Try to make the full path relative so we get a nice, short URL.
# In particular, we don't want temp dir names in test outputs.
Matt Harbison
py3: rename pycompat.getcwd() to encoding.getcwd() (API)...
r39843 cwd = encoding.getcwd()
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if parentpath == cwd:
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 parentpath = b''
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 else:
FUJIWARA Katsunori
unionrepo: use pathutil.normasprefix to ensure os.sep at the end of cwd...
r24835 cwd = pathutil.normasprefix(cwd)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if parentpath.startswith(cwd):
Augie Fackler
formatting: blacken the codebase...
r43346 parentpath = parentpath[len(cwd) :]
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 if path.startswith(b'union:'):
s = path.split(b":", 1)[1].split(b"+", 1)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if len(s) == 1:
repopath, repopath2 = parentpath, s[0]
else:
repopath, repopath2 = s
else:
repopath, repopath2 = parentpath, path
Gregory Szorc
unionrepo: dynamically create repository type from base repository...
r39641
return makeunionrepository(ui, repopath, repopath2)
Augie Fackler
formatting: blacken the codebase...
r43346
Gregory Szorc
unionrepo: dynamically create repository type from base repository...
r39641 def makeunionrepository(ui, repopath1, repopath2):
"""Make a union repository object from 2 local repo paths."""
repo1 = localrepo.instance(ui, repopath1, create=False)
repo2 = localrepo.instance(ui, repopath2, create=False)
Augie Fackler
formatting: byteify all mercurial/ and hgext/ string literals...
r43347 url = b'union:%s+%s' % (
Augie Fackler
formatting: blacken the codebase...
r43346 util.expandpath(repopath1),
util.expandpath(repopath2),
)
Gregory Szorc
unionrepo: dynamically create repository type from base repository...
r39641
class derivedunionrepository(unionrepository, repo1.__class__):
pass
repo = repo1
repo.__class__ = derivedunionrepository
unionrepository.__init__(repo1, repo2, url)
return repo