##// END OF EJS Templates
manifest: avoid corruption by dropping removed files with pure (issue5801)...
manifest: avoid corruption by dropping removed files with pure (issue5801) Previously, removed files would simply be marked by overwriting the first byte with NUL and dropping their entry in `self.position`. But no effort was made to ignore them when compacting the dictionary into text form. This allowed them to slip into the manifest revision, since the code seems to be trying to minimize the string operations by copying as large a chunk as possible. As part of this, compact() walks the existing text based on entries in the `positions` list, and consumed everything up to the next position entry. This typically resulted in a ValueError complaining about unsorted manifest entries. Sometimes it seems that files do get dropped in large repos- it seems to correspond to there being a new entry that would take the same slot. A much more trivial problem is that if the only changes were removals, `_compact()` didn't even run because `__delitem__` doesn't add anything to `self.extradata`. Now there's an explicit variable to flag this, both to allow `_compact()` to run, and to avoid searching the manifest in cases where there are no removals. In practice, this behavior was mostly obscured by the check in fastdelta() which takes a different path that explicitly drops removed files if there are fewer than 1000 changes. However, timeless has a repo where after rebasing tens of commits, a totally different path[1] is taken that bypasses the change count check and hits this problem. [1] https://www.mercurial-scm.org/repo/hg/file/2338bdea4474/mercurial/manifest.py#l1511

File last commit:

r42376:608ab4a3 default
r42569:0546ead3 stable
Show More
unionrepo.py
287 lines | 9.9 KiB | text/x-python | PythonLexer
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 # unionrepo.py - repository class for viewing union of repository changesets
#
# Derived from bundlerepo.py
# Copyright 2006, 2007 Benoit Boissinot <bboissin@gmail.com>
# Copyright 2013 Unity Technologies, Mads Kiilerich <madski@unity3d.com>
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
"""Repository class for "in-memory pull" of one local repository to another,
allowing operations like diff and log with revsets.
"""
Gregory Szorc
unionrepo: use absolute_import
r25988 from __future__ import absolute_import
from .i18n import _
from .node import nullid
from . import (
changelog,
cmdutil,
Matt Harbison
py3: rename pycompat.getcwd() to encoding.getcwd() (API)...
r39843 encoding,
Pierre-Yves David
error: get Abort from 'error' instead of 'util'...
r26587 error,
Gregory Szorc
unionrepo: use absolute_import
r25988 filelog,
localrepo,
manifest,
mdiff,
pathutil,
revlog,
util,
Pierre-Yves David
vfs: use 'vfs' module directly in 'mercurial.unionrepo'...
r31242 vfs as vfsmod,
Gregory Szorc
unionrepo: use absolute_import
r25988 )
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
class unionrevlog(revlog.revlog):
def __init__(self, opener, indexfile, revlog2, linkmapper):
# How it works:
# To retrieve a revision, we just need to know the node id so we can
# look it up in revlog2.
#
# To differentiate a rev in the second revlog from a rev in the revlog,
# we check revision against repotiprev.
Pierre-Yves David
vfs: use 'vfs' module directly in 'mercurial.unionrepo'...
r31242 opener = vfsmod.readonlyvfs(opener)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 revlog.revlog.__init__(self, opener, indexfile)
self.revlog2 = revlog2
n = len(self)
self.repotiprev = n - 1
self.bundlerevs = set() # used by 'bundle()' revset expression
for rev2 in self.revlog2:
rev = self.revlog2.index[rev2]
# rev numbers - in revlog2, very different from self.rev
Yuya Nishihara
unionrepo: fill in uncompressed length of revlog entry...
r38194 _start, _csize, rsize, base, linkrev, p1rev, p2rev, node = rev
Mike Edgar
changegroup: add flags field to cg3 delta header...
r27433 flags = _start & 0xFFFF
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
if linkmapper is None: # link is to same revlog
assert linkrev == rev2 # we never link back
link = n
else: # rev must be mapped from repo2 cl to unified cl by linkmapper
link = linkmapper(linkrev)
Pierre-Yves David
unionrepo: take delta base in account with building unified revlog...
r26230 if linkmapper is not None: # link is to same revlog
base = linkmapper(base)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if node in self.nodemap:
# this happens for the common revlog revisions
self.bundlerevs.add(self.nodemap[node])
continue
p1node = self.revlog2.node(p1rev)
p2node = self.revlog2.node(p2rev)
Yuya Nishihara
unionrepo: fill in uncompressed length of revlog entry...
r38194 # TODO: it's probably wrong to set compressed length to None, but
# I have no idea if csize is valid in the base revlog context.
e = (flags, None, rsize, base,
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 link, self.rev(p1node), self.rev(p2node), node)
Martin von Zweigbergk
index: replace insert(-1, e) method by append(e) method...
r38886 self.index.append(e)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 self.nodemap[node] = n
self.bundlerevs.add(n)
n += 1
def _chunk(self, rev):
if rev <= self.repotiprev:
return revlog.revlog._chunk(self, rev)
return self.revlog2._chunk(self.node(rev))
def revdiff(self, rev1, rev2):
"""return or calculate a delta between two revisions"""
if rev1 > self.repotiprev and rev2 > self.repotiprev:
return self.revlog2.revdiff(
self.revlog2.rev(self.node(rev1)),
self.revlog2.rev(self.node(rev2)))
elif rev1 <= self.repotiprev and rev2 <= self.repotiprev:
Wojciech Lopata
unionrevlog: extract 'baserevision' and 'baserevdiff' methods...
r19630 return self.baserevdiff(rev1, rev2)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
Jun Wu
unionrepo: avoid unnecessary node -> rev conversion
r31724 return mdiff.textdiff(self.revision(rev1), self.revision(rev2))
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
Gregory Szorc
filelog: wrap revlog instead of inheriting it (API)...
r37515 def revision(self, nodeorrev, _df=None, raw=False):
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 """return an uncompressed revision of a given node or revision
number.
"""
if isinstance(nodeorrev, int):
rev = nodeorrev
node = self.node(rev)
else:
node = nodeorrev
rev = self.rev(node)
if node == nullid:
return ""
if rev > self.repotiprev:
text = self.revlog2.revision(node)
Gregory Szorc
revlog: rename _cache to _revisioncache...
r40088 self._revisioncache = (node, rev, text)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 else:
Wojciech Lopata
unionrevlog: extract 'baserevision' and 'baserevdiff' methods...
r19630 text = self.baserevision(rev)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 # already cached
return text
Wojciech Lopata
unionrevlog: extract 'baserevision' and 'baserevdiff' methods...
r19630 def baserevision(self, nodeorrev):
# Revlog subclasses may override 'revision' method to modify format of
# content retrieved from revlog. To use unionrevlog with such class one
# needs to override 'baserevision' and make more specific call here.
return revlog.revlog.revision(self, nodeorrev)
def baserevdiff(self, rev1, rev2):
# Exists for the same purpose as baserevision.
return revlog.revlog.revdiff(self, rev1, rev2)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 def addrevision(self, text, transaction, link, p1=None, p2=None, d=None):
raise NotImplementedError
Yuya Nishihara
revlog: update signature of dummy addgroup() in bundlerepo and unionrepo...
r34216 def addgroup(self, deltas, transaction, addrevisioncb=None):
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 raise NotImplementedError
def strip(self, rev, minlink):
raise NotImplementedError
def checksize(self):
raise NotImplementedError
class unionchangelog(unionrevlog, changelog.changelog):
def __init__(self, opener, opener2):
changelog.changelog.__init__(self, opener)
linkmapper = None
changelog2 = changelog.changelog(opener2)
unionrevlog.__init__(self, opener, self.indexfile, changelog2,
linkmapper)
Wojciech Lopata
unionrevlog: extract 'baserevision' and 'baserevdiff' methods...
r19630 def baserevision(self, nodeorrev):
# Although changelog doesn't override 'revision' method, some extensions
# may replace this class with another that does. Same story with
# manifest and filelog classes.
return changelog.changelog.revision(self, nodeorrev)
def baserevdiff(self, rev1, rev2):
return changelog.changelog.revdiff(self, rev1, rev2)
Durham Goode
manifest: add unionmanifestlog support...
r30374 class unionmanifest(unionrevlog, manifest.manifestrevlog):
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 def __init__(self, opener, opener2, linkmapper):
Durham Goode
manifest: add unionmanifestlog support...
r30374 manifest.manifestrevlog.__init__(self, opener)
manifest2 = manifest.manifestrevlog(opener2)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 unionrevlog.__init__(self, opener, self.indexfile, manifest2,
linkmapper)
Wojciech Lopata
unionrevlog: extract 'baserevision' and 'baserevdiff' methods...
r19630 def baserevision(self, nodeorrev):
Durham Goode
manifest: add unionmanifestlog support...
r30374 return manifest.manifestrevlog.revision(self, nodeorrev)
Wojciech Lopata
unionrevlog: extract 'baserevision' and 'baserevdiff' methods...
r19630
def baserevdiff(self, rev1, rev2):
Durham Goode
manifest: add unionmanifestlog support...
r30374 return manifest.manifestrevlog.revdiff(self, rev1, rev2)
Wojciech Lopata
unionrevlog: extract 'baserevision' and 'baserevdiff' methods...
r19630
Gregory Szorc
filelog: wrap revlog instead of inheriting it (API)...
r37515 class unionfilelog(filelog.filelog):
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 def __init__(self, opener, path, opener2, linkmapper, repo):
filelog.filelog.__init__(self, opener, path)
filelog2 = filelog.filelog(opener2, path)
Gregory Szorc
filelog: wrap revlog instead of inheriting it (API)...
r37515 self._revlog = unionrevlog(opener, self.indexfile,
filelog2._revlog, linkmapper)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 self._repo = repo
Gregory Szorc
filelog: wrap revlog instead of inheriting it (API)...
r37515 self.repotiprev = self._revlog.repotiprev
self.revlog2 = self._revlog.revlog2
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
Wojciech Lopata
unionrevlog: extract 'baserevision' and 'baserevdiff' methods...
r19630 def baserevision(self, nodeorrev):
return filelog.filelog.revision(self, nodeorrev)
def baserevdiff(self, rev1, rev2):
return filelog.filelog.revdiff(self, rev1, rev2)
Mike Edgar
revlog: add "iscensored()" to revlog public API...
r24118 def iscensored(self, rev):
"""Check if a revision is censored."""
if rev <= self.repotiprev:
return filelog.filelog.iscensored(self, rev)
Sean Farley
unionrepo: fix wrong rev being checked in iscensored (issue5024)
r27723 node = self.node(rev)
return self.revlog2.iscensored(self.revlog2.rev(node))
Mike Edgar
revlog: add "iscensored()" to revlog public API...
r24118
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 class unionpeer(localrepo.localpeer):
def canpush(self):
return False
Gregory Szorc
unionrepo: dynamically create repository type from base repository...
r39641 class unionrepository(object):
"""Represents the union of data in 2 repositories.
Instances are not usable if constructed directly. Use ``instance()``
or ``makeunionrepository()`` to create a usable instance.
"""
def __init__(self, repo2, url):
self.repo2 = repo2
self._url = url
Mads Kiilerich
config: set a 'source' in most cases where config don't come from file but code...
r20790 self.ui.setconfig('phases', 'publish', False, 'unionrepo')
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
@localrepo.unfilteredpropertycache
def changelog(self):
Angel Ezquerra
localrepo: remove all external users of localrepo.sopener...
r23878 return unionchangelog(self.svfs, self.repo2.svfs)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
Gregory Szorc
localrepo: pass root manifest into manifestlog.__init__...
r39799 @localrepo.unfilteredpropertycache
def manifestlog(self):
rootstore = unionmanifest(self.svfs, self.repo2.svfs,
self.unfiltered()._clrev)
Martin von Zweigbergk
manifest: accept narrowmatch into constructor instead of getting from repo...
r41067 return manifest.manifestlog(self.svfs, self, rootstore,
self.narrowmatch())
Gregory Szorc
localrepo: pass root manifest into manifestlog.__init__...
r39799
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 def _clrev(self, rev2):
"""map from repo2 changelog rev to temporary rev in self.changelog"""
node = self.repo2.changelog.node(rev2)
return self.changelog.rev(node)
def url(self):
return self._url
def file(self, f):
Angel Ezquerra
localrepo: remove all external users of localrepo.sopener...
r23878 return unionfilelog(self.svfs, f, self.repo2.svfs,
Pierre-Yves David
unionrepo: properly handle hidden linkrev in revlog (issue5070)...
r28222 self.unfiltered()._clrev, self)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
def close(self):
self.repo2.close()
def cancopy(self):
return False
def peer(self):
return unionpeer(self)
def getcwd(self):
Matt Harbison
py3: rename pycompat.getcwd() to encoding.getcwd() (API)...
r39843 return encoding.getcwd() # always outside the repo
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944
Gregory Szorc
hg: allow extra arguments to be passed to repo creation (API)...
r39585 def instance(ui, path, create, intents=None, createopts=None):
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if create:
Pierre-Yves David
error: get Abort from 'error' instead of 'util'...
r26587 raise error.Abort(_('cannot create new union repository'))
configitems: register the 'bundle.mainreporoot' config
r33179 parentpath = ui.config("bundle", "mainreporoot")
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if not parentpath:
# try to find the correct path to the working directory repo
Matt Harbison
py3: rename pycompat.getcwd() to encoding.getcwd() (API)...
r39843 parentpath = cmdutil.findrepo(encoding.getcwd())
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if parentpath is None:
parentpath = ''
if parentpath:
# Try to make the full path relative so we get a nice, short URL.
# In particular, we don't want temp dir names in test outputs.
Matt Harbison
py3: rename pycompat.getcwd() to encoding.getcwd() (API)...
r39843 cwd = encoding.getcwd()
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if parentpath == cwd:
parentpath = ''
else:
FUJIWARA Katsunori
unionrepo: use pathutil.normasprefix to ensure os.sep at the end of cwd...
r24835 cwd = pathutil.normasprefix(cwd)
Mads Kiilerich
unionrepo: read-only operations on a union of two localrepos...
r18944 if parentpath.startswith(cwd):
parentpath = parentpath[len(cwd):]
if path.startswith('union:'):
s = path.split(":", 1)[1].split("+", 1)
if len(s) == 1:
repopath, repopath2 = parentpath, s[0]
else:
repopath, repopath2 = s
else:
repopath, repopath2 = parentpath, path
Gregory Szorc
unionrepo: dynamically create repository type from base repository...
r39641
return makeunionrepository(ui, repopath, repopath2)
def makeunionrepository(ui, repopath1, repopath2):
"""Make a union repository object from 2 local repo paths."""
repo1 = localrepo.instance(ui, repopath1, create=False)
repo2 = localrepo.instance(ui, repopath2, create=False)
url = 'union:%s+%s' % (util.expandpath(repopath1),
util.expandpath(repopath2))
class derivedunionrepository(unionrepository, repo1.__class__):
pass
repo = repo1
repo.__class__ = derivedunionrepository
unionrepository.__init__(repo1, repo2, url)
return repo