simplestorerepo.py
683 lines
| 19.0 KiB
| text/x-python
|
PythonLexer
/ tests / simplestorerepo.py
|
r37355 | # simplestorerepo.py - Extension that swaps in alternate repository storage. | ||
# | ||||
# Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com> | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
|
r37356 | # To use this with the test suite: | ||
# | ||||
# $ HGREPOFEATURES="simplestore" ./run-tests.py \ | ||||
# --extra-config-opt extensions.simplestore=`pwd`/simplestorerepo.py | ||||
|
r37355 | from __future__ import absolute_import | ||
|
r37433 | import stat | ||
|
r37355 | from mercurial.i18n import _ | ||
from mercurial.node import ( | ||||
bin, | ||||
hex, | ||||
nullid, | ||||
nullrev, | ||||
) | ||||
from mercurial.thirdparty import ( | ||||
|
r39267 | attr, | ||
|
r37355 | ) | ||
from mercurial import ( | ||||
ancestor, | ||||
|
r37364 | bundlerepo, | ||
|
r37355 | error, | ||
|
r37433 | extensions, | ||
localrepo, | ||||
|
r37355 | mdiff, | ||
pycompat, | ||||
revlog, | ||||
|
r37433 | store, | ||
|
r37435 | verify, | ||
|
r37355 | ) | ||
|
r43078 | from mercurial.interfaces import ( | ||
repository, | ||||
|
r43079 | util as interfaceutil, | ||
|
r43078 | ) | ||
|
r38565 | from mercurial.utils import ( | ||
|
r41193 | cborutil, | ||
|
r39913 | storageutil, | ||
|
r38565 | ) | ||
r42956 | from mercurial.revlogutils import ( | |||
flagutil, | ||||
) | ||||
|
r37355 | |||
# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for | ||||
# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should | ||||
# be specifying the version(s) of Mercurial they are tested with, or | ||||
# leave the attribute unspecified. | ||||
testedwith = 'ships-with-hg-core' | ||||
|
r37433 | REQUIREMENT = 'testonly-simplestore' | ||
|
r37355 | def validatenode(node): | ||
if isinstance(node, int): | ||||
raise ValueError('expected node; got int') | ||||
if len(node) != 20: | ||||
raise ValueError('expected 20 byte node') | ||||
def validaterev(rev): | ||||
if not isinstance(rev, int): | ||||
raise ValueError('expected int') | ||||
|
r39813 | class simplestoreerror(error.StorageError): | ||
pass | ||||
|
r39267 | @interfaceutil.implementer(repository.irevisiondelta) | ||
|
r41192 | @attr.s(slots=True) | ||
|
r39267 | class simplestorerevisiondelta(object): | ||
node = attr.ib() | ||||
p1node = attr.ib() | ||||
p2node = attr.ib() | ||||
basenode = attr.ib() | ||||
flags = attr.ib() | ||||
baserevisionsize = attr.ib() | ||||
revision = attr.ib() | ||||
delta = attr.ib() | ||||
|
r41192 | linknode = attr.ib(default=None) | ||
@interfaceutil.implementer(repository.iverifyproblem) | ||||
@attr.s(frozen=True) | ||||
class simplefilestoreproblem(object): | ||||
warning = attr.ib(default=None) | ||||
error = attr.ib(default=None) | ||||
node = attr.ib(default=None) | ||||
|
r39267 | |||
|
r38565 | @interfaceutil.implementer(repository.ifilestorage) | ||
r43143 | class filestorage(flagutil.flagprocessorsmixin): | |||
|
r37355 | """Implements storage for a tracked path. | ||
Data is stored in the VFS in a directory corresponding to the tracked | ||||
path. | ||||
Index data is stored in an ``index`` file using CBOR. | ||||
Fulltext data is stored in files having names of the node. | ||||
""" | ||||
r43143 | _flagserrorclass = simplestoreerror | |||
|
r37355 | def __init__(self, svfs, path): | ||
self._svfs = svfs | ||||
self._path = path | ||||
self._storepath = b'/'.join([b'data', path]) | ||||
self._indexpath = b'/'.join([self._storepath, b'index']) | ||||
indexdata = self._svfs.tryread(self._indexpath) | ||||
if indexdata: | ||||
|
r41193 | indexdata = cborutil.decodeall(indexdata) | ||
|
r37355 | |||
self._indexdata = indexdata or [] | ||||
self._indexbynode = {} | ||||
self._indexbyrev = {} | ||||
|
r39896 | self._index = [] | ||
|
r37355 | self._refreshindex() | ||
r43143 | self._flagprocessors = dict(flagutil.flagprocessors) | |||
|
r37355 | def _refreshindex(self): | ||
self._indexbynode.clear() | ||||
self._indexbyrev.clear() | ||||
|
r39896 | self._index = [] | ||
|
r37355 | |||
for i, entry in enumerate(self._indexdata): | ||||
self._indexbynode[entry[b'node']] = entry | ||||
self._indexbyrev[i] = entry | ||||
self._indexbynode[nullid] = { | ||||
b'node': nullid, | ||||
b'p1': nullid, | ||||
b'p2': nullid, | ||||
b'linkrev': nullrev, | ||||
b'flags': 0, | ||||
} | ||||
self._indexbyrev[nullrev] = { | ||||
b'node': nullid, | ||||
b'p1': nullid, | ||||
b'p2': nullid, | ||||
b'linkrev': nullrev, | ||||
b'flags': 0, | ||||
} | ||||
for i, entry in enumerate(self._indexdata): | ||||
p1rev, p2rev = self.parentrevs(self.rev(entry[b'node'])) | ||||
# start, length, rawsize, chainbase, linkrev, p1, p2, node | ||||
|
r39896 | self._index.append((0, 0, 0, -1, entry[b'linkrev'], p1rev, p2rev, | ||
entry[b'node'])) | ||||
|
r37355 | |||
|
r39896 | self._index.append((0, 0, 0, -1, -1, -1, -1, nullid)) | ||
|
r37355 | |||
def __len__(self): | ||||
return len(self._indexdata) | ||||
def __iter__(self): | ||||
return iter(range(len(self))) | ||||
def revs(self, start=0, stop=None): | ||||
step = 1 | ||||
if stop is not None: | ||||
if start > stop: | ||||
step = -1 | ||||
stop += step | ||||
else: | ||||
stop = len(self) | ||||
return range(start, stop, step) | ||||
def parents(self, node): | ||||
validatenode(node) | ||||
if node not in self._indexbynode: | ||||
raise KeyError('unknown node') | ||||
entry = self._indexbynode[node] | ||||
return entry[b'p1'], entry[b'p2'] | ||||
def parentrevs(self, rev): | ||||
p1, p2 = self.parents(self._indexbyrev[rev][b'node']) | ||||
return self.rev(p1), self.rev(p2) | ||||
def rev(self, node): | ||||
validatenode(node) | ||||
|
r37426 | try: | ||
self._indexbynode[node] | ||||
except KeyError: | ||||
raise error.LookupError(node, self._indexpath, _('no node')) | ||||
|
r37355 | |||
for rev, entry in self._indexbyrev.items(): | ||||
if entry[b'node'] == node: | ||||
return rev | ||||
raise error.ProgrammingError('this should not occur') | ||||
def node(self, rev): | ||||
validaterev(rev) | ||||
return self._indexbyrev[rev][b'node'] | ||||
|
r41192 | def hasnode(self, node): | ||
validatenode(node) | ||||
return node in self._indexbynode | ||||
def censorrevision(self, tr, censornode, tombstone=b''): | ||||
raise NotImplementedError('TODO') | ||||
|
r37355 | def lookup(self, node): | ||
if isinstance(node, int): | ||||
return self.node(node) | ||||
if len(node) == 20: | ||||
|
r37426 | self.rev(node) | ||
return node | ||||
|
r37355 | |||
try: | ||||
rev = int(node) | ||||
if '%d' % rev != node: | ||||
raise ValueError | ||||
if rev < 0: | ||||
rev = len(self) + rev | ||||
if rev < 0 or rev >= len(self): | ||||
raise ValueError | ||||
return self.node(rev) | ||||
except (ValueError, OverflowError): | ||||
pass | ||||
if len(node) == 40: | ||||
try: | ||||
rawnode = bin(node) | ||||
self.rev(rawnode) | ||||
return rawnode | ||||
|
r37426 | except TypeError: | ||
|
r37355 | pass | ||
|
r37426 | raise error.LookupError(node, self._path, _('invalid lookup input')) | ||
|
r37355 | |||
def linkrev(self, rev): | ||||
validaterev(rev) | ||||
return self._indexbyrev[rev][b'linkrev'] | ||||
|
r39909 | def _flags(self, rev): | ||
|
r37355 | validaterev(rev) | ||
return self._indexbyrev[rev][b'flags'] | ||||
|
r39270 | def _candelta(self, baserev, rev): | ||
|
r37355 | validaterev(baserev) | ||
validaterev(rev) | ||||
|
r39909 | if ((self._flags(baserev) & revlog.REVIDX_RAWTEXT_CHANGING_FLAGS) | ||
or (self._flags(rev) & revlog.REVIDX_RAWTEXT_CHANGING_FLAGS)): | ||||
|
r37355 | return False | ||
return True | ||||
def checkhash(self, text, node, p1=None, p2=None, rev=None): | ||||
if p1 is None and p2 is None: | ||||
p1, p2 = self.parents(node) | ||||
|
r39913 | if node != storageutil.hashrevisionsha1(text, p1, p2): | ||
|
r39813 | raise simplestoreerror(_("integrity check failed on %s") % | ||
|
r37355 | self._path) | ||
|
r41192 | def revision(self, nodeorrev, raw=False): | ||
if isinstance(nodeorrev, int): | ||||
node = self.node(nodeorrev) | ||||
else: | ||||
node = nodeorrev | ||||
|
r37355 | validatenode(node) | ||
if node == nullid: | ||||
return b'' | ||||
rev = self.rev(node) | ||||
|
r39909 | flags = self._flags(rev) | ||
|
r37355 | |||
path = b'/'.join([self._storepath, hex(node)]) | ||||
rawtext = self._svfs.read(path) | ||||
r43149 | if raw: | |||
validatehash = self._processflagsraw(rawtext, flags) | ||||
text = rawtext | ||||
else: | ||||
r43261 | r = flagutil.processflagsread(self, rawtext, flags) | |||
r43253 | text, validatehash, sidedata = r | |||
|
r37355 | if validatehash: | ||
self.checkhash(text, node, rev=rev) | ||||
return text | ||||
r42950 | def rawdata(self, nodeorrev): | |||
return self.revision(raw=True) | ||||
|
r37355 | def read(self, node): | ||
validatenode(node) | ||||
revision = self.revision(node) | ||||
if not revision.startswith(b'\1\n'): | ||||
return revision | ||||
start = revision.index(b'\1\n', 2) | ||||
return revision[start + 2:] | ||||
def renamed(self, node): | ||||
validatenode(node) | ||||
if self.parents(node)[0] != nullid: | ||||
return False | ||||
fulltext = self.revision(node) | ||||
|
r39914 | m = storageutil.parsemeta(fulltext)[0] | ||
|
r37355 | |||
if m and 'copy' in m: | ||||
return m['copy'], bin(m['copyrev']) | ||||
return False | ||||
def cmp(self, node, text): | ||||
validatenode(node) | ||||
t = text | ||||
if text.startswith(b'\1\n'): | ||||
t = b'\1\n\1\n' + text | ||||
p1, p2 = self.parents(node) | ||||
|
r39913 | if storageutil.hashrevisionsha1(t, p1, p2) == node: | ||
|
r37355 | return False | ||
if self.iscensored(self.rev(node)): | ||||
return text != b'' | ||||
if self.renamed(node): | ||||
t2 = self.read(node) | ||||
return t2 != text | ||||
return True | ||||
def size(self, rev): | ||||
validaterev(rev) | ||||
node = self._indexbyrev[rev][b'node'] | ||||
if self.renamed(node): | ||||
return len(self.read(node)) | ||||
if self.iscensored(rev): | ||||
return 0 | ||||
return len(self.revision(node)) | ||||
def iscensored(self, rev): | ||||
validaterev(rev) | ||||
|
r40083 | return self._flags(rev) & repository.REVISION_FLAG_CENSORED | ||
|
r37355 | |||
def commonancestorsheads(self, a, b): | ||||
validatenode(a) | ||||
validatenode(b) | ||||
a = self.rev(a) | ||||
b = self.rev(b) | ||||
ancestors = ancestor.commonancestorsheads(self.parentrevs, a, b) | ||||
return pycompat.maplist(self.node, ancestors) | ||||
def descendants(self, revs): | ||||
# This is a copy of revlog.descendants() | ||||
first = min(revs) | ||||
if first == nullrev: | ||||
for i in self: | ||||
yield i | ||||
return | ||||
seen = set(revs) | ||||
for i in self.revs(start=first + 1): | ||||
for x in self.parentrevs(i): | ||||
if x != nullrev and x in seen: | ||||
seen.add(i) | ||||
yield i | ||||
break | ||||
# Required by verify. | ||||
def files(self): | ||||
entries = self._svfs.listdir(self._storepath) | ||||
# Strip out undo.backup.* files created as part of transaction | ||||
# recording. | ||||
entries = [f for f in entries if not f.startswith('undo.backup.')] | ||||
return [b'/'.join((self._storepath, f)) for f in entries] | ||||
|
r41192 | def storageinfo(self, exclusivefiles=False, sharedfiles=False, | ||
revisionscount=False, trackedsize=False, | ||||
storedsize=False): | ||||
# TODO do a real implementation of this | ||||
return { | ||||
'exclusivefiles': [], | ||||
'sharedfiles': [], | ||||
'revisionscount': len(self), | ||||
'trackedsize': 0, | ||||
'storedsize': None, | ||||
} | ||||
def verifyintegrity(self, state): | ||||
state['skipread'] = set() | ||||
for rev in self: | ||||
node = self.node(rev) | ||||
try: | ||||
self.revision(node) | ||||
except Exception as e: | ||||
yield simplefilestoreproblem( | ||||
error='unpacking %s: %s' % (node, e), | ||||
node=node) | ||||
state['skipread'].add(node) | ||||
def emitrevisions(self, nodes, nodesorder=None, revisiondata=False, | ||||
assumehaveparentrevisions=False, | ||||
deltamode=repository.CG_DELTAMODE_STD): | ||||
# TODO this will probably break on some ordering options. | ||||
nodes = [n for n in nodes if n != nullid] | ||||
if not nodes: | ||||
return | ||||
for delta in storageutil.emitrevisions( | ||||
self, nodes, nodesorder, simplestorerevisiondelta, | ||||
revisiondata=revisiondata, | ||||
assumehaveparentrevisions=assumehaveparentrevisions, | ||||
deltamode=deltamode): | ||||
yield delta | ||||
|
r37355 | def add(self, text, meta, transaction, linkrev, p1, p2): | ||
if meta or text.startswith(b'\1\n'): | ||||
|
r39914 | text = storageutil.packmeta(meta, text) | ||
|
r37355 | |||
return self.addrevision(text, transaction, linkrev, p1, p2) | ||||
def addrevision(self, text, transaction, linkrev, p1, p2, node=None, | ||||
|
r37454 | flags=revlog.REVIDX_DEFAULT_FLAGS, cachedelta=None): | ||
|
r37355 | validatenode(p1) | ||
validatenode(p2) | ||||
if flags: | ||||
|
r39913 | node = node or storageutil.hashrevisionsha1(text, p1, p2) | ||
|
r37355 | |||
r43260 | rawtext, validatehash = flagutil.processflagswrite(self, text, flags) | |||
|
r37355 | |||
|
r39913 | node = node or storageutil.hashrevisionsha1(text, p1, p2) | ||
|
r37355 | |||
if node in self._indexbynode: | ||||
return node | ||||
if validatehash: | ||||
self.checkhash(rawtext, node, p1=p1, p2=p2) | ||||
|
r37454 | return self._addrawrevision(node, rawtext, transaction, linkrev, p1, p2, | ||
flags) | ||||
def _addrawrevision(self, node, rawtext, transaction, link, p1, p2, flags): | ||||
transaction.addbackup(self._indexpath) | ||||
|
r37355 | path = b'/'.join([self._storepath, hex(node)]) | ||
|
r37454 | self._svfs.write(path, rawtext) | ||
|
r37355 | |||
self._indexdata.append({ | ||||
b'node': node, | ||||
b'p1': p1, | ||||
b'p2': p2, | ||||
|
r37454 | b'linkrev': link, | ||
|
r37355 | b'flags': flags, | ||
}) | ||||
self._reflectindexupdate() | ||||
return node | ||||
def _reflectindexupdate(self): | ||||
self._refreshindex() | ||||
|
r41193 | self._svfs.write(self._indexpath, | ||
''.join(cborutil.streamencode(self._indexdata))) | ||||
|
r37355 | |||
|
r40425 | def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None, | ||
maybemissingparents=False): | ||||
if maybemissingparents: | ||||
raise error.Abort(_('simple store does not support missing parents ' | ||||
'write mode')) | ||||
|
r37355 | nodes = [] | ||
transaction.addbackup(self._indexpath) | ||||
for node, p1, p2, linknode, deltabase, delta, flags in deltas: | ||||
linkrev = linkmapper(linknode) | ||||
|
r37454 | flags = flags or revlog.REVIDX_DEFAULT_FLAGS | ||
|
r37355 | |||
nodes.append(node) | ||||
if node in self._indexbynode: | ||||
continue | ||||
# Need to resolve the fulltext from the delta base. | ||||
if deltabase == nullid: | ||||
text = mdiff.patch(b'', delta) | ||||
else: | ||||
text = mdiff.patch(self.revision(deltabase), delta) | ||||
|
r37454 | self._addrawrevision(node, text, transaction, linkrev, p1, p2, | ||
flags) | ||||
|
r37355 | |||
if addrevisioncb: | ||||
addrevisioncb(self, node) | ||||
|
r41192 | return nodes | ||
|
r37355 | |||
|
r41192 | def _headrevs(self): | ||
# Assume all revisions are heads by default. | ||||
revishead = {rev: True for rev in self._indexbyrev} | ||||
for rev, entry in self._indexbyrev.items(): | ||||
# Unset head flag for all seen parents. | ||||
revishead[self.rev(entry[b'p1'])] = False | ||||
revishead[self.rev(entry[b'p2'])] = False | ||||
return [rev for rev, ishead in sorted(revishead.items()) | ||||
if ishead] | ||||
|
r37355 | |||
def heads(self, start=None, stop=None): | ||||
# This is copied from revlog.py. | ||||
if start is None and stop is None: | ||||
if not len(self): | ||||
return [nullid] | ||||
|
r41192 | return [self.node(r) for r in self._headrevs()] | ||
|
r37355 | |||
if start is None: | ||||
start = nullid | ||||
if stop is None: | ||||
stop = [] | ||||
stoprevs = set([self.rev(n) for n in stop]) | ||||
startrev = self.rev(start) | ||||
reachable = {startrev} | ||||
heads = {startrev} | ||||
parentrevs = self.parentrevs | ||||
for r in self.revs(start=startrev + 1): | ||||
for p in parentrevs(r): | ||||
if p in reachable: | ||||
if r not in stoprevs: | ||||
reachable.add(r) | ||||
heads.add(r) | ||||
if p in heads and p not in stoprevs: | ||||
heads.remove(p) | ||||
return [self.node(r) for r in heads] | ||||
def children(self, node): | ||||
validatenode(node) | ||||
# This is a copy of revlog.children(). | ||||
c = [] | ||||
p = self.rev(node) | ||||
for r in self.revs(start=p + 1): | ||||
prevs = [pr for pr in self.parentrevs(r) if pr != nullrev] | ||||
if prevs: | ||||
for pr in prevs: | ||||
if pr == p: | ||||
c.append(self.node(r)) | ||||
elif p == nullrev: | ||||
c.append(self.node(r)) | ||||
return c | ||||
def getstrippoint(self, minlink): | ||||
|
r41192 | return storageutil.resolvestripinfo( | ||
minlink, len(self) - 1, self._headrevs(), self.linkrev, | ||||
self.parentrevs) | ||||
|
r37355 | |||
def strip(self, minlink, transaction): | ||||
if not len(self): | ||||
return | ||||
rev, _ignored = self.getstrippoint(minlink) | ||||
if rev == len(self): | ||||
return | ||||
# Purge index data starting at the requested revision. | ||||
self._indexdata[rev:] = [] | ||||
self._reflectindexupdate() | ||||
|
r37433 | def issimplestorefile(f, kind, st): | ||
if kind != stat.S_IFREG: | ||||
return False | ||||
if store.isrevlog(f, kind, st): | ||||
return False | ||||
# Ignore transaction undo files. | ||||
if f.startswith('undo.'): | ||||
return False | ||||
# Otherwise assume it belongs to the simple store. | ||||
return True | ||||
class simplestore(store.encodedstore): | ||||
def datafiles(self): | ||||
for x in super(simplestore, self).datafiles(): | ||||
yield x | ||||
# Supplement with non-revlog files. | ||||
extrafiles = self._walk('data', True, filefilter=issimplestorefile) | ||||
for unencoded, encoded, size in extrafiles: | ||||
try: | ||||
unencoded = store.decodefilename(unencoded) | ||||
except KeyError: | ||||
unencoded = None | ||||
yield unencoded, encoded, size | ||||
|
r37355 | def reposetup(ui, repo): | ||
if not repo.local(): | ||||
return | ||||
|
r37364 | if isinstance(repo, bundlerepo.bundlerepository): | ||
raise error.Abort(_('cannot use simple store with bundlerepo')) | ||||
|
r37355 | class simplestorerepo(repo.__class__): | ||
def file(self, f): | ||||
return filestorage(self.svfs, f) | ||||
repo.__class__ = simplestorerepo | ||||
|
r37433 | |||
def featuresetup(ui, supported): | ||||
supported.add(REQUIREMENT) | ||||
|
r41192 | def newreporequirements(orig, ui, createopts): | ||
|
r37433 | """Modifies default requirements for new repos to use the simple store.""" | ||
|
r41192 | requirements = orig(ui, createopts) | ||
|
r37433 | |||
# These requirements are only used to affect creation of the store | ||||
# object. We have our own store. So we can remove them. | ||||
# TODO do this once we feel like taking the test hit. | ||||
#if 'fncache' in requirements: | ||||
# requirements.remove('fncache') | ||||
#if 'dotencode' in requirements: | ||||
# requirements.remove('dotencode') | ||||
requirements.add(REQUIREMENT) | ||||
return requirements | ||||
def makestore(orig, requirements, path, vfstype): | ||||
if REQUIREMENT not in requirements: | ||||
return orig(requirements, path, vfstype) | ||||
return simplestore(path, vfstype) | ||||
|
r37435 | def verifierinit(orig, self, *args, **kwargs): | ||
orig(self, *args, **kwargs) | ||||
# We don't care that files in the store don't align with what is | ||||
# advertised. So suppress these warnings. | ||||
self.warnorphanstorefiles = False | ||||
|
r37433 | def extsetup(ui): | ||
localrepo.featuresetupfuncs.add(featuresetup) | ||||
extensions.wrapfunction(localrepo, 'newreporequirements', | ||||
newreporequirements) | ||||
|
r41192 | extensions.wrapfunction(localrepo, 'makestore', makestore) | ||
|
r37435 | extensions.wrapfunction(verify.verifier, '__init__', verifierinit) | ||