|
|
import threading
|
|
|
|
|
|
from mercurial.node import (
|
|
|
hex,
|
|
|
sha1nodeconstants,
|
|
|
)
|
|
|
from mercurial.pycompat import getattr
|
|
|
from mercurial import (
|
|
|
mdiff,
|
|
|
revlog,
|
|
|
)
|
|
|
from . import (
|
|
|
basestore,
|
|
|
constants,
|
|
|
shallowutil,
|
|
|
)
|
|
|
|
|
|
|
|
|
class ChainIndicies:
|
|
|
"""A static class for easy reference to the delta chain indicies."""
|
|
|
|
|
|
# The filename of this revision delta
|
|
|
NAME = 0
|
|
|
# The mercurial file node for this revision delta
|
|
|
NODE = 1
|
|
|
# The filename of the delta base's revision. This is useful when delta
|
|
|
# between different files (like in the case of a move or copy, we can delta
|
|
|
# against the original file content).
|
|
|
BASENAME = 2
|
|
|
# The mercurial file node for the delta base revision. This is the nullid if
|
|
|
# this delta is a full text.
|
|
|
BASENODE = 3
|
|
|
# The actual delta or full text data.
|
|
|
DATA = 4
|
|
|
|
|
|
|
|
|
class unioncontentstore(basestore.baseunionstore):
|
|
|
def __init__(self, *args, **kwargs):
|
|
|
super(unioncontentstore, self).__init__(*args, **kwargs)
|
|
|
|
|
|
self.stores = args
|
|
|
self.writestore = kwargs.get('writestore')
|
|
|
|
|
|
# If allowincomplete==True then the union store can return partial
|
|
|
# delta chains, otherwise it will throw a KeyError if a full
|
|
|
# deltachain can't be found.
|
|
|
self.allowincomplete = kwargs.get('allowincomplete', False)
|
|
|
|
|
|
def get(self, name, node):
|
|
|
"""Fetches the full text revision contents of the given name+node pair.
|
|
|
If the full text doesn't exist, throws a KeyError.
|
|
|
|
|
|
Under the hood, this uses getdeltachain() across all the stores to build
|
|
|
up a full chain to produce the full text.
|
|
|
"""
|
|
|
chain = self.getdeltachain(name, node)
|
|
|
|
|
|
if chain[-1][ChainIndicies.BASENODE] != sha1nodeconstants.nullid:
|
|
|
# If we didn't receive a full chain, throw
|
|
|
raise KeyError((name, hex(node)))
|
|
|
|
|
|
# The last entry in the chain is a full text, so we start our delta
|
|
|
# applies with that.
|
|
|
fulltext = chain.pop()[ChainIndicies.DATA]
|
|
|
|
|
|
text = fulltext
|
|
|
while chain:
|
|
|
delta = chain.pop()[ChainIndicies.DATA]
|
|
|
text = mdiff.patches(text, [delta])
|
|
|
|
|
|
return text
|
|
|
|
|
|
@basestore.baseunionstore.retriable
|
|
|
def getdelta(self, name, node):
|
|
|
"""Return the single delta entry for the given name/node pair."""
|
|
|
for store in self.stores:
|
|
|
try:
|
|
|
return store.getdelta(name, node)
|
|
|
except KeyError:
|
|
|
pass
|
|
|
|
|
|
raise KeyError((name, hex(node)))
|
|
|
|
|
|
def getdeltachain(self, name, node):
|
|
|
"""Returns the deltachain for the given name/node pair.
|
|
|
|
|
|
Returns an ordered list of:
|
|
|
|
|
|
[(name, node, deltabasename, deltabasenode, deltacontent),...]
|
|
|
|
|
|
where the chain is terminated by a full text entry with a nullid
|
|
|
deltabasenode.
|
|
|
"""
|
|
|
chain = self._getpartialchain(name, node)
|
|
|
while chain[-1][ChainIndicies.BASENODE] != sha1nodeconstants.nullid:
|
|
|
x, x, deltabasename, deltabasenode, x = chain[-1]
|
|
|
try:
|
|
|
morechain = self._getpartialchain(deltabasename, deltabasenode)
|
|
|
chain.extend(morechain)
|
|
|
except KeyError:
|
|
|
# If we allow incomplete chains, don't throw.
|
|
|
if not self.allowincomplete:
|
|
|
raise
|
|
|
break
|
|
|
|
|
|
return chain
|
|
|
|
|
|
@basestore.baseunionstore.retriable
|
|
|
def getmeta(self, name, node):
|
|
|
"""Returns the metadata dict for given node."""
|
|
|
for store in self.stores:
|
|
|
try:
|
|
|
return store.getmeta(name, node)
|
|
|
except KeyError:
|
|
|
pass
|
|
|
raise KeyError((name, hex(node)))
|
|
|
|
|
|
def getmetrics(self):
|
|
|
metrics = [s.getmetrics() for s in self.stores]
|
|
|
return shallowutil.sumdicts(*metrics)
|
|
|
|
|
|
@basestore.baseunionstore.retriable
|
|
|
def _getpartialchain(self, name, node):
|
|
|
"""Returns a partial delta chain for the given name/node pair.
|
|
|
|
|
|
A partial chain is a chain that may not be terminated in a full-text.
|
|
|
"""
|
|
|
for store in self.stores:
|
|
|
try:
|
|
|
return store.getdeltachain(name, node)
|
|
|
except KeyError:
|
|
|
pass
|
|
|
|
|
|
raise KeyError((name, hex(node)))
|
|
|
|
|
|
def add(self, name, node, data):
|
|
|
raise RuntimeError(
|
|
|
b"cannot add content only to remotefilelog contentstore"
|
|
|
)
|
|
|
|
|
|
def getmissing(self, keys):
|
|
|
missing = keys
|
|
|
for store in self.stores:
|
|
|
if missing:
|
|
|
missing = store.getmissing(missing)
|
|
|
return missing
|
|
|
|
|
|
def addremotefilelognode(self, name, node, data):
|
|
|
if self.writestore:
|
|
|
self.writestore.addremotefilelognode(name, node, data)
|
|
|
else:
|
|
|
raise RuntimeError(b"no writable store configured")
|
|
|
|
|
|
def markledger(self, ledger, options=None):
|
|
|
for store in self.stores:
|
|
|
store.markledger(ledger, options)
|
|
|
|
|
|
|
|
|
class remotefilelogcontentstore(basestore.basestore):
|
|
|
def __init__(self, *args, **kwargs):
|
|
|
super(remotefilelogcontentstore, self).__init__(*args, **kwargs)
|
|
|
self._threaddata = threading.local()
|
|
|
|
|
|
def get(self, name, node):
|
|
|
# return raw revision text
|
|
|
data = self._getdata(name, node)
|
|
|
|
|
|
offset, size, flags = shallowutil.parsesizeflags(data)
|
|
|
content = data[offset : offset + size]
|
|
|
|
|
|
ancestormap = shallowutil.ancestormap(data)
|
|
|
p1, p2, linknode, copyfrom = ancestormap[node]
|
|
|
copyrev = None
|
|
|
if copyfrom:
|
|
|
copyrev = hex(p1)
|
|
|
|
|
|
self._updatemetacache(node, size, flags)
|
|
|
|
|
|
# lfs tracks renames in its own metadata, remove hg copy metadata,
|
|
|
# because copy metadata will be re-added by lfs flag processor.
|
|
|
if flags & revlog.REVIDX_EXTSTORED:
|
|
|
copyrev = copyfrom = None
|
|
|
revision = shallowutil.createrevlogtext(content, copyfrom, copyrev)
|
|
|
return revision
|
|
|
|
|
|
def getdelta(self, name, node):
|
|
|
# Since remotefilelog content stores only contain full texts, just
|
|
|
# return that.
|
|
|
revision = self.get(name, node)
|
|
|
return (
|
|
|
revision,
|
|
|
name,
|
|
|
sha1nodeconstants.nullid,
|
|
|
self.getmeta(name, node),
|
|
|
)
|
|
|
|
|
|
def getdeltachain(self, name, node):
|
|
|
# Since remotefilelog content stores just contain full texts, we return
|
|
|
# a fake delta chain that just consists of a single full text revision.
|
|
|
# The nullid in the deltabasenode slot indicates that the revision is a
|
|
|
# fulltext.
|
|
|
revision = self.get(name, node)
|
|
|
return [(name, node, None, sha1nodeconstants.nullid, revision)]
|
|
|
|
|
|
def getmeta(self, name, node):
|
|
|
self._sanitizemetacache()
|
|
|
if node != self._threaddata.metacache[0]:
|
|
|
data = self._getdata(name, node)
|
|
|
offset, size, flags = shallowutil.parsesizeflags(data)
|
|
|
self._updatemetacache(node, size, flags)
|
|
|
return self._threaddata.metacache[1]
|
|
|
|
|
|
def add(self, name, node, data):
|
|
|
raise RuntimeError(
|
|
|
b"cannot add content only to remotefilelog contentstore"
|
|
|
)
|
|
|
|
|
|
def _sanitizemetacache(self):
|
|
|
metacache = getattr(self._threaddata, 'metacache', None)
|
|
|
if metacache is None:
|
|
|
self._threaddata.metacache = (None, None) # (node, meta)
|
|
|
|
|
|
def _updatemetacache(self, node, size, flags):
|
|
|
self._sanitizemetacache()
|
|
|
if node == self._threaddata.metacache[0]:
|
|
|
return
|
|
|
meta = {constants.METAKEYFLAG: flags, constants.METAKEYSIZE: size}
|
|
|
self._threaddata.metacache = (node, meta)
|
|
|
|
|
|
|
|
|
class remotecontentstore:
|
|
|
def __init__(self, ui, fileservice, shared):
|
|
|
self._fileservice = fileservice
|
|
|
# type(shared) is usually remotefilelogcontentstore
|
|
|
self._shared = shared
|
|
|
|
|
|
def get(self, name, node):
|
|
|
self._fileservice.prefetch(
|
|
|
[(name, hex(node))], force=True, fetchdata=True
|
|
|
)
|
|
|
return self._shared.get(name, node)
|
|
|
|
|
|
def getdelta(self, name, node):
|
|
|
revision = self.get(name, node)
|
|
|
return (
|
|
|
revision,
|
|
|
name,
|
|
|
sha1nodeconstants.nullid,
|
|
|
self._shared.getmeta(name, node),
|
|
|
)
|
|
|
|
|
|
def getdeltachain(self, name, node):
|
|
|
# Since our remote content stores just contain full texts, we return a
|
|
|
# fake delta chain that just consists of a single full text revision.
|
|
|
# The nullid in the deltabasenode slot indicates that the revision is a
|
|
|
# fulltext.
|
|
|
revision = self.get(name, node)
|
|
|
return [(name, node, None, sha1nodeconstants.nullid, revision)]
|
|
|
|
|
|
def getmeta(self, name, node):
|
|
|
self._fileservice.prefetch(
|
|
|
[(name, hex(node))], force=True, fetchdata=True
|
|
|
)
|
|
|
return self._shared.getmeta(name, node)
|
|
|
|
|
|
def add(self, name, node, data):
|
|
|
raise RuntimeError(b"cannot add to a remote store")
|
|
|
|
|
|
def getmissing(self, keys):
|
|
|
return keys
|
|
|
|
|
|
def markledger(self, ledger, options=None):
|
|
|
pass
|
|
|
|
|
|
|
|
|
class manifestrevlogstore:
|
|
|
def __init__(self, repo):
|
|
|
self._store = repo.store
|
|
|
self._svfs = repo.svfs
|
|
|
self._revlogs = dict()
|
|
|
self._cl = revlog.revlog(self._svfs, radix=b'00changelog.i')
|
|
|
self._repackstartlinkrev = 0
|
|
|
|
|
|
def get(self, name, node):
|
|
|
return self._revlog(name).rawdata(node)
|
|
|
|
|
|
def getdelta(self, name, node):
|
|
|
revision = self.get(name, node)
|
|
|
return revision, name, self._cl.nullid, self.getmeta(name, node)
|
|
|
|
|
|
def getdeltachain(self, name, node):
|
|
|
revision = self.get(name, node)
|
|
|
return [(name, node, None, self._cl.nullid, revision)]
|
|
|
|
|
|
def getmeta(self, name, node):
|
|
|
rl = self._revlog(name)
|
|
|
rev = rl.rev(node)
|
|
|
return {
|
|
|
constants.METAKEYFLAG: rl.flags(rev),
|
|
|
constants.METAKEYSIZE: rl.rawsize(rev),
|
|
|
}
|
|
|
|
|
|
def getancestors(self, name, node, known=None):
|
|
|
if known is None:
|
|
|
known = set()
|
|
|
if node in known:
|
|
|
return []
|
|
|
|
|
|
rl = self._revlog(name)
|
|
|
ancestors = {}
|
|
|
missing = {node}
|
|
|
for ancrev in rl.ancestors([rl.rev(node)], inclusive=True):
|
|
|
ancnode = rl.node(ancrev)
|
|
|
missing.discard(ancnode)
|
|
|
|
|
|
p1, p2 = rl.parents(ancnode)
|
|
|
if p1 != self._cl.nullid and p1 not in known:
|
|
|
missing.add(p1)
|
|
|
if p2 != self._cl.nullid and p2 not in known:
|
|
|
missing.add(p2)
|
|
|
|
|
|
linknode = self._cl.node(rl.linkrev(ancrev))
|
|
|
ancestors[rl.node(ancrev)] = (p1, p2, linknode, b'')
|
|
|
if not missing:
|
|
|
break
|
|
|
return ancestors
|
|
|
|
|
|
def getnodeinfo(self, name, node):
|
|
|
cl = self._cl
|
|
|
rl = self._revlog(name)
|
|
|
parents = rl.parents(node)
|
|
|
linkrev = rl.linkrev(rl.rev(node))
|
|
|
return (parents[0], parents[1], cl.node(linkrev), None)
|
|
|
|
|
|
def add(self, *args):
|
|
|
raise RuntimeError(b"cannot add to a revlog store")
|
|
|
|
|
|
def _revlog(self, name):
|
|
|
rl = self._revlogs.get(name)
|
|
|
if rl is None:
|
|
|
revlogname = b'00manifesttree'
|
|
|
if name != b'':
|
|
|
revlogname = b'meta/%s/00manifest' % name
|
|
|
rl = revlog.revlog(self._svfs, radix=revlogname)
|
|
|
self._revlogs[name] = rl
|
|
|
return rl
|
|
|
|
|
|
def getmissing(self, keys):
|
|
|
missing = []
|
|
|
for name, node in keys:
|
|
|
mfrevlog = self._revlog(name)
|
|
|
if node not in mfrevlog.nodemap:
|
|
|
missing.append((name, node))
|
|
|
|
|
|
return missing
|
|
|
|
|
|
def setrepacklinkrevrange(self, startrev, endrev):
|
|
|
self._repackstartlinkrev = startrev
|
|
|
self._repackendlinkrev = endrev
|
|
|
|
|
|
def markledger(self, ledger, options=None):
|
|
|
if options and options.get(constants.OPTION_PACKSONLY):
|
|
|
return
|
|
|
treename = b''
|
|
|
rl = revlog.revlog(self._svfs, radix=b'00manifesttree')
|
|
|
startlinkrev = self._repackstartlinkrev
|
|
|
endlinkrev = self._repackendlinkrev
|
|
|
for rev in range(len(rl) - 1, -1, -1):
|
|
|
linkrev = rl.linkrev(rev)
|
|
|
if linkrev < startlinkrev:
|
|
|
break
|
|
|
if linkrev > endlinkrev:
|
|
|
continue
|
|
|
node = rl.node(rev)
|
|
|
ledger.markdataentry(self, treename, node)
|
|
|
ledger.markhistoryentry(self, treename, node)
|
|
|
|
|
|
for t, path, size in self._store.datafiles():
|
|
|
if path[:5] != b'meta/' or path[-2:] != b'.i':
|
|
|
continue
|
|
|
|
|
|
treename = path[5 : -len(b'/00manifest')]
|
|
|
|
|
|
rl = revlog.revlog(self._svfs, indexfile=path[:-2])
|
|
|
for rev in range(len(rl) - 1, -1, -1):
|
|
|
linkrev = rl.linkrev(rev)
|
|
|
if linkrev < startlinkrev:
|
|
|
break
|
|
|
if linkrev > endlinkrev:
|
|
|
continue
|
|
|
node = rl.node(rev)
|
|
|
ledger.markdataentry(self, treename, node)
|
|
|
ledger.markhistoryentry(self, treename, node)
|
|
|
|
|
|
def cleanup(self, ledger):
|
|
|
pass
|
|
|
|
