wrapper.py
548 lines
| 16.7 KiB
| text/x-python
|
PythonLexer
|
r35097 | # wrapper.py - methods wrapping core mercurial logic | ||
# | ||||
# Copyright 2017 Facebook, Inc. | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
import hashlib | ||||
|
r35098 | from mercurial.i18n import _ | ||
|
r47771 | from mercurial.node import bin, hex, short | ||
|
r43359 | from mercurial.pycompat import ( | ||
getattr, | ||||
setattr, | ||||
) | ||||
|
r35098 | |||
|
r35097 | from mercurial import ( | ||
|
r41078 | bundle2, | ||
changegroup, | ||||
cmdutil, | ||||
context, | ||||
|
r35097 | error, | ||
|
r41078 | exchange, | ||
exthelper, | ||||
localrepo, | ||||
|
r35097 | revlog, | ||
|
r41078 | scmutil, | ||
|
r35097 | util, | ||
|
r41078 | vfs as vfsmod, | ||
wireprotov1server, | ||||
|
r35097 | ) | ||
r46662 | from mercurial.upgrade_utils import ( | |||
actions as upgrade_actions, | ||||
engine as upgrade_engine, | ||||
) | ||||
r46661 | ||||
|
r43346 | from mercurial.interfaces import repository | ||
|
r43078 | |||
|
r37102 | from mercurial.utils import ( | ||
|
r39914 | storageutil, | ||
|
r37102 | stringutil, | ||
) | ||||
|
r35364 | from ..largefiles import lfutil | ||
|
r35097 | from . import ( | ||
blobstore, | ||||
pointer, | ||||
) | ||||
|
r41078 | eh = exthelper.exthelper() | ||
|
r43346 | |||
|
r43347 | @eh.wrapfunction(localrepo, b'makefilestorage') | ||
|
r39887 | def localrepomakefilestorage(orig, requirements, features, **kwargs): | ||
if b'lfs' in requirements: | ||||
features.add(repository.REPO_FEATURE_LFS) | ||||
return orig(requirements=requirements, features=features, **kwargs) | ||||
|
r43346 | |||
|
r43347 | @eh.wrapfunction(changegroup, b'allsupportedversions') | ||
|
r35097 | def allsupportedversions(orig, ui): | ||
versions = orig(ui) | ||||
|
r43347 | versions.add(b'03') | ||
|
r35097 | return versions | ||
|
r43346 | |||
|
r43347 | @eh.wrapfunction(wireprotov1server, b'_capabilities') | ||
|
r35522 | def _capabilities(orig, repo, proto): | ||
'''Wrap server command to announce lfs server capability''' | ||||
caps = orig(repo, proto) | ||||
|
r43347 | if util.safehasattr(repo.svfs, b'lfslocalblobstore'): | ||
|
r40360 | # Advertise a slightly different capability when lfs is *required*, so | ||
# that the client knows it MUST load the extension. If lfs is not | ||||
# required on the server, there's no reason to autoload the extension | ||||
# on the client. | ||||
if b'lfs' in repo.requirements: | ||||
|
r43347 | caps.append(b'lfs-serve') | ||
|
r40360 | |||
|
r43347 | caps.append(b'lfs') | ||
|
r35522 | return caps | ||
|
r43346 | |||
|
r35097 | def bypasscheckhash(self, text): | ||
return False | ||||
|
r43346 | |||
|
r35097 | def readfromstore(self, text): | ||
"""Read filelog content from local blobstore transform for flagprocessor. | ||||
Default tranform for flagprocessor, returning contents from blobstore. | ||||
Returns a 2-typle (text, validatehash) where validatehash is True as the | ||||
contents of the blobstore should be checked using checkhash. | ||||
""" | ||||
p = pointer.deserialize(text) | ||||
oid = p.oid() | ||||
store = self.opener.lfslocalblobstore | ||||
if not store.has(oid): | ||||
|
r35584 | p.filename = self.filename | ||
|
r35097 | self.opener.lfsremoteblobstore.readbatch([p], store) | ||
|
r35492 | |||
# The caller will validate the content | ||||
text = store.read(oid, verify=False) | ||||
|
r35097 | |||
# pack hg filelog metadata | ||||
hgmeta = {} | ||||
for k in p.keys(): | ||||
|
r43347 | if k.startswith(b'x-hg-'): | ||
name = k[len(b'x-hg-') :] | ||||
|
r35097 | hgmeta[name] = p[k] | ||
|
r43347 | if hgmeta or text.startswith(b'\1\n'): | ||
|
r39914 | text = storageutil.packmeta(hgmeta, text) | ||
|
r35097 | |||
|
r47443 | return (text, True) | ||
|
r35097 | |||
|
r43346 | |||
|
r47443 | def writetostore(self, text): | ||
|
r35097 | # hg filelog metadata (includes rename, etc) | ||
|
r39914 | hgmeta, offset = storageutil.parsemeta(text) | ||
|
r35097 | if offset and offset > 0: | ||
# lfs blob does not contain hg filelog metadata | ||||
text = text[offset:] | ||||
# git-lfs only supports sha256 | ||||
|
r36622 | oid = hex(hashlib.sha256(text).digest()) | ||
|
r35567 | self.opener.lfslocalblobstore.write(oid, text) | ||
|
r35097 | |||
# replace contents with metadata | ||||
|
r43347 | longoid = b'sha256:%s' % oid | ||
metadata = pointer.gitlfspointer(oid=longoid, size=b'%d' % len(text)) | ||||
|
r35097 | |||
# by default, we expect the content to be binary. however, LFS could also | ||||
# be used for non-binary content. add a special entry for non-binary data. | ||||
# this will be used by filectx.isbinary(). | ||||
|
r37102 | if not stringutil.binary(text): | ||
|
r35097 | # not hg filelog metadata (affecting commit hash), no "x-hg-" prefix | ||
|
r43347 | metadata[b'x-is-binary'] = b'0' | ||
|
r35097 | |||
# translate hg filelog metadata to lfs metadata with "x-hg-" prefix | ||||
if hgmeta is not None: | ||||
|
r49768 | for k, v in hgmeta.items(): | ||
|
r43347 | metadata[b'x-hg-%s' % k] = v | ||
|
r35097 | |||
rawtext = metadata.serialize() | ||||
return (rawtext, False) | ||||
|
r43346 | |||
|
r35097 | def _islfs(rlog, node=None, rev=None): | ||
if rev is None: | ||||
if node is None: | ||||
# both None - likely working copy content where node is not ready | ||||
return False | ||||
|
r44410 | rev = rlog.rev(node) | ||
|
r35097 | else: | ||
|
r44410 | node = rlog.node(rev) | ||
|
r47771 | if node == rlog.nullid: | ||
|
r35097 | return False | ||
|
r44410 | flags = rlog.flags(rev) | ||
|
r35097 | return bool(flags & revlog.REVIDX_EXTSTORED) | ||
|
r43346 | |||
|
r41078 | # Wrapping may also be applied by remotefilelog | ||
|
r43346 | def filelogaddrevision( | ||
orig, | ||||
self, | ||||
text, | ||||
transaction, | ||||
link, | ||||
p1, | ||||
p2, | ||||
cachedelta=None, | ||||
node=None, | ||||
flags=revlog.REVIDX_DEFAULT_FLAGS, | ||||
**kwds | ||||
): | ||||
|
r38199 | # The matcher isn't available if reposetup() wasn't called. | ||
|
r43347 | lfstrack = self._revlog.opener.options.get(b'lfstrack') | ||
|
r35097 | |||
|
r38199 | if lfstrack: | ||
textlen = len(text) | ||||
# exclude hg rename meta from file size | ||||
|
r39914 | meta, offset = storageutil.parsemeta(text) | ||
|
r38199 | if offset: | ||
textlen -= offset | ||||
|
r35666 | |||
|
r39892 | if lfstrack(self._revlog.filename, textlen): | ||
|
r38199 | flags |= revlog.REVIDX_EXTSTORED | ||
|
r35097 | |||
|
r43346 | return orig( | ||
self, | ||||
text, | ||||
transaction, | ||||
link, | ||||
p1, | ||||
p2, | ||||
cachedelta=cachedelta, | ||||
node=node, | ||||
flags=flags, | ||||
**kwds | ||||
) | ||||
|
r35097 | |||
|
r41078 | # Wrapping may also be applied by remotefilelog | ||
|
r35097 | def filelogrenamed(orig, self, node): | ||
|
r44410 | if _islfs(self._revlog, node): | ||
r43043 | rawtext = self._revlog.rawdata(node) | |||
|
r35097 | if not rawtext: | ||
return False | ||||
metadata = pointer.deserialize(rawtext) | ||||
|
r43347 | if b'x-hg-copy' in metadata and b'x-hg-copyrev' in metadata: | ||
return metadata[b'x-hg-copy'], bin(metadata[b'x-hg-copyrev']) | ||||
|
r35097 | else: | ||
return False | ||||
return orig(self, node) | ||||
|
r43346 | |||
|
r41078 | # Wrapping may also be applied by remotefilelog | ||
|
r35097 | def filelogsize(orig, self, rev): | ||
|
r44410 | if _islfs(self._revlog, rev=rev): | ||
|
r35097 | # fast path: use lfs metadata to answer size | ||
r43043 | rawtext = self._revlog.rawdata(rev) | |||
|
r35097 | metadata = pointer.deserialize(rawtext) | ||
|
r43347 | return int(metadata[b'size']) | ||
|
r35097 | return orig(self, rev) | ||
|
r43346 | |||
|
r44529 | @eh.wrapfunction(revlog, b'_verify_revision') | ||
def _verify_revision(orig, rl, skipflags, state, node): | ||||
if _islfs(rl, node=node): | ||||
rawtext = rl.rawdata(node) | ||||
metadata = pointer.deserialize(rawtext) | ||||
# Don't skip blobs that are stored locally, as local verification is | ||||
# relatively cheap and there's no other way to verify the raw data in | ||||
# the revlog. | ||||
if rl.opener.lfslocalblobstore.has(metadata.oid()): | ||||
skipflags &= ~revlog.REVIDX_EXTSTORED | ||||
|
r44530 | elif skipflags & revlog.REVIDX_EXTSTORED: | ||
# The wrapped method will set `skipread`, but there's enough local | ||||
# info to check renames. | ||||
state[b'safe_renamed'].add(node) | ||||
|
r44529 | |||
orig(rl, skipflags, state, node) | ||||
|
r43347 | @eh.wrapfunction(context.basefilectx, b'cmp') | ||
|
r35097 | def filectxcmp(orig, self, fctx): | ||
"""returns True if text is different than fctx""" | ||||
# some fctx (ex. hg-git) is not based on basefilectx and do not have islfs | ||||
if self.islfs() and getattr(fctx, 'islfs', lambda: False)(): | ||||
# fast path: check LFS oid | ||||
p1 = pointer.deserialize(self.rawdata()) | ||||
p2 = pointer.deserialize(fctx.rawdata()) | ||||
return p1.oid() != p2.oid() | ||||
return orig(self, fctx) | ||||
|
r43346 | |||
|
r43347 | @eh.wrapfunction(context.basefilectx, b'isbinary') | ||
|
r35097 | def filectxisbinary(orig, self): | ||
if self.islfs(): | ||||
# fast path: use lfs metadata to answer isbinary | ||||
metadata = pointer.deserialize(self.rawdata()) | ||||
# if lfs metadata says nothing, assume it's binary by default | ||||
|
r43347 | return bool(int(metadata.get(b'x-is-binary', 1))) | ||
|
r35097 | return orig(self) | ||
|
r43346 | |||
|
r35097 | def filectxislfs(self): | ||
|
r44410 | return _islfs(self.filelog()._revlog, self.filenode()) | ||
|
r35097 | |||
|
r43346 | |||
|
r43347 | @eh.wrapfunction(cmdutil, b'_updatecatformatter') | ||
|
r35681 | def _updatecatformatter(orig, fm, ctx, matcher, path, decode): | ||
orig(fm, ctx, matcher, path, decode) | ||||
fm.data(rawdata=ctx[path].rawdata()) | ||||
|
r43346 | |||
|
r43347 | @eh.wrapfunction(scmutil, b'wrapconvertsink') | ||
|
r35170 | def convertsink(orig, sink): | ||
sink = orig(sink) | ||||
|
r43347 | if sink.repotype == b'hg': | ||
|
r43346 | |||
|
r35170 | class lfssink(sink.__class__): | ||
|
r43346 | def putcommit( | ||
self, | ||||
files, | ||||
copies, | ||||
parents, | ||||
commit, | ||||
source, | ||||
revmap, | ||||
full, | ||||
cleanp2, | ||||
): | ||||
|
r35170 | pc = super(lfssink, self).putcommit | ||
|
r43346 | node = pc( | ||
files, | ||||
copies, | ||||
parents, | ||||
commit, | ||||
source, | ||||
revmap, | ||||
full, | ||||
cleanp2, | ||||
) | ||||
|
r35170 | |||
|
r43347 | if b'lfs' not in self.repo.requirements: | ||
|
r35170 | ctx = self.repo[node] | ||
# The file list may contain removed files, so check for | ||||
# membership before assuming it is in the context. | ||||
if any(f in ctx and ctx[f].islfs() for f, n in files): | ||||
|
r43347 | self.repo.requirements.add(b'lfs') | ||
|
r45666 | scmutil.writereporequirements(self.repo) | ||
|
r35170 | |||
return node | ||||
sink.__class__ = lfssink | ||||
return sink | ||||
|
r43346 | |||
|
r41078 | # bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs | ||
# options and blob stores are passed from othervfs to the new readonlyvfs. | ||||
|
r43347 | @eh.wrapfunction(vfsmod.readonlyvfs, b'__init__') | ||
|
r35097 | def vfsinit(orig, self, othervfs): | ||
orig(self, othervfs) | ||||
# copy lfs related options | ||||
for k, v in othervfs.options.items(): | ||||
|
r43347 | if k.startswith(b'lfs'): | ||
|
r35097 | self.options[k] = v | ||
# also copy lfs blobstores. note: this can run before reposetup, so lfs | ||||
# blobstore attributes are not always ready at this time. | ||||
|
r43347 | for name in [b'lfslocalblobstore', b'lfsremoteblobstore']: | ||
|
r35097 | if util.safehasattr(othervfs, name): | ||
setattr(self, name, getattr(othervfs, name)) | ||||
|
r43346 | |||
|
r45632 | def _prefetchfiles(repo, revmatches): | ||
|
r35940 | """Ensure that required LFS blobs are present, fetching them as a group if | ||
|
r36159 | needed.""" | ||
|
r43347 | if not util.safehasattr(repo.svfs, b'lfslocalblobstore'): | ||
|
r38199 | return | ||
|
r35940 | pointers = [] | ||
|
r37780 | oids = set() | ||
|
r35940 | localstore = repo.svfs.lfslocalblobstore | ||
|
r45632 | for rev, match in revmatches: | ||
|
r37780 | ctx = repo[rev] | ||
for f in ctx.walk(match): | ||||
p = pointerfromctx(ctx, f) | ||||
if p and p.oid() not in oids and not localstore.has(p.oid()): | ||||
p.filename = f | ||||
pointers.append(p) | ||||
oids.add(p.oid()) | ||||
|
r35940 | |||
if pointers: | ||||
|
r37536 | # Recalculating the repo store here allows 'paths.default' that is set | ||
# on the repo by a clone command to be used for the update. | ||||
blobstore.remote(repo).readbatch(pointers, localstore) | ||||
|
r35940 | |||
|
r43346 | |||
|
r35097 | def _canskipupload(repo): | ||
|
r38199 | # Skip if this hasn't been passed to reposetup() | ||
|
r43347 | if not util.safehasattr(repo.svfs, b'lfsremoteblobstore'): | ||
|
r38199 | return True | ||
|
r35097 | # if remotestore is a null store, upload is a no-op and can be skipped | ||
return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote) | ||||
|
r43346 | |||
|
r35097 | def candownload(repo): | ||
|
r38199 | # Skip if this hasn't been passed to reposetup() | ||
|
r43347 | if not util.safehasattr(repo.svfs, b'lfsremoteblobstore'): | ||
|
r38199 | return False | ||
|
r35097 | # if remotestore is a null store, downloads will lead to nothing | ||
return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote) | ||||
|
r43346 | |||
|
r35097 | def uploadblobsfromrevs(repo, revs): | ||
|
r46554 | """upload lfs blobs introduced by revs | ||
|
r35097 | |||
Note: also used by other extensions e. g. infinitepush. avoid renaming. | ||||
|
r46554 | """ | ||
|
r35097 | if _canskipupload(repo): | ||
return | ||||
pointers = extractpointers(repo, revs) | ||||
uploadblobs(repo, pointers) | ||||
|
r43346 | |||
|
r35097 | def prepush(pushop): | ||
"""Prepush hook. | ||||
Read through the revisions to push, looking for filelog entries that can be | ||||
deserialized into metadata so that we can block the push on their upload to | ||||
the remote blobstore. | ||||
""" | ||||
return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing) | ||||
|
r43346 | |||
|
r43347 | @eh.wrapfunction(exchange, b'push') | ||
|
r35522 | def push(orig, repo, remote, *args, **kwargs): | ||
|
r37582 | """bail on push if the extension isn't enabled on remote when needed, and | ||
update the remote store based on the destination path.""" | ||||
|
r43347 | if b'lfs' in repo.requirements: | ||
|
r35522 | # If the remote peer is for a local repo, the requirement tests in the | ||
# base class method enforce lfs support. Otherwise, some revisions in | ||||
# this repo use lfs, and the remote repo needs the extension loaded. | ||||
|
r43347 | if not remote.local() and not remote.capable(b'lfs'): | ||
|
r35522 | # This is a copy of the message in exchange.push() when requirements | ||
# are missing between local repos. | ||||
|
r43347 | m = _(b"required features are not supported in the destination: %s") | ||
|
r43346 | raise error.Abort( | ||
|
r43347 | m % b'lfs', hint=_(b'enable the lfs extension on the server') | ||
|
r43346 | ) | ||
|
r37582 | |||
# Repositories where this extension is disabled won't have the field. | ||||
# But if there's a requirement, then the extension must be loaded AND | ||||
# there may be blobs to push. | ||||
remotestore = repo.svfs.lfsremoteblobstore | ||||
try: | ||||
repo.svfs.lfsremoteblobstore = blobstore.remote(repo, remote.url()) | ||||
return orig(repo, remote, *args, **kwargs) | ||||
finally: | ||||
repo.svfs.lfsremoteblobstore = remotestore | ||||
else: | ||||
return orig(repo, remote, *args, **kwargs) | ||||
|
r35522 | |||
|
r43346 | |||
|
r41078 | # when writing a bundle via "hg bundle" command, upload related LFS blobs | ||
|
r43347 | @eh.wrapfunction(bundle2, b'writenewbundle') | ||
|
r43346 | def writenewbundle( | ||
orig, ui, repo, source, filename, bundletype, outgoing, *args, **kwargs | ||||
): | ||||
|
r35097 | """upload LFS blobs added by outgoing revisions on 'hg bundle'""" | ||
uploadblobsfromrevs(repo, outgoing.missing) | ||||
|
r43346 | return orig( | ||
ui, repo, source, filename, bundletype, outgoing, *args, **kwargs | ||||
) | ||||
|
r35097 | |||
def extractpointers(repo, revs): | ||||
"""return a list of lfs pointers added by given revs""" | ||||
|
r43347 | repo.ui.debug(b'lfs: computing set of blobs to upload\n') | ||
|
r35097 | pointers = {} | ||
|
r39306 | |||
|
r39426 | makeprogress = repo.ui.makeprogress | ||
|
r43347 | with makeprogress( | ||
_(b'lfs search'), _(b'changesets'), len(revs) | ||||
) as progress: | ||||
|
r39306 | for r in revs: | ||
ctx = repo[r] | ||||
for p in pointersfromctx(ctx).values(): | ||||
pointers[p.oid()] = p | ||||
progress.increment() | ||||
|
r39985 | return sorted(pointers.values(), key=lambda p: p.oid()) | ||
|
r35097 | |||
|
r43346 | |||
|
r36016 | def pointerfromctx(ctx, f, removed=False): | ||
|
r35939 | """return a pointer for the named file from the given changectx, or None if | ||
|
r36016 | the file isn't LFS. | ||
Optionally, the pointer for a file deleted from the context can be returned. | ||||
Since no such pointer is actually stored, and to distinguish from a non LFS | ||||
file, this pointer is represented by an empty dict. | ||||
""" | ||||
_ctx = ctx | ||||
|
r35939 | if f not in ctx: | ||
|
r36016 | if not removed: | ||
return None | ||||
if f in ctx.p1(): | ||||
_ctx = ctx.p1() | ||||
elif f in ctx.p2(): | ||||
_ctx = ctx.p2() | ||||
else: | ||||
return None | ||||
fctx = _ctx[f] | ||||
|
r44410 | if not _islfs(fctx.filelog()._revlog, fctx.filenode()): | ||
|
r35939 | return None | ||
try: | ||||
|
r36016 | p = pointer.deserialize(fctx.rawdata()) | ||
if ctx == _ctx: | ||||
return p | ||||
return {} | ||||
|
r35939 | except pointer.InvalidPointer as ex: | ||
|
r43346 | raise error.Abort( | ||
|
r43347 | _(b'lfs: corrupted pointer (%s@%s): %s\n') | ||
|
r43346 | % (f, short(_ctx.node()), ex) | ||
) | ||||
|
r35939 | |||
|
r36016 | def pointersfromctx(ctx, removed=False): | ||
"""return a dict {path: pointer} for given single changectx. | ||||
If ``removed`` == True and the LFS file was removed from ``ctx``, the value | ||||
stored for the path is an empty dict. | ||||
""" | ||||
|
r35097 | result = {} | ||
|
r40422 | m = ctx.repo().narrowmatch() | ||
# TODO: consider manifest.fastread() instead | ||||
|
r35097 | for f in ctx.files(): | ||
|
r40422 | if not m(f): | ||
continue | ||||
|
r36016 | p = pointerfromctx(ctx, f, removed=removed) | ||
if p is not None: | ||||
|
r35939 | result[f] = p | ||
|
r35097 | return result | ||
|
r43346 | |||
|
r35097 | def uploadblobs(repo, pointers): | ||
"""upload given pointers from local blobstore""" | ||||
if not pointers: | ||||
return | ||||
remoteblob = repo.svfs.lfsremoteblobstore | ||||
remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore) | ||||
|
r35347 | |||
|
r43346 | |||
|
r46836 | @eh.wrapfunction(upgrade_engine, b'finishdatamigration') | ||
|
r35364 | def upgradefinishdatamigration(orig, ui, srcrepo, dstrepo, requirements): | ||
orig(ui, srcrepo, dstrepo, requirements) | ||||
|
r38199 | # Skip if this hasn't been passed to reposetup() | ||
|
r43347 | if util.safehasattr( | ||
srcrepo.svfs, b'lfslocalblobstore' | ||||
) and util.safehasattr(dstrepo.svfs, b'lfslocalblobstore'): | ||||
|
r38199 | srclfsvfs = srcrepo.svfs.lfslocalblobstore.vfs | ||
dstlfsvfs = dstrepo.svfs.lfslocalblobstore.vfs | ||||
|
r35364 | |||
|
r38199 | for dirpath, dirs, files in srclfsvfs.walk(): | ||
for oid in files: | ||||
|
r43347 | ui.write(_(b'copying lfs blob %s\n') % oid) | ||
|
r38199 | lfutil.link(srclfsvfs.join(oid), dstlfsvfs.join(oid)) | ||
|
r35364 | |||
|
r43346 | |||
r46662 | @eh.wrapfunction(upgrade_actions, b'preservedrequirements') | |||
@eh.wrapfunction(upgrade_actions, b'supporteddestrequirements') | ||||
|
r35347 | def upgraderequirements(orig, repo): | ||
reqs = orig(repo) | ||||
|
r43347 | if b'lfs' in repo.requirements: | ||
reqs.add(b'lfs') | ||||
|
r35347 | return reqs | ||