filelog: store filename directly on revlog instance

This attribute is only used by LFS. It is used by one of the revlog
flag processor functions, which gets an instance of the revlog - not
the file storage type. So, it makes sense to store this attribute on
the revlog instead of the filelog.

With this change, I'm pretty confident that LFS is no longer directly
accessing file storage interface members that are revlog centric. i.e.
it gets us one step closer to eliminating revlog-centric APIs from the
file storage interface!

Differential Revision: https://phab.mercurial-scm.org/D4715

hgext/lfs/wrapper.py

@@ -1,406 +1,406 b''
 # wrapper.py - methods wrapping core mercurial logic
 #
 # Copyright 2017 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import hashlib
 
 from mercurial.i18n import _
 from mercurial.node import bin, hex, nullid, short
 
 from mercurial import (
     error,
     repository,
     revlog,
     util,
 )
 
 from mercurial.utils import (
     stringutil,
 )
 
 from ..largefiles import lfutil
 
 from . import (
     blobstore,
     pointer,
 )
 
 def localrepomakefilestorage(orig, requirements, features, **kwargs):
     if b'lfs' in requirements:
         features.add(repository.REPO_FEATURE_LFS)
 
     return orig(requirements=requirements, features=features, **kwargs)
 
 def allsupportedversions(orig, ui):
     versions = orig(ui)
     versions.add('03')
     return versions
 
 def _capabilities(orig, repo, proto):
     '''Wrap server command to announce lfs server capability'''
     caps = orig(repo, proto)
     if util.safehasattr(repo.svfs, 'lfslocalblobstore'):
         # XXX: change to 'lfs=serve' when separate git server isn't required?
         caps.append('lfs')
     return caps
 
 def bypasscheckhash(self, text):
     return False
 
 def readfromstore(self, text):
     """Read filelog content from local blobstore transform for flagprocessor.
 
     Default tranform for flagprocessor, returning contents from blobstore.
     Returns a 2-typle (text, validatehash) where validatehash is True as the
     contents of the blobstore should be checked using checkhash.
     """
     p = pointer.deserialize(text)
     oid = p.oid()
     store = self.opener.lfslocalblobstore
     if not store.has(oid):
         p.filename = self.filename
         self.opener.lfsremoteblobstore.readbatch([p], store)
 
     # The caller will validate the content
     text = store.read(oid, verify=False)
 
     # pack hg filelog metadata
     hgmeta = {}
     for k in p.keys():
         if k.startswith('x-hg-'):
             name = k[len('x-hg-'):]
             hgmeta[name] = p[k]
     if hgmeta or text.startswith('\1\n'):
         text = revlog.packmeta(hgmeta, text)
 
     return (text, True)
 
 def writetostore(self, text):
     # hg filelog metadata (includes rename, etc)
     hgmeta, offset = revlog.parsemeta(text)
     if offset and offset > 0:
         # lfs blob does not contain hg filelog metadata
         text = text[offset:]
 
     # git-lfs only supports sha256
     oid = hex(hashlib.sha256(text).digest())
     self.opener.lfslocalblobstore.write(oid, text)
 
     # replace contents with metadata
     longoid = 'sha256:%s' % oid
     metadata = pointer.gitlfspointer(oid=longoid, size='%d' % len(text))
 
     # by default, we expect the content to be binary. however, LFS could also
     # be used for non-binary content. add a special entry for non-binary data.
     # this will be used by filectx.isbinary().
     if not stringutil.binary(text):
         # not hg filelog metadata (affecting commit hash), no "x-hg-" prefix
         metadata['x-is-binary'] = '0'
 
     # translate hg filelog metadata to lfs metadata with "x-hg-" prefix
     if hgmeta is not None:
         for k, v in hgmeta.iteritems():
             metadata['x-hg-%s' % k] = v
 
     rawtext = metadata.serialize()
     return (rawtext, False)
 
 def _islfs(rlog, node=None, rev=None):
     if rev is None:
         if node is None:
             # both None - likely working copy content where node is not ready
             return False
         rev = rlog._revlog.rev(node)
     else:
         node = rlog._revlog.node(rev)
     if node == nullid:
         return False
     flags = rlog._revlog.flags(rev)
     return bool(flags & revlog.REVIDX_EXTSTORED)
 
 def filelogaddrevision(orig, self, text, transaction, link, p1, p2,
                        cachedelta=None, node=None,
                        flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds):
     # The matcher isn't available if reposetup() wasn't called.
     lfstrack = self._revlog.opener.options.get('lfstrack')
 
     if lfstrack:
         textlen = len(text)
         # exclude hg rename meta from file size
         meta, offset = revlog.parsemeta(text)
         if offset:
             textlen -= offset
 
-        if lfstrack(self.filename, textlen):
+        if lfstrack(self._revlog.filename, textlen):
             flags |= revlog.REVIDX_EXTSTORED
 
     return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
                 node=node, flags=flags, **kwds)
 
 def filelogrenamed(orig, self, node):
     if _islfs(self, node):
         rawtext = self._revlog.revision(node, raw=True)
         if not rawtext:
             return False
         metadata = pointer.deserialize(rawtext)
         if 'x-hg-copy' in metadata and 'x-hg-copyrev' in metadata:
             return metadata['x-hg-copy'], bin(metadata['x-hg-copyrev'])
         else:
             return False
     return orig(self, node)
 
 def filelogsize(orig, self, rev):
     if _islfs(self, rev=rev):
         # fast path: use lfs metadata to answer size
         rawtext = self._revlog.revision(rev, raw=True)
         metadata = pointer.deserialize(rawtext)
         return int(metadata['size'])
     return orig(self, rev)
 
 def filectxcmp(orig, self, fctx):
     """returns True if text is different than fctx"""
     # some fctx (ex. hg-git) is not based on basefilectx and do not have islfs
     if self.islfs() and getattr(fctx, 'islfs', lambda: False)():
         # fast path: check LFS oid
         p1 = pointer.deserialize(self.rawdata())
         p2 = pointer.deserialize(fctx.rawdata())
         return p1.oid() != p2.oid()
     return orig(self, fctx)
 
 def filectxisbinary(orig, self):
     if self.islfs():
         # fast path: use lfs metadata to answer isbinary
         metadata = pointer.deserialize(self.rawdata())
         # if lfs metadata says nothing, assume it's binary by default
         return bool(int(metadata.get('x-is-binary', 1)))
     return orig(self)
 
 def filectxislfs(self):
     return _islfs(self.filelog(), self.filenode())
 
 def _updatecatformatter(orig, fm, ctx, matcher, path, decode):
     orig(fm, ctx, matcher, path, decode)
     fm.data(rawdata=ctx[path].rawdata())
 
 def convertsink(orig, sink):
     sink = orig(sink)
     if sink.repotype == 'hg':
         class lfssink(sink.__class__):
             def putcommit(self, files, copies, parents, commit, source, revmap,
                           full, cleanp2):
                 pc = super(lfssink, self).putcommit
                 node = pc(files, copies, parents, commit, source, revmap, full,
                           cleanp2)
 
                 if 'lfs' not in self.repo.requirements:
                     ctx = self.repo[node]
 
                     # The file list may contain removed files, so check for
                     # membership before assuming it is in the context.
                     if any(f in ctx and ctx[f].islfs() for f, n in files):
                         self.repo.requirements.add('lfs')
                         self.repo._writerequirements()
 
                         # Permanently enable lfs locally
                         self.repo.vfs.append(
                             'hgrc', util.tonativeeol('\n[extensions]\nlfs=\n'))
 
                 return node
 
         sink.__class__ = lfssink
 
     return sink
 
 def vfsinit(orig, self, othervfs):
     orig(self, othervfs)
     # copy lfs related options
     for k, v in othervfs.options.items():
         if k.startswith('lfs'):
             self.options[k] = v
     # also copy lfs blobstores. note: this can run before reposetup, so lfs
     # blobstore attributes are not always ready at this time.
     for name in ['lfslocalblobstore', 'lfsremoteblobstore']:
         if util.safehasattr(othervfs, name):
             setattr(self, name, getattr(othervfs, name))
 
 def _prefetchfiles(repo, revs, match):
     """Ensure that required LFS blobs are present, fetching them as a group if
     needed."""
     if not util.safehasattr(repo.svfs, 'lfslocalblobstore'):
         return
 
     pointers = []
     oids = set()
     localstore = repo.svfs.lfslocalblobstore
 
     for rev in revs:
         ctx = repo[rev]
         for f in ctx.walk(match):
             p = pointerfromctx(ctx, f)
             if p and p.oid() not in oids and not localstore.has(p.oid()):
                 p.filename = f
                 pointers.append(p)
                 oids.add(p.oid())
 
     if pointers:
         # Recalculating the repo store here allows 'paths.default' that is set
         # on the repo by a clone command to be used for the update.
         blobstore.remote(repo).readbatch(pointers, localstore)
 
 def _canskipupload(repo):
     # Skip if this hasn't been passed to reposetup()
     if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
         return True
 
     # if remotestore is a null store, upload is a no-op and can be skipped
     return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
 
 def candownload(repo):
     # Skip if this hasn't been passed to reposetup()
     if not util.safehasattr(repo.svfs, 'lfsremoteblobstore'):
         return False
 
     # if remotestore is a null store, downloads will lead to nothing
     return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
 
 def uploadblobsfromrevs(repo, revs):
     '''upload lfs blobs introduced by revs
 
     Note: also used by other extensions e. g. infinitepush. avoid renaming.
     '''
     if _canskipupload(repo):
         return
     pointers = extractpointers(repo, revs)
     uploadblobs(repo, pointers)
 
 def prepush(pushop):
     """Prepush hook.
 
     Read through the revisions to push, looking for filelog entries that can be
     deserialized into metadata so that we can block the push on their upload to
     the remote blobstore.
     """
     return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing)
 
 def push(orig, repo, remote, *args, **kwargs):
     """bail on push if the extension isn't enabled on remote when needed, and
     update the remote store based on the destination path."""
     if 'lfs' in repo.requirements:
         # If the remote peer is for a local repo, the requirement tests in the
         # base class method enforce lfs support.  Otherwise, some revisions in
         # this repo use lfs, and the remote repo needs the extension loaded.
         if not remote.local() and not remote.capable('lfs'):
             # This is a copy of the message in exchange.push() when requirements
             # are missing between local repos.
             m = _("required features are not supported in the destination: %s")
             raise error.Abort(m % 'lfs',
                               hint=_('enable the lfs extension on the server'))
 
         # Repositories where this extension is disabled won't have the field.
         # But if there's a requirement, then the extension must be loaded AND
         # there may be blobs to push.
         remotestore = repo.svfs.lfsremoteblobstore
         try:
             repo.svfs.lfsremoteblobstore = blobstore.remote(repo, remote.url())
             return orig(repo, remote, *args, **kwargs)
         finally:
             repo.svfs.lfsremoteblobstore = remotestore
     else:
         return orig(repo, remote, *args, **kwargs)
 
 def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing,
                    *args, **kwargs):
     """upload LFS blobs added by outgoing revisions on 'hg bundle'"""
     uploadblobsfromrevs(repo, outgoing.missing)
     return orig(ui, repo, source, filename, bundletype, outgoing, *args,
                 **kwargs)
 
 def extractpointers(repo, revs):
     """return a list of lfs pointers added by given revs"""
     repo.ui.debug('lfs: computing set of blobs to upload\n')
     pointers = {}
 
     makeprogress = repo.ui.makeprogress
     with makeprogress(_('lfs search'), _('changesets'), len(revs)) as progress:
         for r in revs:
             ctx = repo[r]
             for p in pointersfromctx(ctx).values():
                 pointers[p.oid()] = p
             progress.increment()
         return sorted(pointers.values())
 
 def pointerfromctx(ctx, f, removed=False):
     """return a pointer for the named file from the given changectx, or None if
     the file isn't LFS.
 
     Optionally, the pointer for a file deleted from the context can be returned.
     Since no such pointer is actually stored, and to distinguish from a non LFS
     file, this pointer is represented by an empty dict.
     """
     _ctx = ctx
     if f not in ctx:
         if not removed:
             return None
         if f in ctx.p1():
             _ctx = ctx.p1()
         elif f in ctx.p2():
             _ctx = ctx.p2()
         else:
             return None
     fctx = _ctx[f]
     if not _islfs(fctx.filelog(), fctx.filenode()):
         return None
     try:
         p = pointer.deserialize(fctx.rawdata())
         if ctx == _ctx:
             return p
         return {}
     except pointer.InvalidPointer as ex:
         raise error.Abort(_('lfs: corrupted pointer (%s@%s): %s\n')
                           % (f, short(_ctx.node()), ex))
 
 def pointersfromctx(ctx, removed=False):
     """return a dict {path: pointer} for given single changectx.
 
     If ``removed`` == True and the LFS file was removed from ``ctx``, the value
     stored for the path is an empty dict.
     """
     result = {}
     for f in ctx.files():
         p = pointerfromctx(ctx, f, removed=removed)
         if p is not None:
             result[f] = p
     return result
 
 def uploadblobs(repo, pointers):
     """upload given pointers from local blobstore"""
     if not pointers:
         return
 
     remoteblob = repo.svfs.lfsremoteblobstore
     remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore)
 
 def upgradefinishdatamigration(orig, ui, srcrepo, dstrepo, requirements):
     orig(ui, srcrepo, dstrepo, requirements)
 
     # Skip if this hasn't been passed to reposetup()
     if (util.safehasattr(srcrepo.svfs, 'lfslocalblobstore') and
         util.safehasattr(dstrepo.svfs, 'lfslocalblobstore')):
         srclfsvfs = srcrepo.svfs.lfslocalblobstore.vfs
         dstlfsvfs = dstrepo.svfs.lfslocalblobstore.vfs
 
         for dirpath, dirs, files in srclfsvfs.walk():
             for oid in files:
                 ui.write(_('copying lfs blob %s\n') % oid)
                 lfutil.link(srclfsvfs.join(oid), dstlfsvfs.join(oid))
 
 def upgraderequirements(orig, repo):
     reqs = orig(repo)
     if 'lfs' in repo.requirements:
         reqs.add('lfs')
     return reqs

mercurial/filelog.py

@@ -1,271 +1,262 b''
 # filelog.py - file history class for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 from . import (
     error,
     repository,
     revlog,
 )
 from .utils import (
     interfaceutil,
 )
 
 @interfaceutil.implementer(repository.ifilestorage)
 class filelog(object):
     def __init__(self, opener, path):
         self._revlog = revlog.revlog(opener,
                                      '/'.join(('data', path + '.i')),
                                      censorable=True)
         # Full name of the user visible file, relative to the repository root.
         # Used by LFS.
-        self.filename = path
+        self._revlog.filename = path
         # Used by repo upgrade.
         self.index = self._revlog.index
         # Used by changegroup generation.
         self._generaldelta = self._revlog._generaldelta
 
     def __len__(self):
         return len(self._revlog)
 
     def __iter__(self):
         return self._revlog.__iter__()
 
     def revs(self, start=0, stop=None):
         return self._revlog.revs(start=start, stop=stop)
 
     def parents(self, node):
         return self._revlog.parents(node)
 
     def parentrevs(self, rev):
         return self._revlog.parentrevs(rev)
 
     def rev(self, node):
         return self._revlog.rev(node)
 
     def node(self, rev):
         return self._revlog.node(rev)
 
     def lookup(self, node):
         return self._revlog.lookup(node)
 
     def linkrev(self, rev):
         return self._revlog.linkrev(rev)
 
     # Used by verify.
     def flags(self, rev):
         return self._revlog.flags(rev)
 
     def commonancestorsheads(self, node1, node2):
         return self._revlog.commonancestorsheads(node1, node2)
 
     # Used by dagop.blockdescendants().
     def descendants(self, revs):
         return self._revlog.descendants(revs)
 
     def heads(self, start=None, stop=None):
         return self._revlog.heads(start, stop)
 
     # Used by hgweb, children extension.
     def children(self, node):
         return self._revlog.children(node)
 
     def deltaparent(self, rev):
         return self._revlog.deltaparent(rev)
 
     def iscensored(self, rev):
         return self._revlog.iscensored(rev)
 
     # Used by verify.
     def rawsize(self, rev):
         return self._revlog.rawsize(rev)
 
     # Might be unused.
     def checkhash(self, text, node, p1=None, p2=None, rev=None):
         return self._revlog.checkhash(text, node, p1=p1, p2=p2, rev=rev)
 
     def revision(self, node, _df=None, raw=False):
         return self._revlog.revision(node, _df=_df, raw=raw)
 
     def revdiff(self, rev1, rev2):
         return self._revlog.revdiff(rev1, rev2)
 
     def emitrevisiondeltas(self, requests):
         return self._revlog.emitrevisiondeltas(requests)
 
     def addrevision(self, revisiondata, transaction, linkrev, p1, p2,
                     node=None, flags=revlog.REVIDX_DEFAULT_FLAGS,
                     cachedelta=None):
         return self._revlog.addrevision(revisiondata, transaction, linkrev,
                                     p1, p2, node=node, flags=flags,
                                     cachedelta=cachedelta)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         return self._revlog.addgroup(deltas, linkmapper, transaction,
                                  addrevisioncb=addrevisioncb)
 
     def getstrippoint(self, minlink):
         return self._revlog.getstrippoint(minlink)
 
     def strip(self, minlink, transaction):
         return self._revlog.strip(minlink, transaction)
 
     def censorrevision(self, tr, node, tombstone=b''):
         return self._revlog.censorrevision(node, tombstone=tombstone)
 
     def files(self):
         return self._revlog.files()
 
     def read(self, node):
         t = self.revision(node)
         if not t.startswith('\1\n'):
             return t
         s = t.index('\1\n', 2)
         return t[s + 2:]
 
     def add(self, text, meta, transaction, link, p1=None, p2=None):
         if meta or text.startswith('\1\n'):
             text = revlog.packmeta(meta, text)
         return self.addrevision(text, transaction, link, p1, p2)
 
     def renamed(self, node):
         if self.parents(node)[0] != revlog.nullid:
             return False
         t = self.revision(node)
         m = revlog.parsemeta(t)[0]
         # copy and copyrev occur in pairs. In rare cases due to bugs,
         # one can occur without the other.
         if m and "copy" in m and "copyrev" in m:
             return (m["copy"], revlog.bin(m["copyrev"]))
         return False
 
     def size(self, rev):
         """return the size of a given revision"""
 
         # for revisions with renames, we have to go the slow way
         node = self.node(rev)
         if self.renamed(node):
             return len(self.read(node))
         if self.iscensored(rev):
             return 0
 
         # XXX if self.read(node).startswith("\1\n"), this returns (size+4)
         return self._revlog.size(rev)
 
     def cmp(self, node, text):
         """compare text with a given file revision
 
         returns True if text is different than what is stored.
         """
 
         t = text
         if text.startswith('\1\n'):
             t = '\1\n\1\n' + text
 
         samehashes = not self._revlog.cmp(node, t)
         if samehashes:
             return False
 
         # censored files compare against the empty file
         if self.iscensored(self.rev(node)):
             return text != ''
 
         # renaming a file produces a different hash, even if the data
         # remains unchanged. Check if it's the case (slow):
         if self.renamed(node):
             t2 = self.read(node)
             return t2 != text
 
         return True
 
     def verifyintegrity(self, state):
         return self._revlog.verifyintegrity(state)
 
     # TODO these aren't part of the interface and aren't internal methods.
     # Callers should be fixed to not use them.
 
-    # Used by LFS.
-    @property
-    def filename(self):
-        return self._revlog.filename
-
-    @filename.setter
-    def filename(self, value):
-        self._revlog.filename = value
-
     # Used by bundlefilelog, unionfilelog.
     @property
     def indexfile(self):
         return self._revlog.indexfile
 
     @indexfile.setter
     def indexfile(self, value):
         self._revlog.indexfile = value
 
     # Used by repo upgrade.
     @property
     def opener(self):
         return self._revlog.opener
 
     # Used by repo upgrade.
     def clone(self, tr, destrevlog, **kwargs):
         if not isinstance(destrevlog, filelog):
             raise error.ProgrammingError('expected filelog to clone()')
 
         return self._revlog.clone(tr, destrevlog._revlog, **kwargs)
 
 class narrowfilelog(filelog):
     """Filelog variation to be used with narrow stores."""
 
     def __init__(self, opener, path, narrowmatch):
         super(narrowfilelog, self).__init__(opener, path)
         self._narrowmatch = narrowmatch
 
     def renamed(self, node):
         res = super(narrowfilelog, self).renamed(node)
 
         # Renames that come from outside the narrowspec are problematic
         # because we may lack the base text for the rename. This can result
         # in code attempting to walk the ancestry or compute a diff
         # encountering a missing revision. We address this by silently
         # removing rename metadata if the source file is outside the
         # narrow spec.
         #
         # A better solution would be to see if the base revision is available,
         # rather than assuming it isn't.
         #
         # An even better solution would be to teach all consumers of rename
         # metadata that the base revision may not be available.
         #
         # TODO consider better ways of doing this.
         if res and not self._narrowmatch(res[0]):
             return None
 
         return res
 
     def size(self, rev):
         # Because we have a custom renamed() that may lie, we need to call
         # the base renamed() to report accurate results.
         node = self.node(rev)
         if super(narrowfilelog, self).renamed(node):
             return len(self.read(node))
         else:
             return super(narrowfilelog, self).size(rev)
 
     def cmp(self, node, text):
         different = super(narrowfilelog, self).cmp(node, text)
 
         # Because renamed() may lie, we may get false positives for
         # different content. Check for this by comparing against the original
         # renamed() implementation.
         if different:
             if super(narrowfilelog, self).renamed(node):
                 t2 = self.read(node)
                 return t2 != text
 
         return different