__init__.py
444 lines
| 14.2 KiB
| text/x-python
|
PythonLexer
|
r35097 | # lfs - hash-preserving large file support using Git-LFS protocol | ||
# | ||||
# Copyright 2017 Facebook, Inc. | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
"""lfs - large file support (EXPERIMENTAL) | ||||
|
r35786 | This extension allows large files to be tracked outside of the normal | ||
repository storage and stored on a centralized server, similar to the | ||||
``largefiles`` extension. The ``git-lfs`` protocol is used when | ||||
communicating with the server, so existing git infrastructure can be | ||||
harnessed. Even though the files are stored outside of the repository, | ||||
they are still integrity checked in the same manner as normal files. | ||||
|
r35683 | |||
|
r35786 | The files stored outside of the repository are downloaded on demand, | ||
which reduces the time to clone, and possibly the local disk usage. | ||||
This changes fundamental workflows in a DVCS, so careful thought | ||||
should be given before deploying it. :hg:`convert` can be used to | ||||
convert LFS repositories to normal repositories that no longer | ||||
require this extension, and do so without changing the commit hashes. | ||||
This allows the extension to be disabled if the centralized workflow | ||||
becomes burdensome. However, the pre and post convert clones will | ||||
not be able to communicate with each other unless the extension is | ||||
enabled on both. | ||||
|
r35825 | To start a new repository, or to add LFS files to an existing one, just | ||
create an ``.hglfs`` file as described below in the root directory of | ||||
the repository. Typically, this file should be put under version | ||||
control, so that the settings will propagate to other repositories with | ||||
push and pull. During any commit, Mercurial will consult this file to | ||||
determine if an added or modified file should be stored externally. The | ||||
type of storage depends on the characteristics of the file at each | ||||
commit. A file that is near a size threshold may switch back and forth | ||||
between LFS and normal storage, as needed. | ||||
|
r35786 | |||
Alternately, both normal repositories and largefile controlled | ||||
repositories can be converted to LFS by using :hg:`convert` and the | ||||
``lfs.track`` config option described below. The ``.hglfs`` file | ||||
should then be created and added, to control subsequent LFS selection. | ||||
The hashes are also unchanged in this case. The LFS and non-LFS | ||||
repositories can be distinguished because the LFS repository will | ||||
abort any command if this extension is disabled. | ||||
|
r35683 | |||
|
r35786 | Committed LFS files are held locally, until the repository is pushed. | ||
Prior to pushing the normal repository data, the LFS files that are | ||||
tracked by the outgoing commits are automatically uploaded to the | ||||
configured central server. No LFS files are transferred on | ||||
:hg:`pull` or :hg:`clone`. Instead, the files are downloaded on | ||||
demand as they need to be read, if a cached copy cannot be found | ||||
locally. Both committing and downloading an LFS file will link the | ||||
file to a usercache, to speed up future access. See the `usercache` | ||||
config setting described below. | ||||
|
r43597 | The extension reads its configuration from a versioned ``.hglfs`` | ||
configuration file found in the root of the working directory. The | ||||
``.hglfs`` file uses the same syntax as all other Mercurial | ||||
configuration files. It uses a single section, ``[track]``. | ||||
|
r35786 | |||
|
r43597 | The ``[track]`` section specifies which files are stored as LFS (or | ||
not). Each line is keyed by a file pattern, with a predicate value. | ||||
The first file pattern match is used, so put more specific patterns | ||||
first. The available predicates are ``all()``, ``none()``, and | ||||
``size()``. See "hg help filesets.size" for the latter. | ||||
|
r35786 | |||
|
r43597 | Example versioned ``.hglfs`` file:: | ||
|
r35683 | |||
|
r43597 | [track] | ||
# No Makefile or python file, anywhere, will be LFS | ||||
**Makefile = none() | ||||
**.py = none() | ||||
|
r35683 | |||
|
r43597 | **.zip = all() | ||
**.exe = size(">1MB") | ||||
|
r35786 | |||
|
r43597 | # Catchall for everything not matched above | ||
** = size(">10MB") | ||||
|
r35683 | |||
|
r35097 | Configs:: | ||
[lfs] | ||||
# Remote endpoint. Multiple protocols are supported: | ||||
# - http(s)://user:pass@example.com/path | ||||
# git-lfs endpoint | ||||
# - file:///tmp/path | ||||
# local filesystem, usually for testing | ||||
|
r37582 | # if unset, lfs will assume the remote repository also handles blob storage | ||
# for http(s) URLs. Otherwise, lfs will prompt to set this when it must | ||||
# use this value. | ||||
|
r35097 | # (default: unset) | ||
|
r35786 | url = https://example.com/repo.git/info/lfs | ||
|
r35097 | |||
|
r35636 | # Which files to track in LFS. Path tests are "**.extname" for file | ||
# extensions, and "path:under/some/directory" for path prefix. Both | ||||
|
r35759 | # are relative to the repository root. | ||
|
r35636 | # File size can be tested with the "size()" fileset, and tests can be | ||
# joined with fileset operators. (See "hg help filesets.operators".) | ||||
# | ||||
# Some examples: | ||||
# - all() # everything | ||||
# - none() # nothing | ||||
# - size(">20MB") # larger than 20MB | ||||
# - !**.txt # anything not a *.txt file | ||||
# - **.zip | **.tar.gz | **.7z # some types of compressed files | ||||
|
r35759 | # - path:bin # files under "bin" in the project root | ||
|
r35636 | # - (**.php & size(">2MB")) | (**.js & size(">5MB")) | **.tar.gz | ||
|
r35759 | # | (path:bin & !path:/bin/README) | size(">1GB") | ||
|
r35636 | # (default: none()) | ||
|
r35683 | # | ||
# This is ignored if there is a tracked '.hglfs' file, and this setting | ||||
# will eventually be deprecated and removed. | ||||
|
r35636 | track = size(">10M") | ||
|
r35097 | |||
# how many times to retry before giving up on transferring an object | ||||
retry = 5 | ||||
|
r35281 | |||
# the local directory to store lfs files for sharing across local clones. | ||||
# If not set, the cache is located in an OS specific cache location. | ||||
usercache = /path/to/global/cache | ||||
|
r35097 | """ | ||
from __future__ import absolute_import | ||||
|
r40304 | import sys | ||
|
r35098 | from mercurial.i18n import _ | ||
|
r46729 | from mercurial.node import bin | ||
|
r35098 | |||
|
r35097 | from mercurial import ( | ||
r46369 | bundlecaches, | |||
|
r35683 | config, | ||
|
r41315 | context, | ||
|
r35683 | error, | ||
|
r35097 | extensions, | ||
|
r41078 | exthelper, | ||
|
r35097 | filelog, | ||
|
r38841 | filesetlang, | ||
|
r35167 | localrepo, | ||
|
r48928 | logcmdutil, | ||
|
r35636 | minifileset, | ||
|
r35675 | pycompat, | ||
|
r35097 | revlog, | ||
scmutil, | ||||
|
r36939 | templateutil, | ||
|
r35749 | util, | ||
|
r35097 | ) | ||
|
r43346 | from mercurial.interfaces import repository | ||
|
r43078 | |||
|
r35097 | from . import ( | ||
blobstore, | ||||
|
r37165 | wireprotolfsserver, | ||
|
r35097 | wrapper, | ||
) | ||||
# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for | ||||
# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should | ||||
# be specifying the version(s) of Mercurial they are tested with, or | ||||
# leave the attribute unspecified. | ||||
|
r43347 | testedwith = b'ships-with-hg-core' | ||
|
r35097 | |||
|
r41078 | eh = exthelper.exthelper() | ||
eh.merge(wrapper.eh) | ||||
eh.merge(wireprotolfsserver.eh) | ||||
|
r35099 | |||
|
r41078 | cmdtable = eh.cmdtable | ||
configtable = eh.configtable | ||||
extsetup = eh.finalextsetup | ||||
uisetup = eh.finaluisetup | ||||
|
r41100 | filesetpredicate = eh.filesetpredicate | ||
|
r41078 | reposetup = eh.finalreposetup | ||
|
r41099 | templatekeyword = eh.templatekeyword | ||
|
r41078 | |||
|
r43346 | eh.configitem( | ||
|
r46554 | b'experimental', | ||
b'lfs.serve', | ||||
default=True, | ||||
|
r37265 | ) | ||
|
r43346 | eh.configitem( | ||
|
r46554 | b'experimental', | ||
b'lfs.user-agent', | ||||
default=None, | ||||
|
r35456 | ) | ||
|
r43346 | eh.configitem( | ||
|
r46554 | b'experimental', | ||
b'lfs.disableusercache', | ||||
default=False, | ||||
|
r37580 | ) | ||
|
r43346 | eh.configitem( | ||
|
r46554 | b'experimental', | ||
b'lfs.worker-enable', | ||||
default=True, | ||||
|
r35750 | ) | ||
|
r35456 | |||
|
r43346 | eh.configitem( | ||
|
r46554 | b'lfs', | ||
b'url', | ||||
default=None, | ||||
|
r35099 | ) | ||
|
r43346 | eh.configitem( | ||
|
r46554 | b'lfs', | ||
b'usercache', | ||||
default=None, | ||||
|
r35281 | ) | ||
|
r35636 | # Deprecated | ||
|
r43346 | eh.configitem( | ||
|
r46554 | b'lfs', | ||
b'threshold', | ||||
default=None, | ||||
|
r35099 | ) | ||
|
r43346 | eh.configitem( | ||
|
r46554 | b'lfs', | ||
b'track', | ||||
default=b'none()', | ||||
|
r35636 | ) | ||
|
r43346 | eh.configitem( | ||
|
r46554 | b'lfs', | ||
b'retry', | ||||
default=5, | ||||
|
r35099 | ) | ||
|
r35097 | |||
|
r40304 | lfsprocessor = ( | ||
wrapper.readfromstore, | ||||
wrapper.writetostore, | ||||
wrapper.bypasscheckhash, | ||||
) | ||||
|
r43346 | |||
|
r35167 | def featuresetup(ui, supported): | ||
# don't die on seeing a repo with the lfs requirement | ||||
|
r43347 | supported |= {b'lfs'} | ||
|
r35167 | |||
|
r43346 | |||
|
r41078 | @eh.uisetup | ||
def _uisetup(ui): | ||||
|
r37153 | localrepo.featuresetupfuncs.add(featuresetup) | ||
|
r35167 | |||
|
r43346 | |||
|
r41078 | @eh.reposetup | ||
def _reposetup(ui, repo): | ||||
|
r35097 | # Nothing to do with a remote repo | ||
if not repo.local(): | ||||
return | ||||
repo.svfs.lfslocalblobstore = blobstore.local(repo) | ||||
repo.svfs.lfsremoteblobstore = blobstore.remote(repo) | ||||
|
r35683 | class lfsrepo(repo.__class__): | ||
@localrepo.unfilteredmethod | ||||
|
r42839 | def commitctx(self, ctx, error=False, origctx=None): | ||
|
r43347 | repo.svfs.options[b'lfstrack'] = _trackedmatcher(self) | ||
|
r42839 | return super(lfsrepo, self).commitctx(ctx, error, origctx=origctx) | ||
|
r35683 | |||
repo.__class__ = lfsrepo | ||||
|
r43347 | if b'lfs' not in repo.requirements: | ||
|
r43346 | |||
|
r35167 | def checkrequireslfs(ui, repo, **kwargs): | ||
|
r43347 | if b'lfs' in repo.requirements: | ||
|
r40184 | return 0 | ||
|
r43906 | last = kwargs.get('node_last') | ||
|
r40184 | if last: | ||
|
r46729 | s = repo.set(b'%n:%n', bin(kwargs['node']), bin(last)) | ||
|
r40184 | else: | ||
|
r46729 | s = repo.set(b'%n', bin(kwargs['node'])) | ||
|
r41266 | match = repo._storenarrowmatch | ||
|
r35520 | for ctx in s: | ||
|
r35167 | # TODO: is there a way to just walk the files in the commit? | ||
|
r43346 | if any( | ||
ctx[f].islfs() for f in ctx.files() if f in ctx and match(f) | ||||
): | ||||
|
r43347 | repo.requirements.add(b'lfs') | ||
|
r39887 | repo.features.add(repository.REPO_FEATURE_LFS) | ||
|
r45666 | scmutil.writereporequirements(repo) | ||
|
r43347 | repo.prepushoutgoinghooks.add(b'lfs', wrapper.prepush) | ||
|
r35520 | break | ||
|
r35167 | |||
|
r43347 | ui.setconfig(b'hooks', b'commit.lfs', checkrequireslfs, b'lfs') | ||
ui.setconfig( | ||||
b'hooks', b'pretxnchangegroup.lfs', checkrequireslfs, b'lfs' | ||||
) | ||||
|
r35753 | else: | ||
|
r43347 | repo.prepushoutgoinghooks.add(b'lfs', wrapper.prepush) | ||
|
r35167 | |||
|
r43346 | |||
|
r35898 | def _trackedmatcher(repo): | ||
|
r35682 | """Return a function (path, size) -> bool indicating whether or not to | ||
track a given file with lfs.""" | ||||
|
r43347 | if not repo.wvfs.exists(b'.hglfs'): | ||
|
r35825 | # No '.hglfs' in wdir. Fallback to config for now. | ||
|
r43347 | trackspec = repo.ui.config(b'lfs', b'track') | ||
|
r35683 | |||
|
r35825 | # deprecated config: lfs.threshold | ||
|
r43347 | threshold = repo.ui.configbytes(b'lfs', b'threshold') | ||
|
r35825 | if threshold: | ||
|
r38841 | filesetlang.parse(trackspec) # make sure syntax errors are confined | ||
|
r43347 | trackspec = b"(%s) | size('>%d')" % (trackspec, threshold) | ||
|
r35683 | |||
|
r35825 | return minifileset.compile(trackspec) | ||
|
r35683 | |||
|
r43347 | data = repo.wvfs.tryread(b'.hglfs') | ||
|
r35683 | if not data: | ||
return lambda p, s: False | ||||
# Parse errors here will abort with a message that points to the .hglfs file | ||||
# and line number. | ||||
cfg = config.config() | ||||
|
r43347 | cfg.parse(b'.hglfs', data) | ||
|
r35682 | |||
|
r35683 | try: | ||
|
r43346 | rules = [ | ||
(minifileset.compile(pattern), minifileset.compile(rule)) | ||||
|
r43347 | for pattern, rule in cfg.items(b'track') | ||
|
r43346 | ] | ||
|
r35683 | except error.ParseError as e: | ||
# The original exception gives no indicator that the error is in the | ||||
# .hglfs file, so add that. | ||||
# TODO: See if the line number of the file can be made available. | ||||
|
r43347 | raise error.Abort(_(b'parse error in .hglfs: %s') % e) | ||
|
r35683 | |||
def _match(path, size): | ||||
for pat, rule in rules: | ||||
if pat(path, size): | ||||
return rule(path, size) | ||||
return False | ||||
return _match | ||||
|
r35682 | |||
|
r43346 | |||
|
r41078 | # Called by remotefilelog | ||
|
r35097 | def wrapfilelog(filelog): | ||
wrapfunction = extensions.wrapfunction | ||||
wrapfunction(filelog, 'addrevision', wrapper.filelogaddrevision) | ||||
wrapfunction(filelog, 'renamed', wrapper.filelogrenamed) | ||||
wrapfunction(filelog, 'size', wrapper.filelogsize) | ||||
|
r43346 | |||
|
r43347 | @eh.wrapfunction(localrepo, b'resolverevlogstorevfsoptions') | ||
|
r40304 | def _resolverevlogstorevfsoptions(orig, ui, requirements, features): | ||
opts = orig(ui, requirements, features) | ||||
for name, module in extensions.extensions(ui): | ||||
if module is sys.modules[__name__]: | ||||
if revlog.REVIDX_EXTSTORED in opts[b'flagprocessors']: | ||||
|
r43346 | msg = ( | ||
_(b"cannot register multiple processors on flag '%#x'.") | ||||
% revlog.REVIDX_EXTSTORED | ||||
) | ||||
|
r40304 | raise error.Abort(msg) | ||
opts[b'flagprocessors'][revlog.REVIDX_EXTSTORED] = lfsprocessor | ||||
break | ||||
return opts | ||||
|
r43346 | |||
|
r41078 | @eh.extsetup | ||
def _extsetup(ui): | ||||
|
r35097 | wrapfilelog(filelog.filelog) | ||
|
r41315 | context.basefilectx.islfs = wrapper.filectxislfs | ||
|
r43347 | scmutil.fileprefetchhooks.add(b'lfs', wrapper._prefetchfiles) | ||
|
r35940 | |||
|
r35097 | # Make bundle choose changegroup3 instead of changegroup2. This affects | ||
# "hg bundle" command. Note: it does not cover all bundle formats like | ||||
# "packed1". Using "packed1" with lfs will likely cause trouble. | ||||
r46369 | bundlecaches._bundlespeccontentopts[b"v2"][b"cg.version"] = b"03" | |||
|
r35097 | |||
|
r43346 | |||
|
r43347 | @eh.filesetpredicate(b'lfs()') | ||
|
r36008 | def lfsfileset(mctx, x): | ||
"""File that uses LFS storage.""" | ||||
# i18n: "lfs" is a keyword | ||||
|
r43347 | filesetlang.getargs(x, 0, 0, _(b"lfs takes no arguments")) | ||
|
r38711 | ctx = mctx.ctx | ||
|
r43346 | |||
|
r38711 | def lfsfilep(f): | ||
return wrapper.pointerfromctx(ctx, f, removed=True) is not None | ||||
|
r43346 | |||
|
r43347 | return mctx.predicate(lfsfilep, predrepr=b'<lfs>') | ||
|
r36008 | |||
|
r43346 | |||
|
r43347 | @eh.templatekeyword(b'lfs_files', requires={b'ctx'}) | ||
|
r36616 | def lfsfiles(context, mapping): | ||
|
r36017 | """List of strings. All files modified, added, or removed by this | ||
changeset.""" | ||||
|
r43347 | ctx = context.resource(mapping, b'ctx') | ||
|
r35675 | |||
|
r43346 | pointers = wrapper.pointersfromctx(ctx, removed=True) # {path: pointer} | ||
|
r35675 | files = sorted(pointers.keys()) | ||
|
r35787 | def pointer(v): | ||
|
r35749 | # In the file spec, version is first and the other keys are sorted. | ||
|
r43347 | sortkeyfunc = lambda x: (x[0] != b'version', x) | ||
|
r43375 | items = sorted(pycompat.iteritems(pointers[v]), key=sortkeyfunc) | ||
|
r35749 | return util.sortdict(items) | ||
|
r35675 | makemap = lambda v: { | ||
|
r43347 | b'file': v, | ||
b'lfsoid': pointers[v].oid() if pointers[v] else None, | ||||
b'lfspointer': templateutil.hybriddict(pointer(v)), | ||||
|
r35675 | } | ||
# TODO: make the separator ', '? | ||||
|
r43347 | f = templateutil._showcompatlist(context, mapping, b'lfs_file', files) | ||
|
r36939 | return templateutil.hybrid(f, files, makemap, pycompat.identity) | ||
|
r35097 | |||
|
r43346 | |||
@eh.command( | ||||
|
r43347 | b'debuglfsupload', | ||
[(b'r', b'rev', [], _(b'upload large files introduced by REV'))], | ||||
|
r43346 | ) | ||
|
r35097 | def debuglfsupload(ui, repo, **opts): | ||
"""upload lfs blobs added by the working copy parent or given revisions""" | ||||
|
r43906 | revs = opts.get('rev', []) | ||
|
r48928 | pointers = wrapper.extractpointers(repo, logcmdutil.revrange(repo, revs)) | ||
|
r35097 | wrapper.uploadblobs(repo, pointers) | ||
|
r44528 | |||
@eh.wrapcommand( | ||||
|
r44529 | b'verify', | ||
opts=[(b'', b'no-lfs', None, _(b'skip missing lfs blob content'))], | ||||
|
r44528 | ) | ||
def verify(orig, ui, repo, **opts): | ||||
skipflags = repo.ui.configint(b'verify', b'skipflags') | ||||
no_lfs = opts.pop('no_lfs') | ||||
if skipflags: | ||||
# --lfs overrides the config bit, if set. | ||||
if no_lfs is False: | ||||
skipflags &= ~repository.REVISION_FLAG_EXTSTORED | ||||
else: | ||||
skipflags = 0 | ||||
if no_lfs is True: | ||||
skipflags |= repository.REVISION_FLAG_EXTSTORED | ||||
with ui.configoverride({(b'verify', b'skipflags'): skipflags}): | ||||
return orig(ui, repo, **opts) | ||||