blobstore.py
759 lines
| 26.8 KiB
| text/x-python
|
PythonLexer
|
r35097 | # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages | ||
# | ||||
# Copyright 2017 Facebook, Inc. | ||||
# | ||||
# This software may be used and distributed according to the terms of the | ||||
# GNU General Public License version 2 or any later version. | ||||
from __future__ import absolute_import | ||||
|
r40701 | import contextlib | ||
|
r37535 | import errno | ||
|
r35492 | import hashlib | ||
|
r35097 | import json | ||
import os | ||||
import re | ||||
|
r35491 | import socket | ||
|
r35097 | |||
|
r35098 | from mercurial.i18n import _ | ||
|
r43359 | from mercurial.pycompat import getattr | ||
|
r35098 | |||
|
r35097 | from mercurial import ( | ||
|
r40697 | encoding, | ||
|
r35097 | error, | ||
|
r40711 | node, | ||
|
r35363 | pathutil, | ||
|
r36619 | pycompat, | ||
|
r35097 | url as urlmod, | ||
util, | ||||
vfs as vfsmod, | ||||
|
r35449 | worker, | ||
|
r35097 | ) | ||
|
r43346 | from mercurial.utils import stringutil | ||
|
r40697 | |||
|
r35281 | from ..largefiles import lfutil | ||
|
r35097 | # 64 bytes for SHA256 | ||
|
r36473 | _lfsre = re.compile(br'\A[a-f0-9]{64}\Z') | ||
|
r35097 | |||
|
r43346 | |||
|
r35097 | class lfsvfs(vfsmod.vfs): | ||
def join(self, path): | ||||
"""split the path at first two characters, like: XX/XXXXX...""" | ||||
if not _lfsre.match(path): | ||||
|
r41471 | raise error.ProgrammingError(b'unexpected lfs path: %s' % path) | ||
|
r35097 | return super(lfsvfs, self).join(path[0:2], path[2:]) | ||
|
r35363 | def walk(self, path=None, onerror=None): | ||
|
r35397 | """Yield (dirpath, [], oids) tuple for blobs under path | ||
|
r35363 | |||
Oids only exist in the root of this vfs, so dirpath is always ''. | ||||
""" | ||||
root = os.path.normpath(self.base) | ||||
# when dirpath == root, dirpath[prefixlen:] becomes empty | ||||
# because len(dirpath) < prefixlen. | ||||
prefixlen = len(pathutil.normasprefix(root)) | ||||
oids = [] | ||||
|
r43346 | for dirpath, dirs, files in os.walk( | ||
self.reljoin(self.base, path or b''), onerror=onerror | ||||
): | ||||
|
r35363 | dirpath = dirpath[prefixlen:] | ||
# Silently skip unexpected files and directories | ||||
if len(dirpath) == 2: | ||||
|
r43346 | oids.extend( | ||
[dirpath + f for f in files if _lfsre.match(dirpath + f)] | ||||
) | ||||
|
r35363 | |||
|
r43347 | yield (b'', [], oids) | ||
|
r35363 | |||
|
r43346 | |||
|
r37535 | class nullvfs(lfsvfs): | ||
def __init__(self): | ||||
pass | ||||
def exists(self, oid): | ||||
return False | ||||
def read(self, oid): | ||||
# store.read() calls into here if the blob doesn't exist in its | ||||
# self.vfs. Raise the same error as a normal vfs when asked to read a | ||||
# file that doesn't exist. The only difference is the full file path | ||||
# isn't available in the error. | ||||
|
r43346 | raise IOError( | ||
errno.ENOENT, | ||||
pycompat.sysstr(b'%s: No such file or directory' % oid), | ||||
) | ||||
|
r37535 | |||
def walk(self, path=None, onerror=None): | ||||
|
r41471 | return (b'', [], []) | ||
|
r37535 | |||
def write(self, oid, data): | ||||
pass | ||||
|
r43346 | |||
|
r44596 | class lfsuploadfile(object): | ||
|
r35097 | """a file-like object that supports __len__ and read. | ||
""" | ||||
|
r44596 | def __init__(self, fp): | ||
|
r35097 | self._fp = fp | ||
fp.seek(0, os.SEEK_END) | ||||
self._len = fp.tell() | ||||
fp.seek(0) | ||||
def __len__(self): | ||||
return self._len | ||||
def read(self, size): | ||||
if self._fp is None: | ||||
return b'' | ||||
data = self._fp.read(size) | ||||
|
r44596 | if not data: | ||
|
r35097 | self._fp.close() | ||
self._fp = None | ||||
return data | ||||
|
r43346 | |||
|
r35097 | class local(object): | ||
"""Local blobstore for large file contents. | ||||
This blobstore is used both as a cache and as a staging area for large blobs | ||||
to be uploaded to the remote blobstore. | ||||
""" | ||||
def __init__(self, repo): | ||||
|
r41471 | fullpath = repo.svfs.join(b'lfs/objects') | ||
|
r35097 | self.vfs = lfsvfs(fullpath) | ||
|
r37580 | |||
|
r41471 | if repo.ui.configbool(b'experimental', b'lfs.disableusercache'): | ||
|
r37535 | self.cachevfs = nullvfs() | ||
else: | ||||
|
r41471 | usercache = lfutil._usercachedir(repo.ui, b'lfs') | ||
|
r37580 | self.cachevfs = lfsvfs(usercache) | ||
|
r35489 | self.ui = repo.ui | ||
|
r35097 | |||
|
r35543 | def open(self, oid): | ||
"""Open a read-only file descriptor to the named blob, in either the | ||||
usercache or the local store.""" | ||||
|
r35555 | # The usercache is the most likely place to hold the file. Commit will | ||
# write to both it and the local store, as will anything that downloads | ||||
# the blobs. However, things like clone without an update won't | ||||
# populate the local store. For an init + push of a local clone, | ||||
# the usercache is the only place it _could_ be. If not present, the | ||||
# missing file msg here will indicate the local repo, not the usercache. | ||||
|
r35543 | if self.cachevfs.exists(oid): | ||
|
r41471 | return self.cachevfs(oid, b'rb') | ||
|
r35543 | |||
|
r41471 | return self.vfs(oid, b'rb') | ||
|
r35543 | |||
|
r44544 | def download(self, oid, src, content_length): | ||
|
r35565 | """Read the blob from the remote source in chunks, verify the content, | ||
and write to this local blobstore.""" | ||||
sha256 = hashlib.sha256() | ||||
|
r44544 | size = 0 | ||
|
r35565 | |||
|
r41471 | with self.vfs(oid, b'wb', atomictemp=True) as fp: | ||
|
r35565 | for chunk in util.filechunkiter(src, size=1048576): | ||
fp.write(chunk) | ||||
sha256.update(chunk) | ||||
|
r44544 | size += len(chunk) | ||
# If the server advertised a length longer than what we actually | ||||
# received, then we should expect that the server crashed while | ||||
# producing the response (but the server has no way of telling us | ||||
# that), and we really don't need to try to write the response to | ||||
# the localstore, because it's not going to match the expected. | ||||
if content_length is not None and int(content_length) != size: | ||||
msg = ( | ||||
b"Response length (%s) does not match Content-Length " | ||||
b"header (%d): likely server-side crash" | ||||
) | ||||
raise LfsRemoteError(_(msg) % (size, int(content_length))) | ||||
|
r35565 | |||
|
r40711 | realoid = node.hex(sha256.digest()) | ||
|
r35565 | if realoid != oid: | ||
|
r43346 | raise LfsCorruptionError( | ||
_(b'corrupt remote lfs object: %s') % oid | ||||
) | ||||
|
r35565 | |||
|
r37535 | self._linktousercache(oid) | ||
|
r35565 | |||
|
r35567 | def write(self, oid, data): | ||
"""Write blob to local blobstore. | ||||
|
r35492 | |||
|
r35567 | This should only be called from the filelog during a commit or similar. | ||
As such, there is no need to verify the data. Imports from a remote | ||||
store must use ``download()`` instead.""" | ||||
|
r41471 | with self.vfs(oid, b'wb', atomictemp=True) as fp: | ||
|
r35097 | fp.write(data) | ||
|
r37535 | self._linktousercache(oid) | ||
|
r39491 | def linkfromusercache(self, oid): | ||
"""Link blobs found in the user cache into this store. | ||||
The server module needs to do this when it lets the client know not to | ||||
upload the blob, to ensure it is always available in this store. | ||||
Normally this is done implicitly when the client reads or writes the | ||||
blob, but that doesn't happen when the server tells the client that it | ||||
already has the blob. | ||||
""" | ||||
|
r43346 | if not isinstance(self.cachevfs, nullvfs) and not self.vfs.exists(oid): | ||
|
r41471 | self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) | ||
|
r39491 | lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) | ||
|
r37535 | def _linktousercache(self, oid): | ||
|
r35281 | # XXX: should we verify the content of the cache, and hardlink back to | ||
# the local store on success, but truncate, write and link on failure? | ||||
|
r43346 | if not self.cachevfs.exists(oid) and not isinstance( | ||
self.cachevfs, nullvfs | ||||
): | ||||
|
r41471 | self.ui.note(_(b'lfs: adding %s to the usercache\n') % oid) | ||
|
r35567 | lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid)) | ||
|
r35281 | |||
|
r35492 | def read(self, oid, verify=True): | ||
|
r35097 | """Read blob from local blobstore.""" | ||
|
r35281 | if not self.vfs.exists(oid): | ||
|
r35492 | blob = self._read(self.cachevfs, oid, verify) | ||
|
r35493 | |||
# Even if revlog will verify the content, it needs to be verified | ||||
# now before making the hardlink to avoid propagating corrupt blobs. | ||||
# Don't abort if corruption is detected, because `hg verify` will | ||||
# give more useful info about the corruption- simply don't add the | ||||
# hardlink. | ||||
|
r40711 | if verify or node.hex(hashlib.sha256(blob).digest()) == oid: | ||
|
r41471 | self.ui.note(_(b'lfs: found %s in the usercache\n') % oid) | ||
|
r35493 | lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid)) | ||
|
r35489 | else: | ||
|
r41471 | self.ui.note(_(b'lfs: found %s in the local lfs store\n') % oid) | ||
|
r35492 | blob = self._read(self.vfs, oid, verify) | ||
return blob | ||||
def _read(self, vfs, oid, verify): | ||||
"""Read blob (after verifying) from the given store""" | ||||
blob = vfs.read(oid) | ||||
if verify: | ||||
_verify(oid, blob) | ||||
return blob | ||||
|
r35097 | |||
|
r37163 | def verify(self, oid): | ||
"""Indicate whether or not the hash of the underlying file matches its | ||||
name.""" | ||||
sha256 = hashlib.sha256() | ||||
with self.open(oid) as fp: | ||||
for chunk in util.filechunkiter(fp, size=1048576): | ||||
sha256.update(chunk) | ||||
|
r40711 | return oid == node.hex(sha256.digest()) | ||
|
r37163 | |||
|
r35097 | def has(self, oid): | ||
"""Returns True if the local blobstore contains the requested blob, | ||||
False otherwise.""" | ||||
|
r35281 | return self.cachevfs.exists(oid) or self.vfs.exists(oid) | ||
|
r35097 | |||
|
r43346 | |||
|
r40697 | def _urlerrorreason(urlerror): | ||
'''Create a friendly message for the given URLError to be used in an | ||||
LfsRemoteError message. | ||||
''' | ||||
inst = urlerror | ||||
if isinstance(urlerror.reason, Exception): | ||||
inst = urlerror.reason | ||||
|
r43347 | if util.safehasattr(inst, b'reason'): | ||
|
r43346 | try: # usually it is in the form (errno, strerror) | ||
|
r40697 | reason = inst.reason.args[1] | ||
except (AttributeError, IndexError): | ||||
# it might be anything, for example a string | ||||
reason = inst.reason | ||||
if isinstance(reason, pycompat.unicode): | ||||
# SSLError of Python 2.7.9 contains a unicode | ||||
reason = encoding.unitolocal(reason) | ||||
return reason | ||||
elif getattr(inst, "strerror", None): | ||||
return encoding.strtolocal(inst.strerror) | ||||
else: | ||||
return stringutil.forcebytestr(urlerror) | ||||
|
r43346 | |||
|
r41756 | class lfsauthhandler(util.urlreq.basehandler): | ||
handler_order = 480 # Before HTTPDigestAuthHandler (== 490) | ||||
def http_error_401(self, req, fp, code, msg, headers): | ||||
"""Enforces that any authentication performed is HTTP Basic | ||||
Authentication. No authentication is also acceptable. | ||||
""" | ||||
|
r43906 | authreq = headers.get('www-authenticate', None) | ||
|
r41756 | if authreq: | ||
scheme = authreq.split()[0] | ||||
|
r43906 | if scheme.lower() != 'basic': | ||
|
r41756 | msg = _(b'the server must support Basic Authentication') | ||
|
r43346 | raise util.urlerr.httperror( | ||
req.get_full_url(), | ||||
code, | ||||
encoding.strfromlocal(msg), | ||||
headers, | ||||
fp, | ||||
) | ||||
|
r41756 | return None | ||
|
r43346 | |||
|
r35097 | class _gitlfsremote(object): | ||
def __init__(self, repo, url): | ||||
ui = repo.ui | ||||
self.ui = ui | ||||
baseurl, authinfo = url.authinfo() | ||||
|
r41471 | self.baseurl = baseurl.rstrip(b'/') | ||
useragent = repo.ui.config(b'experimental', b'lfs.user-agent') | ||||
|
r35456 | if not useragent: | ||
|
r41471 | useragent = b'git-lfs/2.3.4 (Mercurial %s)' % util.version() | ||
|
r35455 | self.urlopener = urlmod.opener(ui, authinfo, useragent) | ||
|
r41756 | self.urlopener.add_handler(lfsauthhandler()) | ||
|
r41471 | self.retry = ui.configint(b'lfs', b'retry') | ||
|
r35097 | |||
def writebatch(self, pointers, fromstore): | ||||
"""Batch upload from local to remote blobstore.""" | ||||
|
r41471 | self._batch(_deduplicate(pointers), fromstore, b'upload') | ||
|
r35097 | |||
def readbatch(self, pointers, tostore): | ||||
"""Batch download from remote to local blostore.""" | ||||
|
r41471 | self._batch(_deduplicate(pointers), tostore, b'download') | ||
|
r35097 | |||
def _batchrequest(self, pointers, action): | ||||
"""Get metadata about objects pointed by pointers for given action | ||||
Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} | ||||
See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md | ||||
""" | ||||
|
r43346 | objects = [ | ||
|
r43906 | {'oid': pycompat.strurl(p.oid()), 'size': p.size()} | ||
|
r43346 | for p in pointers | ||
] | ||||
requestdata = pycompat.bytesurl( | ||||
json.dumps( | ||||
|
r43906 | {'objects': objects, 'operation': pycompat.strurl(action),} | ||
|
r43346 | ) | ||
) | ||||
|
r41471 | url = b'%s/objects/batch' % self.baseurl | ||
batchreq = util.urlreq.request(pycompat.strurl(url), data=requestdata) | ||||
|
r43906 | batchreq.add_header('Accept', 'application/vnd.git-lfs+json') | ||
batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json') | ||||
|
r35097 | try: | ||
|
r40701 | with contextlib.closing(self.urlopener.open(batchreq)) as rsp: | ||
rawjson = rsp.read() | ||||
|
r35097 | except util.urlerr.httperror as ex: | ||
|
r40696 | hints = { | ||
|
r43346 | 400: _( | ||
b'check that lfs serving is enabled on %s and "%s" is ' | ||||
b'supported' | ||||
) | ||||
% (self.baseurl, action), | ||||
|
r41471 | 404: _(b'the "lfs.url" config may be used to override %s') | ||
|
r43346 | % self.baseurl, | ||
|
r40696 | } | ||
|
r41471 | hint = hints.get(ex.code, _(b'api=%s, action=%s') % (url, action)) | ||
raise LfsRemoteError( | ||||
_(b'LFS HTTP error: %s') % stringutil.forcebytestr(ex), | ||||
|
r43346 | hint=hint, | ||
) | ||||
|
r40697 | except util.urlerr.urlerror as ex: | ||
|
r43346 | hint = ( | ||
_(b'the "lfs.url" config may be used to override %s') | ||||
% self.baseurl | ||||
) | ||||
raise LfsRemoteError( | ||||
_(b'LFS error: %s') % _urlerrorreason(ex), hint=hint | ||||
) | ||||
|
r35097 | try: | ||
|
r43697 | response = pycompat.json_loads(rawjson) | ||
|
r35097 | except ValueError: | ||
|
r43346 | raise LfsRemoteError( | ||
_(b'LFS server returns invalid JSON: %s') | ||||
% rawjson.encode("utf-8") | ||||
) | ||||
|
r36944 | |||
if self.ui.debugflag: | ||||
|
r41471 | self.ui.debug(b'Status: %d\n' % rsp.status) | ||
|
r36944 | # lfs-test-server and hg serve return headers in different order | ||
|
r41476 | headers = pycompat.bytestr(rsp.info()).strip() | ||
|
r43346 | self.ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) | ||
|
r36944 | |||
|
r43906 | if 'objects' in response: | ||
response['objects'] = sorted( | ||||
response['objects'], key=lambda p: p['oid'] | ||||
|
r43346 | ) | ||
self.ui.debug( | ||||
b'%s\n' | ||||
% pycompat.bytesurl( | ||||
json.dumps( | ||||
response, | ||||
indent=2, | ||||
|
r43906 | separators=('', ': '), | ||
|
r43346 | sort_keys=True, | ||
) | ||||
) | ||||
) | ||||
|
r36944 | |||
|
r41474 | def encodestr(x): | ||
if isinstance(x, pycompat.unicode): | ||||
|
r43361 | return x.encode('utf-8') | ||
|
r41474 | return x | ||
return pycompat.rapply(encodestr, response) | ||||
|
r35097 | |||
|
r35684 | def _checkforservererror(self, pointers, responses, action): | ||
|
r35097 | """Scans errors from objects | ||
|
r35712 | Raises LfsRemoteError if any objects have an error""" | ||
|
r35097 | for response in responses: | ||
|
r35684 | # The server should return 404 when objects cannot be found. Some | ||
# server implementation (ex. lfs-test-server) does not set "error" | ||||
# but just removes "download" from "actions". Treat that case | ||||
# as the same as 404 error. | ||||
|
r41471 | if b'error' not in response: | ||
|
r43346 | if action == b'download' and action not in response.get( | ||
b'actions', [] | ||||
): | ||||
|
r37259 | code = 404 | ||
|
r35713 | else: | ||
|
r37259 | continue | ||
else: | ||||
# An error dict without a code doesn't make much sense, so | ||||
# treat as a server error. | ||||
|
r41471 | code = response.get(b'error').get(b'code', 500) | ||
|
r37259 | |||
ptrmap = {p.oid(): p for p in pointers} | ||||
|
r41471 | p = ptrmap.get(response[b'oid'], None) | ||
|
r37259 | if p: | ||
|
r41471 | filename = getattr(p, 'filename', b'unknown') | ||
|
r37259 | errors = { | ||
|
r41471 | 404: b'The object does not exist', | ||
410: b'The object was removed by the owner', | ||||
422: b'Validation error', | ||||
500: b'Internal server error', | ||||
|
r37259 | } | ||
|
r41471 | msg = errors.get(code, b'status code %d' % code) | ||
|
r43346 | raise LfsRemoteError( | ||
_(b'LFS server error for "%s": %s') % (filename, msg) | ||||
) | ||||
|
r37259 | else: | ||
raise LfsRemoteError( | ||||
|
r41471 | _(b'LFS server error. Unsolicited response for oid %s') | ||
|
r43346 | % response[b'oid'] | ||
) | ||||
|
r35097 | |||
def _extractobjects(self, response, pointers, action): | ||||
"""extract objects from response of the batch API | ||||
response: parsed JSON object returned by batch API | ||||
return response['objects'] filtered by action | ||||
raise if any object has an error | ||||
""" | ||||
# Scan errors from objects - fail early | ||||
|
r41471 | objects = response.get(b'objects', []) | ||
|
r35684 | self._checkforservererror(pointers, objects, action) | ||
|
r35097 | |||
# Filter objects with given action. Practically, this skips uploading | ||||
# objects which exist in the server. | ||||
|
r43346 | filteredobjects = [ | ||
o for o in objects if action in o.get(b'actions', []) | ||||
] | ||||
|
r35097 | |||
return filteredobjects | ||||
|
r35449 | def _basictransfer(self, obj, action, localstore): | ||
|
r35097 | """Download or upload a single object using basic transfer protocol | ||
obj: dict, an object description returned by batch API | ||||
action: string, one of ['upload', 'download'] | ||||
localstore: blobstore.local | ||||
See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\ | ||||
basic-transfers.md | ||||
""" | ||||
|
r41474 | oid = obj[b'oid'] | ||
href = obj[b'actions'][action].get(b'href') | ||||
headers = obj[b'actions'][action].get(b'header', {}).items() | ||||
|
r35097 | |||
|
r41474 | request = util.urlreq.request(pycompat.strurl(href)) | ||
|
r41471 | if action == b'upload': | ||
|
r35097 | # If uploading blobs, read data from local blobstore. | ||
|
r37234 | if not localstore.verify(oid): | ||
|
r43346 | raise error.Abort( | ||
_(b'detected corrupt lfs object: %s') % oid, | ||||
hint=_(b'run hg verify'), | ||||
) | ||||
|
r44596 | request.data = lfsuploadfile(localstore.open(oid)) | ||
|
r41472 | request.get_method = lambda: r'PUT' | ||
|
r43906 | request.add_header('Content-Type', 'application/octet-stream') | ||
request.add_header('Content-Length', len(request.data)) | ||||
|
r35097 | |||
for k, v in headers: | ||||
|
r41474 | request.add_header(pycompat.strurl(k), pycompat.strurl(v)) | ||
|
r35097 | |||
try: | ||||
|
r44543 | with contextlib.closing(self.urlopener.open(request)) as res: | ||
|
r44544 | contentlength = res.info().get(b"content-length") | ||
|
r40701 | ui = self.ui # Shorten debug lines | ||
if self.ui.debugflag: | ||||
|
r44543 | ui.debug(b'Status: %d\n' % res.status) | ||
|
r40701 | # lfs-test-server and hg serve return headers in different | ||
# order | ||||
|
r44543 | headers = pycompat.bytestr(res.info()).strip() | ||
|
r43346 | ui.debug(b'%s\n' % b'\n'.join(sorted(headers.splitlines()))) | ||
|
r36944 | |||
|
r41471 | if action == b'download': | ||
|
r40701 | # If downloading blobs, store downloaded data to local | ||
# blobstore | ||||
|
r44544 | localstore.download(oid, res, contentlength) | ||
|
r40701 | else: | ||
|
r44545 | blocks = [] | ||
|
r40701 | while True: | ||
|
r44543 | data = res.read(1048576) | ||
|
r40701 | if not data: | ||
break | ||||
|
r44545 | blocks.append(data) | ||
response = b"".join(blocks) | ||||
|
r40701 | if response: | ||
|
r41471 | ui.debug(b'lfs %s response: %s' % (action, response)) | ||
|
r35097 | except util.urlerr.httperror as ex: | ||
|
r35752 | if self.ui.debugflag: | ||
|
r43346 | self.ui.debug( | ||
b'%s: %s\n' % (oid, ex.read()) | ||||
) # XXX: also bytes? | ||||
raise LfsRemoteError( | ||||
_(b'LFS HTTP error: %s (oid=%s, action=%s)') | ||||
% (stringutil.forcebytestr(ex), oid, action) | ||||
) | ||||
|
r40697 | except util.urlerr.urlerror as ex: | ||
|
r43346 | hint = _(b'attempted connection to %s') % pycompat.bytesurl( | ||
util.urllibcompat.getfullurl(request) | ||||
) | ||||
raise LfsRemoteError( | ||||
_(b'LFS error: %s') % _urlerrorreason(ex), hint=hint | ||||
) | ||||
|
r35097 | |||
def _batch(self, pointers, localstore, action): | ||||
|
r41471 | if action not in [b'upload', b'download']: | ||
raise error.ProgrammingError(b'invalid Git-LFS action: %s' % action) | ||||
|
r35097 | |||
response = self._batchrequest(pointers, action) | ||||
objects = self._extractobjects(response, pointers, action) | ||||
|
r41471 | total = sum(x.get(b'size', 0) for x in objects) | ||
|
r35449 | sizes = {} | ||
for obj in objects: | ||||
|
r41471 | sizes[obj.get(b'oid')] = obj.get(b'size', 0) | ||
|
r43346 | topic = { | ||
b'upload': _(b'lfs uploading'), | ||||
b'download': _(b'lfs downloading'), | ||||
}[action] | ||||
|
r35494 | if len(objects) > 1: | ||
|
r43346 | self.ui.note( | ||
_(b'lfs: need to transfer %d objects (%s)\n') | ||||
% (len(objects), util.bytecount(total)) | ||||
) | ||||
|
r39426 | |||
|
r35449 | def transfer(chunk): | ||
for obj in chunk: | ||||
|
r41471 | objsize = obj.get(b'size', 0) | ||
|
r35449 | if self.ui.verbose: | ||
|
r41471 | if action == b'download': | ||
msg = _(b'lfs: downloading %s (%s)\n') | ||||
elif action == b'upload': | ||||
msg = _(b'lfs: uploading %s (%s)\n') | ||||
|
r43346 | self.ui.note( | ||
msg % (obj.get(b'oid'), util.bytecount(objsize)) | ||||
) | ||||
|
r35449 | retry = self.retry | ||
while True: | ||||
try: | ||||
self._basictransfer(obj, action, localstore) | ||||
|
r41471 | yield 1, obj.get(b'oid') | ||
|
r35449 | break | ||
|
r35491 | except socket.error as ex: | ||
|
r35449 | if retry > 0: | ||
|
r35494 | self.ui.note( | ||
|
r41471 | _(b'lfs: failed: %r (remaining retry %d)\n') | ||
|
r43346 | % (stringutil.forcebytestr(ex), retry) | ||
) | ||||
|
r35449 | retry -= 1 | ||
continue | ||||
raise | ||||
|
r35750 | # Until https multiplexing gets sorted out | ||
|
r41471 | if self.ui.configbool(b'experimental', b'lfs.worker-enable'): | ||
|
r43346 | oids = worker.worker( | ||
self.ui, | ||||
0.1, | ||||
transfer, | ||||
(), | ||||
sorted(objects, key=lambda o: o.get(b'oid')), | ||||
) | ||||
|
r35750 | else: | ||
|
r41471 | oids = transfer(sorted(objects, key=lambda o: o.get(b'oid'))) | ||
|
r35750 | |||
|
r44534 | with self.ui.makeprogress( | ||
topic, unit=_(b"bytes"), total=total | ||||
) as progress: | ||||
|
r39426 | progress.update(0) | ||
processed = 0 | ||||
blobs = 0 | ||||
for _one, oid in oids: | ||||
processed += sizes[oid] | ||||
blobs += 1 | ||||
progress.update(processed) | ||||
|
r41471 | self.ui.note(_(b'lfs: processed: %s\n') % oid) | ||
|
r35097 | |||
|
r35899 | if blobs > 0: | ||
|
r41471 | if action == b'upload': | ||
|
r43346 | self.ui.status( | ||
_(b'lfs: uploaded %d files (%s)\n') | ||||
% (blobs, util.bytecount(processed)) | ||||
) | ||||
|
r41471 | elif action == b'download': | ||
|
r43346 | self.ui.status( | ||
_(b'lfs: downloaded %d files (%s)\n') | ||||
% (blobs, util.bytecount(processed)) | ||||
) | ||||
|
r35899 | |||
|
r35097 | def __del__(self): | ||
# copied from mercurial/httppeer.py | ||||
urlopener = getattr(self, 'urlopener', None) | ||||
if urlopener: | ||||
for h in urlopener.handlers: | ||||
h.close() | ||||
|
r43346 | getattr(h, "close_all", lambda: None)() | ||
|
r35097 | |||
class _dummyremote(object): | ||||
"""Dummy store storing blobs to temp directory.""" | ||||
def __init__(self, repo, url): | ||||
|
r41471 | fullpath = repo.vfs.join(b'lfs', url.path) | ||
|
r35097 | self.vfs = lfsvfs(fullpath) | ||
def writebatch(self, pointers, fromstore): | ||||
|
r35945 | for p in _deduplicate(pointers): | ||
|
r35492 | content = fromstore.read(p.oid(), verify=True) | ||
|
r41471 | with self.vfs(p.oid(), b'wb', atomictemp=True) as fp: | ||
|
r35097 | fp.write(content) | ||
def readbatch(self, pointers, tostore): | ||||
|
r35945 | for p in _deduplicate(pointers): | ||
|
r41471 | with self.vfs(p.oid(), b'rb') as fp: | ||
|
r44544 | tostore.download(p.oid(), fp, None) | ||
|
r35097 | |||
|
r43346 | |||
|
r35097 | class _nullremote(object): | ||
"""Null store storing blobs to /dev/null.""" | ||||
def __init__(self, repo, url): | ||||
pass | ||||
def writebatch(self, pointers, fromstore): | ||||
pass | ||||
def readbatch(self, pointers, tostore): | ||||
pass | ||||
|
r43346 | |||
|
r35097 | class _promptremote(object): | ||
"""Prompt user to set lfs.url when accessed.""" | ||||
def __init__(self, repo, url): | ||||
pass | ||||
def writebatch(self, pointers, fromstore, ui=None): | ||||
self._prompt() | ||||
def readbatch(self, pointers, tostore, ui=None): | ||||
self._prompt() | ||||
def _prompt(self): | ||||
|
r41471 | raise error.Abort(_(b'lfs.url needs to be configured')) | ||
|
r35097 | |||
|
r43346 | |||
|
r35097 | _storemap = { | ||
|
r41471 | b'https': _gitlfsremote, | ||
b'http': _gitlfsremote, | ||||
b'file': _dummyremote, | ||||
b'null': _nullremote, | ||||
|
r35097 | None: _promptremote, | ||
} | ||||
|
r43346 | |||
|
r35945 | def _deduplicate(pointers): | ||
"""Remove any duplicate oids that exist in the list""" | ||||
reduced = util.sortdict() | ||||
for p in pointers: | ||||
reduced[p.oid()] = p | ||||
return reduced.values() | ||||
|
r43346 | |||
|
r35492 | def _verify(oid, content): | ||
|
r40711 | realoid = node.hex(hashlib.sha256(content).digest()) | ||
|
r35492 | if realoid != oid: | ||
|
r43346 | raise LfsCorruptionError( | ||
_(b'detected corrupt lfs object: %s') % oid, | ||||
hint=_(b'run hg verify'), | ||||
) | ||||
|
r35492 | |||
|
r37582 | def remote(repo, remote=None): | ||
|
r37536 | """remotestore factory. return a store in _storemap depending on config | ||
If ``lfs.url`` is specified, use that remote endpoint. Otherwise, try to | ||||
infer the endpoint, based on the remote repository using the same path | ||||
adjustments as git. As an extension, 'http' is supported as well so that | ||||
``hg serve`` works out of the box. | ||||
https://github.com/git-lfs/git-lfs/blob/master/docs/api/server-discovery.md | ||||
""" | ||||
|
r41471 | lfsurl = repo.ui.config(b'lfs', b'url') | ||
|
r43347 | url = util.url(lfsurl or b'') | ||
|
r37583 | if lfsurl is None: | ||
|
r37582 | if remote: | ||
|
r37583 | path = remote | ||
|
r43347 | elif util.safehasattr(repo, b'_subtoppath'): | ||
|
r37581 | # The pull command sets this during the optional update phase, which | ||
# tells exactly where the pull originated, whether 'paths.default' | ||||
# or explicit. | ||||
|
r37583 | path = repo._subtoppath | ||
|
r37581 | else: | ||
# TODO: investigate 'paths.remote:lfsurl' style path customization, | ||||
# and fall back to inferring from 'paths.remote' if unspecified. | ||||
|
r41471 | path = repo.ui.config(b'paths', b'default') or b'' | ||
|
r37583 | |||
defaulturl = util.url(path) | ||||
|
r37536 | |||
# TODO: support local paths as well. | ||||
# TODO: consider the ssh -> https transformation that git applies | ||||
if defaulturl.scheme in (b'http', b'https'): | ||||
|
r37583 | if defaulturl.path and defaulturl.path[:-1] != b'/': | ||
defaulturl.path += b'/' | ||||
|
r37709 | defaulturl.path = (defaulturl.path or b'') + b'.git/info/lfs' | ||
|
r37536 | |||
url = util.url(bytes(defaulturl)) | ||||
|
r41471 | repo.ui.note(_(b'lfs: assuming remote store: %s\n') % url) | ||
|
r37536 | |||
|
r35097 | scheme = url.scheme | ||
if scheme not in _storemap: | ||||
|
r41471 | raise error.Abort(_(b'lfs: unknown url scheme: %s') % scheme) | ||
|
r35097 | return _storemap[scheme](repo, url) | ||
|
r43346 | |||
|
r39813 | class LfsRemoteError(error.StorageError): | ||
|
r35097 | pass | ||
|
r37710 | |||
|
r43346 | |||
|
r37710 | class LfsCorruptionError(error.Abort): | ||
"""Raised when a corrupt blob is detected, aborting an operation | ||||
It exists to allow specialized handling on the server side.""" | ||||