diff --git a/hgext/lfs/__init__.py b/hgext/lfs/__init__.py new file mode 100644 --- /dev/null +++ b/hgext/lfs/__init__.py @@ -0,0 +1,132 @@ +# lfs - hash-preserving large file support using Git-LFS protocol +# +# Copyright 2017 Facebook, Inc. +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +"""lfs - large file support (EXPERIMENTAL) + +Configs:: + + [lfs] + # Remote endpoint. Multiple protocols are supported: + # - http(s)://user:pass@example.com/path + # git-lfs endpoint + # - file:///tmp/path + # local filesystem, usually for testing + # if unset, lfs will prompt setting this when it must use this value. + # (default: unset) + url = https://example.com/lfs + + # size of a file to make it use LFS + threshold = 10M + + # how many times to retry before giving up on transferring an object + retry = 5 +""" + +from __future__ import absolute_import + +from mercurial import ( + bundle2, + changegroup, + context, + exchange, + extensions, + filelog, + registrar, + revlog, + scmutil, + vfs as vfsmod, +) +from mercurial.i18n import _ + +from . import ( + blobstore, + wrapper, +) + +# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for +# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should +# be specifying the version(s) of Mercurial they are tested with, or +# leave the attribute unspecified. +testedwith = 'ships-with-hg-core' + +cmdtable = {} +command = registrar.command(cmdtable) + +templatekeyword = registrar.templatekeyword() + +def reposetup(ui, repo): + # Nothing to do with a remote repo + if not repo.local(): + return + + threshold = repo.ui.configbytes('lfs', 'threshold', None) + + repo.svfs.options['lfsthreshold'] = threshold + repo.svfs.lfslocalblobstore = blobstore.local(repo) + repo.svfs.lfsremoteblobstore = blobstore.remote(repo) + + # Push hook + repo.prepushoutgoinghooks.add('lfs', wrapper.prepush) + +def wrapfilelog(filelog): + wrapfunction = extensions.wrapfunction + + wrapfunction(filelog, 'addrevision', wrapper.filelogaddrevision) + wrapfunction(filelog, 'renamed', wrapper.filelogrenamed) + wrapfunction(filelog, 'size', wrapper.filelogsize) + +def extsetup(ui): + wrapfilelog(filelog.filelog) + + wrapfunction = extensions.wrapfunction + wrapfunction(changegroup, + 'supportedoutgoingversions', + wrapper.supportedoutgoingversions) + wrapfunction(changegroup, + 'allsupportedversions', + wrapper.allsupportedversions) + + wrapfunction(context.basefilectx, 'cmp', wrapper.filectxcmp) + wrapfunction(context.basefilectx, 'isbinary', wrapper.filectxisbinary) + context.basefilectx.islfs = wrapper.filectxislfs + + revlog.addflagprocessor( + revlog.REVIDX_EXTSTORED, + ( + wrapper.readfromstore, + wrapper.writetostore, + wrapper.bypasscheckhash, + ), + ) + + # Make bundle choose changegroup3 instead of changegroup2. This affects + # "hg bundle" command. Note: it does not cover all bundle formats like + # "packed1". Using "packed1" with lfs will likely cause trouble. + names = [k for k, v in exchange._bundlespeccgversions.items() if v == '02'] + for k in names: + exchange._bundlespeccgversions[k] = '03' + + # bundlerepo uses "vfsmod.readonlyvfs(othervfs)", we need to make sure lfs + # options and blob stores are passed from othervfs to the new readonlyvfs. + wrapfunction(vfsmod.readonlyvfs, '__init__', wrapper.vfsinit) + + # when writing a bundle via "hg bundle" command, upload related LFS blobs + wrapfunction(bundle2, 'writenewbundle', wrapper.writenewbundle) + +@templatekeyword('lfs_files') +def lfsfiles(repo, ctx, **args): + """List of strings. LFS files added or modified by the changeset.""" + pointers = wrapper.pointersfromctx(ctx) # {path: pointer} + return sorted(pointers.keys()) + +@command('debuglfsupload', + [('r', 'rev', [], _('upload large files introduced by REV'))]) +def debuglfsupload(ui, repo, **opts): + """upload lfs blobs added by the working copy parent or given revisions""" + revs = opts.get('rev', []) + pointers = wrapper.extractpointers(repo, scmutil.revrange(repo, revs)) + wrapper.uploadblobs(repo, pointers) diff --git a/hgext/lfs/blobstore.py b/hgext/lfs/blobstore.py new file mode 100644 --- /dev/null +++ b/hgext/lfs/blobstore.py @@ -0,0 +1,346 @@ +# blobstore.py - local and remote (speaking Git-LFS protocol) blob storages +# +# Copyright 2017 Facebook, Inc. +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import + +import json +import os +import re + +from mercurial import ( + error, + url as urlmod, + util, + vfs as vfsmod, +) +from mercurial.i18n import _ + +# 64 bytes for SHA256 +_lfsre = re.compile(r'\A[a-f0-9]{64}\Z') + +class lfsvfs(vfsmod.vfs): + def join(self, path): + """split the path at first two characters, like: XX/XXXXX...""" + if not _lfsre.match(path): + raise error.ProgrammingError('unexpected lfs path: %s' % path) + return super(lfsvfs, self).join(path[0:2], path[2:]) + +class filewithprogress(object): + """a file-like object that supports __len__ and read. + + Useful to provide progress information for how many bytes are read. + """ + + def __init__(self, fp, callback): + self._fp = fp + self._callback = callback # func(readsize) + fp.seek(0, os.SEEK_END) + self._len = fp.tell() + fp.seek(0) + + def __len__(self): + return self._len + + def read(self, size): + if self._fp is None: + return b'' + data = self._fp.read(size) + if data: + if self._callback: + self._callback(len(data)) + else: + self._fp.close() + self._fp = None + return data + +class local(object): + """Local blobstore for large file contents. + + This blobstore is used both as a cache and as a staging area for large blobs + to be uploaded to the remote blobstore. + """ + + def __init__(self, repo): + fullpath = repo.svfs.join('lfs/objects') + self.vfs = lfsvfs(fullpath) + + def write(self, oid, data): + """Write blob to local blobstore.""" + with self.vfs(oid, 'wb', atomictemp=True) as fp: + fp.write(data) + + def read(self, oid): + """Read blob from local blobstore.""" + return self.vfs.read(oid) + + def has(self, oid): + """Returns True if the local blobstore contains the requested blob, + False otherwise.""" + return self.vfs.exists(oid) + +class _gitlfsremote(object): + + def __init__(self, repo, url): + ui = repo.ui + self.ui = ui + baseurl, authinfo = url.authinfo() + self.baseurl = baseurl.rstrip('/') + self.urlopener = urlmod.opener(ui, authinfo) + self.retry = ui.configint('lfs', 'retry', 5) + + def writebatch(self, pointers, fromstore): + """Batch upload from local to remote blobstore.""" + self._batch(pointers, fromstore, 'upload') + + def readbatch(self, pointers, tostore): + """Batch download from remote to local blostore.""" + self._batch(pointers, tostore, 'download') + + def _batchrequest(self, pointers, action): + """Get metadata about objects pointed by pointers for given action + + Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]} + See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md + """ + objects = [{'oid': p.oid(), 'size': p.size()} for p in pointers] + requestdata = json.dumps({ + 'objects': objects, + 'operation': action, + }) + batchreq = util.urlreq.request('%s/objects/batch' % self.baseurl, + data=requestdata) + batchreq.add_header('Accept', 'application/vnd.git-lfs+json') + batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json') + try: + rawjson = self.urlopener.open(batchreq).read() + except util.urlerr.httperror as ex: + raise LfsRemoteError(_('LFS HTTP error: %s (action=%s)') + % (ex, action)) + try: + response = json.loads(rawjson) + except ValueError: + raise LfsRemoteError(_('LFS server returns invalid JSON: %s') + % rawjson) + return response + + def _checkforservererror(self, pointers, responses): + """Scans errors from objects + + Returns LfsRemoteError if any objects has an error""" + for response in responses: + error = response.get('error') + if error: + ptrmap = {p.oid(): p for p in pointers} + p = ptrmap.get(response['oid'], None) + if error['code'] == 404 and p: + filename = getattr(p, 'filename', 'unknown') + raise LfsRemoteError( + _(('LFS server error. Remote object ' + 'for file %s not found: %r')) % (filename, response)) + raise LfsRemoteError(_('LFS server error: %r') % response) + + def _extractobjects(self, response, pointers, action): + """extract objects from response of the batch API + + response: parsed JSON object returned by batch API + return response['objects'] filtered by action + raise if any object has an error + """ + # Scan errors from objects - fail early + objects = response.get('objects', []) + self._checkforservererror(pointers, objects) + + # Filter objects with given action. Practically, this skips uploading + # objects which exist in the server. + filteredobjects = [o for o in objects if action in o.get('actions', [])] + # But for downloading, we want all objects. Therefore missing objects + # should be considered an error. + if action == 'download': + if len(filteredobjects) < len(objects): + missing = [o.get('oid', '?') + for o in objects + if action not in o.get('actions', [])] + raise LfsRemoteError( + _('LFS server claims required objects do not exist:\n%s') + % '\n'.join(missing)) + + return filteredobjects + + def _basictransfer(self, obj, action, localstore, progress=None): + """Download or upload a single object using basic transfer protocol + + obj: dict, an object description returned by batch API + action: string, one of ['upload', 'download'] + localstore: blobstore.local + + See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\ + basic-transfers.md + """ + oid = str(obj['oid']) + + href = str(obj['actions'][action].get('href')) + headers = obj['actions'][action].get('header', {}).items() + + request = util.urlreq.request(href) + if action == 'upload': + # If uploading blobs, read data from local blobstore. + request.data = filewithprogress(localstore.vfs(oid), progress) + request.get_method = lambda: 'PUT' + + for k, v in headers: + request.add_header(k, v) + + response = b'' + try: + req = self.urlopener.open(request) + while True: + data = req.read(1048576) + if not data: + break + if action == 'download' and progress: + progress(len(data)) + response += data + except util.urlerr.httperror as ex: + raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)') + % (ex, oid, action)) + + if action == 'download': + # If downloading blobs, store downloaded data to local blobstore + localstore.write(oid, response) + + def _batch(self, pointers, localstore, action): + if action not in ['upload', 'download']: + raise error.ProgrammingError('invalid Git-LFS action: %s' % action) + + response = self._batchrequest(pointers, action) + prunningsize = [0] + objects = self._extractobjects(response, pointers, action) + total = sum(x.get('size', 0) for x in objects) + topic = {'upload': _('lfs uploading'), + 'download': _('lfs downloading')}[action] + if self.ui.verbose and len(objects) > 1: + self.ui.write(_('lfs: need to transfer %d objects (%s)\n') + % (len(objects), util.bytecount(total))) + self.ui.progress(topic, 0, total=total) + def progress(size): + # advance progress bar by "size" bytes + prunningsize[0] += size + self.ui.progress(topic, prunningsize[0], total=total) + for obj in sorted(objects, key=lambda o: o.get('oid')): + objsize = obj.get('size', 0) + if self.ui.verbose: + if action == 'download': + msg = _('lfs: downloading %s (%s)\n') + elif action == 'upload': + msg = _('lfs: uploading %s (%s)\n') + self.ui.write(msg % (obj.get('oid'), util.bytecount(objsize))) + origrunningsize = prunningsize[0] + retry = self.retry + while True: + prunningsize[0] = origrunningsize + try: + self._basictransfer(obj, action, localstore, + progress=progress) + break + except Exception as ex: + if retry > 0: + if self.ui.verbose: + self.ui.write( + _('lfs: failed: %r (remaining retry %d)\n') + % (ex, retry)) + retry -= 1 + continue + raise + + self.ui.progress(topic, pos=None, total=total) + + def __del__(self): + # copied from mercurial/httppeer.py + urlopener = getattr(self, 'urlopener', None) + if urlopener: + for h in urlopener.handlers: + h.close() + getattr(h, "close_all", lambda : None)() + +class _dummyremote(object): + """Dummy store storing blobs to temp directory.""" + + def __init__(self, repo, url): + fullpath = repo.vfs.join('lfs', url.path) + self.vfs = lfsvfs(fullpath) + + def writebatch(self, pointers, fromstore): + for p in pointers: + content = fromstore.read(p.oid()) + with self.vfs(p.oid(), 'wb', atomictemp=True) as fp: + fp.write(content) + + def readbatch(self, pointers, tostore): + for p in pointers: + content = self.vfs.read(p.oid()) + tostore.write(p.oid(), content) + +class _nullremote(object): + """Null store storing blobs to /dev/null.""" + + def __init__(self, repo, url): + pass + + def writebatch(self, pointers, fromstore): + pass + + def readbatch(self, pointers, tostore): + pass + +class _promptremote(object): + """Prompt user to set lfs.url when accessed.""" + + def __init__(self, repo, url): + pass + + def writebatch(self, pointers, fromstore, ui=None): + self._prompt() + + def readbatch(self, pointers, tostore, ui=None): + self._prompt() + + def _prompt(self): + raise error.Abort(_('lfs.url needs to be configured')) + +_storemap = { + 'https': _gitlfsremote, + 'http': _gitlfsremote, + 'file': _dummyremote, + 'null': _nullremote, + None: _promptremote, +} + +def remote(repo): + """remotestore factory. return a store in _storemap depending on config""" + defaulturl = '' + + # convert deprecated configs to the new url. TODO: remove this if other + # places are migrated to the new url config. + # deprecated config: lfs.remotestore + deprecatedstore = repo.ui.config('lfs', 'remotestore') + if deprecatedstore == 'dummy': + # deprecated config: lfs.remotepath + defaulturl = 'file://' + repo.ui.config('lfs', 'remotepath') + elif deprecatedstore == 'git-lfs': + # deprecated config: lfs.remoteurl + defaulturl = repo.ui.config('lfs', 'remoteurl') + elif deprecatedstore == 'null': + defaulturl = 'null://' + + url = util.url(repo.ui.config('lfs', 'url', defaulturl)) + scheme = url.scheme + if scheme not in _storemap: + raise error.Abort(_('lfs: unknown url scheme: %s') % scheme) + return _storemap[scheme](repo, url) + +class LfsRemoteError(error.RevlogError): + pass diff --git a/hgext/lfs/pointer.py b/hgext/lfs/pointer.py new file mode 100644 --- /dev/null +++ b/hgext/lfs/pointer.py @@ -0,0 +1,72 @@ +# pointer.py - Git-LFS pointer serialization +# +# Copyright 2017 Facebook, Inc. +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import + +import re + +from mercurial import ( + error, +) +from mercurial.i18n import _ + +class InvalidPointer(error.RevlogError): + pass + +class gitlfspointer(dict): + VERSION = 'https://git-lfs.github.com/spec/v1' + + def __init__(self, *args, **kwargs): + self['version'] = self.VERSION + super(gitlfspointer, self).__init__(*args, **kwargs) + + @classmethod + def deserialize(cls, text): + try: + return cls(l.split(' ', 1) for l in text.splitlines()).validate() + except ValueError: # l.split returns 1 item instead of 2 + raise InvalidPointer(_('cannot parse git-lfs text: %r') % text) + + def serialize(self): + sortkeyfunc = lambda x: (x[0] != 'version', x) + items = sorted(self.validate().iteritems(), key=sortkeyfunc) + return ''.join('%s %s\n' % (k, v) for k, v in items) + + def oid(self): + return self['oid'].split(':')[-1] + + def size(self): + return int(self['size']) + + # regular expressions used by _validate + # see https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md + _keyre = re.compile(r'\A[a-z0-9.-]+\Z') + _valuere = re.compile(r'\A[^\n]*\Z') + _requiredre = { + 'size': re.compile(r'\A[0-9]+\Z'), + 'oid': re.compile(r'\Asha256:[0-9a-f]{64}\Z'), + 'version': re.compile(r'\A%s\Z' % re.escape(VERSION)), + } + + def validate(self): + """raise InvalidPointer on error. return self if there is no error""" + requiredcount = 0 + for k, v in self.iteritems(): + if k in self._requiredre: + if not self._requiredre[k].match(v): + raise InvalidPointer(_('unexpected value: %s=%r') % (k, v)) + requiredcount += 1 + elif not self._keyre.match(k): + raise InvalidPointer(_('unexpected key: %s') % k) + if not self._valuere.match(v): + raise InvalidPointer(_('unexpected value: %s=%r') % (k, v)) + if len(self._requiredre) != requiredcount: + miss = sorted(set(self._requiredre.keys()).difference(self.keys())) + raise InvalidPointer(_('missed keys: %s') % ', '.join(miss)) + return self + +deserialize = gitlfspointer.deserialize diff --git a/hgext/lfs/wrapper.py b/hgext/lfs/wrapper.py new file mode 100644 --- /dev/null +++ b/hgext/lfs/wrapper.py @@ -0,0 +1,247 @@ +# wrapper.py - methods wrapping core mercurial logic +# +# Copyright 2017 Facebook, Inc. +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import + +import hashlib + +from mercurial import ( + error, + filelog, + revlog, + util, +) +from mercurial.i18n import _ +from mercurial.node import bin, nullid, short + +from . import ( + blobstore, + pointer, +) + +def supportedoutgoingversions(orig, repo): + versions = orig(repo) + versions.discard('01') + versions.discard('02') + versions.add('03') + return versions + +def allsupportedversions(orig, ui): + versions = orig(ui) + versions.add('03') + return versions + +def bypasscheckhash(self, text): + return False + +def readfromstore(self, text): + """Read filelog content from local blobstore transform for flagprocessor. + + Default tranform for flagprocessor, returning contents from blobstore. + Returns a 2-typle (text, validatehash) where validatehash is True as the + contents of the blobstore should be checked using checkhash. + """ + p = pointer.deserialize(text) + oid = p.oid() + store = self.opener.lfslocalblobstore + if not store.has(oid): + p.filename = getattr(self, 'indexfile', None) + self.opener.lfsremoteblobstore.readbatch([p], store) + text = store.read(oid) + + # pack hg filelog metadata + hgmeta = {} + for k in p.keys(): + if k.startswith('x-hg-'): + name = k[len('x-hg-'):] + hgmeta[name] = p[k] + if hgmeta or text.startswith('\1\n'): + text = filelog.packmeta(hgmeta, text) + + return (text, True) + +def writetostore(self, text): + # hg filelog metadata (includes rename, etc) + hgmeta, offset = filelog.parsemeta(text) + if offset and offset > 0: + # lfs blob does not contain hg filelog metadata + text = text[offset:] + + # git-lfs only supports sha256 + oid = hashlib.sha256(text).hexdigest() + self.opener.lfslocalblobstore.write(oid, text) + + # replace contents with metadata + longoid = 'sha256:%s' % oid + metadata = pointer.gitlfspointer(oid=longoid, size=str(len(text))) + + # by default, we expect the content to be binary. however, LFS could also + # be used for non-binary content. add a special entry for non-binary data. + # this will be used by filectx.isbinary(). + if not util.binary(text): + # not hg filelog metadata (affecting commit hash), no "x-hg-" prefix + metadata['x-is-binary'] = '0' + + # translate hg filelog metadata to lfs metadata with "x-hg-" prefix + if hgmeta is not None: + for k, v in hgmeta.iteritems(): + metadata['x-hg-%s' % k] = v + + rawtext = metadata.serialize() + return (rawtext, False) + +def _islfs(rlog, node=None, rev=None): + if rev is None: + if node is None: + # both None - likely working copy content where node is not ready + return False + rev = rlog.rev(node) + else: + node = rlog.node(rev) + if node == nullid: + return False + flags = rlog.flags(rev) + return bool(flags & revlog.REVIDX_EXTSTORED) + +def filelogaddrevision(orig, self, text, transaction, link, p1, p2, + cachedelta=None, node=None, + flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds): + threshold = self.opener.options['lfsthreshold'] + textlen = len(text) + # exclude hg rename meta from file size + meta, offset = filelog.parsemeta(text) + if offset: + textlen -= offset + + if threshold and textlen > threshold: + flags |= revlog.REVIDX_EXTSTORED + + return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta, + node=node, flags=flags, **kwds) + +def filelogrenamed(orig, self, node): + if _islfs(self, node): + rawtext = self.revision(node, raw=True) + if not rawtext: + return False + metadata = pointer.deserialize(rawtext) + if 'x-hg-copy' in metadata and 'x-hg-copyrev' in metadata: + return metadata['x-hg-copy'], bin(metadata['x-hg-copyrev']) + else: + return False + return orig(self, node) + +def filelogsize(orig, self, rev): + if _islfs(self, rev=rev): + # fast path: use lfs metadata to answer size + rawtext = self.revision(rev, raw=True) + metadata = pointer.deserialize(rawtext) + return int(metadata['size']) + return orig(self, rev) + +def filectxcmp(orig, self, fctx): + """returns True if text is different than fctx""" + # some fctx (ex. hg-git) is not based on basefilectx and do not have islfs + if self.islfs() and getattr(fctx, 'islfs', lambda: False)(): + # fast path: check LFS oid + p1 = pointer.deserialize(self.rawdata()) + p2 = pointer.deserialize(fctx.rawdata()) + return p1.oid() != p2.oid() + return orig(self, fctx) + +def filectxisbinary(orig, self): + if self.islfs(): + # fast path: use lfs metadata to answer isbinary + metadata = pointer.deserialize(self.rawdata()) + # if lfs metadata says nothing, assume it's binary by default + return bool(int(metadata.get('x-is-binary', 1))) + return orig(self) + +def filectxislfs(self): + return _islfs(self.filelog(), self.filenode()) + +def vfsinit(orig, self, othervfs): + orig(self, othervfs) + # copy lfs related options + for k, v in othervfs.options.items(): + if k.startswith('lfs'): + self.options[k] = v + # also copy lfs blobstores. note: this can run before reposetup, so lfs + # blobstore attributes are not always ready at this time. + for name in ['lfslocalblobstore', 'lfsremoteblobstore']: + if util.safehasattr(othervfs, name): + setattr(self, name, getattr(othervfs, name)) + +def _canskipupload(repo): + # if remotestore is a null store, upload is a no-op and can be skipped + return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote) + +def candownload(repo): + # if remotestore is a null store, downloads will lead to nothing + return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote) + +def uploadblobsfromrevs(repo, revs): + '''upload lfs blobs introduced by revs + + Note: also used by other extensions e. g. infinitepush. avoid renaming. + ''' + if _canskipupload(repo): + return + pointers = extractpointers(repo, revs) + uploadblobs(repo, pointers) + +def prepush(pushop): + """Prepush hook. + + Read through the revisions to push, looking for filelog entries that can be + deserialized into metadata so that we can block the push on their upload to + the remote blobstore. + """ + return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing) + +def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing, + *args, **kwargs): + """upload LFS blobs added by outgoing revisions on 'hg bundle'""" + uploadblobsfromrevs(repo, outgoing.missing) + return orig(ui, repo, source, filename, bundletype, outgoing, *args, + **kwargs) + +def extractpointers(repo, revs): + """return a list of lfs pointers added by given revs""" + ui = repo.ui + if ui.debugflag: + ui.write(_('lfs: computing set of blobs to upload\n')) + pointers = {} + for r in revs: + ctx = repo[r] + for p in pointersfromctx(ctx).values(): + pointers[p.oid()] = p + return pointers.values() + +def pointersfromctx(ctx): + """return a dict {path: pointer} for given single changectx""" + result = {} + for f in ctx.files(): + if f not in ctx: + continue + fctx = ctx[f] + if not _islfs(fctx.filelog(), fctx.filenode()): + continue + try: + result[f] = pointer.deserialize(fctx.rawdata()) + except pointer.InvalidPointer as ex: + raise error.Abort(_('lfs: corrupted pointer (%s@%s): %s\n') + % (f, short(ctx.node()), ex)) + return result + +def uploadblobs(repo, pointers): + """upload given pointers from local blobstore""" + if not pointers: + return + + remoteblob = repo.svfs.lfsremoteblobstore + remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore) diff --git a/tests/test-lfs-pointer.py b/tests/test-lfs-pointer.py new file mode 100644 --- /dev/null +++ b/tests/test-lfs-pointer.py @@ -0,0 +1,41 @@ +from __future__ import absolute_import, print_function + +import os +import sys + +# make it runnable using python directly without run-tests.py +sys.path[0:0] = [os.path.join(os.path.dirname(__file__), '..')] + +from hgext.lfs import pointer + +def tryparse(text): + r = {} + try: + r = pointer.deserialize(text) + print('ok') + except Exception as ex: + print(ex) + if r: + text2 = r.serialize() + if text2 != text: + print('reconstructed text differs') + return r + +t = ('version https://git-lfs.github.com/spec/v1\n' + 'oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1' + '258daaa5e2ca24d17e2393\n' + 'size 12345\n' + 'x-foo extra-information\n') + +tryparse('') +tryparse(t) +tryparse(t.replace('git-lfs', 'unknown')) +tryparse(t.replace('v1\n', 'v1\n\n')) +tryparse(t.replace('sha256', 'ahs256')) +tryparse(t.replace('sha256:', '')) +tryparse(t.replace('12345', '0x12345')) +tryparse(t.replace('extra-information', 'extra\0information')) +tryparse(t.replace('extra-information', 'extra\ninformation')) +tryparse(t.replace('x-foo', 'x_foo')) +tryparse(t.replace('oid', 'blobid')) +tryparse(t.replace('size', 'size-bytes').replace('oid', 'object-id')) diff --git a/tests/test-lfs-pointer.py.out b/tests/test-lfs-pointer.py.out new file mode 100644 --- /dev/null +++ b/tests/test-lfs-pointer.py.out @@ -0,0 +1,12 @@ +missed keys: oid, size +ok +unexpected value: version='https://unknown.github.com/spec/v1' +cannot parse git-lfs text: 'version https://git-lfs.github.com/spec/v1\n\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 12345\nx-foo extra-information\n' +unexpected value: oid='ahs256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393' +unexpected value: oid='4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393' +unexpected value: size='0x12345' +ok +cannot parse git-lfs text: 'version https://git-lfs.github.com/spec/v1\noid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\nsize 12345\nx-foo extra\ninformation\n' +unexpected key: x_foo +missed keys: oid +missed keys: oid, size diff --git a/tests/test-lfs-test-server.t b/tests/test-lfs-test-server.t new file mode 100644 --- /dev/null +++ b/tests/test-lfs-test-server.t @@ -0,0 +1,108 @@ +Require lfs-test-server (https://github.com/git-lfs/lfs-test-server) + + $ hash lfs-test-server || { echo 'skipped: missing lfs-test-server'; exit 80; } + + $ LFS_LISTEN="tcp://:$HGPORT" + $ LFS_HOST="localhost:$HGPORT" + $ LFS_PUBLIC=1 + $ export LFS_LISTEN LFS_HOST LFS_PUBLIC + $ lfs-test-server &> lfs-server.log & + $ echo $! >> $DAEMON_PIDS + + $ cat >> $HGRCPATH < [extensions] + > lfs= + > [lfs] + > url=http://foo:bar@$LFS_HOST/ + > threshold=1 + > EOF + + $ hg init repo1 + $ cd repo1 + $ echo THIS-IS-LFS > a + $ hg commit -m a -A a + + $ hg init ../repo2 + $ hg push ../repo2 -v + pushing to ../repo2 + searching for changes + lfs: uploading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes) + 1 changesets found + uncompressed size of bundle content: + * (changelog) (glob) + * (manifests) (glob) + * a (glob) + adding changesets + adding manifests + adding file changes + added 1 changesets with 1 changes to 1 files + + $ cd ../repo2 + $ hg update tip -v + resolving manifests + getting a + lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes) + 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + +When the server has some blobs already + + $ hg mv a b + $ echo ANOTHER-LARGE-FILE > c + $ echo ANOTHER-LARGE-FILE2 > d + $ hg commit -m b-and-c -A b c d + $ hg push ../repo1 -v | grep -v '^ ' + pushing to ../repo1 + searching for changes + lfs: need to transfer 2 objects (39 bytes) + lfs: uploading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes) + lfs: uploading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes) + 1 changesets found + uncompressed size of bundle content: + adding changesets + adding manifests + adding file changes + added 1 changesets with 3 changes to 3 files + + $ hg --repo ../repo1 update tip -v + resolving manifests + getting b + getting c + lfs: downloading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes) + getting d + lfs: downloading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes) + 3 files updated, 0 files merged, 0 files removed, 0 files unresolved + +Check error message when the remote missed a blob: + + $ echo FFFFF > b + $ hg commit -m b -A b + $ echo FFFFF >> b + $ hg commit -m b b + $ rm -rf .hg/store/lfs + $ hg update -C '.^' + abort: LFS server claims required objects do not exist: + 8e6ea5f6c066b44a0efa43bcce86aea73f17e6e23f0663df0251e7524e140a13! + [255] + +Check error message when object does not exist: + + $ hg init test && cd test + $ echo "[extensions]" >> .hg/hgrc + $ echo "lfs=" >> .hg/hgrc + $ echo "[lfs]" >> .hg/hgrc + $ echo "threshold=1" >> .hg/hgrc + $ echo a > a + $ hg add a + $ hg commit -m 'test' + $ echo aaaaa > a + $ hg commit -m 'largefile' + $ hg debugdata .hg/store/data/a.i 1 # verify this is no the file content but includes "oid", the LFS "pointer". + version https://git-lfs.github.com/spec/v1 + oid sha256:bdc26931acfb734b142a8d675f205becf27560dc461f501822de13274fe6fc8a + size 6 + x-is-binary 0 + $ cd .. + $ hg --config 'lfs.url=https://dewey-lfs.vip.facebook.com/lfs' clone test test2 + updating to branch default + abort: LFS server error. Remote object for file data/a.i not found:(.*)! (re) + [255] diff --git a/tests/test-lfs.t b/tests/test-lfs.t new file mode 100644 --- /dev/null +++ b/tests/test-lfs.t @@ -0,0 +1,544 @@ +# Initial setup + + $ cat >> $HGRCPATH << EOF + > [extensions] + > lfs= + > [lfs] + > threshold=1000B + > EOF + + $ LONG=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC + +# Prepare server and enable extension + $ hg init server + $ hg clone -q server client + $ cd client + +# Commit small file + $ echo s > smallfile + $ hg commit -Aqm "add small file" + +# Commit large file + $ echo $LONG > largefile + $ hg commit --traceback -Aqm "add large file" + +# Ensure metadata is stored + $ hg debugdata largefile 0 + version https://git-lfs.github.com/spec/v1 + oid sha256:f11e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b + size 1501 + x-is-binary 0 + +# Check the blobstore is populated + $ find .hg/store/lfs/objects | sort + .hg/store/lfs/objects + .hg/store/lfs/objects/f1 + .hg/store/lfs/objects/f1/1e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b + +# Check the blob stored contains the actual contents of the file + $ cat .hg/store/lfs/objects/f1/1e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC + +# Push changes to the server + + $ hg push + pushing to $TESTTMP/server (glob) + searching for changes + abort: lfs.url needs to be configured + [255] + + $ cat >> $HGRCPATH << EOF + > [lfs] + > url=file:$TESTTMP/dummy-remote/ + > EOF + + $ hg push -v | egrep -v '^(uncompressed| )' + pushing to $TESTTMP/server (glob) + searching for changes + 2 changesets found + adding changesets + adding manifests + adding file changes + added 2 changesets with 2 changes to 2 files + +# Unknown URL scheme + + $ hg push --config lfs.url=ftp://foobar + abort: lfs: unknown url scheme: ftp + [255] + + $ cd ../ + +# Initialize new client (not cloning) and setup extension + $ hg init client2 + $ cd client2 + $ cat >> .hg/hgrc < [paths] + > default = $TESTTMP/server + > EOF + +# Pull from server + $ hg pull default + pulling from $TESTTMP/server (glob) + requesting all changes + adding changesets + adding manifests + adding file changes + added 2 changesets with 2 changes to 2 files + new changesets b29ba743f89d:00c137947d30 + (run 'hg update' to get a working copy) + +# Check the blobstore is not yet populated + $ [ -d .hg/store/lfs/objects ] + [1] + +# Update to the last revision containing the large file + $ hg update + 2 files updated, 0 files merged, 0 files removed, 0 files unresolved + +# Check the blobstore has been populated on update + $ find .hg/store/lfs/objects | sort + .hg/store/lfs/objects + .hg/store/lfs/objects/f1 + .hg/store/lfs/objects/f1/1e77c257047a398492d8d6cb9f6acf3aa7c4384bb23080b43546053e183e4b + +# Check the contents of the file are fetched from blobstore when requested + $ hg cat -r . largefile + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC + +# Check the file has been copied in the working copy + $ cat largefile + AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC + + $ cd .. + +# Check rename, and switch between large and small files + + $ hg init repo3 + $ cd repo3 + $ cat >> .hg/hgrc << EOF + > [lfs] + > threshold=10B + > EOF + + $ echo LONGER-THAN-TEN-BYTES-WILL-TRIGGER-LFS > large + $ echo SHORTER > small + $ hg add . -q + $ hg commit -m 'commit with lfs content' + + $ hg mv large l + $ hg mv small s + $ hg commit -m 'renames' + + $ echo SHORT > l + $ echo BECOME-LARGER-FROM-SHORTER > s + $ hg commit -m 'large to small, small to large' + + $ echo 1 >> l + $ echo 2 >> s + $ hg commit -m 'random modifications' + + $ echo RESTORE-TO-BE-LARGE > l + $ echo SHORTER > s + $ hg commit -m 'switch large and small again' + +# Test lfs_files template + + $ hg log -r 'all()' -T '{rev} {join(lfs_files, ", ")}\n' + 0 large + 1 l + 2 s + 3 s + 4 l + +# Push and pull the above repo + + $ hg --cwd .. init repo4 + $ hg push ../repo4 + pushing to ../repo4 + searching for changes + adding changesets + adding manifests + adding file changes + added 5 changesets with 10 changes to 4 files + + $ hg --cwd .. init repo5 + $ hg --cwd ../repo5 pull ../repo3 + pulling from ../repo3 + requesting all changes + adding changesets + adding manifests + adding file changes + added 5 changesets with 10 changes to 4 files + new changesets fd47a419c4f7:5adf850972b9 + (run 'hg update' to get a working copy) + + $ cd .. + +# Test clone + + $ hg init repo6 + $ cd repo6 + $ cat >> .hg/hgrc << EOF + > [lfs] + > threshold=30B + > EOF + + $ echo LARGE-BECAUSE-IT-IS-MORE-THAN-30-BYTES > large + $ echo SMALL > small + $ hg commit -Aqm 'create a lfs file' large small + $ hg debuglfsupload -r 'all()' -v + + $ cd .. + + $ hg clone repo6 repo7 + updating to branch default + 2 files updated, 0 files merged, 0 files removed, 0 files unresolved + $ cd repo7 + $ cat large + LARGE-BECAUSE-IT-IS-MORE-THAN-30-BYTES + $ cat small + SMALL + + $ cd .. + +# Test rename and status + + $ hg init repo8 + $ cd repo8 + $ cat >> .hg/hgrc << EOF + > [lfs] + > threshold=10B + > EOF + + $ echo THIS-IS-LFS-BECAUSE-10-BYTES > a1 + $ echo SMALL > a2 + $ hg commit -m a -A a1 a2 + $ hg status + $ hg mv a1 b1 + $ hg mv a2 a1 + $ hg mv b1 a2 + $ hg commit -m b + $ hg status + $ HEADER=$'\1\n' + $ printf '%sSTART-WITH-HG-FILELOG-METADATA' "$HEADER" > a2 + $ printf '%sMETA\n' "$HEADER" > a1 + $ hg commit -m meta + $ hg status + $ hg log -T '{rev}: {file_copies} | {file_dels} | {file_adds}\n' + 2: | | + 1: a1 (a2)a2 (a1) | | + 0: | | a1 a2 + + $ for n in a1 a2; do + > for r in 0 1 2; do + > printf '\n%s @ %s\n' $n $r + > hg debugdata $n $r + > done + > done + + a1 @ 0 + version https://git-lfs.github.com/spec/v1 + oid sha256:5bb8341bee63b3649f222b2215bde37322bea075a30575aa685d8f8d21c77024 + size 29 + x-is-binary 0 + + a1 @ 1 + \x01 (esc) + copy: a2 + copyrev: 50470ad23cf937b1f4b9f80bfe54df38e65b50d9 + \x01 (esc) + SMALL + + a1 @ 2 + \x01 (esc) + \x01 (esc) + \x01 (esc) + META + + a2 @ 0 + SMALL + + a2 @ 1 + version https://git-lfs.github.com/spec/v1 + oid sha256:5bb8341bee63b3649f222b2215bde37322bea075a30575aa685d8f8d21c77024 + size 29 + x-hg-copy a1 + x-hg-copyrev be23af27908a582af43e5cda209a5a9b319de8d4 + x-is-binary 0 + + a2 @ 2 + version https://git-lfs.github.com/spec/v1 + oid sha256:876dadc86a8542f9798048f2c47f51dbf8e4359aed883e8ec80c5db825f0d943 + size 32 + x-is-binary 0 + +# Verify commit hashes include rename metadata + + $ hg log -T '{rev}:{node|short} {desc}\n' + 2:0fae949de7fa meta + 1:9cd6bdffdac0 b + 0:7f96794915f7 a + + $ cd .. + +# Test bundle + + $ hg init repo9 + $ cd repo9 + $ cat >> .hg/hgrc << EOF + > [lfs] + > threshold=10B + > [diff] + > git=1 + > EOF + + $ for i in 0 single two three 4; do + > echo 'THIS-IS-LFS-'$i > a + > hg commit -m a-$i -A a + > done + + $ hg update 2 -q + $ echo 'THIS-IS-LFS-2-CHILD' > a + $ hg commit -m branching -q + + $ hg bundle --base 1 bundle.hg -v + 4 changesets found + uncompressed size of bundle content: + * (changelog) (glob) + * (manifests) (glob) + * a (glob) + $ hg --config extensions.strip= strip -r 2 --no-backup --force -q + $ hg -R bundle.hg log -p -T '{rev} {desc}\n' a + 5 branching + diff --git a/a b/a + --- a/a + +++ b/a + @@ -1,1 +1,1 @@ + -THIS-IS-LFS-two + +THIS-IS-LFS-2-CHILD + + 4 a-4 + diff --git a/a b/a + --- a/a + +++ b/a + @@ -1,1 +1,1 @@ + -THIS-IS-LFS-three + +THIS-IS-LFS-4 + + 3 a-three + diff --git a/a b/a + --- a/a + +++ b/a + @@ -1,1 +1,1 @@ + -THIS-IS-LFS-two + +THIS-IS-LFS-three + + 2 a-two + diff --git a/a b/a + --- a/a + +++ b/a + @@ -1,1 +1,1 @@ + -THIS-IS-LFS-single + +THIS-IS-LFS-two + + 1 a-single + diff --git a/a b/a + --- a/a + +++ b/a + @@ -1,1 +1,1 @@ + -THIS-IS-LFS-0 + +THIS-IS-LFS-single + + 0 a-0 + diff --git a/a b/a + new file mode 100644 + --- /dev/null + +++ b/a + @@ -0,0 +1,1 @@ + +THIS-IS-LFS-0 + + $ hg bundle -R bundle.hg --base 1 bundle-again.hg -q + $ hg -R bundle-again.hg log -p -T '{rev} {desc}\n' a + 5 branching + diff --git a/a b/a + --- a/a + +++ b/a + @@ -1,1 +1,1 @@ + -THIS-IS-LFS-two + +THIS-IS-LFS-2-CHILD + + 4 a-4 + diff --git a/a b/a + --- a/a + +++ b/a + @@ -1,1 +1,1 @@ + -THIS-IS-LFS-three + +THIS-IS-LFS-4 + + 3 a-three + diff --git a/a b/a + --- a/a + +++ b/a + @@ -1,1 +1,1 @@ + -THIS-IS-LFS-two + +THIS-IS-LFS-three + + 2 a-two + diff --git a/a b/a + --- a/a + +++ b/a + @@ -1,1 +1,1 @@ + -THIS-IS-LFS-single + +THIS-IS-LFS-two + + 1 a-single + diff --git a/a b/a + --- a/a + +++ b/a + @@ -1,1 +1,1 @@ + -THIS-IS-LFS-0 + +THIS-IS-LFS-single + + 0 a-0 + diff --git a/a b/a + new file mode 100644 + --- /dev/null + +++ b/a + @@ -0,0 +1,1 @@ + +THIS-IS-LFS-0 + + $ cd .. + +# Test isbinary + + $ hg init repo10 + $ cd repo10 + $ cat >> .hg/hgrc << EOF + > [extensions] + > lfs= + > [lfs] + > threshold=1 + > EOF + $ $PYTHON <<'EOF' + > def write(path, content): + > with open(path, 'wb') as f: + > f.write(content) + > write('a', b'\0\0') + > write('b', b'\1\n') + > write('c', b'\1\n\0') + > write('d', b'xx') + > EOF + $ hg add a b c d + $ hg diff --stat + a | Bin + b | 1 + + c | Bin + d | 1 + + 4 files changed, 2 insertions(+), 0 deletions(-) + $ hg commit -m binarytest + $ cat > $TESTTMP/dumpbinary.py << EOF + > def reposetup(ui, repo): + > for n in 'abcd': + > ui.write(('%s: binary=%s\n') % (n, repo['.'][n].isbinary())) + > EOF + $ hg --config extensions.dumpbinary=$TESTTMP/dumpbinary.py id --trace + a: binary=True + b: binary=False + c: binary=True + d: binary=False + b55353847f02 tip + + $ cd .. + +# Test fctx.cmp fastpath - diff without LFS blobs + + $ hg init repo11 + $ cd repo11 + $ cat >> .hg/hgrc < [lfs] + > threshold=1 + > EOF + $ for i in 1 2 3; do + > cp ../repo10/a a + > if [ $i = 3 ]; then + > # make a content-only change + > chmod +x a + > i=2 + > fi + > echo $i >> a + > hg commit -m $i -A a + > done + $ [ -d .hg/store/lfs/objects ] + + $ cd .. + + $ hg clone repo11 repo12 --noupdate + $ cd repo12 + $ hg log --removed -p a -T '{desc}\n' --config diff.nobinary=1 --git + 2 + diff --git a/a b/a + old mode 100644 + new mode 100755 + + 2 + diff --git a/a b/a + Binary file a has changed + + 1 + diff --git a/a b/a + new file mode 100644 + Binary file a has changed + + $ [ -d .hg/store/lfs/objects ] + [1] + + $ cd .. + +# Verify the repos + + $ cat > $TESTTMP/dumpflog.py << EOF + > # print raw revision sizes, flags, and hashes for certain files + > import hashlib + > from mercurial import revlog + > from mercurial.node import short + > def hash(rawtext): + > h = hashlib.sha512() + > h.update(rawtext) + > return h.hexdigest()[:4] + > def reposetup(ui, repo): + > # these 2 files are interesting + > for name in ['l', 's']: + > fl = repo.file(name) + > if len(fl) == 0: + > continue + > sizes = [revlog.revlog.rawsize(fl, i) for i in fl] + > texts = [fl.revision(i, raw=True) for i in fl] + > flags = [fl.flags(i) for i in fl] + > hashes = [hash(t) for t in texts] + > print(' %s: rawsizes=%r flags=%r hashes=%r' + > % (name, sizes, flags, hashes)) + > EOF + + $ for i in client client2 server repo3 repo4 repo5 repo6 repo7 repo8 repo9 \ + > repo10; do + > echo 'repo:' $i + > hg --cwd $i verify --config extensions.dumpflog=$TESTTMP/dumpflog.py -q + > done + repo: client + repo: client2 + repo: server + repo: repo3 + l: rawsizes=[211, 6, 8, 141] flags=[8192, 0, 0, 8192] hashes=['d2b8', '948c', 'cc88', '724d'] + s: rawsizes=[74, 141, 141, 8] flags=[0, 8192, 8192, 0] hashes=['3c80', 'fce0', '874a', '826b'] + repo: repo4 + l: rawsizes=[211, 6, 8, 141] flags=[8192, 0, 0, 8192] hashes=['d2b8', '948c', 'cc88', '724d'] + s: rawsizes=[74, 141, 141, 8] flags=[0, 8192, 8192, 0] hashes=['3c80', 'fce0', '874a', '826b'] + repo: repo5 + l: rawsizes=[211, 6, 8, 141] flags=[8192, 0, 0, 8192] hashes=['d2b8', '948c', 'cc88', '724d'] + s: rawsizes=[74, 141, 141, 8] flags=[0, 8192, 8192, 0] hashes=['3c80', 'fce0', '874a', '826b'] + repo: repo6 + repo: repo7 + repo: repo8 + repo: repo9 + repo: repo10