lfs: improve the client message when the server signals an object error

Two things here.  First, the previous message included a snippet of JSON, which
tends to be long (and in the case of lfs-test-server, has no error message).
Instead, give a concise message where possible, and leave the JSON to a debug
output.  Second, the server can signal issues other than a missing individual
file.  This change shows a corrupt file, but I'm debating letting the corrupt
file get downloaded, because 1) the error code doesn't really fit, and 2) having
it locally makes forensics easier.  Maybe need a config knob for that.

hgext/lfs/blobstore.py

@@ -1,504 +1,515 b''
 # blobstore.py - local and remote (speaking Git-LFS protocol) blob storages
 #
 # Copyright 2017 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import hashlib
 import json
 import os
 import re
 import socket
 
 from mercurial.i18n import _
 
 from mercurial import (
     error,
     pathutil,
     pycompat,
     url as urlmod,
     util,
     vfs as vfsmod,
     worker,
 )
 
 from ..largefiles import lfutil
 
 # 64 bytes for SHA256
 _lfsre = re.compile(br'\A[a-f0-9]{64}\Z')
 
 class lfsvfs(vfsmod.vfs):
     def join(self, path):
         """split the path at first two characters, like: XX/XXXXX..."""
         if not _lfsre.match(path):
             raise error.ProgrammingError('unexpected lfs path: %s' % path)
         return super(lfsvfs, self).join(path[0:2], path[2:])
 
     def walk(self, path=None, onerror=None):
         """Yield (dirpath, [], oids) tuple for blobs under path
 
         Oids only exist in the root of this vfs, so dirpath is always ''.
         """
         root = os.path.normpath(self.base)
         # when dirpath == root, dirpath[prefixlen:] becomes empty
         # because len(dirpath) < prefixlen.
         prefixlen = len(pathutil.normasprefix(root))
         oids = []
 
         for dirpath, dirs, files in os.walk(self.reljoin(self.base, path or ''),
                                             onerror=onerror):
             dirpath = dirpath[prefixlen:]
 
             # Silently skip unexpected files and directories
             if len(dirpath) == 2:
                 oids.extend([dirpath + f for f in files
                              if _lfsre.match(dirpath + f)])
 
         yield ('', [], oids)
 
 class filewithprogress(object):
     """a file-like object that supports __len__ and read.
 
     Useful to provide progress information for how many bytes are read.
     """
 
     def __init__(self, fp, callback):
         self._fp = fp
         self._callback = callback # func(readsize)
         fp.seek(0, os.SEEK_END)
         self._len = fp.tell()
         fp.seek(0)
 
     def __len__(self):
         return self._len
 
     def read(self, size):
         if self._fp is None:
             return b''
         data = self._fp.read(size)
         if data:
             if self._callback:
                 self._callback(len(data))
         else:
             self._fp.close()
             self._fp = None
         return data
 
 class local(object):
     """Local blobstore for large file contents.
 
     This blobstore is used both as a cache and as a staging area for large blobs
     to be uploaded to the remote blobstore.
     """
 
     def __init__(self, repo):
         fullpath = repo.svfs.join('lfs/objects')
         self.vfs = lfsvfs(fullpath)
         usercache = lfutil._usercachedir(repo.ui, 'lfs')
         self.cachevfs = lfsvfs(usercache)
         self.ui = repo.ui
 
     def open(self, oid):
         """Open a read-only file descriptor to the named blob, in either the
         usercache or the local store."""
         # The usercache is the most likely place to hold the file.  Commit will
         # write to both it and the local store, as will anything that downloads
         # the blobs.  However, things like clone without an update won't
         # populate the local store.  For an init + push of a local clone,
         # the usercache is the only place it _could_ be.  If not present, the
         # missing file msg here will indicate the local repo, not the usercache.
         if self.cachevfs.exists(oid):
             return self.cachevfs(oid, 'rb')
 
         return self.vfs(oid, 'rb')
 
     def download(self, oid, src):
         """Read the blob from the remote source in chunks, verify the content,
         and write to this local blobstore."""
         sha256 = hashlib.sha256()
 
         with self.vfs(oid, 'wb', atomictemp=True) as fp:
             for chunk in util.filechunkiter(src, size=1048576):
                 fp.write(chunk)
                 sha256.update(chunk)
 
             realoid = sha256.hexdigest()
             if realoid != oid:
                 raise error.Abort(_('corrupt remote lfs object: %s') % oid)
 
         # XXX: should we verify the content of the cache, and hardlink back to
         # the local store on success, but truncate, write and link on failure?
         if not self.cachevfs.exists(oid):
             self.ui.note(_('lfs: adding %s to the usercache\n') % oid)
             lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid))
 
     def write(self, oid, data):
         """Write blob to local blobstore.
 
         This should only be called from the filelog during a commit or similar.
         As such, there is no need to verify the data.  Imports from a remote
         store must use ``download()`` instead."""
         with self.vfs(oid, 'wb', atomictemp=True) as fp:
             fp.write(data)
 
         # XXX: should we verify the content of the cache, and hardlink back to
         # the local store on success, but truncate, write and link on failure?
         if not self.cachevfs.exists(oid):
             self.ui.note(_('lfs: adding %s to the usercache\n') % oid)
             lfutil.link(self.vfs.join(oid), self.cachevfs.join(oid))
 
     def read(self, oid, verify=True):
         """Read blob from local blobstore."""
         if not self.vfs.exists(oid):
             blob = self._read(self.cachevfs, oid, verify)
 
             # Even if revlog will verify the content, it needs to be verified
             # now before making the hardlink to avoid propagating corrupt blobs.
             # Don't abort if corruption is detected, because `hg verify` will
             # give more useful info about the corruption- simply don't add the
             # hardlink.
             if verify or hashlib.sha256(blob).hexdigest() == oid:
                 self.ui.note(_('lfs: found %s in the usercache\n') % oid)
                 lfutil.link(self.cachevfs.join(oid), self.vfs.join(oid))
         else:
             self.ui.note(_('lfs: found %s in the local lfs store\n') % oid)
             blob = self._read(self.vfs, oid, verify)
         return blob
 
     def _read(self, vfs, oid, verify):
         """Read blob (after verifying) from the given store"""
         blob = vfs.read(oid)
         if verify:
             _verify(oid, blob)
         return blob
 
     def verify(self, oid):
         """Indicate whether or not the hash of the underlying file matches its
         name."""
         sha256 = hashlib.sha256()
 
         with self.open(oid) as fp:
             for chunk in util.filechunkiter(fp, size=1048576):
                 sha256.update(chunk)
 
         return oid == sha256.hexdigest()
 
     def has(self, oid):
         """Returns True if the local blobstore contains the requested blob,
         False otherwise."""
         return self.cachevfs.exists(oid) or self.vfs.exists(oid)
 
 class _gitlfsremote(object):
 
     def __init__(self, repo, url):
         ui = repo.ui
         self.ui = ui
         baseurl, authinfo = url.authinfo()
         self.baseurl = baseurl.rstrip('/')
         useragent = repo.ui.config('experimental', 'lfs.user-agent')
         if not useragent:
             useragent = 'git-lfs/2.3.4 (Mercurial %s)' % util.version()
         self.urlopener = urlmod.opener(ui, authinfo, useragent)
         self.retry = ui.configint('lfs', 'retry')
 
     def writebatch(self, pointers, fromstore):
         """Batch upload from local to remote blobstore."""
         self._batch(_deduplicate(pointers), fromstore, 'upload')
 
     def readbatch(self, pointers, tostore):
         """Batch download from remote to local blostore."""
         self._batch(_deduplicate(pointers), tostore, 'download')
 
     def _batchrequest(self, pointers, action):
         """Get metadata about objects pointed by pointers for given action
 
         Return decoded JSON object like {'objects': [{'oid': '', 'size': 1}]}
         See https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
         """
         objects = [{'oid': p.oid(), 'size': p.size()} for p in pointers]
         requestdata = json.dumps({
             'objects': objects,
             'operation': action,
         })
         batchreq = util.urlreq.request('%s/objects/batch' % self.baseurl,
                                        data=requestdata)
         batchreq.add_header('Accept', 'application/vnd.git-lfs+json')
         batchreq.add_header('Content-Type', 'application/vnd.git-lfs+json')
         try:
             rsp = self.urlopener.open(batchreq)
             rawjson = rsp.read()
         except util.urlerr.httperror as ex:
             raise LfsRemoteError(_('LFS HTTP error: %s (action=%s)')
                                  % (ex, action))
         try:
             response = json.loads(rawjson)
         except ValueError:
             raise LfsRemoteError(_('LFS server returns invalid JSON: %s')
                                  % rawjson)
 
         if self.ui.debugflag:
             self.ui.debug('Status: %d\n' % rsp.status)
             # lfs-test-server and hg serve return headers in different order
             self.ui.debug('%s\n'
                           % '\n'.join(sorted(str(rsp.info()).splitlines())))
 
             if 'objects' in response:
                 response['objects'] = sorted(response['objects'],
                                              key=lambda p: p['oid'])
             self.ui.debug('%s\n'
                           % json.dumps(response, indent=2,
                                        separators=('', ': '), sort_keys=True))
 
         return response
 
     def _checkforservererror(self, pointers, responses, action):
         """Scans errors from objects
 
         Raises LfsRemoteError if any objects have an error"""
         for response in responses:
             # The server should return 404 when objects cannot be found. Some
             # server implementation (ex. lfs-test-server)  does not set "error"
             # but just removes "download" from "actions". Treat that case
             # as the same as 404 error.
-            notfound = (response.get('error', {}).get('code') == 404
-                        or (action == 'download'
-                            and action not in response.get('actions', [])))
-            if notfound:
-                ptrmap = {p.oid(): p for p in pointers}
-                p = ptrmap.get(response['oid'], None)
-                if p:
-                    filename = getattr(p, 'filename', 'unknown')
-                    raise LfsRemoteError(
-                        _(('LFS server error. Remote object '
-                          'for "%s" not found: %r')) % (filename, response))
+            if 'error' not in response:
+                if (action == 'download'
+                    and action not in response.get('actions', [])):
+                    code = 404
                 else:
-                    raise LfsRemoteError(
-                        _('LFS server error. Unsolicited response for oid %s')
-                        % response['oid'])
-            if 'error' in response:
-                raise LfsRemoteError(_('LFS server error: %r') % response)
+                    continue
+            else:
+                # An error dict without a code doesn't make much sense, so
+                # treat as a server error.
+                code = response.get('error').get('code', 500)
+
+            ptrmap = {p.oid(): p for p in pointers}
+            p = ptrmap.get(response['oid'], None)
+            if p:
+                filename = getattr(p, 'filename', 'unknown')
+                errors = {
+                    404: 'The object does not exist',
+                    410: 'The object was removed by the owner',
+                    422: 'Validation error',
+                    500: 'Internal server error',
+                }
+                msg = errors.get(code, 'status code %d' % code)
+                raise LfsRemoteError(_('LFS server error for "%s": %s')
+                                     % (filename, msg))
+            else:
+                raise LfsRemoteError(
+                    _('LFS server error. Unsolicited response for oid %s')
+                    % response['oid'])
 
     def _extractobjects(self, response, pointers, action):
         """extract objects from response of the batch API
 
         response: parsed JSON object returned by batch API
         return response['objects'] filtered by action
         raise if any object has an error
         """
         # Scan errors from objects - fail early
         objects = response.get('objects', [])
         self._checkforservererror(pointers, objects, action)
 
         # Filter objects with given action. Practically, this skips uploading
         # objects which exist in the server.
         filteredobjects = [o for o in objects if action in o.get('actions', [])]
 
         return filteredobjects
 
     def _basictransfer(self, obj, action, localstore):
         """Download or upload a single object using basic transfer protocol
 
         obj: dict, an object description returned by batch API
         action: string, one of ['upload', 'download']
         localstore: blobstore.local
 
         See https://github.com/git-lfs/git-lfs/blob/master/docs/api/\
         basic-transfers.md
         """
         oid = pycompat.bytestr(obj['oid'])
 
         href = pycompat.bytestr(obj['actions'][action].get('href'))
         headers = obj['actions'][action].get('header', {}).items()
 
         request = util.urlreq.request(href)
         if action == 'upload':
             # If uploading blobs, read data from local blobstore.
             if not localstore.verify(oid):
                 raise error.Abort(_('detected corrupt lfs object: %s') % oid,
                                   hint=_('run hg verify'))
             request.data = filewithprogress(localstore.open(oid), None)
             request.get_method = lambda: 'PUT'
 
         for k, v in headers:
             request.add_header(k, v)
 
         response = b''
         try:
             req = self.urlopener.open(request)
 
             if self.ui.debugflag:
                 self.ui.debug('Status: %d\n' % req.status)
                 # lfs-test-server and hg serve return headers in different order
                 self.ui.debug('%s\n'
                               % '\n'.join(sorted(str(req.info()).splitlines())))
 
             if action == 'download':
                 # If downloading blobs, store downloaded data to local blobstore
                 localstore.download(oid, req)
             else:
                 while True:
                     data = req.read(1048576)
                     if not data:
                         break
                     response += data
                 if response:
                     self.ui.debug('lfs %s response: %s' % (action, response))
         except util.urlerr.httperror as ex:
             if self.ui.debugflag:
                 self.ui.debug('%s: %s\n' % (oid, ex.read()))
             raise LfsRemoteError(_('HTTP error: %s (oid=%s, action=%s)')
                                  % (ex, oid, action))
 
     def _batch(self, pointers, localstore, action):
         if action not in ['upload', 'download']:
             raise error.ProgrammingError('invalid Git-LFS action: %s' % action)
 
         response = self._batchrequest(pointers, action)
         objects = self._extractobjects(response, pointers, action)
         total = sum(x.get('size', 0) for x in objects)
         sizes = {}
         for obj in objects:
             sizes[obj.get('oid')] = obj.get('size', 0)
         topic = {'upload': _('lfs uploading'),
                  'download': _('lfs downloading')}[action]
         if len(objects) > 1:
             self.ui.note(_('lfs: need to transfer %d objects (%s)\n')
                          % (len(objects), util.bytecount(total)))
         self.ui.progress(topic, 0, total=total)
         def transfer(chunk):
             for obj in chunk:
                 objsize = obj.get('size', 0)
                 if self.ui.verbose:
                     if action == 'download':
                         msg = _('lfs: downloading %s (%s)\n')
                     elif action == 'upload':
                         msg = _('lfs: uploading %s (%s)\n')
                     self.ui.note(msg % (obj.get('oid'),
                                  util.bytecount(objsize)))
                 retry = self.retry
                 while True:
                     try:
                         self._basictransfer(obj, action, localstore)
                         yield 1, obj.get('oid')
                         break
                     except socket.error as ex:
                         if retry > 0:
                             self.ui.note(
                                 _('lfs: failed: %r (remaining retry %d)\n')
                                 % (ex, retry))
                             retry -= 1
                             continue
                         raise
 
         # Until https multiplexing gets sorted out
         if self.ui.configbool('experimental', 'lfs.worker-enable'):
             oids = worker.worker(self.ui, 0.1, transfer, (),
                                  sorted(objects, key=lambda o: o.get('oid')))
         else:
             oids = transfer(sorted(objects, key=lambda o: o.get('oid')))
 
         processed = 0
         blobs = 0
         for _one, oid in oids:
             processed += sizes[oid]
             blobs += 1
             self.ui.progress(topic, processed, total=total)
             self.ui.note(_('lfs: processed: %s\n') % oid)
         self.ui.progress(topic, pos=None, total=total)
 
         if blobs > 0:
             if action == 'upload':
                 self.ui.status(_('lfs: uploaded %d files (%s)\n')
                                % (blobs, util.bytecount(processed)))
             # TODO: coalesce the download requests, and comment this in
             #elif action == 'download':
             #    self.ui.status(_('lfs: downloaded %d files (%s)\n')
             #                   % (blobs, util.bytecount(processed)))
 
     def __del__(self):
         # copied from mercurial/httppeer.py
         urlopener = getattr(self, 'urlopener', None)
         if urlopener:
             for h in urlopener.handlers:
                 h.close()
                 getattr(h, "close_all", lambda : None)()
 
 class _dummyremote(object):
     """Dummy store storing blobs to temp directory."""
 
     def __init__(self, repo, url):
         fullpath = repo.vfs.join('lfs', url.path)
         self.vfs = lfsvfs(fullpath)
 
     def writebatch(self, pointers, fromstore):
         for p in _deduplicate(pointers):
             content = fromstore.read(p.oid(), verify=True)
             with self.vfs(p.oid(), 'wb', atomictemp=True) as fp:
                 fp.write(content)
 
     def readbatch(self, pointers, tostore):
         for p in _deduplicate(pointers):
             with self.vfs(p.oid(), 'rb') as fp:
                 tostore.download(p.oid(), fp)
 
 class _nullremote(object):
     """Null store storing blobs to /dev/null."""
 
     def __init__(self, repo, url):
         pass
 
     def writebatch(self, pointers, fromstore):
         pass
 
     def readbatch(self, pointers, tostore):
         pass
 
 class _promptremote(object):
     """Prompt user to set lfs.url when accessed."""
 
     def __init__(self, repo, url):
         pass
 
     def writebatch(self, pointers, fromstore, ui=None):
         self._prompt()
 
     def readbatch(self, pointers, tostore, ui=None):
         self._prompt()
 
     def _prompt(self):
         raise error.Abort(_('lfs.url needs to be configured'))
 
 _storemap = {
     'https': _gitlfsremote,
     'http': _gitlfsremote,
     'file': _dummyremote,
     'null': _nullremote,
     None: _promptremote,
 }
 
 def _deduplicate(pointers):
     """Remove any duplicate oids that exist in the list"""
     reduced = util.sortdict()
     for p in pointers:
         reduced[p.oid()] = p
     return reduced.values()
 
 def _verify(oid, content):
     realoid = hashlib.sha256(content).hexdigest()
     if realoid != oid:
         raise error.Abort(_('detected corrupt lfs object: %s') % oid,
                           hint=_('run hg verify'))
 
 def remote(repo):
     """remotestore factory. return a store in _storemap depending on config"""
     url = util.url(repo.ui.config('lfs', 'url') or '')
     scheme = url.scheme
     if scheme not in _storemap:
         raise error.Abort(_('lfs: unknown url scheme: %s') % scheme)
     return _storemap[scheme](repo, url)
 
 class LfsRemoteError(error.RevlogError):
     pass

tests/test-lfs-test-server.t

@@ -1,924 +1,924 b''
 #testcases git-server hg-server
 
 #if git-server
 #require lfs-test-server
 #else
 #require serve
 #endif
 
 #if git-server
   $ LFS_LISTEN="tcp://:$HGPORT"
   $ LFS_HOST="localhost:$HGPORT"
   $ LFS_PUBLIC=1
   $ export LFS_LISTEN LFS_HOST LFS_PUBLIC
 #else
   $ LFS_HOST="localhost:$HGPORT/.git/info/lfs"
 #endif
 
 #if no-windows git-server
   $ lfs-test-server &> lfs-server.log &
   $ echo $! >> $DAEMON_PIDS
 #endif
 
 #if windows git-server
   $ cat >> $TESTTMP/spawn.py <<EOF
   > import os
   > import subprocess
   > import sys
   > 
   > for path in os.environ["PATH"].split(os.pathsep):
   >     exe = os.path.join(path, 'lfs-test-server.exe')
   >     if os.path.exists(exe):
   >         with open('lfs-server.log', 'wb') as out:
   >             p = subprocess.Popen(exe, stdout=out, stderr=out)
   >             sys.stdout.write('%s\n' % p.pid)
   >             sys.exit(0)
   > sys.exit(1)
   > EOF
   $ $PYTHON $TESTTMP/spawn.py >> $DAEMON_PIDS
 #endif
 
   $ cat >> $HGRCPATH <<EOF
   > [extensions]
   > lfs=
   > [lfs]
   > url=http://foo:bar@$LFS_HOST
   > track=all()
   > [web]
   > push_ssl = False
   > allow-push = *
   > EOF
 
 Use a separate usercache, otherwise the server sees what the client commits, and
 never requests a transfer.
 
 #if hg-server
   $ hg init server
   $ hg --config "lfs.usercache=$TESTTMP/servercache" -R server serve -d \
   >    -p $HGPORT --pid-file=hg.pid -A $TESTTMP/access.log -E $TESTTMP/errors.log
   $ cat hg.pid >> $DAEMON_PIDS
 #endif
 
   $ hg init repo1
   $ cd repo1
   $ echo THIS-IS-LFS > a
   $ hg commit -m a -A a
 
 A push can be serviced directly from the usercache if it isn't in the local
 store.
 
   $ hg init ../repo2
   $ mv .hg/store/lfs .hg/store/lfs_
   $ hg push ../repo2 --debug
   http auth: user foo, password ***
   pushing to ../repo2
   http auth: user foo, password ***
   query 1; heads
   searching for changes
   1 total queries in *s (glob)
   listing keys for "phases"
   checking for updated bookmarks
   listing keys for "bookmarks"
   lfs: computing set of blobs to upload
   Status: 200
   Content-Length: 309 (git-server !)
   Content-Length: 350 (hg-server !)
   Content-Type: application/vnd.git-lfs+json
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   {
     "objects": [
       {
         "actions": {
           "upload": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/objects/31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b" (git-server !)
             "href": "http://localhost:$HGPORT/.hg/lfs/objects/31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b" (hg-server !)
           }
         }
         "oid": "31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b"
         "size": 12
       }
     ]
     "transfer": "basic" (hg-server !)
   }
   lfs: uploading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
   Status: 200 (git-server !)
   Status: 201 (hg-server !)
   Content-Length: 0
   Content-Type: text/plain; charset=utf-8
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
   lfs: uploaded 1 files (12 bytes)
   1 changesets found
   list of changesets:
   99a7098854a3984a5c9eab0fc7a2906697b7cb5c
   bundle2-output-bundle: "HG20", 4 parts total
   bundle2-output-part: "replycaps" * bytes payload (glob)
   bundle2-output-part: "check:heads" streamed payload
   bundle2-output-part: "changegroup" (params: 1 mandatory) streamed payload
   bundle2-output-part: "phase-heads" 24 bytes payload
   bundle2-input-bundle: with-transaction
   bundle2-input-part: "replycaps" supported
   bundle2-input-part: total payload size * (glob)
   bundle2-input-part: "check:heads" supported
   bundle2-input-part: total payload size 20
   bundle2-input-part: "changegroup" (params: 1 mandatory) supported
   adding changesets
   add changeset 99a7098854a3
   adding manifests
   adding file changes
   adding a revisions
   added 1 changesets with 1 changes to 1 files
   calling hook pretxnchangegroup.lfs: hgext.lfs.checkrequireslfs
   bundle2-input-part: total payload size 617
   bundle2-input-part: "phase-heads" supported
   bundle2-input-part: total payload size 24
   bundle2-input-bundle: 3 parts total
   updating the branch cache
   bundle2-output-bundle: "HG20", 1 parts total
   bundle2-output-part: "reply:changegroup" (advisory) (params: 0 advisory) empty payload
   bundle2-input-bundle: no-transaction
   bundle2-input-part: "reply:changegroup" (advisory) (params: 0 advisory) supported
   bundle2-input-bundle: 0 parts total
   listing keys for "phases"
   $ mv .hg/store/lfs_ .hg/store/lfs
 
 Clear the cache to force a download
   $ rm -rf `hg config lfs.usercache`
   $ cd ../repo2
   $ hg update tip --debug
   http auth: user foo, password ***
   resolving manifests
    branchmerge: False, force: False, partial: False
    ancestor: 000000000000, local: 000000000000+, remote: 99a7098854a3
   Status: 200
   Content-Length: 311 (git-server !)
   Content-Length: 352 (hg-server !)
   Content-Type: application/vnd.git-lfs+json
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   {
     "objects": [
       {
         "actions": {
           "download": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/*/31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b" (glob)
           }
         }
         "oid": "31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b"
         "size": 12
       }
     ]
     "transfer": "basic" (hg-server !)
   }
   lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
   Status: 200
   Content-Length: 12
   Content-Type: text/plain; charset=utf-8 (git-server !)
   Content-Type: application/octet-stream (hg-server !)
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache
   lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
    a: remote created -> g
   getting a
   lfs: found 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b in the local lfs store
   1 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
 When the server has some blobs already.  `hg serve` doesn't offer to upload
 blobs that it already knows about.  Note that lfs-test-server is simply
 toggling the action to 'download'.  The Batch API spec says it should omit the
 actions property completely.
 
   $ hg mv a b
   $ echo ANOTHER-LARGE-FILE > c
   $ echo ANOTHER-LARGE-FILE2 > d
   $ hg commit -m b-and-c -A b c d
   $ hg push ../repo1 --debug
   http auth: user foo, password ***
   pushing to ../repo1
   http auth: user foo, password ***
   query 1; heads
   searching for changes
   all remote heads known locally
   listing keys for "phases"
   checking for updated bookmarks
   listing keys for "bookmarks"
   listing keys for "bookmarks"
   lfs: computing set of blobs to upload
   Status: 200
   Content-Length: 901 (git-server !)
   Content-Length: 755 (hg-server !)
   Content-Type: application/vnd.git-lfs+json
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   {
     "objects": [
       {
         "actions": { (git-server !)
           "download": { (git-server !)
             "expires_at": "$ISO_8601_DATE_TIME$" (git-server !)
             "header": { (git-server !)
               "Accept": "application/vnd.git-lfs" (git-server !)
             } (git-server !)
             "href": "http://localhost:$HGPORT/objects/31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b" (git-server !)
           } (git-server !)
         } (git-server !)
         "oid": "31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b"
         "size": 12
       }
       {
         "actions": {
           "upload": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/*/37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19" (glob)
           }
         }
         "oid": "37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19"
         "size": 20
       }
       {
         "actions": {
           "upload": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/*/d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998" (glob)
           }
         }
         "oid": "d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998"
         "size": 19
       }
     ]
     "transfer": "basic" (hg-server !)
   }
   lfs: need to transfer 2 objects (39 bytes)
   lfs: uploading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes)
   Status: 200 (git-server !)
   Status: 201 (hg-server !)
   Content-Length: 0
   Content-Type: text/plain; charset=utf-8
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   lfs: processed: 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19
   lfs: uploading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes)
   Status: 200 (git-server !)
   Status: 201 (hg-server !)
   Content-Length: 0
   Content-Type: text/plain; charset=utf-8
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   lfs: processed: d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
   lfs: uploaded 2 files (39 bytes)
   1 changesets found
   list of changesets:
   dfca2c9e2ef24996aa61ba2abd99277d884b3d63
   bundle2-output-bundle: "HG20", 5 parts total
   bundle2-output-part: "replycaps" * bytes payload (glob)
   bundle2-output-part: "check:phases" 24 bytes payload
   bundle2-output-part: "check:heads" streamed payload
   bundle2-output-part: "changegroup" (params: 1 mandatory) streamed payload
   bundle2-output-part: "phase-heads" 24 bytes payload
   bundle2-input-bundle: with-transaction
   bundle2-input-part: "replycaps" supported
   bundle2-input-part: total payload size * (glob)
   bundle2-input-part: "check:phases" supported
   bundle2-input-part: total payload size 24
   bundle2-input-part: "check:heads" supported
   bundle2-input-part: total payload size 20
   bundle2-input-part: "changegroup" (params: 1 mandatory) supported
   adding changesets
   add changeset dfca2c9e2ef2
   adding manifests
   adding file changes
   adding b revisions
   adding c revisions
   adding d revisions
   added 1 changesets with 3 changes to 3 files
   bundle2-input-part: total payload size 1315
   bundle2-input-part: "phase-heads" supported
   bundle2-input-part: total payload size 24
   bundle2-input-bundle: 4 parts total
   updating the branch cache
   bundle2-output-bundle: "HG20", 1 parts total
   bundle2-output-part: "reply:changegroup" (advisory) (params: 0 advisory) empty payload
   bundle2-input-bundle: no-transaction
   bundle2-input-part: "reply:changegroup" (advisory) (params: 0 advisory) supported
   bundle2-input-bundle: 0 parts total
   listing keys for "phases"
 
 Clear the cache to force a download
   $ rm -rf `hg config lfs.usercache`
   $ hg --repo ../repo1 update tip --debug
   http auth: user foo, password ***
   resolving manifests
    branchmerge: False, force: False, partial: False
    ancestor: 99a7098854a3, local: 99a7098854a3+, remote: dfca2c9e2ef2
   Status: 200
   Content-Length: 608 (git-server !)
   Content-Length: 670 (hg-server !)
   Content-Type: application/vnd.git-lfs+json
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   {
     "objects": [
       {
         "actions": {
           "download": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/*/37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19" (glob)
           }
         }
         "oid": "37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19"
         "size": 20
       }
       {
         "actions": {
           "download": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/*/d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998" (glob)
           }
         }
         "oid": "d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998"
         "size": 19
       }
     ]
     "transfer": "basic" (hg-server !)
   }
   lfs: need to transfer 2 objects (39 bytes)
   lfs: downloading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes)
   Status: 200
   Content-Length: 20
   Content-Type: text/plain; charset=utf-8 (git-server !)
   Content-Type: application/octet-stream (hg-server !)
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   lfs: adding 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 to the usercache
   lfs: processed: 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19
   lfs: downloading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes)
   Status: 200
   Content-Length: 19
   Content-Type: text/plain; charset=utf-8 (git-server !)
   Content-Type: application/octet-stream (hg-server !)
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   lfs: adding d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 to the usercache
   lfs: processed: d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
    b: remote created -> g
   getting b
   lfs: found 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b in the local lfs store
    c: remote created -> g
   getting c
   lfs: found d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 in the local lfs store
    d: remote created -> g
   getting d
   lfs: found 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 in the local lfs store
   3 files updated, 0 files merged, 0 files removed, 0 files unresolved
 
 Test a corrupt file download, but clear the cache first to force a download.
 `hg serve` indicates a corrupt file without transferring it, unlike
 lfs-test-server.
 
   $ rm -rf `hg config lfs.usercache`
 #if git-server
   $ cp $TESTTMP/lfs-content/d1/1e/1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 blob
   $ echo 'damage' > $TESTTMP/lfs-content/d1/1e/1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
 #else
   $ cp $TESTTMP/server/.hg/store/lfs/objects/d1/1e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 blob
   $ echo 'damage' > $TESTTMP/server/.hg/store/lfs/objects/d1/1e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
 #endif
   $ rm ../repo1/.hg/store/lfs/objects/d1/1e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
   $ rm ../repo1/*
 
 TODO: give the proper error indication from `hg serve`
 
   $ hg --repo ../repo1 update -C tip --debug
   http auth: user foo, password ***
   resolving manifests
    branchmerge: False, force: True, partial: False
    ancestor: dfca2c9e2ef2+, local: dfca2c9e2ef2+, remote: dfca2c9e2ef2
   Status: 200
   Content-Length: 311 (git-server !)
   Content-Length: 183 (hg-server !)
   Content-Type: application/vnd.git-lfs+json
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   {
     "objects": [
       {
         "actions": { (git-server !)
           "download": { (git-server !)
             "expires_at": "$ISO_8601_DATE_TIME$" (git-server !)
             "header": { (git-server !)
               "Accept": "application/vnd.git-lfs" (git-server !)
             } (git-server !)
             "href": "http://localhost:$HGPORT/objects/d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998" (git-server !)
           } (git-server !)
         "error": { (hg-server !)
           "code": 422 (hg-server !)
           "message": "The object is corrupt" (hg-server !)
         }
         "oid": "d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998"
         "size": 19
       }
     ]
     "transfer": "basic" (hg-server !)
   }
   lfs: downloading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes) (git-server !)
   Status: 200 (git-server !)
   Content-Length: 7 (git-server !)
   Content-Type: text/plain; charset=utf-8 (git-server !)
   Date: $HTTP_DATE$ (git-server !)
   abort: corrupt remote lfs object: d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (git-server !)
-  abort: LFS server error. Remote object for "c" not found: *! (glob) (hg-server !)
+  abort: LFS server error for "c": Validation error! (hg-server !)
   [255]
 
 The corrupted blob is not added to the usercache or local store
 
   $ test -f ../repo1/.hg/store/lfs/objects/d1/1e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
   [1]
   $ test -f `hg config lfs.usercache`/d1/1e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
   [1]
 #if git-server
   $ cp blob $TESTTMP/lfs-content/d1/1e/1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
 #else
   $ cp blob $TESTTMP/server/.hg/store/lfs/objects/d1/1e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
 #endif
 
 Test a corrupted file upload
 
   $ echo 'another lfs blob' > b
   $ hg ci -m 'another blob'
   $ echo 'damage' > .hg/store/lfs/objects/e6/59058e26b07b39d2a9c7145b3f99b41f797b6621c8076600e9cb7ee88291f0
   $ hg push --debug ../repo1
   http auth: user foo, password ***
   pushing to ../repo1
   http auth: user foo, password ***
   query 1; heads
   searching for changes
   all remote heads known locally
   listing keys for "phases"
   checking for updated bookmarks
   listing keys for "bookmarks"
   listing keys for "bookmarks"
   lfs: computing set of blobs to upload
   Status: 200
   Content-Length: 309 (git-server !)
   Content-Length: 350 (hg-server !)
   Content-Type: application/vnd.git-lfs+json
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   {
     "objects": [
       {
         "actions": {
           "upload": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/*/e659058e26b07b39d2a9c7145b3f99b41f797b6621c8076600e9cb7ee88291f0" (glob)
           }
         }
         "oid": "e659058e26b07b39d2a9c7145b3f99b41f797b6621c8076600e9cb7ee88291f0"
         "size": 17
       }
     ]
     "transfer": "basic" (hg-server !)
   }
   lfs: uploading e659058e26b07b39d2a9c7145b3f99b41f797b6621c8076600e9cb7ee88291f0 (17 bytes)
   abort: detected corrupt lfs object: e659058e26b07b39d2a9c7145b3f99b41f797b6621c8076600e9cb7ee88291f0
   (run hg verify)
   [255]
 
 Archive will prefetch blobs in a group
 
   $ rm -rf .hg/store/lfs `hg config lfs.usercache`
   $ hg archive --debug -r 1 ../archive
   http auth: user foo, password ***
   Status: 200
   Content-Length: 905 (git-server !)
   Content-Length: 988 (hg-server !)
   Content-Type: application/vnd.git-lfs+json
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   {
     "objects": [
       {
         "actions": {
           "download": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/*/31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b" (glob)
           }
         }
         "oid": "31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b"
         "size": 12
       }
       {
         "actions": {
           "download": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/*/37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19" (glob)
           }
         }
         "oid": "37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19"
         "size": 20
       }
       {
         "actions": {
           "download": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/*/d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998" (glob)
           }
         }
         "oid": "d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998"
         "size": 19
       }
     ]
     "transfer": "basic" (hg-server !)
   }
   lfs: need to transfer 3 objects (51 bytes)
   lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
   Status: 200
   Content-Length: 12
   Content-Type: text/plain; charset=utf-8 (git-server !)
   Content-Type: application/octet-stream (hg-server !)
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache
   lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
   lfs: downloading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes)
   Status: 200
   Content-Length: 20
   Content-Type: text/plain; charset=utf-8 (git-server !)
   Content-Type: application/octet-stream (hg-server !)
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   lfs: adding 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 to the usercache
   lfs: processed: 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19
   lfs: downloading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes)
   Status: 200
   Content-Length: 19
   Content-Type: text/plain; charset=utf-8 (git-server !)
   Content-Type: application/octet-stream (hg-server !)
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   lfs: adding d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 to the usercache
   lfs: processed: d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
   lfs: found 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b in the local lfs store
   lfs: found 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b in the local lfs store
   lfs: found d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 in the local lfs store
   lfs: found 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 in the local lfs store
   $ find ../archive | sort
   ../archive
   ../archive/.hg_archival.txt
   ../archive/a
   ../archive/b
   ../archive/c
   ../archive/d
 
 Cat will prefetch blobs in a group
 
   $ rm -rf .hg/store/lfs `hg config lfs.usercache`
   $ hg cat --debug -r 1 a b c
   http auth: user foo, password ***
   Status: 200
   Content-Length: 608 (git-server !)
   Content-Length: 670 (hg-server !)
   Content-Type: application/vnd.git-lfs+json
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   {
     "objects": [
       {
         "actions": {
           "download": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/*/31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b" (glob)
           }
         }
         "oid": "31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b"
         "size": 12
       }
       {
         "actions": {
           "download": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/*/d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998" (glob)
           }
         }
         "oid": "d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998"
         "size": 19
       }
     ]
     "transfer": "basic" (hg-server !)
   }
   lfs: need to transfer 2 objects (31 bytes)
   lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
   Status: 200
   Content-Length: 12
   Content-Type: text/plain; charset=utf-8 (git-server !)
   Content-Type: application/octet-stream (hg-server !)
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache
   lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
   lfs: downloading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes)
   Status: 200
   Content-Length: 19
   Content-Type: text/plain; charset=utf-8 (git-server !)
   Content-Type: application/octet-stream (hg-server !)
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   lfs: adding d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 to the usercache
   lfs: processed: d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
   lfs: found 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b in the local lfs store
   THIS-IS-LFS
   lfs: found 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b in the local lfs store
   THIS-IS-LFS
   lfs: found d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 in the local lfs store
   ANOTHER-LARGE-FILE
 
 Revert will prefetch blobs in a group
 
   $ rm -rf .hg/store/lfs
   $ rm -rf `hg config lfs.usercache`
   $ rm *
   $ hg revert --all -r 1 --debug
   http auth: user foo, password ***
   adding a
   reverting b
   reverting c
   reverting d
   Status: 200
   Content-Length: 905 (git-server !)
   Content-Length: 988 (hg-server !)
   Content-Type: application/vnd.git-lfs+json
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   {
     "objects": [
       {
         "actions": {
           "download": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/*/31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b" (glob)
           }
         }
         "oid": "31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b"
         "size": 12
       }
       {
         "actions": {
           "download": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/*/37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19" (glob)
           }
         }
         "oid": "37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19"
         "size": 20
       }
       {
         "actions": {
           "download": {
             "expires_at": "$ISO_8601_DATE_TIME$"
             "header": {
               "Accept": "application/vnd.git-lfs"
             }
             "href": "http://localhost:$HGPORT/*/d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998" (glob)
           }
         }
         "oid": "d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998"
         "size": 19
       }
     ]
     "transfer": "basic" (hg-server !)
   }
   lfs: need to transfer 3 objects (51 bytes)
   lfs: downloading 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b (12 bytes)
   Status: 200
   Content-Length: 12
   Content-Type: text/plain; charset=utf-8 (git-server !)
   Content-Type: application/octet-stream (hg-server !)
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   lfs: adding 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b to the usercache
   lfs: processed: 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b
   lfs: downloading 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 (20 bytes)
   Status: 200
   Content-Length: 20
   Content-Type: text/plain; charset=utf-8 (git-server !)
   Content-Type: application/octet-stream (hg-server !)
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   lfs: adding 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 to the usercache
   lfs: processed: 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19
   lfs: downloading d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 (19 bytes)
   Status: 200
   Content-Length: 19
   Content-Type: text/plain; charset=utf-8 (git-server !)
   Content-Type: application/octet-stream (hg-server !)
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   lfs: adding d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 to the usercache
   lfs: processed: d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998
   lfs: found 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b in the local lfs store
   lfs: found d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 in the local lfs store
   lfs: found 37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 in the local lfs store
   lfs: found 31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b in the local lfs store
 
 Check error message when the remote missed a blob:
 
   $ echo FFFFF > b
   $ hg commit -m b -A b
   $ echo FFFFF >> b
   $ hg commit -m b b
   $ rm -rf .hg/store/lfs
   $ rm -rf `hg config lfs.usercache`
   $ hg update -C '.^' --debug
   http auth: user foo, password ***
   resolving manifests
    branchmerge: False, force: True, partial: False
    ancestor: 62fdbaf221c6+, local: 62fdbaf221c6+, remote: ef0564edf47e
   Status: 200
   Content-Length: 308 (git-server !)
   Content-Length: 186 (hg-server !)
   Content-Type: application/vnd.git-lfs+json
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   {
     "objects": [
       {
         "actions": { (git-server !)
           "upload": { (git-server !)
             "expires_at": "$ISO_8601_DATE_TIME$" (git-server !)
             "header": { (git-server !)
               "Accept": "application/vnd.git-lfs" (git-server !)
             } (git-server !)
             "href": "http://localhost:$HGPORT/objects/8e6ea5f6c066b44a0efa43bcce86aea73f17e6e23f0663df0251e7524e140a13" (git-server !)
           } (git-server !)
         "error": { (hg-server !)
           "code": 404 (hg-server !)
           "message": "The object does not exist" (hg-server !)
         }
         "oid": "8e6ea5f6c066b44a0efa43bcce86aea73f17e6e23f0663df0251e7524e140a13"
         "size": 6
       }
     ]
     "transfer": "basic" (hg-server !)
   }
-  abort: LFS server error. Remote object for "b" not found:(.*)! (re)
+  abort: LFS server error for "b": The object does not exist!
   [255]
 
 Check error message when object does not exist:
 
   $ cd $TESTTMP
   $ hg init test && cd test
   $ echo "[extensions]" >> .hg/hgrc
   $ echo "lfs=" >> .hg/hgrc
   $ echo "[lfs]" >> .hg/hgrc
   $ echo "threshold=1" >> .hg/hgrc
   $ echo a > a
   $ hg add a
   $ hg commit -m 'test'
   $ echo aaaaa > a
   $ hg commit -m 'largefile'
   $ hg debugdata .hg/store/data/a.i 1 # verify this is no the file content but includes "oid", the LFS "pointer".
   version https://git-lfs.github.com/spec/v1
   oid sha256:bdc26931acfb734b142a8d675f205becf27560dc461f501822de13274fe6fc8a
   size 6
   x-is-binary 0
   $ cd ..
   $ rm -rf `hg config lfs.usercache`
 
 (Restart the server in a different location so it no longer has the content)
 
   $ $PYTHON $RUNTESTDIR/killdaemons.py $DAEMON_PIDS
 
 #if hg-server
   $ cat $TESTTMP/access.log $TESTTMP/errors.log
   $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "PUT /.hg/lfs/objects/31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b HTTP/1.1" 201 - (glob)
   $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "PUT /.hg/lfs/objects/37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 HTTP/1.1" 201 - (glob)
   $LOCALIP - - [$LOGDATE$] "PUT /.hg/lfs/objects/d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 HTTP/1.1" 201 - (glob)
   $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/31cf46fbc4ecd458a0943c5b4881f1f5a6dd36c53d6167d5b69ac45149b38e5b HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/37a65ab78d5ecda767e8622c248b5dbff1e68b1678ab0e730d5eb8601ec8ad19 HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "GET /.hg/lfs/objects/d11e1a642b60813aee592094109b406089b8dff4cb157157f753418ec7857998 HTTP/1.1" 200 - (glob)
   $LOCALIP - - [$LOGDATE$] "POST /.git/info/lfs/objects/batch HTTP/1.1" 200 - (glob)
 #endif
 
   $ rm $DAEMON_PIDS
   $ mkdir $TESTTMP/lfs-server2
   $ cd $TESTTMP/lfs-server2
 #if no-windows git-server
   $ lfs-test-server &> lfs-server.log &
   $ echo $! >> $DAEMON_PIDS
 #endif
 
 #if windows git-server
   $ $PYTHON $TESTTMP/spawn.py >> $DAEMON_PIDS
 #endif
 
 #if hg-server
   $ hg init server2
   $ hg --config "lfs.usercache=$TESTTMP/servercache2" -R server2 serve -d \
   >    -p $HGPORT --pid-file=hg.pid -A $TESTTMP/access.log -E $TESTTMP/errors.log
   $ cat hg.pid >> $DAEMON_PIDS
 #endif
 
   $ cd $TESTTMP
   $ hg --debug clone test test2
   http auth: user foo, password ***
   linked 6 files
   http auth: user foo, password ***
   updating to branch default
   resolving manifests
    branchmerge: False, force: False, partial: False
    ancestor: 000000000000, local: 000000000000+, remote: d2a338f184a8
   Status: 200
   Content-Length: 308 (git-server !)
   Content-Length: 186 (hg-server !)
   Content-Type: application/vnd.git-lfs+json
   Date: $HTTP_DATE$
   Server: testing stub value (hg-server !)
   {
     "objects": [
       {
         "actions": { (git-server !)
           "upload": { (git-server !)
             "expires_at": "$ISO_8601_DATE_TIME$" (git-server !)
             "header": { (git-server !)
               "Accept": "application/vnd.git-lfs" (git-server !)
             } (git-server !)
             "href": "http://localhost:$HGPORT/objects/bdc26931acfb734b142a8d675f205becf27560dc461f501822de13274fe6fc8a" (git-server !)
           } (git-server !)
         "error": { (hg-server !)
           "code": 404 (hg-server !)
           "message": "The object does not exist" (hg-server !)
         }
         "oid": "bdc26931acfb734b142a8d675f205becf27560dc461f501822de13274fe6fc8a"
         "size": 6
       }
     ]
     "transfer": "basic" (hg-server !)
   }
-  abort: LFS server error. Remote object for "a" not found:(.*)! (re)
+  abort: LFS server error for "a": The object does not exist!
   [255]
 
   $ $PYTHON $RUNTESTDIR/killdaemons.py $DAEMON_PIDS