fix(git-lfs): fixed security problem with allowing off-chain attacks to replace OID data without validating hash for already present oids.

- Fixes  RCCE-141

vcsserver/git_lfs/app.py

@@ -1,303 +1,314 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
-
+import hashlib
 import re
 import logging
 
 from gunicorn.http.errors import NoMoreData
 from pyramid.config import Configurator
 from pyramid.response import Response, FileIter
 from pyramid.httpexceptions import (
     HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
     HTTPUnprocessableEntity)
 
 from vcsserver.lib.ext_json import json
 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
 from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
 from vcsserver.lib.str_utils import safe_int
 
 log = logging.getLogger(__name__)
 
 
 GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs'  # +json ?
 GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
 
 
 def write_response_error(http_exception, text=None):
     content_type = GIT_LFS_CONTENT_TYPE + '+json'
     _exception = http_exception(content_type=content_type)
     _exception.content_type = content_type
     if text:
         _exception.body = json.dumps({'message': text})
     log.debug('LFS: writing response of type %s to client with text:%s',
               http_exception, text)
     return _exception
 
 
 class AuthHeaderRequired:
     """
     Decorator to check if request has proper auth-header
     """
 
     def __call__(self, func):
         return get_cython_compat_decorator(self.__wrapper, func)
 
     def __wrapper(self, func, *fargs, **fkwargs):
         request = fargs[1]
         auth = request.authorization
         if not auth:
             log.debug('No auth header found, returning 403')
             return write_response_error(HTTPForbidden)
         return func(*fargs[1:], **fkwargs)
 
 
 # views
 
 def lfs_objects(request):
     # indicate not supported, V1 API
     log.warning('LFS: v1 api not supported, reporting it back to client')
     return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
 
 
 @AuthHeaderRequired()
 def lfs_objects_batch(request):
     """
     The client sends the following information to the Batch endpoint to transfer some objects:
 
         operation - Should be download or upload.
         transfers - An optional Array of String identifiers for transfer
             adapters that the client has configured. If omitted, the basic
             transfer adapter MUST be assumed by the server.
         objects - An Array of objects to download.
         oid - String OID of the LFS object.
         size - Integer byte size of the LFS object. Must be at least zero.
     """
     request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
     auth = request.authorization
     repo = request.matchdict.get('repo')
     data = request.json
     operation = data.get('operation')
     http_scheme = request.registry.git_lfs_http_scheme
 
     if operation not in ('download', 'upload'):
         log.debug('LFS: unsupported operation:%s', operation)
         return write_response_error(
             HTTPBadRequest, f'unsupported operation mode: `{operation}`')
 
     if 'objects' not in data:
         log.debug('LFS: missing objects data')
         return write_response_error(
             HTTPBadRequest, 'missing objects data')
 
     log.debug('LFS: handling operation of type: %s', operation)
 
     objects = []
     for o in data['objects']:
         try:
             oid = o['oid']
             obj_size = o['size']
         except KeyError:
             log.exception('LFS, failed to extract data')
             return write_response_error(
                 HTTPBadRequest, 'unsupported data in objects')
 
         obj_data = {'oid': oid}
         if http_scheme == 'http':
             # Note(marcink): when using http, we might have a custom port
             # so we skip setting it to http, url dispatch then wont generate a port in URL
             # for development we need this
             http_scheme = None
 
         obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid,
                                      _scheme=http_scheme)
         obj_verify_href = request.route_url('lfs_objects_verify', repo=repo,
                                             _scheme=http_scheme)
         store = LFSOidStore(
             oid, repo, store_location=request.registry.git_lfs_store_path)
         handler = OidHandler(
             store, repo, auth, oid, obj_size, obj_data,
             obj_href, obj_verify_href)
 
         # this verifies also OIDs
         actions, errors = handler.exec_operation(operation)
         if errors:
             log.warning('LFS: got following errors: %s', errors)
             obj_data['errors'] = errors
 
         if actions:
             obj_data['actions'] = actions
 
         obj_data['size'] = obj_size
         obj_data['authenticated'] = True
         objects.append(obj_data)
 
     result = {'objects': objects, 'transfer': 'basic'}
     log.debug('LFS Response %s', safe_result(result))
 
     return result
 
 
 def lfs_objects_oid_upload(request):
     request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
     repo = request.matchdict.get('repo')
     oid = request.matchdict.get('oid')
     store = LFSOidStore(
         oid, repo, store_location=request.registry.git_lfs_store_path)
     engine = store.get_engine(mode='wb')
     log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
 
+    # validate if OID is not by any chance already in the store
+    if store.has_oid():
+        log.debug('LFS: oid %s exists in store', oid)
+        return {'upload': 'ok', 'state': 'in-store'}
+
     body = request.environ['wsgi.input']
 
+    digest = hashlib.sha256()
     with engine as f:
         blksize = 64 * 1024  # 64kb
         while True:
             # read in chunks as stream comes in from Gunicorn
             # this is a specific Gunicorn support function.
             # might work differently on waitress
             try:
                 chunk = body.read(blksize)
             except NoMoreData:
                 chunk = None
 
             if not chunk:
                 break
+            f.write(chunk)
+            digest.update(chunk)
 
-            f.write(chunk)
+        hex_digest = digest.hexdigest()
+        digest_check = hex_digest == oid
+        if not digest_check:
+            engine.cleanup() # trigger cleanup so we don't save mismatch OID into the store
+            return write_response_error(
+                HTTPBadRequest, f'oid {oid} does not match expected sha {hex_digest}')
 
-    return {'upload': 'ok'}
+    return {'upload': 'ok', 'state': 'written'}
 
 
 def lfs_objects_oid_download(request):
     repo = request.matchdict.get('repo')
     oid = request.matchdict.get('oid')
 
     store = LFSOidStore(
         oid, repo, store_location=request.registry.git_lfs_store_path)
     if not store.has_oid():
         log.debug('LFS: oid %s does not exists in store', oid)
         return write_response_error(
             HTTPNotFound, f'requested file with oid `{oid}` not found in store')
 
     # TODO(marcink): support range header ?
     # Range: bytes=0-, `bytes=(\d+)\-.*`
 
     f = open(store.oid_path, 'rb')
     response = Response(
         content_type='application/octet-stream', app_iter=FileIter(f))
     response.headers.add('X-RC-LFS-Response-Oid', str(oid))
     return response
 
 
 def lfs_objects_verify(request):
     request.response.content_type = GIT_LFS_CONTENT_TYPE + '+json'
     repo = request.matchdict.get('repo')
 
     data = request.json
     oid = data.get('oid')
     size = safe_int(data.get('size'))
 
     if not (oid and size):
         return write_response_error(
             HTTPBadRequest, 'missing oid and size in request data')
 
     store = LFSOidStore(
         oid, repo, store_location=request.registry.git_lfs_store_path)
     if not store.has_oid():
         log.debug('LFS: oid %s does not exists in store', oid)
         return write_response_error(
             HTTPNotFound, f'oid `{oid}` does not exists in store')
 
     store_size = store.size_oid()
     if store_size != size:
-        msg = 'requested file size mismatch store size:{} requested:{}'.format(
-            store_size, size)
-        return write_response_error(
-            HTTPUnprocessableEntity, msg)
+        msg = f'requested file size mismatch store size:{store_size} requested:{size}'
+        return write_response_error(HTTPUnprocessableEntity, msg)
 
-    return {'message': {'size': 'ok', 'in_store': 'ok'}}
+    return {'message': {'size': store_size, 'oid': oid}}
 
 
 def lfs_objects_lock(request):
     return write_response_error(
         HTTPNotImplemented, 'GIT LFS locking api not supported')
 
 
 def not_found(request):
     return write_response_error(
         HTTPNotFound, 'request path not found')
 
 
 def lfs_disabled(request):
     return write_response_error(
         HTTPNotImplemented, 'GIT LFS disabled for this repo')
 
 
 def git_lfs_app(config):
 
     # v1 API deprecation endpoint
     config.add_route('lfs_objects',
                      '/{repo:.*?[^/]}/info/lfs/objects')
     config.add_view(lfs_objects, route_name='lfs_objects',
                     request_method='POST', renderer='json')
 
     # locking API
     config.add_route('lfs_objects_lock',
                      '/{repo:.*?[^/]}/info/lfs/locks')
     config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
                     request_method=('POST', 'GET'), renderer='json')
 
     config.add_route('lfs_objects_lock_verify',
                      '/{repo:.*?[^/]}/info/lfs/locks/verify')
     config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
                     request_method=('POST', 'GET'), renderer='json')
 
     # batch API
     config.add_route('lfs_objects_batch',
                      '/{repo:.*?[^/]}/info/lfs/objects/batch')
     config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
                     request_method='POST', renderer='json')
 
     # oid upload/download API
     config.add_route('lfs_objects_oid',
                      '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
     config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
                     request_method='PUT', renderer='json')
     config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
                     request_method='GET', renderer='json')
 
     # verification API
     config.add_route('lfs_objects_verify',
                      '/{repo:.*?[^/]}/info/lfs/verify')
     config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
                     request_method='POST', renderer='json')
 
     # not found handler for API
     config.add_notfound_view(not_found, renderer='json')
 
 
 def create_app(git_lfs_enabled, git_lfs_store_path, git_lfs_http_scheme):
     config = Configurator()
     if git_lfs_enabled:
         config.include(git_lfs_app)
         config.registry.git_lfs_store_path = git_lfs_store_path
         config.registry.git_lfs_http_scheme = git_lfs_http_scheme
     else:
         # not found handler for API, reporting disabled LFS support
         config.add_notfound_view(lfs_disabled, renderer='json')
 
     app = config.make_wsgi_app()
     return app

vcsserver/git_lfs/lib.py

@@ -1,177 +1,185 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import shutil
 import logging
 from collections import OrderedDict
 
 log = logging.getLogger(__name__)
 
 
 class OidHandler:
 
     def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
                  obj_verify_href=None):
         self.current_store = store
         self.repo_name = repo_name
         self.auth = auth
         self.oid = oid
         self.obj_size = obj_size
         self.obj_data = obj_data
         self.obj_href = obj_href
         self.obj_verify_href = obj_verify_href
 
     def get_store(self, mode=None):
         return self.current_store
 
     def get_auth(self):
         """returns auth header for re-use in upload/download"""
         return " ".join(self.auth)
 
     def download(self):
 
         store = self.get_store()
         response = None
         has_errors = None
 
         if not store.has_oid():
             # error reply back to client that something is wrong with dl
             err_msg = f'object: {store.oid} does not exist in store'
             has_errors = OrderedDict(
                 error=OrderedDict(
                     code=404,
                     message=err_msg
                 )
             )
 
         download_action = OrderedDict(
             href=self.obj_href,
             header=OrderedDict([("Authorization", self.get_auth())])
         )
         if not has_errors:
             response = OrderedDict(download=download_action)
         return response, has_errors
 
     def upload(self, skip_existing=True):
         """
         Write upload action for git-lfs server
         """
 
         store = self.get_store()
         response = None
         has_errors = None
 
         # verify if we have the OID before, if we do, reply with empty
         if store.has_oid():
             log.debug('LFS: store already has oid %s', store.oid)
 
             # validate size
             store_size = store.size_oid()
             size_match = store_size == self.obj_size
             if not size_match:
                 log.warning(
                     'LFS: size mismatch for oid:%s, in store:%s expected: %s',
                     self.oid, store_size, self.obj_size)
             elif skip_existing:
                 log.debug('LFS: skipping further action as oid is existing')
                 return response, has_errors
 
         chunked = ("Transfer-Encoding", "chunked")
         upload_action = OrderedDict(
             href=self.obj_href,
             header=OrderedDict([("Authorization", self.get_auth()), chunked])
         )
         if not has_errors:
             response = OrderedDict(upload=upload_action)
             # if specified in handler, return the verification endpoint
             if self.obj_verify_href:
                 verify_action = OrderedDict(
                     href=self.obj_verify_href,
                     header=OrderedDict([("Authorization", self.get_auth())])
                 )
                 response['verify'] = verify_action
         return response, has_errors
 
     def exec_operation(self, operation, *args, **kwargs):
         handler = getattr(self, operation)
         log.debug('LFS: handling request using %s handler', handler)
         return handler(*args, **kwargs)
 
 
 class LFSOidStore:
 
     def __init__(self, oid, repo, store_location=None):
         self.oid = oid
         self.repo = repo
         defined_store_path = store_location or self.get_default_store()
         self.store_suffix = f"/objects/{oid[:2]}/{oid[2:4]}"
         self.store_path = f"{defined_store_path.rstrip('/')}{self.store_suffix}"
         self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
         self.oid_path = os.path.join(self.store_path, oid)
         self.fd = None
 
     def get_engine(self, mode):
         """
         engine = .get_engine(mode='wb')
         with engine as f:
             f.write('...')
         """
 
         class StoreEngine:
+            _cleanup = None
             def __init__(self, mode, store_path, oid_path, tmp_oid_path):
                 self.mode = mode
                 self.store_path = store_path
                 self.oid_path = oid_path
                 self.tmp_oid_path = tmp_oid_path
 
+            def cleanup(self):
+                self._cleanup = True
+
             def __enter__(self):
                 if not os.path.isdir(self.store_path):
                     os.makedirs(self.store_path)
 
                 # TODO(marcink): maybe write metadata here with size/oid ?
                 fd = open(self.tmp_oid_path, self.mode)
                 self.fd = fd
                 return fd
 
             def __exit__(self, exc_type, exc_value, traceback):
-                # close tmp file, and rename to final destination
                 self.fd.close()
-                shutil.move(self.tmp_oid_path, self.oid_path)
+
+                if self._cleanup is None:
+                    # close tmp file, and rename to final destination
+                    shutil.move(self.tmp_oid_path, self.oid_path)
+                else:
+                    os.remove(self.tmp_oid_path)
 
         return StoreEngine(
             mode, self.store_path, self.oid_path, self.tmp_oid_path)
 
     def get_default_store(self):
         """
         Default store, consistent with defaults of Mercurial large files store
         which is /home/username/.cache/largefiles
         """
         user_home = os.path.expanduser("~")
         return os.path.join(user_home, '.cache', 'lfs-store')
 
     def has_oid(self):
         return os.path.exists(os.path.join(self.store_path, self.oid))
 
     def size_oid(self):
         size = -1
 
         if self.has_oid():
             oid = os.path.join(self.store_path, self.oid)
             size = os.stat(oid).st_size
 
         return size

vcsserver/git_lfs/tests/test_lfs_app.py

@@ -1,274 +1,310 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import pytest
 from webtest.app import TestApp as WebObTestApp
 
 from vcsserver.lib.ext_json import json
 from vcsserver.lib.str_utils import safe_bytes
 from vcsserver.git_lfs.app import create_app
 from vcsserver.git_lfs.lib import LFSOidStore
 
 
 @pytest.fixture(scope='function')
 def git_lfs_app(tmpdir):
     custom_app = WebObTestApp(create_app(
         git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
         git_lfs_http_scheme='http'))
     custom_app._store = str(tmpdir)
     return custom_app
 
 
 @pytest.fixture(scope='function')
 def git_lfs_https_app(tmpdir):
     custom_app = WebObTestApp(create_app(
         git_lfs_enabled=True, git_lfs_store_path=str(tmpdir),
         git_lfs_http_scheme='https'))
     custom_app._store = str(tmpdir)
     return custom_app
 
 
 @pytest.fixture()
 def http_auth():
     return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
 
 
 class TestLFSApplication:
 
     def test_app_wrong_path(self, git_lfs_app):
         git_lfs_app.get('/repo/info/lfs/xxx', status=404)
 
     def test_app_deprecated_endpoint(self, git_lfs_app):
         response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
         assert response.status_code == 501
         assert json.loads(response.text) == {'message': 'LFS: v1 api not supported'}
 
     def test_app_lock_verify_api_not_available(self, git_lfs_app):
         response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
         assert response.status_code == 501
         assert json.loads(response.text) == {
             'message': 'GIT LFS locking api not supported'}
 
     def test_app_lock_api_not_available(self, git_lfs_app):
         response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
         assert response.status_code == 501
         assert json.loads(response.text) == {
             'message': 'GIT LFS locking api not supported'}
 
     def test_app_batch_api_missing_auth(self, git_lfs_app):
         git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params={}, status=403)
 
     def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params={}, status=400,
             extra_environ=http_auth)
         assert json.loads(response.text) == {
             'message': 'unsupported operation mode: `None`'}
 
     def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params={'operation': 'download'},
             status=400, extra_environ=http_auth)
         assert json.loads(response.text) == {
             'message': 'missing objects data'}
 
     def test_app_batch_api_unsupported_data_in_objects(
             self, git_lfs_app, http_auth):
         params = {'operation': 'download',
                   'objects': [{}]}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params=params, status=400,
             extra_environ=http_auth)
         assert json.loads(response.text) == {
             'message': 'unsupported data in objects'}
 
     def test_app_batch_api_download_missing_object(
             self, git_lfs_app, http_auth):
-        params = {'operation': 'download',
-                  'objects': [{'oid': '123', 'size': '1024'}]}
+        params = {
+            'operation': 'download',
+            'objects': [{'oid': '123', 'size': '1024'}]
+        }
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params=params,
             extra_environ=http_auth)
 
         expected_objects = [
-            {'authenticated': True,
-             'errors': {'error': {
-                 'code': 404,
-                 'message': 'object: 123 does not exist in store'}},
-             'oid': '123',
-             'size': '1024'}
+            {
+                'oid': '123',
+                'size': '1024',
+                'authenticated': True,
+                'errors': {'error': {'code': 404, 'message': 'object: 123 does not exist in store'}},
+            }
         ]
+
         assert json.loads(response.text) == {
-            'objects': expected_objects, 'transfer': 'basic'}
+            'objects': expected_objects,
+            'transfer': 'basic'
+        }
 
     def test_app_batch_api_download(self, git_lfs_app, http_auth):
         oid = '456'
         oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
             f.write(safe_bytes('OID_CONTENT'))
 
         params = {'operation': 'download',
                   'objects': [{'oid': oid, 'size': '1024'}]}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params=params,
             extra_environ=http_auth)
 
         expected_objects = [
             {'authenticated': True,
              'actions': {
                  'download': {
                      'header': {'Authorization': 'Basic XXXXX'},
                      'href': 'http://localhost/repo/info/lfs/objects/456'},
              },
              'oid': '456',
              'size': '1024'}
         ]
         assert json.loads(response.text) == {
-            'objects': expected_objects, 'transfer': 'basic'}
+            'objects': expected_objects,
+            'transfer': 'basic'
+        }
 
     def test_app_batch_api_upload(self, git_lfs_app, http_auth):
         params = {'operation': 'upload',
                   'objects': [{'oid': '123', 'size': '1024'}]}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/objects/batch', params=params,
             extra_environ=http_auth)
         expected_objects = [
-            {'authenticated': True,
-             'actions': {
-                 'upload': {
-                     'header': {'Authorization': 'Basic XXXXX',
-                                 'Transfer-Encoding': 'chunked'},
-                     'href': 'http://localhost/repo/info/lfs/objects/123'},
-                 'verify': {
-                     'header': {'Authorization': 'Basic XXXXX'},
-                     'href': 'http://localhost/repo/info/lfs/verify'}
-             },
-             'oid': '123',
-             'size': '1024'}
+            {
+                'authenticated': True,
+                'actions': {
+                    'upload': {
+                        'header': {
+                            'Authorization': 'Basic XXXXX',
+                            'Transfer-Encoding': 'chunked'
+                        },
+                        'href': 'http://localhost/repo/info/lfs/objects/123'
+                    },
+                    'verify': {
+                        'header': {
+                            'Authorization': 'Basic XXXXX'
+                        },
+                        'href': 'http://localhost/repo/info/lfs/verify'
+                    }
+                },
+                'oid': '123',
+                'size': '1024'
+            }
         ]
         assert json.loads(response.text) == {
-            'objects': expected_objects, 'transfer': 'basic'}
+            'objects': expected_objects,
+            'transfer': 'basic'
+        }
 
     def test_app_batch_api_upload_for_https(self, git_lfs_https_app, http_auth):
         params = {'operation': 'upload',
                   'objects': [{'oid': '123', 'size': '1024'}]}
         response = git_lfs_https_app.post_json(
             '/repo/info/lfs/objects/batch', params=params,
             extra_environ=http_auth)
         expected_objects = [
             {'authenticated': True,
              'actions': {
                  'upload': {
                      'header': {'Authorization': 'Basic XXXXX',
-                                 'Transfer-Encoding': 'chunked'},
+                                'Transfer-Encoding': 'chunked'},
                      'href': 'https://localhost/repo/info/lfs/objects/123'},
                  'verify': {
                      'header': {'Authorization': 'Basic XXXXX'},
                      'href': 'https://localhost/repo/info/lfs/verify'}
              },
              'oid': '123',
              'size': '1024'}
         ]
         assert json.loads(response.text) == {
             'objects': expected_objects, 'transfer': 'basic'}
 
     def test_app_verify_api_missing_data(self, git_lfs_app):
         params = {'oid': 'missing'}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/verify', params=params,
             status=400)
 
         assert json.loads(response.text) == {
             'message': 'missing oid and size in request data'}
 
     def test_app_verify_api_missing_obj(self, git_lfs_app):
         params = {'oid': 'missing', 'size': '1024'}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/verify', params=params,
             status=404)
 
         assert json.loads(response.text) == {
-            'message': 'oid `missing` does not exists in store'}
+            'message': 'oid `missing` does not exists in store'
+        }
 
     def test_app_verify_api_size_mismatch(self, git_lfs_app):
         oid = 'existing'
         oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
             f.write(safe_bytes('OID_CONTENT'))
 
         params = {'oid': oid, 'size': '1024'}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/verify', params=params, status=422)
 
         assert json.loads(response.text) == {
-            'message': 'requested file size mismatch '
-                        'store size:11 requested:1024'}
+            'message': 'requested file size mismatch store size:11 requested:1024'
+        }
 
     def test_app_verify_api(self, git_lfs_app):
         oid = 'existing'
         oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
             f.write(safe_bytes('OID_CONTENT'))
 
         params = {'oid': oid, 'size': 11}
         response = git_lfs_app.post_json(
             '/repo/info/lfs/verify', params=params)
 
         assert json.loads(response.text) == {
-            'message': {'size': 'ok', 'in_store': 'ok'}}
+            'message': {'size': 11, 'oid': oid}
+        }
 
     def test_app_download_api_oid_not_existing(self, git_lfs_app):
         oid = 'missing'
 
-        response = git_lfs_app.get(
-            '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
+        response = git_lfs_app.get(f'/repo/info/lfs/objects/{oid}', status=404)
 
         assert json.loads(response.text) == {
             'message': 'requested file with oid `missing` not found in store'}
 
     def test_app_download_api(self, git_lfs_app):
         oid = 'existing'
         oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         if not os.path.isdir(os.path.dirname(oid_path)):
             os.makedirs(os.path.dirname(oid_path))
         with open(oid_path, 'wb') as f:
             f.write(safe_bytes('OID_CONTENT'))
 
-        response = git_lfs_app.get(
-            '/repo/info/lfs/objects/{oid}'.format(oid=oid))
+        response = git_lfs_app.get(f'/repo/info/lfs/objects/{oid}')
         assert response
 
     def test_app_upload(self, git_lfs_app):
-        oid = 'uploaded'
+        oid = '65f23e22a9bfedda96929b3cfcb8b6d2fdd34a2e877ddb81f45d79ab05710e12'
 
         response = git_lfs_app.put(
-            '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
+            f'/repo/info/lfs/objects/{oid}', params='CONTENT')
 
-        assert json.loads(response.text) == {'upload': 'ok'}
+        assert json.loads(response.text) == {'upload': 'ok', 'state': 'written'}
 
         # verify that we actually wrote that OID
         oid_path = LFSOidStore(oid=oid, repo=None, store_location=git_lfs_app._store).oid_path
         assert os.path.isfile(oid_path)
         assert 'CONTENT' == open(oid_path).read()
+
+        response = git_lfs_app.put(
+            f'/repo/info/lfs/objects/{oid}', params='CONTENT')
+
+        assert json.loads(response.text) == {'upload': 'ok', 'state': 'in-store'}
+
+
+    def test_app_upload_wrong_sha(self, git_lfs_app):
+        oid = 'i-am-a-wrong-sha'
+
+        response = git_lfs_app.put(f'/repo/info/lfs/objects/{oid}', params='CONTENT', status=400)
+
+        assert json.loads(response.text) == {
+            'message': 'oid i-am-a-wrong-sha does not match expected sha '
+                       '65f23e22a9bfedda96929b3cfcb8b6d2fdd34a2e877ddb81f45d79ab05710e12'}
+
+        # check this OID wasn't written to store
+        response = git_lfs_app.get(f'/repo/info/lfs/objects/{oid}', status=404)
+        assert json.loads(response.text) == {'message': 'requested file with oid `i-am-a-wrong-sha` not found in store'}

vcsserver/git_lfs/tests/test_lib.py

@@ -1,142 +1,143 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import pytest
 from vcsserver.lib.str_utils import safe_bytes
 from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
 
 
 @pytest.fixture()
 def lfs_store(tmpdir):
     repo = 'test'
-    oid = '123456789'
+    oid = '65f23e22a9bfedda96929b3cfcb8b6d2fdd34a2e877ddb81f45d79ab05710e12'
     store = LFSOidStore(oid=oid, repo=repo, store_location=str(tmpdir))
     return store
 
 
 @pytest.fixture()
 def oid_handler(lfs_store):
     store = lfs_store
     repo = store.repo
     oid = store.oid
 
     oid_handler = OidHandler(
         store=store, repo_name=repo, auth=('basic', 'xxxx'),
         oid=oid,
         obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
         obj_verify_href='http://localhost/verify')
     return oid_handler
 
 
 class TestOidHandler:
 
     @pytest.mark.parametrize('exec_action', [
         'download',
         'upload',
     ])
     def test_exec_action(self, exec_action, oid_handler):
         handler = oid_handler.exec_operation(exec_action)
         assert handler
 
     def test_exec_action_undefined(self, oid_handler):
         with pytest.raises(AttributeError):
             oid_handler.exec_operation('wrong')
 
     def test_download_oid_not_existing(self, oid_handler):
         response, has_errors = oid_handler.exec_operation('download')
 
         assert response is None
         assert has_errors['error'] == {
             'code': 404,
-            'message': 'object: 123456789 does not exist in store'}
+            'message': 'object: 65f23e22a9bfedda96929b3cfcb8b6d2fdd34a2e877ddb81f45d79ab05710e12 does not exist in store'
+        }
 
     def test_download_oid(self, oid_handler):
         store = oid_handler.get_store()
         if not os.path.isdir(os.path.dirname(store.oid_path)):
             os.makedirs(os.path.dirname(store.oid_path))
 
         with open(store.oid_path, 'wb') as f:
             f.write(safe_bytes('CONTENT'))
 
         response, has_errors = oid_handler.exec_operation('download')
 
         assert has_errors is None
         assert response['download'] == {
             'header': {'Authorization': 'basic xxxx'},
             'href': 'http://localhost/handle_oid'
         }
 
     def test_upload_oid_that_exists(self, oid_handler):
         store = oid_handler.get_store()
         if not os.path.isdir(os.path.dirname(store.oid_path)):
             os.makedirs(os.path.dirname(store.oid_path))
 
         with open(store.oid_path, 'wb') as f:
             f.write(safe_bytes('CONTENT'))
         oid_handler.obj_size = 7
         response, has_errors = oid_handler.exec_operation('upload')
         assert has_errors is None
         assert response is None
 
     def test_upload_oid_that_exists_but_has_wrong_size(self, oid_handler):
         store = oid_handler.get_store()
         if not os.path.isdir(os.path.dirname(store.oid_path)):
             os.makedirs(os.path.dirname(store.oid_path))
 
         with open(store.oid_path, 'wb') as f:
             f.write(safe_bytes('CONTENT'))
 
         oid_handler.obj_size = 10240
         response, has_errors = oid_handler.exec_operation('upload')
         assert has_errors is None
         assert response['upload'] == {
             'header': {'Authorization': 'basic xxxx',
                        'Transfer-Encoding': 'chunked'},
             'href': 'http://localhost/handle_oid',
         }
 
     def test_upload_oid(self, oid_handler):
         response, has_errors = oid_handler.exec_operation('upload')
         assert has_errors is None
         assert response['upload'] == {
             'header': {'Authorization': 'basic xxxx',
                        'Transfer-Encoding': 'chunked'},
             'href': 'http://localhost/handle_oid'
         }
 
 
 class TestLFSStore:
     def test_write_oid(self, lfs_store):
         oid_location = lfs_store.oid_path
 
         assert not os.path.isfile(oid_location)
 
         engine = lfs_store.get_engine(mode='wb')
         with engine as f:
             f.write(safe_bytes('CONTENT'))
 
         assert os.path.isfile(oid_location)
 
     def test_detect_has_oid(self, lfs_store):
 
         assert lfs_store.has_oid() is False
         engine = lfs_store.get_engine(mode='wb')
         with engine as f:
             f.write(safe_bytes('CONTENT'))
 
         assert lfs_store.has_oid() is True