git-lfs: added vcsserver handling of git-lfs objects.

- fixes  #4235

vcsserver/git_lfs/__init__.py

@@ -0,0 +1,19 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+
+from app import create_app

vcsserver/git_lfs/app.py

@@ -0,0 +1,276 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import re
+import logging
+from wsgiref.util import FileWrapper
+
+import simplejson as json
+from pyramid.config import Configurator
+from pyramid.response import Response, FileIter
+from pyramid.httpexceptions import (
+    HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
+    HTTPUnprocessableEntity)
+
+from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
+from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
+from vcsserver.utils import safe_int
+
+log = logging.getLogger(__name__)
+
+
+GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ?
+GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
+
+
+def write_response_error(http_exception, text=None):
+    content_type = 'application/json'
+    _exception = http_exception(content_type=content_type)
+    _exception.content_type = content_type
+    if text:
+        _exception.body = json.dumps({'message': text})
+    log.debug('LFS: writing response of type %s to client with text:%s',
+              http_exception, text)
+    return _exception
+
+
+class AuthHeaderRequired(object):
+    """
+    Decorator to check if request has proper auth-header
+    """
+
+    def __call__(self, func):
+        return get_cython_compat_decorator(self.__wrapper, func)
+
+    def __wrapper(self, func, *fargs, **fkwargs):
+        request = fargs[1]
+        auth = request.authorization
+        if not auth:
+            return write_response_error(HTTPForbidden)
+        return func(*fargs[1:], **fkwargs)
+
+
+# views
+
+def lfs_objects(request):
+    # indicate not supported, V1 API
+    log.warning('LFS: v1 api not supported, reporting it back to client')
+    return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
+
+
+@AuthHeaderRequired()
+def lfs_objects_batch(request):
+    """
+    The client sends the following information to the Batch endpoint to transfer some objects:
+
+        operation - Should be download or upload.
+        transfers - An optional Array of String identifiers for transfer
+            adapters that the client has configured. If omitted, the basic
+            transfer adapter MUST be assumed by the server.
+        objects - An Array of objects to download.
+        oid - String OID of the LFS object.
+        size - Integer byte size of the LFS object. Must be at least zero.
+    """
+    auth = request.authorization
+
+    repo = request.matchdict.get('repo')
+
+    data = request.json
+    operation = data.get('operation')
+    if operation not in ('download', 'upload'):
+        log.debug('LFS: unsupported operation:%s', operation)
+        return write_response_error(
+            HTTPBadRequest, 'unsupported operation mode: `%s`' % operation)
+
+    if 'objects' not in data:
+        log.debug('LFS: missing objects data')
+        return write_response_error(
+            HTTPBadRequest, 'missing objects data')
+
+    log.debug('LFS: handling operation of type: %s', operation)
+
+    objects = []
+    for o in data['objects']:
+        try:
+            oid = o['oid']
+            obj_size = o['size']
+        except KeyError:
+            log.exception('LFS, failed to extract data')
+            return write_response_error(
+                HTTPBadRequest, 'unsupported data in objects')
+
+        obj_data = {'oid': oid}
+
+        obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid)
+        obj_verify_href = request.route_url('lfs_objects_verify', repo=repo)
+        store = LFSOidStore(
+            repo, oid, store_location=request.registry.git_lfs_store_path)
+        handler = OidHandler(
+            store, repo, auth, oid, obj_size, obj_data,
+            obj_href, obj_verify_href)
+
+        # this verifies also OIDs
+        actions, errors = handler.exec_operation(operation)
+        if errors:
+            log.warning('LFS: got following errors: %s', errors)
+            obj_data['errors'] = errors
+
+        if actions:
+            obj_data['actions'] = actions
+
+        obj_data['size'] = obj_size
+        obj_data['authenticated'] = True
+        objects.append(obj_data)
+
+    result = {'objects': objects, 'transfer': 'basic'}
+    log.debug('LFS Response %s', safe_result(result))
+
+    return result
+
+
+def lfs_objects_oid_upload(request):
+    repo = request.matchdict.get('repo')
+    oid = request.matchdict.get('oid')
+    store = LFSOidStore(
+        repo, oid, store_location=request.registry.git_lfs_store_path)
+    engine = store.get_engine(mode='wb')
+    log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
+    with engine as f:
+        for chunk in FileWrapper(request.body_file_seekable, blksize=64 * 1024):
+            f.write(chunk)
+
+    return {'upload': 'ok'}
+
+
+def lfs_objects_oid_download(request):
+    repo = request.matchdict.get('repo')
+    oid = request.matchdict.get('oid')
+
+    store = LFSOidStore(
+        repo, oid, store_location=request.registry.git_lfs_store_path)
+    if not store.has_oid():
+        log.debug('LFS: oid %s does not exists in store', oid)
+        return write_response_error(
+            HTTPNotFound, 'requested file with oid `%s` not found in store' % oid)
+
+    # TODO(marcink): support range header ?
+    # Range: bytes=0-, `bytes=(\d+)\-.*`
+
+    f = open(store.oid_path, 'rb')
+    response = Response(
+        content_type='application/octet-stream', app_iter=FileIter(f))
+    response.headers.add('X-RC-LFS-Response-Oid', str(oid))
+    return response
+
+
+def lfs_objects_verify(request):
+    repo = request.matchdict.get('repo')
+
+    data = request.json
+    oid = data.get('oid')
+    size = safe_int(data.get('size'))
+
+    if not (oid and size):
+        return write_response_error(
+            HTTPBadRequest, 'missing oid and size in request data')
+
+    store = LFSOidStore(repo, oid,
+                        store_location=request.registry.git_lfs_store_path)
+    if not store.has_oid():
+        log.debug('LFS: oid %s does not exists in store', oid)
+        return write_response_error(
+            HTTPNotFound, 'oid `%s` does not exists in store' % oid)
+
+    store_size = store.size_oid()
+    if store_size != size:
+        msg = 'requested file size mismatch store size:%s requested:%s' % (
+            store_size, size)
+        return write_response_error(
+            HTTPUnprocessableEntity, msg)
+
+    return {'message': {'size': 'ok', 'in_store': 'ok'}}
+
+
+def lfs_objects_lock(request):
+    return write_response_error(
+        HTTPNotImplemented, 'GIT LFS locking api not supported')
+
+
+def not_found(request):
+    return write_response_error(
+        HTTPNotFound, 'request path not found')
+
+
+def lfs_disabled(request):
+    return write_response_error(
+        HTTPNotImplemented, 'GIT LFS disabled for this repo')
+
+
+def git_lfs_app(config):
+
+    # v1 API deprecation endpoint
+    config.add_route('lfs_objects',
+                     '/{repo:.*?[^/]}/info/lfs/objects')
+    config.add_view(lfs_objects, route_name='lfs_objects',
+                    request_method='POST', renderer='json')
+
+    # locking API
+    config.add_route('lfs_objects_lock',
+                     '/{repo:.*?[^/]}/info/lfs/locks')
+    config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
+                    request_method=('POST', 'GET'), renderer='json')
+
+    config.add_route('lfs_objects_lock_verify',
+                     '/{repo:.*?[^/]}/info/lfs/locks/verify')
+    config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
+                    request_method=('POST', 'GET'), renderer='json')
+
+    # batch API
+    config.add_route('lfs_objects_batch',
+                     '/{repo:.*?[^/]}/info/lfs/objects/batch')
+    config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
+                    request_method='POST', renderer='json')
+
+    # oid upload/download API
+    config.add_route('lfs_objects_oid',
+                     '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
+    config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
+                    request_method='PUT', renderer='json')
+    config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
+                    request_method='GET', renderer='json')
+
+    # verification API
+    config.add_route('lfs_objects_verify',
+                     '/{repo:.*?[^/]}/info/lfs/verify')
+    config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
+                    request_method='POST', renderer='json')
+
+    # not found handler for API
+    config.add_notfound_view(not_found, renderer='json')
+
+
+def create_app(git_lfs_enabled, git_lfs_store_path):
+    config = Configurator()
+    if git_lfs_enabled:
+        config.include(git_lfs_app)
+        config.registry.git_lfs_store_path = git_lfs_store_path
+    else:
+        # not found handler for API, reporting disabled LFS support
+        config.add_notfound_view(lfs_disabled, renderer='json')
+
+    app = config.make_wsgi_app()
+    return app

vcsserver/git_lfs/lib.py

@@ -0,0 +1,167 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import os
+import shutil
+import logging
+from collections import OrderedDict
+
+log = logging.getLogger(__name__)
+
+
+class OidHandler(object):
+
+    def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
+                 obj_verify_href=None):
+        self.current_store = store
+        self.repo_name = repo_name
+        self.auth = auth
+        self.oid = oid
+        self.obj_size = obj_size
+        self.obj_data = obj_data
+        self.obj_href = obj_href
+        self.obj_verify_href = obj_verify_href
+
+    def get_store(self, mode=None):
+        return self.current_store
+
+    def get_auth(self):
+        """returns auth header for re-use in upload/download"""
+        return " ".join(self.auth)
+
+    def download(self):
+
+        store = self.get_store()
+        response = None
+        has_errors = None
+
+        if not store.has_oid():
+            # error reply back to client that something is wrong with dl
+            err_msg = 'object: {} does not exist in store'.format(store.oid)
+            has_errors = OrderedDict(
+                error=OrderedDict(
+                    code=404,
+                    message=err_msg
+                )
+            )
+
+        download_action = OrderedDict(
+            href=self.obj_href,
+            header=OrderedDict([("Authorization", self.get_auth())])
+        )
+        if not has_errors:
+            response = OrderedDict(download=download_action)
+        return response, has_errors
+
+    def upload(self, skip_existing=True):
+        """
+        Write upload action for git-lfs server
+        """
+
+        store = self.get_store()
+        response = None
+        has_errors = None
+
+        # verify if we have the OID before, if we do, reply with empty
+        if store.has_oid():
+            log.debug('LFS: store already has oid %s', store.oid)
+            if skip_existing:
+                log.debug('LFS: skipping further action as oid is existing')
+                return response, has_errors
+
+        upload_action = OrderedDict(
+            href=self.obj_href,
+            header=OrderedDict([("Authorization", self.get_auth())])
+        )
+        if not has_errors:
+            response = OrderedDict(upload=upload_action)
+            # if specified in handler, return the verification endpoint
+            if self.obj_verify_href:
+                verify_action = OrderedDict(
+                    href=self.obj_verify_href,
+                    header=OrderedDict([("Authorization", self.get_auth())])
+                )
+                response['verify'] = verify_action
+        return response, has_errors
+
+    def exec_operation(self, operation, *args, **kwargs):
+        handler = getattr(self, operation)
+        log.debug('LFS: handling request using %s handler', handler)
+        return handler(*args, **kwargs)
+
+
+class LFSOidStore(object):
+
+    def __init__(self, repo, oid, store_location=None):
+        self._store = store_location or self.get_default_store()
+        self.oid = oid
+        self.repo = repo
+        self.store_path = os.path.join(self._store, repo)
+        self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
+        self.oid_path = os.path.join(self.store_path, oid)
+        self.fd = None
+
+    def get_engine(self, mode):
+        """
+        engine = .get_engine(mode='wb')
+        with engine as f:
+            f.write('...')
+        """
+
+        class StoreEngine(object):
+            def __init__(self, mode, store_path, oid_path, tmp_oid_path):
+                self.mode = mode
+                self.store_path = store_path
+                self.oid_path = oid_path
+                self.tmp_oid_path = tmp_oid_path
+
+            def __enter__(self):
+                if not os.path.isdir(self.store_path):
+                    os.makedirs(self.store_path)
+
+                # TODO(marcink): maybe write metadata here with size/oid ?
+                fd = open(self.tmp_oid_path, self.mode)
+                self.fd = fd
+                return fd
+
+            def __exit__(self, exc_type, exc_value, traceback):
+                # close tmp file, and rename to final destination
+                self.fd.close()
+                shutil.move(self.tmp_oid_path, self.oid_path)
+
+        return StoreEngine(
+            mode, self.store_path, self.oid_path, self.tmp_oid_path)
+
+    def get_default_store(self):
+        """
+        Default store, consistent with defaults of Mercurial large files store
+        which is /home/username/.cache/largefiles
+        """
+        user_home = os.path.expanduser("~")
+        return os.path.join(user_home, '.cache', 'lfs-store')
+
+    def has_oid(self):
+        return os.path.exists(os.path.join(self.store_path, self.oid))
+
+    def size_oid(self):
+        size = -1
+
+        if self.has_oid():
+            oid = os.path.join(self.store_path, self.oid)
+            size = os.stat(oid).st_size
+
+        return size
  No newline at end of file

vcsserver/git_lfs/tests/__init__.py

@@ -0,0 +1,16 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA

vcsserver/git_lfs/tests/test_lfs_app.py

@@ -0,0 +1,233 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import os
+import pytest
+from webtest.app import TestApp as WebObTestApp
+
+from vcsserver.git_lfs.app import create_app
+
+
[e-mail](scope='function')
+def git_lfs_app(tmpdir):
+    custom_app = WebObTestApp(create_app(
+        git_lfs_enabled=True, git_lfs_store_path=str(tmpdir)))
+    custom_app._store = str(tmpdir)
+    return custom_app
+
+
[e-mail]()
+def http_auth():
+    return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
+
+
+class TestLFSApplication(object):
+
+    def test_app_wrong_path(self, git_lfs_app):
+        git_lfs_app.get('/repo/info/lfs/xxx', status=404)
+
+    def test_app_deprecated_endpoint(self, git_lfs_app):
+        response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
+        assert response.status_code == 501
+        assert response.json == {u'message': u'LFS: v1 api not supported'}
+
+    def test_app_lock_verify_api_not_available(self, git_lfs_app):
+        response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
+        assert response.status_code == 501
+        assert response.json == {
+            u'message': u'GIT LFS locking api not supported'}
+
+    def test_app_lock_api_not_available(self, git_lfs_app):
+        response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
+        assert response.status_code == 501
+        assert response.json == {
+            u'message': u'GIT LFS locking api not supported'}
+
+    def test_app_batch_api_missing_auth(self, git_lfs_app,):
+        git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params={}, status=403)
+
+    def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params={}, status=400,
+            extra_environ=http_auth)
+        assert response.json == {
+            u'message': u'unsupported operation mode: `None`'}
+
+    def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params={'operation': 'download'},
+            status=400, extra_environ=http_auth)
+        assert response.json == {
+            u'message': u'missing objects data'}
+
+    def test_app_batch_api_unsupported_data_in_objects(
+            self, git_lfs_app, http_auth):
+        params = {'operation': 'download',
+                  'objects': [{}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params, status=400,
+            extra_environ=http_auth)
+        assert response.json == {
+            u'message': u'unsupported data in objects'}
+
+    def test_app_batch_api_download_missing_object(
+            self, git_lfs_app, http_auth):
+        params = {'operation': 'download',
+                  'objects': [{'oid': '123', 'size': '1024'}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params,
+            extra_environ=http_auth)
+
+        expected_objects = [
+            {u'authenticated': True,
+             u'errors': {u'error': {
+                 u'code': 404,
+                 u'message': u'object: 123 does not exist in store'}},
+             u'oid': u'123',
+             u'size': u'1024'}
+        ]
+        assert response.json == {
+            'objects': expected_objects, 'transfer': 'basic'}
+
+    def test_app_batch_api_download(self, git_lfs_app, http_auth):
+        oid = '456'
+        oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
+        os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        params = {'operation': 'download',
+                  'objects': [{'oid': oid, 'size': '1024'}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params,
+            extra_environ=http_auth)
+
+        expected_objects = [
+            {u'authenticated': True,
+             u'actions': {
+                 u'download': {
+                     u'header': {u'Authorization': u'Basic XXXXX'},
+                     u'href': u'http://localhost/repo/info/lfs/objects/456'},
+             },
+             u'oid': u'456',
+             u'size': u'1024'}
+        ]
+        assert response.json == {
+            'objects': expected_objects, 'transfer': 'basic'}
+
+    def test_app_batch_api_upload(self, git_lfs_app, http_auth):
+        params = {'operation': 'upload',
+                  'objects': [{'oid': '123', 'size': '1024'}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params,
+            extra_environ=http_auth)
+        expected_objects = [
+            {u'authenticated': True,
+             u'actions': {
+                 u'upload': {
+                     u'header': {u'Authorization': u'Basic XXXXX'},
+                     u'href': u'http://localhost/repo/info/lfs/objects/123'},
+                 u'verify': {
+                     u'header': {u'Authorization': u'Basic XXXXX'},
+                     u'href': u'http://localhost/repo/info/lfs/verify'}
+             },
+             u'oid': u'123',
+             u'size': u'1024'}
+        ]
+        assert response.json == {
+            'objects': expected_objects, 'transfer': 'basic'}
+
+    def test_app_verify_api_missing_data(self, git_lfs_app):
+        params = {'oid': 'missing',}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params,
+            status=400)
+
+        assert response.json == {
+            u'message': u'missing oid and size in request data'}
+
+    def test_app_verify_api_missing_obj(self, git_lfs_app):
+        params = {'oid': 'missing', 'size': '1024'}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params,
+            status=404)
+
+        assert response.json == {
+            u'message': u'oid `missing` does not exists in store'}
+
+    def test_app_verify_api_size_mismatch(self, git_lfs_app):
+        oid = 'existing'
+        oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
+        os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        params = {'oid': oid, 'size': '1024'}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params, status=422)
+
+        assert response.json == {
+            u'message': u'requested file size mismatch '
+                        u'store size:11 requested:1024'}
+
+    def test_app_verify_api(self, git_lfs_app):
+        oid = 'existing'
+        oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
+        os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        params = {'oid': oid, 'size': 11}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params)
+
+        assert response.json == {
+            u'message': {u'size': u'ok', u'in_store': u'ok'}}
+
+    def test_app_download_api_oid_not_existing(self, git_lfs_app):
+        oid = 'missing'
+
+        response = git_lfs_app.get(
+            '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
+
+        assert response.json == {
+            u'message': u'requested file with oid `missing` not found in store'}
+
+    def test_app_download_api(self, git_lfs_app):
+        oid = 'existing'
+        oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
+        os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        response = git_lfs_app.get(
+            '/repo/info/lfs/objects/{oid}'.format(oid=oid))
+        assert response
+
+    def test_app_upload(self, git_lfs_app):
+        oid = 'uploaded'
+
+        response = git_lfs_app.put(
+            '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
+
+        assert response.json == {u'upload': u'ok'}
+
+        # verify that we actually wrote that OID
+        oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
+        assert os.path.isfile(oid_path)
+        assert 'CONTENT' == open(oid_path).read()

vcsserver/git_lfs/tests/test_lib.py

@@ -0,0 +1,121 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import os
+import pytest
+from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
+
+
[e-mail]()
+def lfs_store(tmpdir):
+    repo = 'test'
+    oid = '123456789'
+    store = LFSOidStore(repo=repo, oid=oid, store_location=str(tmpdir))
+    return store
+
+
[e-mail]()
+def oid_handler(lfs_store):
+    store = lfs_store
+    repo = store.repo
+    oid = store.oid
+
+    oid_handler = OidHandler(
+        store=store, repo_name=repo, auth=('basic', 'xxxx'),
+        oid=oid,
+        obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
+        obj_verify_href='http://localhost/verify')
+    return oid_handler
+
+
+class TestOidHandler(object):
+
+    @pytest.mark.parametrize('exec_action', [
+        'download',
+        'upload',
+    ])
+    def test_exec_action(self, exec_action, oid_handler):
+        handler = oid_handler.exec_operation(exec_action)
+        assert handler
+
+    def test_exec_action_undefined(self, oid_handler):
+        with pytest.raises(AttributeError):
+            oid_handler.exec_operation('wrong')
+
+    def test_download_oid_not_existing(self, oid_handler):
+        response, has_errors = oid_handler.exec_operation('download')
+
+        assert response is None
+        assert has_errors['error'] == {
+            'code': 404,
+            'message': 'object: 123456789 does not exist in store'}
+
+    def test_download_oid(self, oid_handler):
+        store = oid_handler.get_store()
+
+        os.makedirs(os.path.dirname(store.oid_path))
+        with open(store.oid_path, 'wb') as f:
+            f.write('CONTENT')
+
+        response, has_errors = oid_handler.exec_operation('download')
+
+        assert has_errors is None
+        assert response['download'] == {
+            'header': {'Authorization': 'basic xxxx'},
+            'href': 'http://localhost/handle_oid'
+        }
+
+    def test_upload_oid_that_exists(self, oid_handler):
+        store = oid_handler.get_store()
+
+        os.makedirs(os.path.dirname(store.oid_path))
+        with open(store.oid_path, 'wb') as f:
+            f.write('CONTENT')
+
+        response, has_errors = oid_handler.exec_operation('upload')
+        assert has_errors is None
+        assert response is None
+
+    def test_upload_oid(self, oid_handler):
+        response, has_errors = oid_handler.exec_operation('upload')
+        assert has_errors is None
+        assert response['upload'] == {
+            'header': {'Authorization': 'basic xxxx'},
+            'href': 'http://localhost/handle_oid'
+        }
+
+
+class TestLFSStore(object):
+    def test_write_oid(self, lfs_store):
+        oid_location = lfs_store.oid_path
+
+        assert not os.path.isfile(oid_location)
+
+        engine = lfs_store.get_engine(mode='wb')
+        with engine as f:
+            f.write('CONTENT')
+
+        assert os.path.isfile(oid_location)
+
+    def test_detect_has_oid(self, lfs_store):
+
+        assert lfs_store.has_oid() is False
+        engine = lfs_store.get_engine(mode='wb')
+        with engine as f:
+            f.write('CONTENT')
+
+        assert lfs_store.has_oid() is True
  No newline at end of file

vcsserver/git_lfs/utils.py

@@ -0,0 +1,50 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+import copy
+from functools import wraps
+
+
+def get_cython_compat_decorator(wrapper, func):
+    """
+    Creates a cython compatible decorator. The previously used
+    decorator.decorator() function seems to be incompatible with cython.
+
+    :param wrapper: __wrapper method of the decorator class
+    :param func: decorated function
+    """
+    @wraps(func)
+    def local_wrapper(*args, **kwds):
+        return wrapper(func, *args, **kwds)
+    local_wrapper.__wrapped__ = func
+    return local_wrapper
+
+
+def safe_result(result):
+    """clean result for better representation in logs"""
+    clean_copy = copy.deepcopy(result)
+
+    try:
+        if 'objects' in clean_copy:
+            for oid_data in clean_copy['objects']:
+                if 'actions' in oid_data:
+                    for action_name, data in oid_data['actions'].items():
+                        if 'header' in data:
+                            data['header'] = {'Authorization': '*****'}
+    except Exception:
+        return result
+
+    return clean_copy

vcsserver/http_main.py

@@ -30,6 +30,7 b' from pyramid.config import Configurator'
 from pyramid.wsgi import wsgiapp
 
 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
+from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
 from vcsserver.echo_stub.echo_app import EchoApp
 from vcsserver.exceptions import HTTPRepoLocked
@@ -40,11 +41,13 b' try:'
 except ImportError:
     GitFactory = None
     GitRemote = None
+
 try:
     from vcsserver.hg import MercurialFactory, HgRemote
 except ImportError:
     MercurialFactory = None
     HgRemote = None
+
 try:
     from vcsserver.svn import SubversionFactory, SvnRemote
 except ImportError:
@@ -362,9 +365,31 b' class HTTPApplication(object):'
                 config = msgpack.unpackb(packed_config)
 
                 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
-                app = scm_app.create_git_wsgi_app(
-                    repo_path, repo_name, config)
+                content_type = environ.get('CONTENT_TYPE', '')
+
+                path = environ['PATH_INFO']
+                is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
+                log.debug(
+                    'LFS: Detecting if request `%s` is LFS server path based '
+                    'on content type:`%s`, is_lfs:%s',
+                    path, content_type, is_lfs_request)
+
+                if not is_lfs_request:
+                    # fallback detection by path
+                    if GIT_LFS_PROTO_PAT.match(path):
+                        is_lfs_request = True
+                    log.debug(
+                        'LFS: fallback detection by path of: `%s`, is_lfs:%s',
+                        path, is_lfs_request)
+
+                if is_lfs_request:
+                    app = scm_app.create_git_lfs_wsgi_app(
+                        repo_path, repo_name, config)
+                else:
+                    app = scm_app.create_git_wsgi_app(
+                        repo_path, repo_name, config)
                 return app(environ, start_response)
+
             return _git_stream
 
     def is_vcs_view(self, context, request):

vcsserver/scm_app.py

@@ -15,8 +15,8 b''
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
+import os
 import logging
-import os
 
 import mercurial
 import mercurial.error
@@ -25,7 +25,7 b' import mercurial.hgweb.hgweb_mod'
 import mercurial.hgweb.protocol
 import webob.exc
 
-from vcsserver import pygrack, exceptions, settings
+from vcsserver import pygrack, exceptions, settings, git_lfs
 
 
 log = logging.getLogger(__name__)
@@ -132,6 +132,9 b' def create_hg_wsgi_app(repo_path, repo_n'
 
 
 class GitHandler(object):
+    """
+    Handler for Git operations like push/pull etc
+    """
     def __init__(self, repo_location, repo_name, git_path, update_server_info,
                  extras):
         if not os.path.isdir(repo_location):
@@ -172,3 +175,35 b' def create_git_wsgi_app(repo_path, repo_'
         repo_path, repo_name, git_path, update_server_info, config)
 
     return app
+
+
+class GitLFSHandler(object):
+    """
+    Handler for Git LFS operations
+    """
+
+    def __init__(self, repo_location, repo_name, git_path, update_server_info,
+                 extras):
+        if not os.path.isdir(repo_location):
+            raise OSError(repo_location)
+        self.content_path = repo_location
+        self.repo_name = repo_name
+        self.repo_location = repo_location
+        self.extras = extras
+        self.git_path = git_path
+        self.update_server_info = update_server_info
+
+    def get_app(self, git_lfs_enabled, git_lfs_store_path):
+        app = git_lfs.create_app(git_lfs_enabled, git_lfs_store_path)
+        return app
+
+
+def create_git_lfs_wsgi_app(repo_path, repo_name, config):
+    git_path = settings.GIT_EXECUTABLE
+    update_server_info = config.pop('git_update_server_info')
+    git_lfs_enabled = config.pop('git_lfs_enabled')
+    git_lfs_store_path = config.pop('git_lfs_store_path')
+    app = GitLFSHandler(
+        repo_path, repo_name, git_path, update_server_info, config)
+
+    return app.get_app(git_lfs_enabled, git_lfs_store_path)

vcsserver/utils.py

@@ -16,8 +16,23 b''
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 
+def safe_int(val, default=None):
+    """
+    Returns int() of val if val is not convertable to int use default
+    instead
 
-# TODO: johbo: That's a copy from rhodecode
+    :param val:
+    :param default:
+    """
+
+    try:
+        val = int(val)
+    except (ValueError, TypeError):
+        val = default
+
+    return val
+
+
 def safe_str(unicode_, to_encoding=['utf8']):
     """
     safe str function. Does few trick to turn unicode_ into string