git-lfs: added vcsserver handling of git-lfs objects.

- fixes  #4235

vcsserver/git_lfs/__init__.py

@@ -0,0 +1,19 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+
+from app import create_app

vcsserver/git_lfs/app.py

@@ -0,0 +1,276 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import re
+import logging
+from wsgiref.util import FileWrapper
+
+import simplejson as json
+from pyramid.config import Configurator
+from pyramid.response import Response, FileIter
+from pyramid.httpexceptions import (
+    HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
+    HTTPUnprocessableEntity)
+
+from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
+from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
+from vcsserver.utils import safe_int
+
+log = logging.getLogger(__name__)
+
+
+GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ?
+GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
+
+
+def write_response_error(http_exception, text=None):
+    content_type = 'application/json'
+    _exception = http_exception(content_type=content_type)
+    _exception.content_type = content_type
+    if text:
+        _exception.body = json.dumps({'message': text})
+    log.debug('LFS: writing response of type %s to client with text:%s',
+              http_exception, text)
+    return _exception
+
+
+class AuthHeaderRequired(object):
+    """
+    Decorator to check if request has proper auth-header
+    """
+
+    def __call__(self, func):
+        return get_cython_compat_decorator(self.__wrapper, func)
+
+    def __wrapper(self, func, *fargs, **fkwargs):
+        request = fargs[1]
+        auth = request.authorization
+        if not auth:
+            return write_response_error(HTTPForbidden)
+        return func(*fargs[1:], **fkwargs)
+
+
+# views
+
+def lfs_objects(request):
+    # indicate not supported, V1 API
+    log.warning('LFS: v1 api not supported, reporting it back to client')
+    return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
+
+
+@AuthHeaderRequired()
+def lfs_objects_batch(request):
+    """
+    The client sends the following information to the Batch endpoint to transfer some objects:
+
+        operation - Should be download or upload.
+        transfers - An optional Array of String identifiers for transfer
+            adapters that the client has configured. If omitted, the basic
+            transfer adapter MUST be assumed by the server.
+        objects - An Array of objects to download.
+        oid - String OID of the LFS object.
+        size - Integer byte size of the LFS object. Must be at least zero.
+    """
+    auth = request.authorization
+
+    repo = request.matchdict.get('repo')
+
+    data = request.json
+    operation = data.get('operation')
+    if operation not in ('download', 'upload'):
+        log.debug('LFS: unsupported operation:%s', operation)
+        return write_response_error(
+            HTTPBadRequest, 'unsupported operation mode: `%s`' % operation)
+
+    if 'objects' not in data:
+        log.debug('LFS: missing objects data')
+        return write_response_error(
+            HTTPBadRequest, 'missing objects data')
+
+    log.debug('LFS: handling operation of type: %s', operation)
+
+    objects = []
+    for o in data['objects']:
+        try:
+            oid = o['oid']
+            obj_size = o['size']
+        except KeyError:
+            log.exception('LFS, failed to extract data')
+            return write_response_error(
+                HTTPBadRequest, 'unsupported data in objects')
+
+        obj_data = {'oid': oid}
+
+        obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid)
+        obj_verify_href = request.route_url('lfs_objects_verify', repo=repo)
+        store = LFSOidStore(
+            repo, oid, store_location=request.registry.git_lfs_store_path)
+        handler = OidHandler(
+            store, repo, auth, oid, obj_size, obj_data,
+            obj_href, obj_verify_href)
+
+        # this verifies also OIDs
+        actions, errors = handler.exec_operation(operation)
+        if errors:
+            log.warning('LFS: got following errors: %s', errors)
+            obj_data['errors'] = errors
+
+        if actions:
+            obj_data['actions'] = actions
+
+        obj_data['size'] = obj_size
+        obj_data['authenticated'] = True
+        objects.append(obj_data)
+
+    result = {'objects': objects, 'transfer': 'basic'}
+    log.debug('LFS Response %s', safe_result(result))
+
+    return result
+
+
+def lfs_objects_oid_upload(request):
+    repo = request.matchdict.get('repo')
+    oid = request.matchdict.get('oid')
+    store = LFSOidStore(
+        repo, oid, store_location=request.registry.git_lfs_store_path)
+    engine = store.get_engine(mode='wb')
+    log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
+    with engine as f:
+        for chunk in FileWrapper(request.body_file_seekable, blksize=64 * 1024):
+            f.write(chunk)
+
+    return {'upload': 'ok'}
+
+
+def lfs_objects_oid_download(request):
+    repo = request.matchdict.get('repo')
+    oid = request.matchdict.get('oid')
+
+    store = LFSOidStore(
+        repo, oid, store_location=request.registry.git_lfs_store_path)
+    if not store.has_oid():
+        log.debug('LFS: oid %s does not exists in store', oid)
+        return write_response_error(
+            HTTPNotFound, 'requested file with oid `%s` not found in store' % oid)
+
+    # TODO(marcink): support range header ?
+    # Range: bytes=0-, `bytes=(\d+)\-.*`
+
+    f = open(store.oid_path, 'rb')
+    response = Response(
+        content_type='application/octet-stream', app_iter=FileIter(f))
+    response.headers.add('X-RC-LFS-Response-Oid', str(oid))
+    return response
+
+
+def lfs_objects_verify(request):
+    repo = request.matchdict.get('repo')
+
+    data = request.json
+    oid = data.get('oid')
+    size = safe_int(data.get('size'))
+
+    if not (oid and size):
+        return write_response_error(
+            HTTPBadRequest, 'missing oid and size in request data')
+
+    store = LFSOidStore(repo, oid,
+                        store_location=request.registry.git_lfs_store_path)
+    if not store.has_oid():
+        log.debug('LFS: oid %s does not exists in store', oid)
+        return write_response_error(
+            HTTPNotFound, 'oid `%s` does not exists in store' % oid)
+
+    store_size = store.size_oid()
+    if store_size != size:
+        msg = 'requested file size mismatch store size:%s requested:%s' % (
+            store_size, size)
+        return write_response_error(
+            HTTPUnprocessableEntity, msg)
+
+    return {'message': {'size': 'ok', 'in_store': 'ok'}}
+
+
+def lfs_objects_lock(request):
+    return write_response_error(
+        HTTPNotImplemented, 'GIT LFS locking api not supported')
+
+
+def not_found(request):
+    return write_response_error(
+        HTTPNotFound, 'request path not found')
+
+
+def lfs_disabled(request):
+    return write_response_error(
+        HTTPNotImplemented, 'GIT LFS disabled for this repo')
+
+
+def git_lfs_app(config):
+
+    # v1 API deprecation endpoint
+    config.add_route('lfs_objects',
+                     '/{repo:.*?[^/]}/info/lfs/objects')
+    config.add_view(lfs_objects, route_name='lfs_objects',
+                    request_method='POST', renderer='json')
+
+    # locking API
+    config.add_route('lfs_objects_lock',
+                     '/{repo:.*?[^/]}/info/lfs/locks')
+    config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
+                    request_method=('POST', 'GET'), renderer='json')
+
+    config.add_route('lfs_objects_lock_verify',
+                     '/{repo:.*?[^/]}/info/lfs/locks/verify')
+    config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
+                    request_method=('POST', 'GET'), renderer='json')
+
+    # batch API
+    config.add_route('lfs_objects_batch',
+                     '/{repo:.*?[^/]}/info/lfs/objects/batch')
+    config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
+                    request_method='POST', renderer='json')
+
+    # oid upload/download API
+    config.add_route('lfs_objects_oid',
+                     '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
+    config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
+                    request_method='PUT', renderer='json')
+    config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
+                    request_method='GET', renderer='json')
+
+    # verification API
+    config.add_route('lfs_objects_verify',
+                     '/{repo:.*?[^/]}/info/lfs/verify')
+    config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
+                    request_method='POST', renderer='json')
+
+    # not found handler for API
+    config.add_notfound_view(not_found, renderer='json')
+
+
+def create_app(git_lfs_enabled, git_lfs_store_path):
+    config = Configurator()
+    if git_lfs_enabled:
+        config.include(git_lfs_app)
+        config.registry.git_lfs_store_path = git_lfs_store_path
+    else:
+        # not found handler for API, reporting disabled LFS support
+        config.add_notfound_view(lfs_disabled, renderer='json')
+
+    app = config.make_wsgi_app()
+    return app

vcsserver/git_lfs/lib.py

@@ -0,0 +1,167 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import os
+import shutil
+import logging
+from collections import OrderedDict
+
+log = logging.getLogger(__name__)
+
+
+class OidHandler(object):
+
+    def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
+                 obj_verify_href=None):
+        self.current_store = store
+        self.repo_name = repo_name
+        self.auth = auth
+        self.oid = oid
+        self.obj_size = obj_size
+        self.obj_data = obj_data
+        self.obj_href = obj_href
+        self.obj_verify_href = obj_verify_href
+
+    def get_store(self, mode=None):
+        return self.current_store
+
+    def get_auth(self):
+        """returns auth header for re-use in upload/download"""
+        return " ".join(self.auth)
+
+    def download(self):
+
+        store = self.get_store()
+        response = None
+        has_errors = None
+
+        if not store.has_oid():
+            # error reply back to client that something is wrong with dl
+            err_msg = 'object: {} does not exist in store'.format(store.oid)
+            has_errors = OrderedDict(
+                error=OrderedDict(
+                    code=404,
+                    message=err_msg
+                )
+            )
+
+        download_action = OrderedDict(
+            href=self.obj_href,
+            header=OrderedDict([("Authorization", self.get_auth())])
+        )
+        if not has_errors:
+            response = OrderedDict(download=download_action)
+        return response, has_errors
+
+    def upload(self, skip_existing=True):
+        """
+        Write upload action for git-lfs server
+        """
+
+        store = self.get_store()
+        response = None
+        has_errors = None
+
+        # verify if we have the OID before, if we do, reply with empty
+        if store.has_oid():
+            log.debug('LFS: store already has oid %s', store.oid)
+            if skip_existing:
+                log.debug('LFS: skipping further action as oid is existing')
+                return response, has_errors
+
+        upload_action = OrderedDict(
+            href=self.obj_href,
+            header=OrderedDict([("Authorization", self.get_auth())])
+        )
+        if not has_errors:
+            response = OrderedDict(upload=upload_action)
+            # if specified in handler, return the verification endpoint
+            if self.obj_verify_href:
+                verify_action = OrderedDict(
+                    href=self.obj_verify_href,
+                    header=OrderedDict([("Authorization", self.get_auth())])
+                )
+                response['verify'] = verify_action
+        return response, has_errors
+
+    def exec_operation(self, operation, *args, **kwargs):
+        handler = getattr(self, operation)
+        log.debug('LFS: handling request using %s handler', handler)
+        return handler(*args, **kwargs)
+
+
+class LFSOidStore(object):
+
+    def __init__(self, repo, oid, store_location=None):
+        self._store = store_location or self.get_default_store()
+        self.oid = oid
+        self.repo = repo
+        self.store_path = os.path.join(self._store, repo)
+        self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
+        self.oid_path = os.path.join(self.store_path, oid)
+        self.fd = None
+
+    def get_engine(self, mode):
+        """
+        engine = .get_engine(mode='wb')
+        with engine as f:
+            f.write('...')
+        """
+
+        class StoreEngine(object):
+            def __init__(self, mode, store_path, oid_path, tmp_oid_path):
+                self.mode = mode
+                self.store_path = store_path
+                self.oid_path = oid_path
+                self.tmp_oid_path = tmp_oid_path
+
+            def __enter__(self):
+                if not os.path.isdir(self.store_path):
+                    os.makedirs(self.store_path)
+
+                # TODO(marcink): maybe write metadata here with size/oid ?
+                fd = open(self.tmp_oid_path, self.mode)
+                self.fd = fd
+                return fd
+
+            def __exit__(self, exc_type, exc_value, traceback):
+                # close tmp file, and rename to final destination
+                self.fd.close()
+                shutil.move(self.tmp_oid_path, self.oid_path)
+
+        return StoreEngine(
+            mode, self.store_path, self.oid_path, self.tmp_oid_path)
+
+    def get_default_store(self):
+        """
+        Default store, consistent with defaults of Mercurial large files store
+        which is /home/username/.cache/largefiles
+        """
+        user_home = os.path.expanduser("~")
+        return os.path.join(user_home, '.cache', 'lfs-store')
+
+    def has_oid(self):
+        return os.path.exists(os.path.join(self.store_path, self.oid))
+
+    def size_oid(self):
+        size = -1
+
+        if self.has_oid():
+            oid = os.path.join(self.store_path, self.oid)
+            size = os.stat(oid).st_size
+
+        return size
  No newline at end of file

vcsserver/git_lfs/tests/__init__.py

@@ -0,0 +1,16 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA

vcsserver/git_lfs/tests/test_lfs_app.py

@@ -0,0 +1,233 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import os
+import pytest
+from webtest.app import TestApp as WebObTestApp
+
+from vcsserver.git_lfs.app import create_app
+
+
[e-mail](scope='function')
+def git_lfs_app(tmpdir):
+    custom_app = WebObTestApp(create_app(
+        git_lfs_enabled=True, git_lfs_store_path=str(tmpdir)))
+    custom_app._store = str(tmpdir)
+    return custom_app
+
+
[e-mail]()
+def http_auth():
+    return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
+
+
+class TestLFSApplication(object):
+
+    def test_app_wrong_path(self, git_lfs_app):
+        git_lfs_app.get('/repo/info/lfs/xxx', status=404)
+
+    def test_app_deprecated_endpoint(self, git_lfs_app):
+        response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
+        assert response.status_code == 501
+        assert response.json == {u'message': u'LFS: v1 api not supported'}
+
+    def test_app_lock_verify_api_not_available(self, git_lfs_app):
+        response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
+        assert response.status_code == 501
+        assert response.json == {
+            u'message': u'GIT LFS locking api not supported'}
+
+    def test_app_lock_api_not_available(self, git_lfs_app):
+        response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
+        assert response.status_code == 501
+        assert response.json == {
+            u'message': u'GIT LFS locking api not supported'}
+
+    def test_app_batch_api_missing_auth(self, git_lfs_app,):
+        git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params={}, status=403)
+
+    def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params={}, status=400,
+            extra_environ=http_auth)
+        assert response.json == {
+            u'message': u'unsupported operation mode: `None`'}
+
+    def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params={'operation': 'download'},
+            status=400, extra_environ=http_auth)
+        assert response.json == {
+            u'message': u'missing objects data'}
+
+    def test_app_batch_api_unsupported_data_in_objects(
+            self, git_lfs_app, http_auth):
+        params = {'operation': 'download',
+                  'objects': [{}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params, status=400,
+            extra_environ=http_auth)
+        assert response.json == {
+            u'message': u'unsupported data in objects'}
+
+    def test_app_batch_api_download_missing_object(
+            self, git_lfs_app, http_auth):
+        params = {'operation': 'download',
+                  'objects': [{'oid': '123', 'size': '1024'}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params,
+            extra_environ=http_auth)
+
+        expected_objects = [
+            {u'authenticated': True,
+             u'errors': {u'error': {
+                 u'code': 404,
+                 u'message': u'object: 123 does not exist in store'}},
+             u'oid': u'123',
+             u'size': u'1024'}
+        ]
+        assert response.json == {
+            'objects': expected_objects, 'transfer': 'basic'}
+
+    def test_app_batch_api_download(self, git_lfs_app, http_auth):
+        oid = '456'
+        oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
+        os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        params = {'operation': 'download',
+                  'objects': [{'oid': oid, 'size': '1024'}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params,
+            extra_environ=http_auth)
+
+        expected_objects = [
+            {u'authenticated': True,
+             u'actions': {
+                 u'download': {
+                     u'header': {u'Authorization': u'Basic XXXXX'},
+                     u'href': u'http://localhost/repo/info/lfs/objects/456'},
+             },
+             u'oid': u'456',
+             u'size': u'1024'}
+        ]
+        assert response.json == {
+            'objects': expected_objects, 'transfer': 'basic'}
+
+    def test_app_batch_api_upload(self, git_lfs_app, http_auth):
+        params = {'operation': 'upload',
+                  'objects': [{'oid': '123', 'size': '1024'}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params,
+            extra_environ=http_auth)
+        expected_objects = [
+            {u'authenticated': True,
+             u'actions': {
+                 u'upload': {
+                     u'header': {u'Authorization': u'Basic XXXXX'},
+                     u'href': u'http://localhost/repo/info/lfs/objects/123'},
+                 u'verify': {
+                     u'header': {u'Authorization': u'Basic XXXXX'},
+                     u'href': u'http://localhost/repo/info/lfs/verify'}
+             },
+             u'oid': u'123',
+             u'size': u'1024'}
+        ]
+        assert response.json == {
+            'objects': expected_objects, 'transfer': 'basic'}
+
+    def test_app_verify_api_missing_data(self, git_lfs_app):
+        params = {'oid': 'missing',}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params,
+            status=400)
+
+        assert response.json == {
+            u'message': u'missing oid and size in request data'}
+
+    def test_app_verify_api_missing_obj(self, git_lfs_app):
+        params = {'oid': 'missing', 'size': '1024'}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params,
+            status=404)
+
+        assert response.json == {
+            u'message': u'oid `missing` does not exists in store'}
+
+    def test_app_verify_api_size_mismatch(self, git_lfs_app):
+        oid = 'existing'
+        oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
+        os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        params = {'oid': oid, 'size': '1024'}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params, status=422)
+
+        assert response.json == {
+            u'message': u'requested file size mismatch '
+                        u'store size:11 requested:1024'}
+
+    def test_app_verify_api(self, git_lfs_app):
+        oid = 'existing'
+        oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
+        os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        params = {'oid': oid, 'size': 11}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params)
+
+        assert response.json == {
+            u'message': {u'size': u'ok', u'in_store': u'ok'}}
+
+    def test_app_download_api_oid_not_existing(self, git_lfs_app):
+        oid = 'missing'
+
+        response = git_lfs_app.get(
+            '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
+
+        assert response.json == {
+            u'message': u'requested file with oid `missing` not found in store'}
+
+    def test_app_download_api(self, git_lfs_app):
+        oid = 'existing'
+        oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
+        os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        response = git_lfs_app.get(
+            '/repo/info/lfs/objects/{oid}'.format(oid=oid))
+        assert response
+
+    def test_app_upload(self, git_lfs_app):
+        oid = 'uploaded'
+
+        response = git_lfs_app.put(
+            '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
+
+        assert response.json == {u'upload': u'ok'}
+
+        # verify that we actually wrote that OID
+        oid_path = os.path.join(git_lfs_app._store, 'repo', oid)
+        assert os.path.isfile(oid_path)
+        assert 'CONTENT' == open(oid_path).read()

vcsserver/git_lfs/tests/test_lib.py

@@ -0,0 +1,121 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import os
+import pytest
+from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
+
+
[e-mail]()
+def lfs_store(tmpdir):
+    repo = 'test'
+    oid = '123456789'
+    store = LFSOidStore(repo=repo, oid=oid, store_location=str(tmpdir))
+    return store
+
+
[e-mail]()
+def oid_handler(lfs_store):
+    store = lfs_store
+    repo = store.repo
+    oid = store.oid
+
+    oid_handler = OidHandler(
+        store=store, repo_name=repo, auth=('basic', 'xxxx'),
+        oid=oid,
+        obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
+        obj_verify_href='http://localhost/verify')
+    return oid_handler
+
+
+class TestOidHandler(object):
+
+    @pytest.mark.parametrize('exec_action', [
+        'download',
+        'upload',
+    ])
+    def test_exec_action(self, exec_action, oid_handler):
+        handler = oid_handler.exec_operation(exec_action)
+        assert handler
+
+    def test_exec_action_undefined(self, oid_handler):
+        with pytest.raises(AttributeError):
+            oid_handler.exec_operation('wrong')
+
+    def test_download_oid_not_existing(self, oid_handler):
+        response, has_errors = oid_handler.exec_operation('download')
+
+        assert response is None
+        assert has_errors['error'] == {
+            'code': 404,
+            'message': 'object: 123456789 does not exist in store'}
+
+    def test_download_oid(self, oid_handler):
+        store = oid_handler.get_store()
+
+        os.makedirs(os.path.dirname(store.oid_path))
+        with open(store.oid_path, 'wb') as f:
+            f.write('CONTENT')
+
+        response, has_errors = oid_handler.exec_operation('download')
+
+        assert has_errors is None
+        assert response['download'] == {
+            'header': {'Authorization': 'basic xxxx'},
+            'href': 'http://localhost/handle_oid'
+        }
+
+    def test_upload_oid_that_exists(self, oid_handler):
+        store = oid_handler.get_store()
+
+        os.makedirs(os.path.dirname(store.oid_path))
+        with open(store.oid_path, 'wb') as f:
+            f.write('CONTENT')
+
+        response, has_errors = oid_handler.exec_operation('upload')
+        assert has_errors is None
+        assert response is None
+
+    def test_upload_oid(self, oid_handler):
+        response, has_errors = oid_handler.exec_operation('upload')
+        assert has_errors is None
+        assert response['upload'] == {
+            'header': {'Authorization': 'basic xxxx'},
+            'href': 'http://localhost/handle_oid'
+        }
+
+
+class TestLFSStore(object):
+    def test_write_oid(self, lfs_store):
+        oid_location = lfs_store.oid_path
+
+        assert not os.path.isfile(oid_location)
+
+        engine = lfs_store.get_engine(mode='wb')
+        with engine as f:
+            f.write('CONTENT')
+
+        assert os.path.isfile(oid_location)
+
+    def test_detect_has_oid(self, lfs_store):
+
+        assert lfs_store.has_oid() is False
+        engine = lfs_store.get_engine(mode='wb')
+        with engine as f:
+            f.write('CONTENT')
+
+        assert lfs_store.has_oid() is True
  No newline at end of file

vcsserver/git_lfs/utils.py

@@ -0,0 +1,50 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+import copy
+from functools import wraps
+
+
+def get_cython_compat_decorator(wrapper, func):
+    """
+    Creates a cython compatible decorator. The previously used
+    decorator.decorator() function seems to be incompatible with cython.
+
+    :param wrapper: __wrapper method of the decorator class
+    :param func: decorated function
+    """
+    @wraps(func)
+    def local_wrapper(*args, **kwds):
+        return wrapper(func, *args, **kwds)
+    local_wrapper.__wrapped__ = func
+    return local_wrapper
+
+
+def safe_result(result):
+    """clean result for better representation in logs"""
+    clean_copy = copy.deepcopy(result)
+
+    try:
+        if 'objects' in clean_copy:
+            for oid_data in clean_copy['objects']:
+                if 'actions' in oid_data:
+                    for action_name, data in oid_data['actions'].items():
+                        if 'header' in data:
+                            data['header'] = {'Authorization': '*****'}
+    except Exception:
+        return result
+
+    return clean_copy

vcsserver/http_main.py

@@ -1,409 +1,434 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2017 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import base64
 import locale
 import logging
 import uuid
 import wsgiref.util
 import traceback
 from itertools import chain
 
 import msgpack
 from beaker.cache import CacheManager
 from beaker.util import parse_cache_config_options
 from pyramid.config import Configurator
 from pyramid.wsgi import wsgiapp
 
 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
+from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
 from vcsserver.echo_stub.echo_app import EchoApp
 from vcsserver.exceptions import HTTPRepoLocked
 from vcsserver.server import VcsServer
 
 try:
     from vcsserver.git import GitFactory, GitRemote
 except ImportError:
     GitFactory = None
     GitRemote = None
+
 try:
     from vcsserver.hg import MercurialFactory, HgRemote
 except ImportError:
     MercurialFactory = None
     HgRemote = None
+
 try:
     from vcsserver.svn import SubversionFactory, SvnRemote
 except ImportError:
     SubversionFactory = None
     SvnRemote = None
 
 log = logging.getLogger(__name__)
 
 
 class VCS(object):
     def __init__(self, locale=None, cache_config=None):
         self.locale = locale
         self.cache_config = cache_config
         self._configure_locale()
         self._initialize_cache()
 
         if GitFactory and GitRemote:
             git_repo_cache = self.cache.get_cache_region(
                 'git', region='repo_object')
             git_factory = GitFactory(git_repo_cache)
             self._git_remote = GitRemote(git_factory)
         else:
             log.info("Git client import failed")
 
         if MercurialFactory and HgRemote:
             hg_repo_cache = self.cache.get_cache_region(
                 'hg', region='repo_object')
             hg_factory = MercurialFactory(hg_repo_cache)
             self._hg_remote = HgRemote(hg_factory)
         else:
             log.info("Mercurial client import failed")
 
         if SubversionFactory and SvnRemote:
             svn_repo_cache = self.cache.get_cache_region(
                 'svn', region='repo_object')
             svn_factory = SubversionFactory(svn_repo_cache)
             self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
         else:
             log.info("Subversion client import failed")
 
         self._vcsserver = VcsServer()
 
     def _initialize_cache(self):
         cache_config = parse_cache_config_options(self.cache_config)
         log.info('Initializing beaker cache: %s' % cache_config)
         self.cache = CacheManager(**cache_config)
 
     def _configure_locale(self):
         if self.locale:
             log.info('Settings locale: `LC_ALL` to %s' % self.locale)
         else:
             log.info(
                 'Configuring locale subsystem based on environment variables')
         try:
             # If self.locale is the empty string, then the locale
             # module will use the environment variables. See the
             # documentation of the package `locale`.
             locale.setlocale(locale.LC_ALL, self.locale)
 
             language_code, encoding = locale.getlocale()
             log.info(
                 'Locale set to language code "%s" with encoding "%s".',
                 language_code, encoding)
         except locale.Error:
             log.exception(
                 'Cannot set locale, not configuring the locale system')
 
 
 class WsgiProxy(object):
     def __init__(self, wsgi):
         self.wsgi = wsgi
 
     def __call__(self, environ, start_response):
         input_data = environ['wsgi.input'].read()
         input_data = msgpack.unpackb(input_data)
 
         error = None
         try:
             data, status, headers = self.wsgi.handle(
                 input_data['environment'], input_data['input_data'],
                 *input_data['args'], **input_data['kwargs'])
         except Exception as e:
             data, status, headers = [], None, None
             error = {
                 'message': str(e),
                 '_vcs_kind': getattr(e, '_vcs_kind', None)
             }
 
         start_response(200, {})
         return self._iterator(error, status, headers, data)
 
     def _iterator(self, error, status, headers, data):
         initial_data = [
             error,
             status,
             headers,
         ]
 
         for d in chain(initial_data, data):
             yield msgpack.packb(d)
 
 
 class HTTPApplication(object):
     ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
 
     remote_wsgi = remote_wsgi
     _use_echo_app = False
 
     def __init__(self, settings=None, global_config=None):
         self.config = Configurator(settings=settings)
         self.global_config = global_config
 
         locale = settings.get('locale', '') or 'en_US.UTF-8'
         vcs = VCS(locale=locale, cache_config=settings)
         self._remotes = {
             'hg': vcs._hg_remote,
             'git': vcs._git_remote,
             'svn': vcs._svn_remote,
             'server': vcs._vcsserver,
         }
         if settings.get('dev.use_echo_app', 'false').lower() == 'true':
             self._use_echo_app = True
             log.warning("Using EchoApp for VCS operations.")
             self.remote_wsgi = remote_wsgi_stub
         self._configure_settings(settings)
         self._configure()
 
     def _configure_settings(self, app_settings):
         """
         Configure the settings module.
         """
         git_path = app_settings.get('git_path', None)
         if git_path:
             settings.GIT_EXECUTABLE = git_path
 
     def _configure(self):
         self.config.add_renderer(
             name='msgpack',
             factory=self._msgpack_renderer_factory)
 
         self.config.add_route('service', '/_service')
         self.config.add_route('status', '/status')
         self.config.add_route('hg_proxy', '/proxy/hg')
         self.config.add_route('git_proxy', '/proxy/git')
         self.config.add_route('vcs', '/{backend}')
         self.config.add_route('stream_git', '/stream/git/*repo_name')
         self.config.add_route('stream_hg', '/stream/hg/*repo_name')
 
         self.config.add_view(
             self.status_view, route_name='status', renderer='json')
         self.config.add_view(
             self.service_view, route_name='service', renderer='msgpack')
 
         self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
         self.config.add_view(self.git_proxy(), route_name='git_proxy')
         self.config.add_view(
             self.vcs_view, route_name='vcs', renderer='msgpack',
             custom_predicates=[self.is_vcs_view])
 
         self.config.add_view(self.hg_stream(), route_name='stream_hg')
         self.config.add_view(self.git_stream(), route_name='stream_git')
 
         def notfound(request):
             return {'status': '404 NOT FOUND'}
         self.config.add_notfound_view(notfound, renderer='json')
 
         self.config.add_view(self.handle_vcs_exception, context=Exception)
 
         self.config.add_tween(
             'vcsserver.tweens.RequestWrapperTween',
         )
 
     def wsgi_app(self):
         return self.config.make_wsgi_app()
 
     def vcs_view(self, request):
         remote = self._remotes[request.matchdict['backend']]
         payload = msgpack.unpackb(request.body, use_list=True)
         method = payload.get('method')
         params = payload.get('params')
         wire = params.get('wire')
         args = params.get('args')
         kwargs = params.get('kwargs')
         if wire:
             try:
                 wire['context'] = uuid.UUID(wire['context'])
             except KeyError:
                 pass
             args.insert(0, wire)
 
         log.debug('method called:%s with kwargs:%s', method, kwargs)
         try:
             resp = getattr(remote, method)(*args, **kwargs)
         except Exception as e:
             tb_info = traceback.format_exc()
 
             type_ = e.__class__.__name__
             if type_ not in self.ALLOWED_EXCEPTIONS:
                 type_ = None
 
             resp = {
                 'id': payload.get('id'),
                 'error': {
                     'message': e.message,
                     'traceback': tb_info,
                     'type': type_
                 }
             }
             try:
                 resp['error']['_vcs_kind'] = e._vcs_kind
             except AttributeError:
                 pass
         else:
             resp = {
                 'id': payload.get('id'),
                 'result': resp
             }
 
         return resp
 
     def status_view(self, request):
         return {'status': 'OK'}
 
     def service_view(self, request):
         import vcsserver
         import ConfigParser as configparser
 
         payload = msgpack.unpackb(request.body, use_list=True)
 
         try:
             path = self.global_config['__file__']
             config = configparser.ConfigParser()
             config.read(path)
             parsed_ini = config
             if parsed_ini.has_section('server:main'):
                 parsed_ini = dict(parsed_ini.items('server:main'))
         except Exception:
             log.exception('Failed to read .ini file for display')
             parsed_ini = {}
 
         resp = {
             'id': payload.get('id'),
             'result': dict(
                 version=vcsserver.__version__,
                 config=parsed_ini,
                 payload=payload,
             )
         }
         return resp
 
     def _msgpack_renderer_factory(self, info):
         def _render(value, system):
             value = msgpack.packb(value)
             request = system.get('request')
             if request is not None:
                 response = request.response
                 ct = response.content_type
                 if ct == response.default_content_type:
                     response.content_type = 'application/x-msgpack'
             return value
         return _render
 
     def hg_proxy(self):
         @wsgiapp
         def _hg_proxy(environ, start_response):
             app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
             return app(environ, start_response)
         return _hg_proxy
 
     def git_proxy(self):
         @wsgiapp
         def _git_proxy(environ, start_response):
             app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
             return app(environ, start_response)
         return _git_proxy
 
     def hg_stream(self):
         if self._use_echo_app:
             @wsgiapp
             def _hg_stream(environ, start_response):
                 app = EchoApp('fake_path', 'fake_name', None)
                 return app(environ, start_response)
             return _hg_stream
         else:
             @wsgiapp
             def _hg_stream(environ, start_response):
                 repo_path = environ['HTTP_X_RC_REPO_PATH']
                 repo_name = environ['HTTP_X_RC_REPO_NAME']
                 packed_config = base64.b64decode(
                     environ['HTTP_X_RC_REPO_CONFIG'])
                 config = msgpack.unpackb(packed_config)
                 app = scm_app.create_hg_wsgi_app(
                     repo_path, repo_name, config)
 
                 # Consitent path information for hgweb
                 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
                 environ['REPO_NAME'] = repo_name
                 return app(environ, ResponseFilter(start_response))
             return _hg_stream
 
     def git_stream(self):
         if self._use_echo_app:
             @wsgiapp
             def _git_stream(environ, start_response):
                 app = EchoApp('fake_path', 'fake_name', None)
                 return app(environ, start_response)
             return _git_stream
         else:
             @wsgiapp
             def _git_stream(environ, start_response):
                 repo_path = environ['HTTP_X_RC_REPO_PATH']
                 repo_name = environ['HTTP_X_RC_REPO_NAME']
                 packed_config = base64.b64decode(
                     environ['HTTP_X_RC_REPO_CONFIG'])
                 config = msgpack.unpackb(packed_config)
 
                 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
-                app = scm_app.create_git_wsgi_app(
-                    repo_path, repo_name, config)
+                content_type = environ.get('CONTENT_TYPE', '')
+
+                path = environ['PATH_INFO']
+                is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
+                log.debug(
+                    'LFS: Detecting if request `%s` is LFS server path based '
+                    'on content type:`%s`, is_lfs:%s',
+                    path, content_type, is_lfs_request)
+
+                if not is_lfs_request:
+                    # fallback detection by path
+                    if GIT_LFS_PROTO_PAT.match(path):
+                        is_lfs_request = True
+                    log.debug(
+                        'LFS: fallback detection by path of: `%s`, is_lfs:%s',
+                        path, is_lfs_request)
+
+                if is_lfs_request:
+                    app = scm_app.create_git_lfs_wsgi_app(
+                        repo_path, repo_name, config)
+                else:
+                    app = scm_app.create_git_wsgi_app(
+                        repo_path, repo_name, config)
                 return app(environ, start_response)
+
             return _git_stream
 
     def is_vcs_view(self, context, request):
         """
         View predicate that returns true if given backend is supported by
         defined remotes.
         """
         backend = request.matchdict.get('backend')
         return backend in self._remotes
 
     def handle_vcs_exception(self, exception, request):
         _vcs_kind = getattr(exception, '_vcs_kind', '')
         if _vcs_kind == 'repo_locked':
             # Get custom repo-locked status code if present.
             status_code = request.headers.get('X-RC-Locked-Status-Code')
             return HTTPRepoLocked(
                 title=exception.message, status_code=status_code)
 
         # Re-raise exception if we can not handle it.
         log.exception(
             'error occurred handling this request for path: %s', request.path)
         raise exception
 
 
 class ResponseFilter(object):
 
     def __init__(self, start_response):
         self._start_response = start_response
 
     def __call__(self, status, response_headers, exc_info=None):
         headers = tuple(
             (h, v) for h, v in response_headers
             if not wsgiref.util.is_hop_by_hop(h))
         return self._start_response(status, headers, exc_info)
 
 
 def main(global_config, **settings):
     if MercurialFactory:
         hgpatches.patch_largefiles_capabilities()
         hgpatches.patch_subrepo_type_mapping()
     app = HTTPApplication(settings=settings, global_config=global_config)
     return app.wsgi_app()

vcsserver/scm_app.py

@@ -1,174 +1,209 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2017 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
+import os
 import logging
-import os
 
 import mercurial
 import mercurial.error
 import mercurial.hgweb.common
 import mercurial.hgweb.hgweb_mod
 import mercurial.hgweb.protocol
 import webob.exc
 
-from vcsserver import pygrack, exceptions, settings
+from vcsserver import pygrack, exceptions, settings, git_lfs
 
 
 log = logging.getLogger(__name__)
 
 
 # propagated from mercurial documentation
 HG_UI_SECTIONS = [
     'alias', 'auth', 'decode/encode', 'defaults', 'diff', 'email', 'extensions',
     'format', 'merge-patterns', 'merge-tools', 'hooks', 'http_proxy', 'smtp',
     'patch', 'paths', 'profiling', 'server', 'trusted', 'ui', 'web',
 ]
 
 
 class HgWeb(mercurial.hgweb.hgweb_mod.hgweb):
     """Extension of hgweb that simplifies some functions."""
 
     def _get_view(self, repo):
         """Views are not supported."""
         return repo
 
     def loadsubweb(self):
         """The result is only used in the templater method which is not used."""
         return None
 
     def run(self):
         """Unused function so raise an exception if accidentally called."""
         raise NotImplementedError
 
     def templater(self, req):
         """Function used in an unreachable code path.
 
         This code is unreachable because we guarantee that the HTTP request,
         corresponds to a Mercurial command. See the is_hg method. So, we are
         never going to get a user-visible url.
         """
         raise NotImplementedError
 
     def archivelist(self, nodeid):
         """Unused function so raise an exception if accidentally called."""
         raise NotImplementedError
 
     def run_wsgi(self, req):
         """Check the request has a valid command, failing fast otherwise."""
         cmd = req.form.get('cmd', [''])[0]
         if not mercurial.hgweb.protocol.iscmd(cmd):
             req.respond(
                 mercurial.hgweb.common.ErrorResponse(
                     mercurial.hgweb.common.HTTP_BAD_REQUEST),
                 mercurial.hgweb.protocol.HGTYPE
             )
             return ['']
 
         return super(HgWeb, self).run_wsgi(req)
 
 
 def make_hg_ui_from_config(repo_config):
     baseui = mercurial.ui.ui()
 
     # clean the baseui object
     baseui._ocfg = mercurial.config.config()
     baseui._ucfg = mercurial.config.config()
     baseui._tcfg = mercurial.config.config()
 
     for section, option, value in repo_config:
         baseui.setconfig(section, option, value)
 
     # make our hgweb quiet so it doesn't print output
     baseui.setconfig('ui', 'quiet', 'true')
 
     return baseui
 
 
 def update_hg_ui_from_hgrc(baseui, repo_path):
     path = os.path.join(repo_path, '.hg', 'hgrc')
 
     if not os.path.isfile(path):
         log.debug('hgrc file is not present at %s, skipping...', path)
         return
     log.debug('reading hgrc from %s', path)
     cfg = mercurial.config.config()
     cfg.read(path)
     for section in HG_UI_SECTIONS:
         for k, v in cfg.items(section):
             log.debug('settings ui from file: [%s] %s=%s', section, k, v)
             baseui.setconfig(section, k, v)
 
 
 def create_hg_wsgi_app(repo_path, repo_name, config):
     """
     Prepares a WSGI application to handle Mercurial requests.
 
     :param config: is a list of 3-item tuples representing a ConfigObject
         (it is the serialized version of the config object).
     """
     log.debug("Creating Mercurial WSGI application")
 
     baseui = make_hg_ui_from_config(config)
     update_hg_ui_from_hgrc(baseui, repo_path)
 
     try:
         return HgWeb(repo_path, name=repo_name, baseui=baseui)
     except mercurial.error.RequirementError as exc:
         raise exceptions.RequirementException(exc)
 
 
 class GitHandler(object):
+    """
+    Handler for Git operations like push/pull etc
+    """
     def __init__(self, repo_location, repo_name, git_path, update_server_info,
                  extras):
         if not os.path.isdir(repo_location):
             raise OSError(repo_location)
         self.content_path = repo_location
         self.repo_name = repo_name
         self.repo_location = repo_location
         self.extras = extras
         self.git_path = git_path
         self.update_server_info = update_server_info
 
     def __call__(self, environ, start_response):
         app = webob.exc.HTTPNotFound()
         candidate_paths = (
             self.content_path, os.path.join(self.content_path, '.git'))
 
         for content_path in candidate_paths:
             try:
                 app = pygrack.GitRepository(
                     self.repo_name, content_path, self.git_path,
                     self.update_server_info, self.extras)
                 break
             except OSError:
                 continue
 
         return app(environ, start_response)
 
 
 def create_git_wsgi_app(repo_path, repo_name, config):
     """
     Creates a WSGI application to handle Git requests.
 
     :param config: is a dictionary holding the extras.
     """
     git_path = settings.GIT_EXECUTABLE
     update_server_info = config.pop('git_update_server_info')
     app = GitHandler(
         repo_path, repo_name, git_path, update_server_info, config)
 
     return app
+
+
+class GitLFSHandler(object):
+    """
+    Handler for Git LFS operations
+    """
+
+    def __init__(self, repo_location, repo_name, git_path, update_server_info,
+                 extras):
+        if not os.path.isdir(repo_location):
+            raise OSError(repo_location)
+        self.content_path = repo_location
+        self.repo_name = repo_name
+        self.repo_location = repo_location
+        self.extras = extras
+        self.git_path = git_path
+        self.update_server_info = update_server_info
+
+    def get_app(self, git_lfs_enabled, git_lfs_store_path):
+        app = git_lfs.create_app(git_lfs_enabled, git_lfs_store_path)
+        return app
+
+
+def create_git_lfs_wsgi_app(repo_path, repo_name, config):
+    git_path = settings.GIT_EXECUTABLE
+    update_server_info = config.pop('git_update_server_info')
+    git_lfs_enabled = config.pop('git_lfs_enabled')
+    git_lfs_store_path = config.pop('git_lfs_store_path')
+    app = GitLFSHandler(
+        repo_path, repo_name, git_path, update_server_info, config)
+
+    return app.get_app(git_lfs_enabled, git_lfs_store_path)

vcsserver/utils.py

@@ -1,57 +1,72 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2017 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 
+def safe_int(val, default=None):
+    """
+    Returns int() of val if val is not convertable to int use default
+    instead
 
-# TODO: johbo: That's a copy from rhodecode
+    :param val:
+    :param default:
+    """
+
+    try:
+        val = int(val)
+    except (ValueError, TypeError):
+        val = default
+
+    return val
+
+
 def safe_str(unicode_, to_encoding=['utf8']):
     """
     safe str function. Does few trick to turn unicode_ into string
 
     In case of UnicodeEncodeError, we try to return it with encoding detected
     by chardet library if it fails fallback to string with errors replaced
 
     :param unicode_: unicode to encode
     :rtype: str
     :returns: str object
     """
 
     # if it's not basestr cast to str
     if not isinstance(unicode_, basestring):
         return str(unicode_)
 
     if isinstance(unicode_, str):
         return unicode_
 
     if not isinstance(to_encoding, (list, tuple)):
         to_encoding = [to_encoding]
 
     for enc in to_encoding:
         try:
             return unicode_.encode(enc)
         except UnicodeEncodeError:
             pass
 
     try:
         import chardet
         encoding = chardet.detect(unicode_)['encoding']
         if encoding is None:
             raise UnicodeEncodeError()
 
         return unicode_.encode(encoding)
     except (ImportError, UnicodeEncodeError):
         return unicode_.encode(to_encoding[0], 'replace')