release: Merge default into stable for release preparation

vcsserver/git_lfs/__init__.py

@@ -0,0 +1,19 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+
+from app import create_app

vcsserver/git_lfs/app.py

@@ -0,0 +1,276 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import re
+import logging
+from wsgiref.util import FileWrapper
+
+import simplejson as json
+from pyramid.config import Configurator
+from pyramid.response import Response, FileIter
+from pyramid.httpexceptions import (
+    HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
+    HTTPUnprocessableEntity)
+
+from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
+from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
+from vcsserver.utils import safe_int
+
+log = logging.getLogger(__name__)
+
+
+GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ?
+GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
+
+
+def write_response_error(http_exception, text=None):
+    content_type = 'application/json'
+    _exception = http_exception(content_type=content_type)
+    _exception.content_type = content_type
+    if text:
+        _exception.body = json.dumps({'message': text})
+    log.debug('LFS: writing response of type %s to client with text:%s',
+              http_exception, text)
+    return _exception
+
+
+class AuthHeaderRequired(object):
+    """
+    Decorator to check if request has proper auth-header
+    """
+
+    def __call__(self, func):
+        return get_cython_compat_decorator(self.__wrapper, func)
+
+    def __wrapper(self, func, *fargs, **fkwargs):
+        request = fargs[1]
+        auth = request.authorization
+        if not auth:
+            return write_response_error(HTTPForbidden)
+        return func(*fargs[1:], **fkwargs)
+
+
+# views
+
+def lfs_objects(request):
+    # indicate not supported, V1 API
+    log.warning('LFS: v1 api not supported, reporting it back to client')
+    return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
+
+
+@AuthHeaderRequired()
+def lfs_objects_batch(request):
+    """
+    The client sends the following information to the Batch endpoint to transfer some objects:
+
+        operation - Should be download or upload.
+        transfers - An optional Array of String identifiers for transfer
+            adapters that the client has configured. If omitted, the basic
+            transfer adapter MUST be assumed by the server.
+        objects - An Array of objects to download.
+        oid - String OID of the LFS object.
+        size - Integer byte size of the LFS object. Must be at least zero.
+    """
+    auth = request.authorization
+
+    repo = request.matchdict.get('repo')
+
+    data = request.json
+    operation = data.get('operation')
+    if operation not in ('download', 'upload'):
+        log.debug('LFS: unsupported operation:%s', operation)
+        return write_response_error(
+            HTTPBadRequest, 'unsupported operation mode: `%s`' % operation)
+
+    if 'objects' not in data:
+        log.debug('LFS: missing objects data')
+        return write_response_error(
+            HTTPBadRequest, 'missing objects data')
+
+    log.debug('LFS: handling operation of type: %s', operation)
+
+    objects = []
+    for o in data['objects']:
+        try:
+            oid = o['oid']
+            obj_size = o['size']
+        except KeyError:
+            log.exception('LFS, failed to extract data')
+            return write_response_error(
+                HTTPBadRequest, 'unsupported data in objects')
+
+        obj_data = {'oid': oid}
+
+        obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid)
+        obj_verify_href = request.route_url('lfs_objects_verify', repo=repo)
+        store = LFSOidStore(
+            oid, repo, store_location=request.registry.git_lfs_store_path)
+        handler = OidHandler(
+            store, repo, auth, oid, obj_size, obj_data,
+            obj_href, obj_verify_href)
+
+        # this verifies also OIDs
+        actions, errors = handler.exec_operation(operation)
+        if errors:
+            log.warning('LFS: got following errors: %s', errors)
+            obj_data['errors'] = errors
+
+        if actions:
+            obj_data['actions'] = actions
+
+        obj_data['size'] = obj_size
+        obj_data['authenticated'] = True
+        objects.append(obj_data)
+
+    result = {'objects': objects, 'transfer': 'basic'}
+    log.debug('LFS Response %s', safe_result(result))
+
+    return result
+
+
+def lfs_objects_oid_upload(request):
+    repo = request.matchdict.get('repo')
+    oid = request.matchdict.get('oid')
+    store = LFSOidStore(
+        oid, repo, store_location=request.registry.git_lfs_store_path)
+    engine = store.get_engine(mode='wb')
+    log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
+    with engine as f:
+        for chunk in FileWrapper(request.body_file_seekable, blksize=64 * 1024):
+            f.write(chunk)
+
+    return {'upload': 'ok'}
+
+
+def lfs_objects_oid_download(request):
+    repo = request.matchdict.get('repo')
+    oid = request.matchdict.get('oid')
+
+    store = LFSOidStore(
+        oid, repo, store_location=request.registry.git_lfs_store_path)
+    if not store.has_oid():
+        log.debug('LFS: oid %s does not exists in store', oid)
+        return write_response_error(
+            HTTPNotFound, 'requested file with oid `%s` not found in store' % oid)
+
+    # TODO(marcink): support range header ?
+    # Range: bytes=0-, `bytes=(\d+)\-.*`
+
+    f = open(store.oid_path, 'rb')
+    response = Response(
+        content_type='application/octet-stream', app_iter=FileIter(f))
+    response.headers.add('X-RC-LFS-Response-Oid', str(oid))
+    return response
+
+
+def lfs_objects_verify(request):
+    repo = request.matchdict.get('repo')
+
+    data = request.json
+    oid = data.get('oid')
+    size = safe_int(data.get('size'))
+
+    if not (oid and size):
+        return write_response_error(
+            HTTPBadRequest, 'missing oid and size in request data')
+
+    store = LFSOidStore(
+        oid, repo, store_location=request.registry.git_lfs_store_path)
+    if not store.has_oid():
+        log.debug('LFS: oid %s does not exists in store', oid)
+        return write_response_error(
+            HTTPNotFound, 'oid `%s` does not exists in store' % oid)
+
+    store_size = store.size_oid()
+    if store_size != size:
+        msg = 'requested file size mismatch store size:%s requested:%s' % (
+            store_size, size)
+        return write_response_error(
+            HTTPUnprocessableEntity, msg)
+
+    return {'message': {'size': 'ok', 'in_store': 'ok'}}
+
+
+def lfs_objects_lock(request):
+    return write_response_error(
+        HTTPNotImplemented, 'GIT LFS locking api not supported')
+
+
+def not_found(request):
+    return write_response_error(
+        HTTPNotFound, 'request path not found')
+
+
+def lfs_disabled(request):
+    return write_response_error(
+        HTTPNotImplemented, 'GIT LFS disabled for this repo')
+
+
+def git_lfs_app(config):
+
+    # v1 API deprecation endpoint
+    config.add_route('lfs_objects',
+                     '/{repo:.*?[^/]}/info/lfs/objects')
+    config.add_view(lfs_objects, route_name='lfs_objects',
+                    request_method='POST', renderer='json')
+
+    # locking API
+    config.add_route('lfs_objects_lock',
+                     '/{repo:.*?[^/]}/info/lfs/locks')
+    config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
+                    request_method=('POST', 'GET'), renderer='json')
+
+    config.add_route('lfs_objects_lock_verify',
+                     '/{repo:.*?[^/]}/info/lfs/locks/verify')
+    config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
+                    request_method=('POST', 'GET'), renderer='json')
+
+    # batch API
+    config.add_route('lfs_objects_batch',
+                     '/{repo:.*?[^/]}/info/lfs/objects/batch')
+    config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
+                    request_method='POST', renderer='json')
+
+    # oid upload/download API
+    config.add_route('lfs_objects_oid',
+                     '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
+    config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
+                    request_method='PUT', renderer='json')
+    config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
+                    request_method='GET', renderer='json')
+
+    # verification API
+    config.add_route('lfs_objects_verify',
+                     '/{repo:.*?[^/]}/info/lfs/verify')
+    config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
+                    request_method='POST', renderer='json')
+
+    # not found handler for API
+    config.add_notfound_view(not_found, renderer='json')
+
+
+def create_app(git_lfs_enabled, git_lfs_store_path):
+    config = Configurator()
+    if git_lfs_enabled:
+        config.include(git_lfs_app)
+        config.registry.git_lfs_store_path = git_lfs_store_path
+    else:
+        # not found handler for API, reporting disabled LFS support
+        config.add_notfound_view(lfs_disabled, renderer='json')
+
+    app = config.make_wsgi_app()
+    return app

vcsserver/git_lfs/lib.py

@@ -0,0 +1,166 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import os
+import shutil
+import logging
+from collections import OrderedDict
+
+log = logging.getLogger(__name__)
+
+
+class OidHandler(object):
+
+    def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
+                 obj_verify_href=None):
+        self.current_store = store
+        self.repo_name = repo_name
+        self.auth = auth
+        self.oid = oid
+        self.obj_size = obj_size
+        self.obj_data = obj_data
+        self.obj_href = obj_href
+        self.obj_verify_href = obj_verify_href
+
+    def get_store(self, mode=None):
+        return self.current_store
+
+    def get_auth(self):
+        """returns auth header for re-use in upload/download"""
+        return " ".join(self.auth)
+
+    def download(self):
+
+        store = self.get_store()
+        response = None
+        has_errors = None
+
+        if not store.has_oid():
+            # error reply back to client that something is wrong with dl
+            err_msg = 'object: {} does not exist in store'.format(store.oid)
+            has_errors = OrderedDict(
+                error=OrderedDict(
+                    code=404,
+                    message=err_msg
+                )
+            )
+
+        download_action = OrderedDict(
+            href=self.obj_href,
+            header=OrderedDict([("Authorization", self.get_auth())])
+        )
+        if not has_errors:
+            response = OrderedDict(download=download_action)
+        return response, has_errors
+
+    def upload(self, skip_existing=True):
+        """
+        Write upload action for git-lfs server
+        """
+
+        store = self.get_store()
+        response = None
+        has_errors = None
+
+        # verify if we have the OID before, if we do, reply with empty
+        if store.has_oid():
+            log.debug('LFS: store already has oid %s', store.oid)
+            if skip_existing:
+                log.debug('LFS: skipping further action as oid is existing')
+                return response, has_errors
+
+        upload_action = OrderedDict(
+            href=self.obj_href,
+            header=OrderedDict([("Authorization", self.get_auth())])
+        )
+        if not has_errors:
+            response = OrderedDict(upload=upload_action)
+            # if specified in handler, return the verification endpoint
+            if self.obj_verify_href:
+                verify_action = OrderedDict(
+                    href=self.obj_verify_href,
+                    header=OrderedDict([("Authorization", self.get_auth())])
+                )
+                response['verify'] = verify_action
+        return response, has_errors
+
+    def exec_operation(self, operation, *args, **kwargs):
+        handler = getattr(self, operation)
+        log.debug('LFS: handling request using %s handler', handler)
+        return handler(*args, **kwargs)
+
+
+class LFSOidStore(object):
+
+    def __init__(self, oid, repo, store_location=None):
+        self.oid = oid
+        self.repo = repo
+        self.store_path = store_location or self.get_default_store()
+        self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
+        self.oid_path = os.path.join(self.store_path, oid)
+        self.fd = None
+
+    def get_engine(self, mode):
+        """
+        engine = .get_engine(mode='wb')
+        with engine as f:
+            f.write('...')
+        """
+
+        class StoreEngine(object):
+            def __init__(self, mode, store_path, oid_path, tmp_oid_path):
+                self.mode = mode
+                self.store_path = store_path
+                self.oid_path = oid_path
+                self.tmp_oid_path = tmp_oid_path
+
+            def __enter__(self):
+                if not os.path.isdir(self.store_path):
+                    os.makedirs(self.store_path)
+
+                # TODO(marcink): maybe write metadata here with size/oid ?
+                fd = open(self.tmp_oid_path, self.mode)
+                self.fd = fd
+                return fd
+
+            def __exit__(self, exc_type, exc_value, traceback):
+                # close tmp file, and rename to final destination
+                self.fd.close()
+                shutil.move(self.tmp_oid_path, self.oid_path)
+
+        return StoreEngine(
+            mode, self.store_path, self.oid_path, self.tmp_oid_path)
+
+    def get_default_store(self):
+        """
+        Default store, consistent with defaults of Mercurial large files store
+        which is /home/username/.cache/largefiles
+        """
+        user_home = os.path.expanduser("~")
+        return os.path.join(user_home, '.cache', 'lfs-store')
+
+    def has_oid(self):
+        return os.path.exists(os.path.join(self.store_path, self.oid))
+
+    def size_oid(self):
+        size = -1
+
+        if self.has_oid():
+            oid = os.path.join(self.store_path, self.oid)
+            size = os.stat(oid).st_size
+
+        return size

vcsserver/git_lfs/tests/__init__.py

@@ -0,0 +1,16 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA

vcsserver/git_lfs/tests/test_lfs_app.py

@@ -0,0 +1,237 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import os
+import pytest
+from webtest.app import TestApp as WebObTestApp
+
+from vcsserver.git_lfs.app import create_app
+
+
[e-mail](scope='function')
+def git_lfs_app(tmpdir):
+    custom_app = WebObTestApp(create_app(
+        git_lfs_enabled=True, git_lfs_store_path=str(tmpdir)))
+    custom_app._store = str(tmpdir)
+    return custom_app
+
+
[e-mail]()
+def http_auth():
+    return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
+
+
+class TestLFSApplication(object):
+
+    def test_app_wrong_path(self, git_lfs_app):
+        git_lfs_app.get('/repo/info/lfs/xxx', status=404)
+
+    def test_app_deprecated_endpoint(self, git_lfs_app):
+        response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
+        assert response.status_code == 501
+        assert response.json == {u'message': u'LFS: v1 api not supported'}
+
+    def test_app_lock_verify_api_not_available(self, git_lfs_app):
+        response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
+        assert response.status_code == 501
+        assert response.json == {
+            u'message': u'GIT LFS locking api not supported'}
+
+    def test_app_lock_api_not_available(self, git_lfs_app):
+        response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
+        assert response.status_code == 501
+        assert response.json == {
+            u'message': u'GIT LFS locking api not supported'}
+
+    def test_app_batch_api_missing_auth(self, git_lfs_app,):
+        git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params={}, status=403)
+
+    def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params={}, status=400,
+            extra_environ=http_auth)
+        assert response.json == {
+            u'message': u'unsupported operation mode: `None`'}
+
+    def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params={'operation': 'download'},
+            status=400, extra_environ=http_auth)
+        assert response.json == {
+            u'message': u'missing objects data'}
+
+    def test_app_batch_api_unsupported_data_in_objects(
+            self, git_lfs_app, http_auth):
+        params = {'operation': 'download',
+                  'objects': [{}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params, status=400,
+            extra_environ=http_auth)
+        assert response.json == {
+            u'message': u'unsupported data in objects'}
+
+    def test_app_batch_api_download_missing_object(
+            self, git_lfs_app, http_auth):
+        params = {'operation': 'download',
+                  'objects': [{'oid': '123', 'size': '1024'}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params,
+            extra_environ=http_auth)
+
+        expected_objects = [
+            {u'authenticated': True,
+             u'errors': {u'error': {
+                 u'code': 404,
+                 u'message': u'object: 123 does not exist in store'}},
+             u'oid': u'123',
+             u'size': u'1024'}
+        ]
+        assert response.json == {
+            'objects': expected_objects, 'transfer': 'basic'}
+
+    def test_app_batch_api_download(self, git_lfs_app, http_auth):
+        oid = '456'
+        oid_path = os.path.join(git_lfs_app._store, oid)
+        if not os.path.isdir(os.path.dirname(oid_path)):
+            os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        params = {'operation': 'download',
+                  'objects': [{'oid': oid, 'size': '1024'}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params,
+            extra_environ=http_auth)
+
+        expected_objects = [
+            {u'authenticated': True,
+             u'actions': {
+                 u'download': {
+                     u'header': {u'Authorization': u'Basic XXXXX'},
+                     u'href': u'http://localhost/repo/info/lfs/objects/456'},
+             },
+             u'oid': u'456',
+             u'size': u'1024'}
+        ]
+        assert response.json == {
+            'objects': expected_objects, 'transfer': 'basic'}
+
+    def test_app_batch_api_upload(self, git_lfs_app, http_auth):
+        params = {'operation': 'upload',
+                  'objects': [{'oid': '123', 'size': '1024'}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params,
+            extra_environ=http_auth)
+        expected_objects = [
+            {u'authenticated': True,
+             u'actions': {
+                 u'upload': {
+                     u'header': {u'Authorization': u'Basic XXXXX'},
+                     u'href': u'http://localhost/repo/info/lfs/objects/123'},
+                 u'verify': {
+                     u'header': {u'Authorization': u'Basic XXXXX'},
+                     u'href': u'http://localhost/repo/info/lfs/verify'}
+             },
+             u'oid': u'123',
+             u'size': u'1024'}
+        ]
+        assert response.json == {
+            'objects': expected_objects, 'transfer': 'basic'}
+
+    def test_app_verify_api_missing_data(self, git_lfs_app):
+        params = {'oid': 'missing',}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params,
+            status=400)
+
+        assert response.json == {
+            u'message': u'missing oid and size in request data'}
+
+    def test_app_verify_api_missing_obj(self, git_lfs_app):
+        params = {'oid': 'missing', 'size': '1024'}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params,
+            status=404)
+
+        assert response.json == {
+            u'message': u'oid `missing` does not exists in store'}
+
+    def test_app_verify_api_size_mismatch(self, git_lfs_app):
+        oid = 'existing'
+        oid_path = os.path.join(git_lfs_app._store, oid)
+        if not os.path.isdir(os.path.dirname(oid_path)):
+            os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        params = {'oid': oid, 'size': '1024'}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params, status=422)
+
+        assert response.json == {
+            u'message': u'requested file size mismatch '
+                        u'store size:11 requested:1024'}
+
+    def test_app_verify_api(self, git_lfs_app):
+        oid = 'existing'
+        oid_path = os.path.join(git_lfs_app._store, oid)
+        if not os.path.isdir(os.path.dirname(oid_path)):
+            os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        params = {'oid': oid, 'size': 11}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params)
+
+        assert response.json == {
+            u'message': {u'size': u'ok', u'in_store': u'ok'}}
+
+    def test_app_download_api_oid_not_existing(self, git_lfs_app):
+        oid = 'missing'
+
+        response = git_lfs_app.get(
+            '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
+
+        assert response.json == {
+            u'message': u'requested file with oid `missing` not found in store'}
+
+    def test_app_download_api(self, git_lfs_app):
+        oid = 'existing'
+        oid_path = os.path.join(git_lfs_app._store, oid)
+        if not os.path.isdir(os.path.dirname(oid_path)):
+            os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        response = git_lfs_app.get(
+            '/repo/info/lfs/objects/{oid}'.format(oid=oid))
+        assert response
+
+    def test_app_upload(self, git_lfs_app):
+        oid = 'uploaded'
+
+        response = git_lfs_app.put(
+            '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
+
+        assert response.json == {u'upload': u'ok'}
+
+        # verify that we actually wrote that OID
+        oid_path = os.path.join(git_lfs_app._store, oid)
+        assert os.path.isfile(oid_path)
+        assert 'CONTENT' == open(oid_path).read()

vcsserver/git_lfs/tests/test_lib.py

@@ -0,0 +1,123 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import os
+import pytest
+from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
+
+
[e-mail]()
+def lfs_store(tmpdir):
+    repo = 'test'
+    oid = '123456789'
+    store = LFSOidStore(oid=oid, repo=repo, store_location=str(tmpdir))
+    return store
+
+
[e-mail]()
+def oid_handler(lfs_store):
+    store = lfs_store
+    repo = store.repo
+    oid = store.oid
+
+    oid_handler = OidHandler(
+        store=store, repo_name=repo, auth=('basic', 'xxxx'),
+        oid=oid,
+        obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
+        obj_verify_href='http://localhost/verify')
+    return oid_handler
+
+
+class TestOidHandler(object):
+
+    @pytest.mark.parametrize('exec_action', [
+        'download',
+        'upload',
+    ])
+    def test_exec_action(self, exec_action, oid_handler):
+        handler = oid_handler.exec_operation(exec_action)
+        assert handler
+
+    def test_exec_action_undefined(self, oid_handler):
+        with pytest.raises(AttributeError):
+            oid_handler.exec_operation('wrong')
+
+    def test_download_oid_not_existing(self, oid_handler):
+        response, has_errors = oid_handler.exec_operation('download')
+
+        assert response is None
+        assert has_errors['error'] == {
+            'code': 404,
+            'message': 'object: 123456789 does not exist in store'}
+
+    def test_download_oid(self, oid_handler):
+        store = oid_handler.get_store()
+        if not os.path.isdir(os.path.dirname(store.oid_path)):
+            os.makedirs(os.path.dirname(store.oid_path))
+
+        with open(store.oid_path, 'wb') as f:
+            f.write('CONTENT')
+
+        response, has_errors = oid_handler.exec_operation('download')
+
+        assert has_errors is None
+        assert response['download'] == {
+            'header': {'Authorization': 'basic xxxx'},
+            'href': 'http://localhost/handle_oid'
+        }
+
+    def test_upload_oid_that_exists(self, oid_handler):
+        store = oid_handler.get_store()
+        if not os.path.isdir(os.path.dirname(store.oid_path)):
+            os.makedirs(os.path.dirname(store.oid_path))
+
+        with open(store.oid_path, 'wb') as f:
+            f.write('CONTENT')
+
+        response, has_errors = oid_handler.exec_operation('upload')
+        assert has_errors is None
+        assert response is None
+
+    def test_upload_oid(self, oid_handler):
+        response, has_errors = oid_handler.exec_operation('upload')
+        assert has_errors is None
+        assert response['upload'] == {
+            'header': {'Authorization': 'basic xxxx'},
+            'href': 'http://localhost/handle_oid'
+        }
+
+
+class TestLFSStore(object):
+    def test_write_oid(self, lfs_store):
+        oid_location = lfs_store.oid_path
+
+        assert not os.path.isfile(oid_location)
+
+        engine = lfs_store.get_engine(mode='wb')
+        with engine as f:
+            f.write('CONTENT')
+
+        assert os.path.isfile(oid_location)
+
+    def test_detect_has_oid(self, lfs_store):
+
+        assert lfs_store.has_oid() is False
+        engine = lfs_store.get_engine(mode='wb')
+        with engine as f:
+            f.write('CONTENT')
+
+        assert lfs_store.has_oid() is True
  No newline at end of file

vcsserver/git_lfs/utils.py

@@ -0,0 +1,50 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+import copy
+from functools import wraps
+
+
+def get_cython_compat_decorator(wrapper, func):
+    """
+    Creates a cython compatible decorator. The previously used
+    decorator.decorator() function seems to be incompatible with cython.
+
+    :param wrapper: __wrapper method of the decorator class
+    :param func: decorated function
+    """
+    @wraps(func)
+    def local_wrapper(*args, **kwds):
+        return wrapper(func, *args, **kwds)
+    local_wrapper.__wrapped__ = func
+    return local_wrapper
+
+
+def safe_result(result):
+    """clean result for better representation in logs"""
+    clean_copy = copy.deepcopy(result)
+
+    try:
+        if 'objects' in clean_copy:
+            for oid_data in clean_copy['objects']:
+                if 'actions' in oid_data:
+                    for action_name, data in oid_data['actions'].items():
+                        if 'header' in data:
+                            data['header'] = {'Authorization': '*****'}
+    except Exception:
+        return result
+
+    return clean_copy

.bumpversion.cfg

@@ -1,5 +1,5 b''
 [bumpversion]
-current_version = 4.6.1
+current_version = 4.7.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:vcsserver/VERSION]

.release.cfg

@@ -5,12 +5,10 b' done = false'
 done = true
 
 [task:fixes_on_stable]
-done = true
 
 [task:pip2nix_generated]
-done = true
 
 [release]
-state = prepared
-version = 4.6.1
+state = in_progress
+version = 4.7.0
 

configs/production_http.ini

@@ -15,10 +15,8 b' port = 9900'
 ##########################
 ## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini
 use = egg:gunicorn#main
-## Sets the number of process workers. You must set `instance_id = *`
-## when this option is set to more than one worker, recommended
+## Sets the number of process workers. Recommended
 ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
-## The `instance_id = *` must be set in the [app:main] section below
 workers = 2
 ## process name
 proc_name = rhodecode_vcsserver

default.nix

@@ -89,6 +89,7 b' let'
       name = "rhodecode-vcsserver-${version}";
       releaseName = "RhodeCodeVCSServer-${version}";
       src = rhodecode-vcsserver-src;
+      dontStrip = true; # prevent strip, we don't need it.
 
       propagatedBuildInputs = attrs.propagatedBuildInputs ++ ([
         pkgs.git

pkgs/python-packages.nix

@@ -159,13 +159,13 b''
     };
   };
   decorator = super.buildPythonPackage {
-    name = "decorator-4.0.10";
+    name = "decorator-4.0.11";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/13/8a/4eed41e338e8dcc13ca41c94b142d4d20c0de684ee5065523fee406ce76f/decorator-4.0.10.tar.gz";
-      md5 = "434b57fdc3230c500716c5aff8896100";
+      url = "https://pypi.python.org/packages/cc/ac/5a16f1fc0506ff72fcc8fd4e858e3a1c231f224ab79bb7c4c9b2094cc570/decorator-4.0.11.tar.gz";
+      md5 = "73644c8f0bd4983d1b6a34b49adec0ae";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "new BSD License"; } ];
@@ -315,13 +315,13 b''
     };
   };
   mercurial = super.buildPythonPackage {
-    name = "mercurial-4.0.2";
+    name = "mercurial-4.1.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/85/1b/0296aacd697228974a473d2508f013532f987ed6b1bacfe5abd6d5be6332/mercurial-4.0.2.tar.gz";
-      md5 = "fa72a08e2723e4fa2a21c4e66437f3fa";
+      url = "https://pypi.python.org/packages/88/c1/f0501fd67f5e69346da41ee0bd7b2619ce4bbc9854bb645074c418b9941f/mercurial-4.1.2.tar.gz";
+      md5 = "934c99808bdc8385e074b902d59b0d93";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl1 pkgs.lib.licenses.gpl2Plus ];
@@ -445,13 +445,13 b''
     };
   };
   pyramid = super.buildPythonPackage {
-    name = "pyramid-1.6.1";
+    name = "pyramid-1.7.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools WebOb repoze.lru zope.interface zope.deprecation venusian translationstring PasteDeploy];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/30/b3/fcc4a2a4800cbf21989e00454b5828cf1f7fe35c63e0810b350e56d4c475/pyramid-1.6.1.tar.gz";
-      md5 = "b18688ff3cc33efdbb098a35b45dd122";
+      url = "https://pypi.python.org/packages/33/91/55f5c661f8923902cd1f68d75f2b937c45e7682857356cf18f0be5493899/pyramid-1.7.4.tar.gz";
+      md5 = "6ef1dfdcff9136d04490410757c4c446";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
@@ -588,7 +588,7 b''
     };
   };
   rhodecode-vcsserver = super.buildPythonPackage {
-    name = "rhodecode-vcsserver-4.6.1";
+    name = "rhodecode-vcsserver-4.7.0";
     buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage configobj];
     doCheck = true;
     propagatedBuildInputs = with self; [Beaker configobj dulwich hgsubversion infrae.cache mercurial msgpack-python pyramid pyramid-jinja2 pyramid-mako repoze.lru simplejson subprocess32 subvertpy six translationstring WebOb wheel zope.deprecation zope.interface ipdb ipython gevent greenlet gunicorn waitress Pyro4 serpent pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage];

requirements.txt

@@ -1,15 +1,16 b''
-# core
+## core
 setuptools==30.1.0
 
 Beaker==1.7.0
 configobj==5.0.6
+decorator==4.0.11
 dulwich==0.13.0
 hgsubversion==1.8.6
 infrae.cache==1.0.1
-mercurial==4.0.2
+mercurial==4.1.2
 msgpack-python==0.4.8
-pyramid==1.6.1
 pyramid-jinja2==2.5
+pyramid==1.7.4
 pyramid-mako==1.0.2
 repoze.lru==0.6
 simplejson==3.7.2
@@ -28,7 +29,6 b' zope.interface==4.1.3'
 ## debug
 ipdb==0.10.1
 ipython==5.1.0
-
 # http servers
 gevent==1.1.2
 greenlet==0.4.10

vcsserver/VERSION

@@ -1,1 +1,1 b''
-4.6.1
  No newline at end of file
+4.7.0
  No newline at end of file

vcsserver/base.py

@@ -15,6 +15,8 b''
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
+import sys
+import traceback
 import logging
 import urlparse
 
@@ -80,3 +82,17 b' def obfuscate_qs(query_string):'
 
     return '&'.join('{}{}'.format(
         k, '={}'.format(v) if v else '') for k, v in parsed)
+
+
+def raise_from_original(new_type):
+    """
+    Raise a new exception type with original args and traceback.
+    """
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+
+    traceback.format_exception(exc_type, exc_value, exc_traceback)
+
+    try:
+        raise new_type(*exc_value.args), None, exc_traceback
+    finally:
+        del exc_traceback

vcsserver/git.py

@@ -35,10 +35,10 b' from dulwich.server import update_server'
 
 from vcsserver import exceptions, settings, subprocessio
 from vcsserver.utils import safe_str
-from vcsserver.base import RepoFactory, obfuscate_qs
+from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
 from vcsserver.hgcompat import (
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
-
+from vcsserver.git_lfs.lib import LFSOidStore
 
 DIR_STAT = stat.S_IFDIR
 FILE_MODE = stat.S_IFMT
@@ -58,6 +58,14 b' def reraise_safe_exceptions(func):'
             raise exceptions.LookupException(e.message)
         except (HangupException, UnexpectedCommandError) as e:
             raise exceptions.VcsException(e.message)
+        except Exception as e:
+            # NOTE(marcink): becuase of how dulwich handles some exceptions
+            # (KeyError on empty repos), we cannot track this and catch all
+            # exceptions, it's an exceptions from other handlers
+            #if not hasattr(e, '_vcs_kind'):
+                #log.exception("Unhandled exception in git remote call")
+                #raise_from_original(exceptions.UnhandledException)
+            raise
     return wrapper
 
 
@@ -97,6 +105,11 b' class GitRemote(object):'
             "_commit": self.revision,
         }
 
+    def _wire_to_config(self, wire):
+        if 'config' in wire:
+            return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
+        return {}
+
     def _assign_ref(self, wire, ref, commit_id):
         repo = self._factory.repo(wire)
         repo[ref] = commit_id
@@ -133,6 +146,56 b' class GitRemote(object):'
         blob = repo[sha]
         return blob.raw_length()
 
+    def _parse_lfs_pointer(self, raw_content):
+
+        spec_string = 'version https://git-lfs.github.com/spec'
+        if raw_content and raw_content.startswith(spec_string):
+            pattern = re.compile(r"""
+            (?:\n)?
+            ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
+            ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
+            ^size[ ](?P<oid_size>[0-9]+)\n
+            (?:\n)?
+            """, re.VERBOSE | re.MULTILINE)
+            match = pattern.match(raw_content)
+            if match:
+                return match.groupdict()
+
+        return {}
+
+    @reraise_safe_exceptions
+    def is_large_file(self, wire, sha):
+        repo = self._factory.repo(wire)
+        blob = repo[sha]
+        return self._parse_lfs_pointer(blob.as_raw_string())
+
+    @reraise_safe_exceptions
+    def in_largefiles_store(self, wire, oid):
+        repo = self._factory.repo(wire)
+        conf = self._wire_to_config(wire)
+
+        store_location = conf.get('vcs_git_lfs_store_location')
+        if store_location:
+            repo_name = repo.path
+            store = LFSOidStore(
+                oid=oid, repo=repo_name, store_location=store_location)
+            return store.has_oid()
+
+        return False
+
+    @reraise_safe_exceptions
+    def store_path(self, wire, oid):
+        repo = self._factory.repo(wire)
+        conf = self._wire_to_config(wire)
+
+        store_location = conf.get('vcs_git_lfs_store_location')
+        if store_location:
+            repo_name = repo.path
+            store = LFSOidStore(
+                oid=oid, repo=repo_name, store_location=store_location)
+            return store.oid_path
+        raise ValueError('Unable to fetch oid with path {}'.format(oid))
+
     @reraise_safe_exceptions
     def bulk_request(self, wire, rev, pre_load):
         result = {}

vcsserver/hg.py

@@ -18,7 +18,6 b''
 import io
 import logging
 import stat
-import sys
 import urllib
 import urllib2
 
@@ -26,9 +25,10 b' from hgext import largefiles, rebase'
 from hgext.strip import strip as hgext_strip
 from mercurial import commands
 from mercurial import unionrepo
+from mercurial import verify
 
 from vcsserver import exceptions
-from vcsserver.base import RepoFactory, obfuscate_qs
+from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
 from vcsserver.hgcompat import (
     archival, bin, clone, config as hgconfig, diffopts, hex,
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
@@ -91,17 +91,6 b' def reraise_safe_exceptions(func):'
     return wrapper
 
 
-def raise_from_original(new_type):
-    """
-    Raise a new exception type with original args and traceback.
-    """
-    _, original, traceback = sys.exc_info()
-    try:
-        raise new_type(*original.args), None, traceback
-    finally:
-        del traceback
-
-
 class MercurialFactory(RepoFactory):
 
     def _create_config(self, config, hooks=True):
@@ -496,7 +485,7 b' class HgRemote(object):'
         return largefiles.lfutil.isstandin(path)
 
     @reraise_safe_exceptions
-    def in_store(self, wire, sha):
+    def in_largefiles_store(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.instore(repo, sha)
 
@@ -598,6 +587,21 b' class HgRemote(object):'
             repo.baseui, repo, ctx.node(), update=update, backup=backup)
 
     @reraise_safe_exceptions
+    def verify(self, wire,):
+        repo = self._factory.repo(wire)
+        baseui = self._factory._create_config(wire['config'])
+        baseui.setconfig('ui', 'quiet', 'false')
+        output = io.BytesIO()
+
+        def write(data, **unused_kwargs):
+            output.write(data)
+        baseui.write = write
+
+        repo.ui = baseui
+        verify.verify(repo)
+        return output.getvalue()
+
+    @reraise_safe_exceptions
     def tag(self, wire, name, revision, message, local, user,
             tag_time, tag_timezone):
         repo = self._factory.repo(wire)
@@ -674,12 +678,10 b' class HgRemote(object):'
     @reraise_safe_exceptions
     def ancestor(self, wire, revision1, revision2):
         repo = self._factory.repo(wire)
-        baseui = self._factory._create_config(wire['config'])
-        output = io.BytesIO()
-        baseui.write = output.write
-        commands.debugancestor(baseui, repo, revision1, revision2)
-
-        return output.getvalue()
+        changelog = repo.changelog
+        lookup = repo.lookup
+        a = changelog.ancestor(lookup(revision1), lookup(revision2))
+        return hex(a)
 
     @reraise_safe_exceptions
     def push(self, wire, revisions, dest_path, hooks=True,

vcsserver/hooks.py

@@ -17,12 +17,14 b''
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
+import io
+import sys
+import json
+import logging
 import collections
 import importlib
-import io
-import json
 import subprocess
-import sys
+
 from httplib import HTTPConnection
 
 
@@ -33,6 +35,8 b' import simplejson as json'
 
 from vcsserver import exceptions
 
+log = logging.getLogger(__name__)
+
 
 class HooksHttpClient(object):
     connection = None
@@ -105,6 +109,11 b' class GitMessageWriter(RemoteMessageWrit'
 
 def _handle_exception(result):
     exception_class = result.get('exception')
+    exception_traceback = result.get('exception_traceback')
+
+    if exception_traceback:
+        log.error('Got traceback from remote call:%s', exception_traceback)
+
     if exception_class == 'HTTPLockedRC':
         raise exceptions.RepositoryLockedException(*result['exception_args'])
     elif exception_class == 'RepositoryError':
@@ -152,26 +161,42 b' def post_pull(ui, repo, **kwargs):'
     return _call_hook('post_pull', _extras_from_ui(ui), HgMessageWriter(ui))
 
 
-def pre_push(ui, repo, **kwargs):
-    return _call_hook('pre_push', _extras_from_ui(ui), HgMessageWriter(ui))
+def pre_push(ui, repo, node=None, **kwargs):
+    extras = _extras_from_ui(ui)
+
+    rev_data = []
+    if node and kwargs.get('hooktype') == 'pretxnchangegroup':
+        branches = collections.defaultdict(list)
+        for commit_id, branch in _rev_range_hash(repo, node, with_branch=True):
+            branches[branch].append(commit_id)
+
+        for branch, commits in branches.iteritems():
+            old_rev = kwargs.get('node_last') or commits[0]
+            rev_data.append({
+                'old_rev': old_rev,
+                'new_rev': commits[-1],
+                'ref': '',
+                'type': 'branch',
+                'name': branch,
+            })
+
+    extras['commit_ids'] = rev_data
+    return _call_hook('pre_push', extras, HgMessageWriter(ui))
 
 
-# N.B.(skreft): the two functions below were taken and adapted from
-# rhodecode.lib.vcs.remote.handle_git_pre_receive
-# They are required to compute the commit_ids
-def _get_revs(repo, rev_opt):
-    revs = [rev for rev in mercurial.scmutil.revrange(repo, rev_opt)]
-    if len(revs) == 0:
-        return (mercurial.node.nullrev, mercurial.node.nullrev)
+def _rev_range_hash(repo, node, with_branch=False):
 
-    return max(revs), min(revs)
-
+    commits = []
+    for rev in xrange(repo[node], len(repo)):
+        ctx = repo[rev]
+        commit_id = mercurial.node.hex(ctx.node())
+        branch = ctx.branch()
+        if with_branch:
+            commits.append((commit_id, branch))
+        else:
+            commits.append(commit_id)
 
-def _rev_range_hash(repo, node):
-    stop, start = _get_revs(repo, [node + ':'])
-    revs = [mercurial.node.hex(repo[r].node()) for r in xrange(start, stop + 1)]
-
-    return revs
+    return commits
 
 
 def post_push(ui, repo, node, **kwargs):
@@ -257,7 +282,23 b' def git_post_pull(extras):'
     return HookResponse(status, stdout.getvalue())
 
 
-def git_pre_receive(unused_repo_path, unused_revs, env):
+def _parse_git_ref_lines(revision_lines):
+    rev_data = []
+    for revision_line in revision_lines or []:
+        old_rev, new_rev, ref = revision_line.strip().split(' ')
+        ref_data = ref.split('/', 2)
+        if ref_data[1] in ('tags', 'heads'):
+            rev_data.append({
+                'old_rev': old_rev,
+                'new_rev': new_rev,
+                'ref': ref,
+                'type': ref_data[1],
+                'name': ref_data[2],
+            })
+    return rev_data
+
+
+def git_pre_receive(unused_repo_path, revision_lines, env):
     """
     Pre push hook.
 
@@ -268,8 +309,10 b' def git_pre_receive(unused_repo_path, un'
     :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
+    rev_data = _parse_git_ref_lines(revision_lines)
     if 'push' not in extras['hooks']:
         return 0
+    extras['commit_ids'] = rev_data
     return _call_hook('pre_push', extras, GitMessageWriter())
 
 
@@ -277,7 +320,7 b' def _run_command(arguments):'
     """
     Run the specified command and return the stdout.
 
-    :param arguments: sequence of program arugments (including the program name)
+    :param arguments: sequence of program arguments (including the program name)
     :type arguments: list[str]
     """
     # TODO(skreft): refactor this method and all the other similar ones.
@@ -308,18 +351,7 b' def git_post_receive(unused_repo_path, r'
     if 'push' not in extras['hooks']:
         return 0
 
-    rev_data = []
-    for revision_line in revision_lines:
-        old_rev, new_rev, ref = revision_line.strip().split(' ')
-        ref_data = ref.split('/', 2)
-        if ref_data[1] in ('tags', 'heads'):
-            rev_data.append({
-                'old_rev': old_rev,
-                'new_rev': new_rev,
-                'ref': ref,
-                'type': ref_data[1],
-                'name': ref_data[2],
-            })
+    rev_data = _parse_git_ref_lines(revision_lines)
 
     git_revs = []
 
@@ -339,7 +371,7 b' def git_post_receive(unused_repo_path, r'
                 except Exception:
                     cmd = ['git', 'symbolic-ref', 'HEAD',
                            'refs/heads/%s' % push_ref['name']]
-                    print "Setting default branch to %s" % push_ref['name']
+                    print("Setting default branch to %s" % push_ref['name'])
                     _run_command(cmd)
 
                 cmd = ['git', 'for-each-ref', '--format=%(refname)',

vcsserver/http_main.py

@@ -30,6 +30,7 b' from pyramid.config import Configurator'
 from pyramid.wsgi import wsgiapp
 
 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
+from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
 from vcsserver.echo_stub.echo_app import EchoApp
 from vcsserver.exceptions import HTTPRepoLocked
@@ -40,11 +41,13 b' try:'
 except ImportError:
     GitFactory = None
     GitRemote = None
+
 try:
     from vcsserver.hg import MercurialFactory, HgRemote
 except ImportError:
     MercurialFactory = None
     HgRemote = None
+
 try:
     from vcsserver.svn import SubversionFactory, SvnRemote
 except ImportError:
@@ -153,8 +156,10 b' class HTTPApplication(object):'
     remote_wsgi = remote_wsgi
     _use_echo_app = False
 
-    def __init__(self, settings=None):
+    def __init__(self, settings=None, global_config=None):
         self.config = Configurator(settings=settings)
+        self.global_config = global_config
+
         locale = settings.get('locale', '') or 'en_US.UTF-8'
         vcs = VCS(locale=locale, cache_config=settings)
         self._remotes = {
@@ -209,12 +214,7 b' class HTTPApplication(object):'
             return {'status': '404 NOT FOUND'}
         self.config.add_notfound_view(notfound, renderer='json')
 
-        self.config.add_view(
-            self.handle_vcs_exception, context=Exception,
-            custom_predicates=[self.is_vcs_exception])
-
-        self.config.add_view(
-            self.general_error_handler, context=Exception)
+        self.config.add_view(self.handle_vcs_exception, context=Exception)
 
         self.config.add_tween(
             'vcsserver.tweens.RequestWrapperTween',
@@ -273,12 +273,26 b' class HTTPApplication(object):'
 
     def service_view(self, request):
         import vcsserver
+        import ConfigParser as configparser
+
         payload = msgpack.unpackb(request.body, use_list=True)
+
+        try:
+            path = self.global_config['__file__']
+            config = configparser.ConfigParser()
+            config.read(path)
+            parsed_ini = config
+            if parsed_ini.has_section('server:main'):
+                parsed_ini = dict(parsed_ini.items('server:main'))
+        except Exception:
+            log.exception('Failed to read .ini file for display')
+            parsed_ini = {}
+
         resp = {
             'id': payload.get('id'),
             'result': dict(
                 version=vcsserver.__version__,
-                config={},
+                config=parsed_ini,
                 payload=payload,
             )
         }
@@ -351,9 +365,31 b' class HTTPApplication(object):'
                 config = msgpack.unpackb(packed_config)
 
                 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
+                content_type = environ.get('CONTENT_TYPE', '')
+
+                path = environ['PATH_INFO']
+                is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
+                log.debug(
+                    'LFS: Detecting if request `%s` is LFS server path based '
+                    'on content type:`%s`, is_lfs:%s',
+                    path, content_type, is_lfs_request)
+
+                if not is_lfs_request:
+                    # fallback detection by path
+                    if GIT_LFS_PROTO_PAT.match(path):
+                        is_lfs_request = True
+                    log.debug(
+                        'LFS: fallback detection by path of: `%s`, is_lfs:%s',
+                        path, is_lfs_request)
+
+                if is_lfs_request:
+                    app = scm_app.create_git_lfs_wsgi_app(
+                        repo_path, repo_name, config)
+                else:
                 app = scm_app.create_git_wsgi_app(
                     repo_path, repo_name, config)
                 return app(environ, start_response)
+
             return _git_stream
 
     def is_vcs_view(self, context, request):
@@ -364,27 +400,17 b' class HTTPApplication(object):'
         backend = request.matchdict.get('backend')
         return backend in self._remotes
 
-    def is_vcs_exception(self, context, request):
-        """
-        View predicate that returns true if the context object is a VCS
-        exception.
-        """
-        return hasattr(context, '_vcs_kind')
-
     def handle_vcs_exception(self, exception, request):
-        if exception._vcs_kind == 'repo_locked':
+        _vcs_kind = getattr(exception, '_vcs_kind', '')
+        if _vcs_kind == 'repo_locked':
             # Get custom repo-locked status code if present.
             status_code = request.headers.get('X-RC-Locked-Status-Code')
             return HTTPRepoLocked(
                 title=exception.message, status_code=status_code)
 
         # Re-raise exception if we can not handle it.
-        raise exception
-
-    def general_error_handler(self, exception, request):
         log.exception(
-            'error occurred handling this request for path: %s',
-            request.path)
+            'error occurred handling this request for path: %s', request.path)
         raise exception
 
 
@@ -404,5 +430,5 b' def main(global_config, **settings):'
     if MercurialFactory:
         hgpatches.patch_largefiles_capabilities()
         hgpatches.patch_subrepo_type_mapping()
-    app = HTTPApplication(settings=settings)
+    app = HTTPApplication(settings=settings, global_config=global_config)
     return app.wsgi_app()

vcsserver/scm_app.py

@@ -15,8 +15,8 b''
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
+import os
 import logging
-import os
 
 import mercurial
 import mercurial.error
@@ -25,7 +25,7 b' import mercurial.hgweb.hgweb_mod'
 import mercurial.hgweb.protocol
 import webob.exc
 
-from vcsserver import pygrack, exceptions, settings
+from vcsserver import pygrack, exceptions, settings, git_lfs
 
 
 log = logging.getLogger(__name__)
@@ -132,6 +132,9 b' def create_hg_wsgi_app(repo_path, repo_n'
 
 
 class GitHandler(object):
+    """
+    Handler for Git operations like push/pull etc
+    """
     def __init__(self, repo_location, repo_name, git_path, update_server_info,
                  extras):
         if not os.path.isdir(repo_location):
@@ -172,3 +175,35 b' def create_git_wsgi_app(repo_path, repo_'
         repo_path, repo_name, git_path, update_server_info, config)
 
     return app
+
+
+class GitLFSHandler(object):
+    """
+    Handler for Git LFS operations
+    """
+
+    def __init__(self, repo_location, repo_name, git_path, update_server_info,
+                 extras):
+        if not os.path.isdir(repo_location):
+            raise OSError(repo_location)
+        self.content_path = repo_location
+        self.repo_name = repo_name
+        self.repo_location = repo_location
+        self.extras = extras
+        self.git_path = git_path
+        self.update_server_info = update_server_info
+
+    def get_app(self, git_lfs_enabled, git_lfs_store_path):
+        app = git_lfs.create_app(git_lfs_enabled, git_lfs_store_path)
+        return app
+
+
+def create_git_lfs_wsgi_app(repo_path, repo_name, config):
+    git_path = settings.GIT_EXECUTABLE
+    update_server_info = config.pop('git_update_server_info')
+    git_lfs_enabled = config.pop('git_lfs_enabled')
+    git_lfs_store_path = config.pop('git_lfs_store_path')
+    app = GitLFSHandler(
+        repo_path, repo_name, git_path, update_server_info, config)
+
+    return app.get_app(git_lfs_enabled, git_lfs_store_path)

vcsserver/svn.py

@@ -33,7 +33,7 b' import svn.repos'
 
 from vcsserver import svn_diff
 from vcsserver import exceptions
-from vcsserver.base import RepoFactory
+from vcsserver.base import RepoFactory, raise_from_original
 
 
 log = logging.getLogger(__name__)
@@ -62,17 +62,6 b' def reraise_safe_exceptions(func):'
     return wrapper
 
 
-def raise_from_original(new_type):
-    """
-    Raise a new exception type with original args and traceback.
-    """
-    _, original, traceback = sys.exc_info()
-    try:
-        raise new_type(*original.args), None, traceback
-    finally:
-        del traceback
-
-
 class SubversionFactory(RepoFactory):
 
     def _create_repo(self, wire, create, compatible_version):
@@ -388,6 +377,10 b' class SvnRemote(object):'
                 "Path might not exist %s, %s" % (path1, path2))
             return ""
 
+    @reraise_safe_exceptions
+    def is_large_file(self, wire, path):
+        return False
+
 
 class SvnDiffer(object):
     """

vcsserver/utils.py

@@ -16,8 +16,23 b''
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 
+def safe_int(val, default=None):
+    """
+    Returns int() of val if val is not convertable to int use default
+    instead
 
-# TODO: johbo: That's a copy from rhodecode
+    :param val:
+    :param default:
+    """
+
+    try:
+        val = int(val)
+    except (ValueError, TypeError):
+        val = default
+
+    return val
+
+
 def safe_str(unicode_, to_encoding=['utf8']):
     """
     safe str function. Does few trick to turn unicode_ into string