release: Merge default into stable for release preparation

vcsserver/git_lfs/__init__.py

@@ -0,0 +1,19 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+
+from app import create_app

vcsserver/git_lfs/app.py

@@ -0,0 +1,276 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import re
+import logging
+from wsgiref.util import FileWrapper
+
+import simplejson as json
+from pyramid.config import Configurator
+from pyramid.response import Response, FileIter
+from pyramid.httpexceptions import (
+    HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden,
+    HTTPUnprocessableEntity)
+
+from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
+from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator
+from vcsserver.utils import safe_int
+
+log = logging.getLogger(__name__)
+
+
+GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ?
+GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))')
+
+
+def write_response_error(http_exception, text=None):
+    content_type = 'application/json'
+    _exception = http_exception(content_type=content_type)
+    _exception.content_type = content_type
+    if text:
+        _exception.body = json.dumps({'message': text})
+    log.debug('LFS: writing response of type %s to client with text:%s',
+              http_exception, text)
+    return _exception
+
+
+class AuthHeaderRequired(object):
+    """
+    Decorator to check if request has proper auth-header
+    """
+
+    def __call__(self, func):
+        return get_cython_compat_decorator(self.__wrapper, func)
+
+    def __wrapper(self, func, *fargs, **fkwargs):
+        request = fargs[1]
+        auth = request.authorization
+        if not auth:
+            return write_response_error(HTTPForbidden)
+        return func(*fargs[1:], **fkwargs)
+
+
+# views
+
+def lfs_objects(request):
+    # indicate not supported, V1 API
+    log.warning('LFS: v1 api not supported, reporting it back to client')
+    return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported')
+
+
+@AuthHeaderRequired()
+def lfs_objects_batch(request):
+    """
+    The client sends the following information to the Batch endpoint to transfer some objects:
+
+        operation - Should be download or upload.
+        transfers - An optional Array of String identifiers for transfer
+            adapters that the client has configured. If omitted, the basic
+            transfer adapter MUST be assumed by the server.
+        objects - An Array of objects to download.
+        oid - String OID of the LFS object.
+        size - Integer byte size of the LFS object. Must be at least zero.
+    """
+    auth = request.authorization
+
+    repo = request.matchdict.get('repo')
+
+    data = request.json
+    operation = data.get('operation')
+    if operation not in ('download', 'upload'):
+        log.debug('LFS: unsupported operation:%s', operation)
+        return write_response_error(
+            HTTPBadRequest, 'unsupported operation mode: `%s`' % operation)
+
+    if 'objects' not in data:
+        log.debug('LFS: missing objects data')
+        return write_response_error(
+            HTTPBadRequest, 'missing objects data')
+
+    log.debug('LFS: handling operation of type: %s', operation)
+
+    objects = []
+    for o in data['objects']:
+        try:
+            oid = o['oid']
+            obj_size = o['size']
+        except KeyError:
+            log.exception('LFS, failed to extract data')
+            return write_response_error(
+                HTTPBadRequest, 'unsupported data in objects')
+
+        obj_data = {'oid': oid}
+
+        obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid)
+        obj_verify_href = request.route_url('lfs_objects_verify', repo=repo)
+        store = LFSOidStore(
+            oid, repo, store_location=request.registry.git_lfs_store_path)
+        handler = OidHandler(
+            store, repo, auth, oid, obj_size, obj_data,
+            obj_href, obj_verify_href)
+
+        # this verifies also OIDs
+        actions, errors = handler.exec_operation(operation)
+        if errors:
+            log.warning('LFS: got following errors: %s', errors)
+            obj_data['errors'] = errors
+
+        if actions:
+            obj_data['actions'] = actions
+
+        obj_data['size'] = obj_size
+        obj_data['authenticated'] = True
+        objects.append(obj_data)
+
+    result = {'objects': objects, 'transfer': 'basic'}
+    log.debug('LFS Response %s', safe_result(result))
+
+    return result
+
+
+def lfs_objects_oid_upload(request):
+    repo = request.matchdict.get('repo')
+    oid = request.matchdict.get('oid')
+    store = LFSOidStore(
+        oid, repo, store_location=request.registry.git_lfs_store_path)
+    engine = store.get_engine(mode='wb')
+    log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid)
+    with engine as f:
+        for chunk in FileWrapper(request.body_file_seekable, blksize=64 * 1024):
+            f.write(chunk)
+
+    return {'upload': 'ok'}
+
+
+def lfs_objects_oid_download(request):
+    repo = request.matchdict.get('repo')
+    oid = request.matchdict.get('oid')
+
+    store = LFSOidStore(
+        oid, repo, store_location=request.registry.git_lfs_store_path)
+    if not store.has_oid():
+        log.debug('LFS: oid %s does not exists in store', oid)
+        return write_response_error(
+            HTTPNotFound, 'requested file with oid `%s` not found in store' % oid)
+
+    # TODO(marcink): support range header ?
+    # Range: bytes=0-, `bytes=(\d+)\-.*`
+
+    f = open(store.oid_path, 'rb')
+    response = Response(
+        content_type='application/octet-stream', app_iter=FileIter(f))
+    response.headers.add('X-RC-LFS-Response-Oid', str(oid))
+    return response
+
+
+def lfs_objects_verify(request):
+    repo = request.matchdict.get('repo')
+
+    data = request.json
+    oid = data.get('oid')
+    size = safe_int(data.get('size'))
+
+    if not (oid and size):
+        return write_response_error(
+            HTTPBadRequest, 'missing oid and size in request data')
+
+    store = LFSOidStore(
+        oid, repo, store_location=request.registry.git_lfs_store_path)
+    if not store.has_oid():
+        log.debug('LFS: oid %s does not exists in store', oid)
+        return write_response_error(
+            HTTPNotFound, 'oid `%s` does not exists in store' % oid)
+
+    store_size = store.size_oid()
+    if store_size != size:
+        msg = 'requested file size mismatch store size:%s requested:%s' % (
+            store_size, size)
+        return write_response_error(
+            HTTPUnprocessableEntity, msg)
+
+    return {'message': {'size': 'ok', 'in_store': 'ok'}}
+
+
+def lfs_objects_lock(request):
+    return write_response_error(
+        HTTPNotImplemented, 'GIT LFS locking api not supported')
+
+
+def not_found(request):
+    return write_response_error(
+        HTTPNotFound, 'request path not found')
+
+
+def lfs_disabled(request):
+    return write_response_error(
+        HTTPNotImplemented, 'GIT LFS disabled for this repo')
+
+
+def git_lfs_app(config):
+
+    # v1 API deprecation endpoint
+    config.add_route('lfs_objects',
+                     '/{repo:.*?[^/]}/info/lfs/objects')
+    config.add_view(lfs_objects, route_name='lfs_objects',
+                    request_method='POST', renderer='json')
+
+    # locking API
+    config.add_route('lfs_objects_lock',
+                     '/{repo:.*?[^/]}/info/lfs/locks')
+    config.add_view(lfs_objects_lock, route_name='lfs_objects_lock',
+                    request_method=('POST', 'GET'), renderer='json')
+
+    config.add_route('lfs_objects_lock_verify',
+                     '/{repo:.*?[^/]}/info/lfs/locks/verify')
+    config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify',
+                    request_method=('POST', 'GET'), renderer='json')
+
+    # batch API
+    config.add_route('lfs_objects_batch',
+                     '/{repo:.*?[^/]}/info/lfs/objects/batch')
+    config.add_view(lfs_objects_batch, route_name='lfs_objects_batch',
+                    request_method='POST', renderer='json')
+
+    # oid upload/download API
+    config.add_route('lfs_objects_oid',
+                     '/{repo:.*?[^/]}/info/lfs/objects/{oid}')
+    config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid',
+                    request_method='PUT', renderer='json')
+    config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid',
+                    request_method='GET', renderer='json')
+
+    # verification API
+    config.add_route('lfs_objects_verify',
+                     '/{repo:.*?[^/]}/info/lfs/verify')
+    config.add_view(lfs_objects_verify, route_name='lfs_objects_verify',
+                    request_method='POST', renderer='json')
+
+    # not found handler for API
+    config.add_notfound_view(not_found, renderer='json')
+
+
+def create_app(git_lfs_enabled, git_lfs_store_path):
+    config = Configurator()
+    if git_lfs_enabled:
+        config.include(git_lfs_app)
+        config.registry.git_lfs_store_path = git_lfs_store_path
+    else:
+        # not found handler for API, reporting disabled LFS support
+        config.add_notfound_view(lfs_disabled, renderer='json')
+
+    app = config.make_wsgi_app()
+    return app

vcsserver/git_lfs/lib.py

@@ -0,0 +1,166 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import os
+import shutil
+import logging
+from collections import OrderedDict
+
+log = logging.getLogger(__name__)
+
+
+class OidHandler(object):
+
+    def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href,
+                 obj_verify_href=None):
+        self.current_store = store
+        self.repo_name = repo_name
+        self.auth = auth
+        self.oid = oid
+        self.obj_size = obj_size
+        self.obj_data = obj_data
+        self.obj_href = obj_href
+        self.obj_verify_href = obj_verify_href
+
+    def get_store(self, mode=None):
+        return self.current_store
+
+    def get_auth(self):
+        """returns auth header for re-use in upload/download"""
+        return " ".join(self.auth)
+
+    def download(self):
+
+        store = self.get_store()
+        response = None
+        has_errors = None
+
+        if not store.has_oid():
+            # error reply back to client that something is wrong with dl
+            err_msg = 'object: {} does not exist in store'.format(store.oid)
+            has_errors = OrderedDict(
+                error=OrderedDict(
+                    code=404,
+                    message=err_msg
+                )
+            )
+
+        download_action = OrderedDict(
+            href=self.obj_href,
+            header=OrderedDict([("Authorization", self.get_auth())])
+        )
+        if not has_errors:
+            response = OrderedDict(download=download_action)
+        return response, has_errors
+
+    def upload(self, skip_existing=True):
+        """
+        Write upload action for git-lfs server
+        """
+
+        store = self.get_store()
+        response = None
+        has_errors = None
+
+        # verify if we have the OID before, if we do, reply with empty
+        if store.has_oid():
+            log.debug('LFS: store already has oid %s', store.oid)
+            if skip_existing:
+                log.debug('LFS: skipping further action as oid is existing')
+                return response, has_errors
+
+        upload_action = OrderedDict(
+            href=self.obj_href,
+            header=OrderedDict([("Authorization", self.get_auth())])
+        )
+        if not has_errors:
+            response = OrderedDict(upload=upload_action)
+            # if specified in handler, return the verification endpoint
+            if self.obj_verify_href:
+                verify_action = OrderedDict(
+                    href=self.obj_verify_href,
+                    header=OrderedDict([("Authorization", self.get_auth())])
+                )
+                response['verify'] = verify_action
+        return response, has_errors
+
+    def exec_operation(self, operation, *args, **kwargs):
+        handler = getattr(self, operation)
+        log.debug('LFS: handling request using %s handler', handler)
+        return handler(*args, **kwargs)
+
+
+class LFSOidStore(object):
+
+    def __init__(self, oid, repo, store_location=None):
+        self.oid = oid
+        self.repo = repo
+        self.store_path = store_location or self.get_default_store()
+        self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp')
+        self.oid_path = os.path.join(self.store_path, oid)
+        self.fd = None
+
+    def get_engine(self, mode):
+        """
+        engine = .get_engine(mode='wb')
+        with engine as f:
+            f.write('...')
+        """
+
+        class StoreEngine(object):
+            def __init__(self, mode, store_path, oid_path, tmp_oid_path):
+                self.mode = mode
+                self.store_path = store_path
+                self.oid_path = oid_path
+                self.tmp_oid_path = tmp_oid_path
+
+            def __enter__(self):
+                if not os.path.isdir(self.store_path):
+                    os.makedirs(self.store_path)
+
+                # TODO(marcink): maybe write metadata here with size/oid ?
+                fd = open(self.tmp_oid_path, self.mode)
+                self.fd = fd
+                return fd
+
+            def __exit__(self, exc_type, exc_value, traceback):
+                # close tmp file, and rename to final destination
+                self.fd.close()
+                shutil.move(self.tmp_oid_path, self.oid_path)
+
+        return StoreEngine(
+            mode, self.store_path, self.oid_path, self.tmp_oid_path)
+
+    def get_default_store(self):
+        """
+        Default store, consistent with defaults of Mercurial large files store
+        which is /home/username/.cache/largefiles
+        """
+        user_home = os.path.expanduser("~")
+        return os.path.join(user_home, '.cache', 'lfs-store')
+
+    def has_oid(self):
+        return os.path.exists(os.path.join(self.store_path, self.oid))
+
+    def size_oid(self):
+        size = -1
+
+        if self.has_oid():
+            oid = os.path.join(self.store_path, self.oid)
+            size = os.stat(oid).st_size
+
+        return size

vcsserver/git_lfs/tests/__init__.py

@@ -0,0 +1,16 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA

vcsserver/git_lfs/tests/test_lfs_app.py

@@ -0,0 +1,237 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import os
+import pytest
+from webtest.app import TestApp as WebObTestApp
+
+from vcsserver.git_lfs.app import create_app
+
+
[e-mail](scope='function')
+def git_lfs_app(tmpdir):
+    custom_app = WebObTestApp(create_app(
+        git_lfs_enabled=True, git_lfs_store_path=str(tmpdir)))
+    custom_app._store = str(tmpdir)
+    return custom_app
+
+
[e-mail]()
+def http_auth():
+    return {'HTTP_AUTHORIZATION': "Basic XXXXX"}
+
+
+class TestLFSApplication(object):
+
+    def test_app_wrong_path(self, git_lfs_app):
+        git_lfs_app.get('/repo/info/lfs/xxx', status=404)
+
+    def test_app_deprecated_endpoint(self, git_lfs_app):
+        response = git_lfs_app.post('/repo/info/lfs/objects', status=501)
+        assert response.status_code == 501
+        assert response.json == {u'message': u'LFS: v1 api not supported'}
+
+    def test_app_lock_verify_api_not_available(self, git_lfs_app):
+        response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501)
+        assert response.status_code == 501
+        assert response.json == {
+            u'message': u'GIT LFS locking api not supported'}
+
+    def test_app_lock_api_not_available(self, git_lfs_app):
+        response = git_lfs_app.post('/repo/info/lfs/locks', status=501)
+        assert response.status_code == 501
+        assert response.json == {
+            u'message': u'GIT LFS locking api not supported'}
+
+    def test_app_batch_api_missing_auth(self, git_lfs_app,):
+        git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params={}, status=403)
+
+    def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth):
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params={}, status=400,
+            extra_environ=http_auth)
+        assert response.json == {
+            u'message': u'unsupported operation mode: `None`'}
+
+    def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth):
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params={'operation': 'download'},
+            status=400, extra_environ=http_auth)
+        assert response.json == {
+            u'message': u'missing objects data'}
+
+    def test_app_batch_api_unsupported_data_in_objects(
+            self, git_lfs_app, http_auth):
+        params = {'operation': 'download',
+                  'objects': [{}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params, status=400,
+            extra_environ=http_auth)
+        assert response.json == {
+            u'message': u'unsupported data in objects'}
+
+    def test_app_batch_api_download_missing_object(
+            self, git_lfs_app, http_auth):
+        params = {'operation': 'download',
+                  'objects': [{'oid': '123', 'size': '1024'}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params,
+            extra_environ=http_auth)
+
+        expected_objects = [
+            {u'authenticated': True,
+             u'errors': {u'error': {
+                 u'code': 404,
+                 u'message': u'object: 123 does not exist in store'}},
+             u'oid': u'123',
+             u'size': u'1024'}
+        ]
+        assert response.json == {
+            'objects': expected_objects, 'transfer': 'basic'}
+
+    def test_app_batch_api_download(self, git_lfs_app, http_auth):
+        oid = '456'
+        oid_path = os.path.join(git_lfs_app._store, oid)
+        if not os.path.isdir(os.path.dirname(oid_path)):
+            os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        params = {'operation': 'download',
+                  'objects': [{'oid': oid, 'size': '1024'}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params,
+            extra_environ=http_auth)
+
+        expected_objects = [
+            {u'authenticated': True,
+             u'actions': {
+                 u'download': {
+                     u'header': {u'Authorization': u'Basic XXXXX'},
+                     u'href': u'http://localhost/repo/info/lfs/objects/456'},
+             },
+             u'oid': u'456',
+             u'size': u'1024'}
+        ]
+        assert response.json == {
+            'objects': expected_objects, 'transfer': 'basic'}
+
+    def test_app_batch_api_upload(self, git_lfs_app, http_auth):
+        params = {'operation': 'upload',
+                  'objects': [{'oid': '123', 'size': '1024'}]}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/objects/batch', params=params,
+            extra_environ=http_auth)
+        expected_objects = [
+            {u'authenticated': True,
+             u'actions': {
+                 u'upload': {
+                     u'header': {u'Authorization': u'Basic XXXXX'},
+                     u'href': u'http://localhost/repo/info/lfs/objects/123'},
+                 u'verify': {
+                     u'header': {u'Authorization': u'Basic XXXXX'},
+                     u'href': u'http://localhost/repo/info/lfs/verify'}
+             },
+             u'oid': u'123',
+             u'size': u'1024'}
+        ]
+        assert response.json == {
+            'objects': expected_objects, 'transfer': 'basic'}
+
+    def test_app_verify_api_missing_data(self, git_lfs_app):
+        params = {'oid': 'missing',}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params,
+            status=400)
+
+        assert response.json == {
+            u'message': u'missing oid and size in request data'}
+
+    def test_app_verify_api_missing_obj(self, git_lfs_app):
+        params = {'oid': 'missing', 'size': '1024'}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params,
+            status=404)
+
+        assert response.json == {
+            u'message': u'oid `missing` does not exists in store'}
+
+    def test_app_verify_api_size_mismatch(self, git_lfs_app):
+        oid = 'existing'
+        oid_path = os.path.join(git_lfs_app._store, oid)
+        if not os.path.isdir(os.path.dirname(oid_path)):
+            os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        params = {'oid': oid, 'size': '1024'}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params, status=422)
+
+        assert response.json == {
+            u'message': u'requested file size mismatch '
+                        u'store size:11 requested:1024'}
+
+    def test_app_verify_api(self, git_lfs_app):
+        oid = 'existing'
+        oid_path = os.path.join(git_lfs_app._store, oid)
+        if not os.path.isdir(os.path.dirname(oid_path)):
+            os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        params = {'oid': oid, 'size': 11}
+        response = git_lfs_app.post_json(
+            '/repo/info/lfs/verify', params=params)
+
+        assert response.json == {
+            u'message': {u'size': u'ok', u'in_store': u'ok'}}
+
+    def test_app_download_api_oid_not_existing(self, git_lfs_app):
+        oid = 'missing'
+
+        response = git_lfs_app.get(
+            '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404)
+
+        assert response.json == {
+            u'message': u'requested file with oid `missing` not found in store'}
+
+    def test_app_download_api(self, git_lfs_app):
+        oid = 'existing'
+        oid_path = os.path.join(git_lfs_app._store, oid)
+        if not os.path.isdir(os.path.dirname(oid_path)):
+            os.makedirs(os.path.dirname(oid_path))
+        with open(oid_path, 'wb') as f:
+            f.write('OID_CONTENT')
+
+        response = git_lfs_app.get(
+            '/repo/info/lfs/objects/{oid}'.format(oid=oid))
+        assert response
+
+    def test_app_upload(self, git_lfs_app):
+        oid = 'uploaded'
+
+        response = git_lfs_app.put(
+            '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT')
+
+        assert response.json == {u'upload': u'ok'}
+
+        # verify that we actually wrote that OID
+        oid_path = os.path.join(git_lfs_app._store, oid)
+        assert os.path.isfile(oid_path)
+        assert 'CONTENT' == open(oid_path).read()

vcsserver/git_lfs/tests/test_lib.py

@@ -0,0 +1,123 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import os
+import pytest
+from vcsserver.git_lfs.lib import OidHandler, LFSOidStore
+
+
[e-mail]()
+def lfs_store(tmpdir):
+    repo = 'test'
+    oid = '123456789'
+    store = LFSOidStore(oid=oid, repo=repo, store_location=str(tmpdir))
+    return store
+
+
[e-mail]()
+def oid_handler(lfs_store):
+    store = lfs_store
+    repo = store.repo
+    oid = store.oid
+
+    oid_handler = OidHandler(
+        store=store, repo_name=repo, auth=('basic', 'xxxx'),
+        oid=oid,
+        obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid',
+        obj_verify_href='http://localhost/verify')
+    return oid_handler
+
+
+class TestOidHandler(object):
+
+    @pytest.mark.parametrize('exec_action', [
+        'download',
+        'upload',
+    ])
+    def test_exec_action(self, exec_action, oid_handler):
+        handler = oid_handler.exec_operation(exec_action)
+        assert handler
+
+    def test_exec_action_undefined(self, oid_handler):
+        with pytest.raises(AttributeError):
+            oid_handler.exec_operation('wrong')
+
+    def test_download_oid_not_existing(self, oid_handler):
+        response, has_errors = oid_handler.exec_operation('download')
+
+        assert response is None
+        assert has_errors['error'] == {
+            'code': 404,
+            'message': 'object: 123456789 does not exist in store'}
+
+    def test_download_oid(self, oid_handler):
+        store = oid_handler.get_store()
+        if not os.path.isdir(os.path.dirname(store.oid_path)):
+            os.makedirs(os.path.dirname(store.oid_path))
+
+        with open(store.oid_path, 'wb') as f:
+            f.write('CONTENT')
+
+        response, has_errors = oid_handler.exec_operation('download')
+
+        assert has_errors is None
+        assert response['download'] == {
+            'header': {'Authorization': 'basic xxxx'},
+            'href': 'http://localhost/handle_oid'
+        }
+
+    def test_upload_oid_that_exists(self, oid_handler):
+        store = oid_handler.get_store()
+        if not os.path.isdir(os.path.dirname(store.oid_path)):
+            os.makedirs(os.path.dirname(store.oid_path))
+
+        with open(store.oid_path, 'wb') as f:
+            f.write('CONTENT')
+
+        response, has_errors = oid_handler.exec_operation('upload')
+        assert has_errors is None
+        assert response is None
+
+    def test_upload_oid(self, oid_handler):
+        response, has_errors = oid_handler.exec_operation('upload')
+        assert has_errors is None
+        assert response['upload'] == {
+            'header': {'Authorization': 'basic xxxx'},
+            'href': 'http://localhost/handle_oid'
+        }
+
+
+class TestLFSStore(object):
+    def test_write_oid(self, lfs_store):
+        oid_location = lfs_store.oid_path
+
+        assert not os.path.isfile(oid_location)
+
+        engine = lfs_store.get_engine(mode='wb')
+        with engine as f:
+            f.write('CONTENT')
+
+        assert os.path.isfile(oid_location)
+
+    def test_detect_has_oid(self, lfs_store):
+
+        assert lfs_store.has_oid() is False
+        engine = lfs_store.get_engine(mode='wb')
+        with engine as f:
+            f.write('CONTENT')
+
+        assert lfs_store.has_oid() is True
  No newline at end of file

vcsserver/git_lfs/utils.py

@@ -0,0 +1,50 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2017 RodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+import copy
+from functools import wraps
+
+
+def get_cython_compat_decorator(wrapper, func):
+    """
+    Creates a cython compatible decorator. The previously used
+    decorator.decorator() function seems to be incompatible with cython.
+
+    :param wrapper: __wrapper method of the decorator class
+    :param func: decorated function
+    """
+    @wraps(func)
+    def local_wrapper(*args, **kwds):
+        return wrapper(func, *args, **kwds)
+    local_wrapper.__wrapped__ = func
+    return local_wrapper
+
+
+def safe_result(result):
+    """clean result for better representation in logs"""
+    clean_copy = copy.deepcopy(result)
+
+    try:
+        if 'objects' in clean_copy:
+            for oid_data in clean_copy['objects']:
+                if 'actions' in oid_data:
+                    for action_name, data in oid_data['actions'].items():
+                        if 'header' in data:
+                            data['header'] = {'Authorization': '*****'}
+    except Exception:
+        return result
+
+    return clean_copy

.bumpversion.cfg

@@ -1,6 +1,6 b''
 [bumpversion]
-current_version = 4.6.1
+current_version = 4.7.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:vcsserver/VERSION]
 

.release.cfg

@@ -1,16 +1,14 b''
 [DEFAULT]
 done = false
 
 [task:bump_version]
 done = true
 
 [task:fixes_on_stable]
-done = true
 
 [task:pip2nix_generated]
-done = true
 
 [release]
-state = prepared
-version = 4.6.1
+state = in_progress
+version = 4.7.0
 

configs/production_http.ini

@@ -1,111 +1,109 b''
 ################################################################################
 # RhodeCode VCSServer with HTTP Backend - configuration                        #
 #                                                                              #
 ################################################################################
 
 
 [server:main]
 ## COMMON ##
 host = 127.0.0.1
 port = 9900
 
 
 ##########################
 ## GUNICORN WSGI SERVER ##
 ##########################
 ## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini
 use = egg:gunicorn#main
-## Sets the number of process workers. You must set `instance_id = *`
-## when this option is set to more than one worker, recommended
+## Sets the number of process workers. Recommended
 ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
-## The `instance_id = *` must be set in the [app:main] section below
 workers = 2
 ## process name
 proc_name = rhodecode_vcsserver
 ## type of worker class, one of sync, gevent
 ## recommended for bigger setup is using of of other than sync one
 worker_class = sync
 ## The maximum number of simultaneous clients. Valid only for Gevent
 #worker_connections = 10
 ## max number of requests that worker will handle before being gracefully
 ## restarted, could prevent memory leaks
 max_requests = 1000
 max_requests_jitter = 30
 ## amount of time a worker can spend with handling a request before it
 ## gets killed and restarted. Set to 6hrs
 timeout = 21600
 
 
 [app:main]
 use = egg:rhodecode-vcsserver
 
 pyramid.default_locale_name = en
 pyramid.includes =
 
 ## default locale used by VCS systems
 locale = en_US.UTF-8
 
 # cache regions, please don't change
 beaker.cache.regions = repo_object
 beaker.cache.repo_object.type = memorylru
 beaker.cache.repo_object.max_items = 100
 # cache auto-expires after N seconds
 beaker.cache.repo_object.expire = 300
 beaker.cache.repo_object.enabled = true
 
 
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, vcsserver, pyro4, beaker
 
 [handlers]
 keys = console
 
 [formatters]
 keys = generic
 
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_vcsserver]
 level = DEBUG
 handlers =
 qualname = vcsserver
 propagate = 1
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker
 propagate = 1
 
 [logger_pyro4]
 level = DEBUG
 handlers =
 qualname = Pyro4
 propagate = 1
 
 
 ##############
 ## HANDLERS ##
 ##############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = DEBUG
 formatter = generic
 
 ################
 ## FORMATTERS ##
 ################
 
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
  No newline at end of file

default.nix

@@ -1,153 +1,154 b''
 # Nix environment for the community edition
 #
 # This shall be as lean as possible, just producing the rhodecode-vcsserver
 # derivation. For advanced tweaks to pimp up the development environment we use
 # "shell.nix" so that it does not have to clutter this file.
 
 { pkgs ? (import <nixpkgs> {})
 , pythonPackages ? "python27Packages"
 , pythonExternalOverrides ? self: super: {}
 , doCheck ? true
 }:
 
 let pkgs_ = pkgs; in
 
 let
   pkgs = pkgs_.overridePackages (self: super: {
     # bump GIT version
     git = pkgs.lib.overrideDerivation pkgs_.git (oldAttrs: {
       name = "git-2.9.3";
       src = pkgs.fetchurl {
         url = "https://www.kernel.org/pub/software/scm/git/git-2.9.3.tar.xz";
         sha256 = "0qzs681a64k3shh5p0rg41l1z16fbk5sj0xga45k34hp1hsp654z";
       };
 
     });
 
     # Override subversion derivation to
     #  - activate python bindings
     subversion = let
       subversionWithPython = super.subversion.override {
         httpSupport = true;
         pythonBindings = true;
         python = self.python27Packages.python;
       };
 
     in
 
     pkgs.lib.overrideDerivation subversionWithPython (oldAttrs: {
       patches = (oldAttrs.patches or []) ++
         pkgs.lib.optionals pkgs.stdenv.isDarwin [
           # johbo: "import svn.client" fails on darwin currently.
           ./pkgs/subversion-1.9.4-darwin.patch
         ];
     });
 
   });
 
   inherit (pkgs.lib) fix extends;
   basePythonPackages = with builtins; if isAttrs pythonPackages
     then pythonPackages
     else getAttr pythonPackages pkgs;
 
   elem = builtins.elem;
   basename = path: with pkgs.lib; last (splitString "/" path);
   startsWith = prefix: full: let
     actualPrefix = builtins.substring 0 (builtins.stringLength prefix) full;
   in actualPrefix == prefix;
 
   src-filter = path: type: with pkgs.lib;
     let
       ext = last (splitString "." path);
     in
       !elem (basename path) [".hg" ".git" "__pycache__" ".eggs"
         "node_modules" "build" "data" "tmp"] &&
       !elem ext ["egg-info" "pyc"] &&
       !startsWith "result" path;
 
   rhodecode-vcsserver-src = builtins.filterSource src-filter ./.;
 
   pythonGeneratedPackages = self: basePythonPackages.override (a: {
     inherit self;
   }) // (scopedImport {
     self = self;
     super = basePythonPackages;
     inherit pkgs;
     inherit (pkgs) fetchurl fetchgit;
   } ./pkgs/python-packages.nix);
 
   pythonOverrides = import ./pkgs/python-packages-overrides.nix {
     inherit basePythonPackages pkgs;
   };
 
   version = builtins.readFile ./vcsserver/VERSION;
 
   pythonLocalOverrides = self: super: {
     rhodecode-vcsserver = super.rhodecode-vcsserver.override (attrs: {
       inherit doCheck version;
 
       name = "rhodecode-vcsserver-${version}";
       releaseName = "RhodeCodeVCSServer-${version}";
       src = rhodecode-vcsserver-src;
+      dontStrip = true; # prevent strip, we don't need it.
 
       propagatedBuildInputs = attrs.propagatedBuildInputs ++ ([
         pkgs.git
         pkgs.subversion
       ]);
 
       # TODO: johbo: Make a nicer way to expose the parts. Maybe
       # pkgs/default.nix?
       passthru = {
         pythonPackages = self;
       };
 
       # Add VCSServer bin directory to path so that tests can find 'vcsserver'.
       preCheck = ''
         export PATH="$out/bin:$PATH"
       '';
 
       # put custom attrs here
       checkPhase = ''
         runHook preCheck
         PYTHONHASHSEED=random py.test -p no:sugar -vv --cov-config=.coveragerc --cov=vcsserver --cov-report=term-missing vcsserver
         runHook postCheck
       '';
 
       postInstall = ''
         echo "Writing meta information for rccontrol to nix-support/rccontrol"
         mkdir -p $out/nix-support/rccontrol
         cp -v vcsserver/VERSION $out/nix-support/rccontrol/version
         echo "DONE: Meta information for rccontrol written"
 
         ln -s ${self.pyramid}/bin/* $out/bin/
         ln -s ${self.gunicorn}/bin/gunicorn $out/bin/
 
         # Symlink version control utilities
         #
         # We ensure that always the correct version is available as a symlink.
         # So that users calling them via the profile path will always use the
         # correct version.
         ln -s ${pkgs.git}/bin/git $out/bin
         ln -s ${self.mercurial}/bin/hg $out/bin
         ln -s ${pkgs.subversion}/bin/svn* $out/bin
 
         for file in $out/bin/*; do
           wrapProgram $file \
             --set PATH $PATH \
             --set PYTHONPATH $PYTHONPATH \
             --set PYTHONHASHSEED random
         done
       '';
 
     });
   };
 
   # Apply all overrides and fix the final package set
   myPythonPackages =
     (fix
     (extends pythonExternalOverrides
     (extends pythonLocalOverrides
     (extends pythonOverrides
              pythonGeneratedPackages))));
 
 in myPythonPackages.rhodecode-vcsserver

pkgs/python-packages.nix

@@ -1,812 +1,812 b''
 # Generated by pip2nix 0.4.0
 # See https://github.com/johbo/pip2nix
 
 {
   Beaker = super.buildPythonPackage {
     name = "Beaker-1.7.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/97/8e/409d2e7c009b8aa803dc9e6f239f1db7c3cdf578249087a404e7c27a505d/Beaker-1.7.0.tar.gz";
       md5 = "386be3f7fe427358881eee4622b428b3";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   Jinja2 = super.buildPythonPackage {
     name = "Jinja2-2.8";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [MarkupSafe];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f2/2f/0b98b06a345a761bec91a079ccae392d282690c2d8272e708f4d10829e22/Jinja2-2.8.tar.gz";
       md5 = "edb51693fe22c53cee5403775c71a99e";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   Mako = super.buildPythonPackage {
     name = "Mako-1.0.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [MarkupSafe];
     src = fetchurl {
       url = "https://pypi.python.org/packages/56/4b/cb75836863a6382199aefb3d3809937e21fa4cb0db15a4f4ba0ecc2e7e8e/Mako-1.0.6.tar.gz";
       md5 = "a28e22a339080316b2acc352b9ee631c";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   MarkupSafe = super.buildPythonPackage {
     name = "MarkupSafe-0.23";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c0/41/bae1254e0396c0cc8cf1751cb7d9afc90a602353695af5952530482c963f/MarkupSafe-0.23.tar.gz";
       md5 = "f5ab3deee4c37cd6a922fb81e730da6e";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   PasteDeploy = super.buildPythonPackage {
     name = "PasteDeploy-1.5.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/0f/90/8e20cdae206c543ea10793cbf4136eb9a8b3f417e04e40a29d72d9922cbd/PasteDeploy-1.5.2.tar.gz";
       md5 = "352b7205c78c8de4987578d19431af3b";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   Pyro4 = super.buildPythonPackage {
     name = "Pyro4-4.41";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [serpent];
     src = fetchurl {
       url = "https://pypi.python.org/packages/56/2b/89b566b4bf3e7f8ba790db2d1223852f8cb454c52cab7693dd41f608ca2a/Pyro4-4.41.tar.gz";
       md5 = "ed69e9bfafa9c06c049a87cb0c4c2b6c";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   WebOb = super.buildPythonPackage {
     name = "WebOb-1.3.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/16/78/adfc0380b8a0d75b2d543fa7085ba98a573b1ae486d9def88d172b81b9fa/WebOb-1.3.1.tar.gz";
       md5 = "20918251c5726956ba8fef22d1556177";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   WebTest = super.buildPythonPackage {
     name = "WebTest-1.4.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [WebOb];
     src = fetchurl {
       url = "https://pypi.python.org/packages/51/3d/84fd0f628df10b30c7db87895f56d0158e5411206b721ca903cb51bfd948/WebTest-1.4.3.zip";
       md5 = "631ce728bed92c681a4020a36adbc353";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   backports.shutil-get-terminal-size = super.buildPythonPackage {
     name = "backports.shutil-get-terminal-size-1.0.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/ec/9c/368086faa9c016efce5da3e0e13ba392c9db79e3ab740b763fe28620b18b/backports.shutil_get_terminal_size-1.0.0.tar.gz";
       md5 = "03267762480bd86b50580dc19dff3c66";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   configobj = super.buildPythonPackage {
     name = "configobj-5.0.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six];
     src = fetchurl {
       url = "https://pypi.python.org/packages/64/61/079eb60459c44929e684fa7d9e2fdca403f67d64dd9dbac27296be2e0fab/configobj-5.0.6.tar.gz";
       md5 = "e472a3a1c2a67bb0ec9b5d54c13a47d6";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   cov-core = super.buildPythonPackage {
     name = "cov-core-1.15.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [coverage];
     src = fetchurl {
       url = "https://pypi.python.org/packages/4b/87/13e75a47b4ba1be06f29f6d807ca99638bedc6b57fa491cd3de891ca2923/cov-core-1.15.0.tar.gz";
       md5 = "f519d4cb4c4e52856afb14af52919fe6";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   coverage = super.buildPythonPackage {
     name = "coverage-3.7.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/09/4f/89b06c7fdc09687bca507dc411c342556ef9c5a3b26756137a4878ff19bf/coverage-3.7.1.tar.gz";
       md5 = "c47b36ceb17eaff3ecfab3bcd347d0df";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   decorator = super.buildPythonPackage {
-    name = "decorator-4.0.10";
+    name = "decorator-4.0.11";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/13/8a/4eed41e338e8dcc13ca41c94b142d4d20c0de684ee5065523fee406ce76f/decorator-4.0.10.tar.gz";
-      md5 = "434b57fdc3230c500716c5aff8896100";
+      url = "https://pypi.python.org/packages/cc/ac/5a16f1fc0506ff72fcc8fd4e858e3a1c231f224ab79bb7c4c9b2094cc570/decorator-4.0.11.tar.gz";
+      md5 = "73644c8f0bd4983d1b6a34b49adec0ae";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal { fullName = "new BSD License"; } ];
     };
   };
   dulwich = super.buildPythonPackage {
     name = "dulwich-0.13.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/84/95/732d280eee829dacc954e8109f97b47abcadcca472c2ab013e1635eb4792/dulwich-0.13.0.tar.gz";
       md5 = "6dede0626657c2bd08f48ca1221eea91";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl2Plus ];
     };
   };
   enum34 = super.buildPythonPackage {
     name = "enum34-1.1.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz";
       md5 = "5f13a0841a61f7fc295c514490d120d0";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   gevent = super.buildPythonPackage {
     name = "gevent-1.1.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [greenlet];
     src = fetchurl {
       url = "https://pypi.python.org/packages/43/8f/cb3224a0e6ab663547f45c10d0651cfd52633fde4283bf68d627084df8cc/gevent-1.1.2.tar.gz";
       md5 = "bb32a2f852a4997138014d5007215c6e";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   gprof2dot = super.buildPythonPackage {
     name = "gprof2dot-2016.10.13";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/a0/e0/73c71baed306f0402a00a94ffc7b2be94ad1296dfcb8b46912655b93154c/gprof2dot-2016.10.13.tar.gz";
       md5 = "0125401f15fd2afe1df686a76c64a4fd";
     };
     meta = {
       license = [ { fullName = "LGPL"; } ];
     };
   };
   greenlet = super.buildPythonPackage {
     name = "greenlet-0.4.10";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/67/62/ca2a95648666eaa2ffeb6a9b3964f21d419ae27f82f2e66b53da5b943fc4/greenlet-0.4.10.zip";
       md5 = "bed0c4b3b896702131f4d5c72f87c41d";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   gunicorn = super.buildPythonPackage {
     name = "gunicorn-19.6.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/84/ce/7ea5396efad1cef682bbc4068e72a0276341d9d9d0f501da609fab9fcb80/gunicorn-19.6.0.tar.gz";
       md5 = "338e5e8a83ea0f0625f768dba4597530";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   hgsubversion = super.buildPythonPackage {
     name = "hgsubversion-1.8.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [mercurial subvertpy];
     src = fetchurl {
       url = "https://pypi.python.org/packages/ce/97/032e5093ad250e9908cea04395cbddb6902d587f712a79b53b2d778bdfdd/hgsubversion-1.8.6.tar.gz";
       md5 = "9310cb266031cf8d0779885782a84a5b";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl1 ];
     };
   };
   infrae.cache = super.buildPythonPackage {
     name = "infrae.cache-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [Beaker repoze.lru];
     src = fetchurl {
       url = "https://pypi.python.org/packages/bb/f0/e7d5e984cf6592fd2807dc7bc44a93f9d18e04e6a61f87fdfb2622422d74/infrae.cache-1.0.1.tar.gz";
       md5 = "b09076a766747e6ed2a755cc62088e32";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
   ipdb = super.buildPythonPackage {
     name = "ipdb-0.10.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [ipython setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/eb/0a/0a37dc19572580336ad3813792c0d18c8d7117c2d66fc63c501f13a7a8f8/ipdb-0.10.1.tar.gz";
       md5 = "4aeab65f633ddc98ebdb5eebf08dc713";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   ipython = super.buildPythonPackage {
     name = "ipython-5.1.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools decorator pickleshare simplegeneric traitlets prompt-toolkit pygments pexpect backports.shutil-get-terminal-size pathlib2 pexpect];
     src = fetchurl {
       url = "https://pypi.python.org/packages/89/63/a9292f7cd9d0090a0f995e1167f3f17d5889dcbc9a175261719c513b9848/ipython-5.1.0.tar.gz";
       md5 = "47c8122420f65b58784cb4b9b4af35e3";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   ipython-genutils = super.buildPythonPackage {
     name = "ipython-genutils-0.1.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/71/b7/a64c71578521606edbbce15151358598f3dfb72a3431763edc2baf19e71f/ipython_genutils-0.1.0.tar.gz";
       md5 = "9a8afbe0978adbcbfcb3b35b2d015a56";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   mercurial = super.buildPythonPackage {
-    name = "mercurial-4.0.2";
+    name = "mercurial-4.1.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/85/1b/0296aacd697228974a473d2508f013532f987ed6b1bacfe5abd6d5be6332/mercurial-4.0.2.tar.gz";
-      md5 = "fa72a08e2723e4fa2a21c4e66437f3fa";
+      url = "https://pypi.python.org/packages/88/c1/f0501fd67f5e69346da41ee0bd7b2619ce4bbc9854bb645074c418b9941f/mercurial-4.1.2.tar.gz";
+      md5 = "934c99808bdc8385e074b902d59b0d93";
     };
     meta = {
       license = [ pkgs.lib.licenses.gpl1 pkgs.lib.licenses.gpl2Plus ];
     };
   };
   mock = super.buildPythonPackage {
     name = "mock-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/15/45/30273ee91feb60dabb8fbb2da7868520525f02cf910279b3047182feed80/mock-1.0.1.zip";
       md5 = "869f08d003c289a97c1a6610faf5e913";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   msgpack-python = super.buildPythonPackage {
     name = "msgpack-python-0.4.8";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/21/27/8a1d82041c7a2a51fcc73675875a5f9ea06c2663e02fcfeb708be1d081a0/msgpack-python-0.4.8.tar.gz";
       md5 = "dcd854fb41ee7584ebbf35e049e6be98";
     };
     meta = {
       license = [ pkgs.lib.licenses.asl20 ];
     };
   };
   pathlib2 = super.buildPythonPackage {
     name = "pathlib2-2.1.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c9/27/8448b10d8440c08efeff0794adf7d0ed27adb98372c70c7b38f3947d4749/pathlib2-2.1.0.tar.gz";
       md5 = "38e4f58b4d69dfcb9edb49a54a8b28d2";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   pexpect = super.buildPythonPackage {
     name = "pexpect-4.2.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [ptyprocess];
     src = fetchurl {
       url = "https://pypi.python.org/packages/e8/13/d0b0599099d6cd23663043a2a0bb7c61e58c6ba359b2656e6fb000ef5b98/pexpect-4.2.1.tar.gz";
       md5 = "3694410001a99dff83f0b500a1ca1c95";
     };
     meta = {
       license = [ pkgs.lib.licenses.isc { fullName = "ISC License (ISCL)"; } ];
     };
   };
   pickleshare = super.buildPythonPackage {
     name = "pickleshare-0.7.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pathlib2];
     src = fetchurl {
       url = "https://pypi.python.org/packages/69/fe/dd137d84daa0fd13a709e448138e310d9ea93070620c9db5454e234af525/pickleshare-0.7.4.tar.gz";
       md5 = "6a9e5dd8dfc023031f6b7b3f824cab12";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   prompt-toolkit = super.buildPythonPackage {
     name = "prompt-toolkit-1.0.9";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six wcwidth];
     src = fetchurl {
       url = "https://pypi.python.org/packages/83/14/5ac258da6c530eca02852ee25c7a9ff3ca78287bb4c198d0d0055845d856/prompt_toolkit-1.0.9.tar.gz";
       md5 = "a39f91a54308fb7446b1a421c11f227c";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   ptyprocess = super.buildPythonPackage {
     name = "ptyprocess-0.5.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/db/d7/b465161910f3d1cef593c5e002bff67e0384898f597f1a7fdc8db4c02bf6/ptyprocess-0.5.1.tar.gz";
       md5 = "94e537122914cc9ec9c1eadcd36e73a1";
     };
     meta = {
       license = [  ];
     };
   };
   py = super.buildPythonPackage {
     name = "py-1.4.31";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f4/9a/8dfda23f36600dd701c6722316ba8a3ab4b990261f83e7d3ffc6dfedf7ef/py-1.4.31.tar.gz";
       md5 = "5d2c63c56dc3f2115ec35c066ecd582b";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   pygments = super.buildPythonPackage {
     name = "pygments-2.2.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/71/2a/2e4e77803a8bd6408a2903340ac498cb0a2181811af7c9ec92cb70b0308a/Pygments-2.2.0.tar.gz";
       md5 = "13037baca42f16917cbd5ad2fab50844";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   pyramid = super.buildPythonPackage {
-    name = "pyramid-1.6.1";
+    name = "pyramid-1.7.4";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools WebOb repoze.lru zope.interface zope.deprecation venusian translationstring PasteDeploy];
     src = fetchurl {
-      url = "https://pypi.python.org/packages/30/b3/fcc4a2a4800cbf21989e00454b5828cf1f7fe35c63e0810b350e56d4c475/pyramid-1.6.1.tar.gz";
-      md5 = "b18688ff3cc33efdbb098a35b45dd122";
+      url = "https://pypi.python.org/packages/33/91/55f5c661f8923902cd1f68d75f2b937c45e7682857356cf18f0be5493899/pyramid-1.7.4.tar.gz";
+      md5 = "6ef1dfdcff9136d04490410757c4c446";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   pyramid-jinja2 = super.buildPythonPackage {
     name = "pyramid-jinja2-2.5";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pyramid zope.deprecation Jinja2 MarkupSafe];
     src = fetchurl {
       url = "https://pypi.python.org/packages/a1/80/595e26ffab7deba7208676b6936b7e5a721875710f982e59899013cae1ed/pyramid_jinja2-2.5.tar.gz";
       md5 = "07cb6547204ac5e6f0b22a954ccee928";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   pyramid-mako = super.buildPythonPackage {
     name = "pyramid-mako-1.0.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pyramid Mako];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f1/92/7e69bcf09676d286a71cb3bbb887b16595b96f9ba7adbdc239ffdd4b1eb9/pyramid_mako-1.0.2.tar.gz";
       md5 = "ee25343a97eb76bd90abdc2a774eb48a";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   pytest = super.buildPythonPackage {
     name = "pytest-3.0.5";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [py];
     src = fetchurl {
       url = "https://pypi.python.org/packages/a8/87/b7ca49efe52d2b4169f2bfc49aa5e384173c4619ea8e635f123a0dac5b75/pytest-3.0.5.tar.gz";
       md5 = "cefd527b59332688bf5db4a10aa8a7cb";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   pytest-catchlog = super.buildPythonPackage {
     name = "pytest-catchlog-1.2.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [py pytest];
     src = fetchurl {
       url = "https://pypi.python.org/packages/f2/2b/2faccdb1a978fab9dd0bf31cca9f6847fbe9184a0bdcc3011ac41dd44191/pytest-catchlog-1.2.2.zip";
       md5 = "09d890c54c7456c818102b7ff8c182c8";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   pytest-cov = super.buildPythonPackage {
     name = "pytest-cov-2.4.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pytest coverage];
     src = fetchurl {
       url = "https://pypi.python.org/packages/00/c0/2bfd1fcdb9d407b8ac8185b1cb5ff458105c6b207a9a7f0e13032de9828f/pytest-cov-2.4.0.tar.gz";
       md5 = "2fda09677d232acc99ec1b3c5831e33f";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal pkgs.lib.licenses.mit ];
     };
   };
   pytest-profiling = super.buildPythonPackage {
     name = "pytest-profiling-1.2.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [six pytest gprof2dot];
     src = fetchurl {
       url = "https://pypi.python.org/packages/73/e8/804681323bac0bc45c520ec34185ba8469008942266d0074699b204835c1/pytest-profiling-1.2.2.tar.gz";
       md5 = "0a16d7dda2d23b91e9730fa4558cf728";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   pytest-runner = super.buildPythonPackage {
     name = "pytest-runner-2.9";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/11/d4/c335ddf94463e451109e3494e909765c3e5205787b772e3b25ee8601b86a/pytest-runner-2.9.tar.gz";
       md5 = "2212a2e34404b0960b2fdc2c469247b2";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   pytest-sugar = super.buildPythonPackage {
     name = "pytest-sugar-0.7.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pytest termcolor];
     src = fetchurl {
       url = "https://pypi.python.org/packages/03/97/05d988b4fa870e7373e8ee4582408543b9ca2bd35c3c67b569369c6f9c49/pytest-sugar-0.7.1.tar.gz";
       md5 = "7400f7c11f3d572b2c2a3b60352d35fe";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   pytest-timeout = super.buildPythonPackage {
     name = "pytest-timeout-1.2.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [pytest];
     src = fetchurl {
       url = "https://pypi.python.org/packages/cc/b7/b2a61365ea6b6d2e8881360ae7ed8dad0327ad2df89f2f0be4a02304deb2/pytest-timeout-1.2.0.tar.gz";
       md5 = "83607d91aa163562c7ee835da57d061d";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit { fullName = "DFSG approved"; } ];
     };
   };
   repoze.lru = super.buildPythonPackage {
     name = "repoze.lru-0.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/6e/1e/aa15cc90217e086dc8769872c8778b409812ff036bf021b15795638939e4/repoze.lru-0.6.tar.gz";
       md5 = "2c3b64b17a8e18b405f55d46173e14dd";
     };
     meta = {
       license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   rhodecode-vcsserver = super.buildPythonPackage {
-    name = "rhodecode-vcsserver-4.6.1";
+    name = "rhodecode-vcsserver-4.7.0";
     buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage configobj];
     doCheck = true;
     propagatedBuildInputs = with self; [Beaker configobj dulwich hgsubversion infrae.cache mercurial msgpack-python pyramid pyramid-jinja2 pyramid-mako repoze.lru simplejson subprocess32 subvertpy six translationstring WebOb wheel zope.deprecation zope.interface ipdb ipython gevent greenlet gunicorn waitress Pyro4 serpent pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage];
     src = ./.;
     meta = {
       license = [ { fullName = "GPL V3"; } { fullName = "GNU General Public License v3 or later (GPLv3+)"; } ];
     };
   };
   serpent = super.buildPythonPackage {
     name = "serpent-1.15";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/7b/38/b2b27673a882ff2ea5871bb3e3e6b496ebbaafd1612e51990ffb158b9254/serpent-1.15.tar.gz";
       md5 = "e27b1aad5c218e16442f52abb7c7053a";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   setuptools = super.buildPythonPackage {
     name = "setuptools-30.1.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/1e/43/002c8616db9a3e7be23c2556e39b90a32bb40ba0dc652de1999d5334d372/setuptools-30.1.0.tar.gz";
       md5 = "cac497f42e5096ac8df29e38d3f81c3e";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   simplegeneric = super.buildPythonPackage {
     name = "simplegeneric-0.8.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/3d/57/4d9c9e3ae9a255cd4e1106bb57e24056d3d0709fc01b2e3e345898e49d5b/simplegeneric-0.8.1.zip";
       md5 = "f9c1fab00fd981be588fc32759f474e3";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
   simplejson = super.buildPythonPackage {
     name = "simplejson-3.7.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/6d/89/7f13f099344eea9d6722779a1f165087cb559598107844b1ac5dbd831fb1/simplejson-3.7.2.tar.gz";
       md5 = "a5fc7d05d4cb38492285553def5d4b46";
     };
     meta = {
       license = [ { fullName = "Academic Free License (AFL)"; } pkgs.lib.licenses.mit ];
     };
   };
   six = super.buildPythonPackage {
     name = "six-1.9.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/16/64/1dc5e5976b17466fd7d712e59cbe9fb1e18bec153109e5ba3ed6c9102f1a/six-1.9.0.tar.gz";
       md5 = "476881ef4012262dfc8adc645ee786c4";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   subprocess32 = super.buildPythonPackage {
     name = "subprocess32-3.2.6";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/28/8d/33ccbff51053f59ae6c357310cac0e79246bbed1d345ecc6188b176d72c3/subprocess32-3.2.6.tar.gz";
       md5 = "754c5ab9f533e764f931136974b618f1";
     };
     meta = {
       license = [ pkgs.lib.licenses.psfl ];
     };
   };
   subvertpy = super.buildPythonPackage {
     name = "subvertpy-0.9.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://code.rhodecode.com/upstream/subvertpy/archive/subvertpy-0.9.3.tar.gz?md5=4e49da2fe07608239cc9a80a7bb8f33c";
       md5 = "4e49da2fe07608239cc9a80a7bb8f33c";
     };
     meta = {
       license = [ pkgs.lib.licenses.lgpl21Plus ];
     };
   };
   termcolor = super.buildPythonPackage {
     name = "termcolor-1.1.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/8a/48/a76be51647d0eb9f10e2a4511bf3ffb8cc1e6b14e9e4fab46173aa79f981/termcolor-1.1.0.tar.gz";
       md5 = "043e89644f8909d462fbbfa511c768df";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   traitlets = super.buildPythonPackage {
     name = "traitlets-4.3.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [ipython-genutils six decorator enum34];
     src = fetchurl {
       url = "https://pypi.python.org/packages/b1/d6/5b5aa6d5c474691909b91493da1e8972e309c9f01ecfe4aeafd272eb3234/traitlets-4.3.1.tar.gz";
       md5 = "dd0b1b6e5d31ce446d55a4b5e5083c98";
     };
     meta = {
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
   translationstring = super.buildPythonPackage {
     name = "translationstring-1.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/5e/eb/bee578cc150b44c653b63f5ebe258b5d0d812ddac12497e5f80fcad5d0b4/translationstring-1.3.tar.gz";
       md5 = "a4b62e0f3c189c783a1685b3027f7c90";
     };
     meta = {
       license = [ { fullName = "BSD-like (http://repoze.org/license.html)"; } ];
     };
   };
   venusian = super.buildPythonPackage {
     name = "venusian-1.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/86/20/1948e0dfc4930ddde3da8c33612f6a5717c0b4bc28f591a5c5cf014dd390/venusian-1.0.tar.gz";
       md5 = "dccf2eafb7113759d60c86faf5538756";
     };
     meta = {
       license = [ { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ];
     };
   };
   waitress = super.buildPythonPackage {
     name = "waitress-1.0.1";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/78/7d/84d11b96c3f60164dec3bef4a859a03aeae0231aa93f57fbe0d05fa4ff36/waitress-1.0.1.tar.gz";
       md5 = "dda92358a7569669086155923a46e57c";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
   wcwidth = super.buildPythonPackage {
     name = "wcwidth-0.1.7";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/55/11/e4a2bb08bb450fdbd42cc709dd40de4ed2c472cf0ccb9e64af22279c5495/wcwidth-0.1.7.tar.gz";
       md5 = "b3b6a0a08f0c8a34d1de8cf44150a4ad";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   wheel = super.buildPythonPackage {
     name = "wheel-0.29.0";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/wheel-0.29.0.tar.gz";
       md5 = "555a67e4507cedee23a0deb9651e452f";
     };
     meta = {
       license = [ pkgs.lib.licenses.mit ];
     };
   };
   zope.deprecation = super.buildPythonPackage {
     name = "zope.deprecation-4.1.2";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/c1/d3/3919492d5e57d8dd01b36f30b34fc8404a30577392b1eb817c303499ad20/zope.deprecation-4.1.2.tar.gz";
       md5 = "e9a663ded58f4f9f7881beb56cae2782";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
   zope.interface = super.buildPythonPackage {
     name = "zope.interface-4.1.3";
     buildInputs = with self; [];
     doCheck = false;
     propagatedBuildInputs = with self; [setuptools];
     src = fetchurl {
       url = "https://pypi.python.org/packages/9d/81/2509ca3c6f59080123c1a8a97125eb48414022618cec0e64eb1313727bfe/zope.interface-4.1.3.tar.gz";
       md5 = "9ae3d24c0c7415deb249dd1a132f0f79";
     };
     meta = {
       license = [ pkgs.lib.licenses.zpt21 ];
     };
   };
 
 ### Test requirements
 
   
 }

requirements.txt

@@ -1,43 +1,43 b''
-# core
+## core
 setuptools==30.1.0
 
 Beaker==1.7.0
 configobj==5.0.6
+decorator==4.0.11
 dulwich==0.13.0
 hgsubversion==1.8.6
 infrae.cache==1.0.1
-mercurial==4.0.2
+mercurial==4.1.2
 msgpack-python==0.4.8
-pyramid==1.6.1
 pyramid-jinja2==2.5
+pyramid==1.7.4
 pyramid-mako==1.0.2
 repoze.lru==0.6
 simplejson==3.7.2
 subprocess32==3.2.6
 
 # Custom subvertpy that is not available on pypi.
 https://code.rhodecode.com/upstream/subvertpy/archive/subvertpy-0.9.3.tar.gz?md5=4e49da2fe07608239cc9a80a7bb8f33c#egg=subvertpy==0.9.3
 
 six==1.9.0
 translationstring==1.3
 WebOb==1.3.1
 wheel==0.29.0
 zope.deprecation==4.1.2
 zope.interface==4.1.3
 
 ## debug
 ipdb==0.10.1
 ipython==5.1.0
-
 # http servers
 gevent==1.1.2
 greenlet==0.4.10
 gunicorn==19.6.0
 waitress==1.0.1
 
 # Pyro/Deprecated TODO(Marcink): remove in 4.7 release.
 Pyro4==4.41
 serpent==1.15
 
 ## test related requirements
 -r requirements_test.txt

vcsserver/VERSION

@@ -1,1 +1,1 b''
-4.6.1
  No newline at end of file
+4.7.0
  No newline at end of file

vcsserver/base.py

@@ -1,82 +1,98 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2017 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
+import sys
+import traceback
 import logging
 import urlparse
 
 log = logging.getLogger(__name__)
 
 
 class RepoFactory(object):
     """
     Utility to create instances of repository
 
     It provides internal caching of the `repo` object based on
     the :term:`call context`.
     """
 
     def __init__(self, repo_cache):
         self._cache = repo_cache
 
     def _create_config(self, path, config):
         config = {}
         return config
 
     def _create_repo(self, wire, create):
         raise NotImplementedError()
 
     def repo(self, wire, create=False):
         """
         Get a repository instance for the given path.
 
         Uses internally the low level beaker API since the decorators introduce
         significant overhead.
         """
         def create_new_repo():
             return self._create_repo(wire, create)
 
         return self._repo(wire, create_new_repo)
 
     def _repo(self, wire, createfunc):
         context = wire.get('context', None)
         cache = wire.get('cache', True)
 
         if context and cache:
             cache_key = (context, wire['path'])
             log.debug(
                 'FETCH %s@%s repo object from cache. Context: %s',
                 self.__class__.__name__, wire['path'], context)
             return self._cache.get(key=cache_key, createfunc=createfunc)
         else:
             log.debug(
                 'INIT %s@%s repo object based on wire %s. Context: %s',
                 self.__class__.__name__, wire['path'], wire, context)
             return createfunc()
 
 
 def obfuscate_qs(query_string):
     if query_string is None:
         return None
 
     parsed = []
     for k, v in urlparse.parse_qsl(query_string, keep_blank_values=True):
         if k in ['auth_token', 'api_key']:
             v = "*****"
         parsed.append((k, v))
 
     return '&'.join('{}{}'.format(
         k, '={}'.format(v) if v else '') for k, v in parsed)
+
+
+def raise_from_original(new_type):
+    """
+    Raise a new exception type with original args and traceback.
+    """
+    exc_type, exc_value, exc_traceback = sys.exc_info()
+
+    traceback.format_exception(exc_type, exc_value, exc_traceback)
+
+    try:
+        raise new_type(*exc_value.args), None, exc_traceback
+    finally:
+        del exc_traceback

vcsserver/git.py

@@ -1,581 +1,644 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2017 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import logging
 import os
 import posixpath as vcspath
 import re
 import stat
 import urllib
 import urllib2
 from functools import wraps
 
 from dulwich import index, objects
 from dulwich.client import HttpGitClient, LocalGitClient
 from dulwich.errors import (
     NotGitRepository, ChecksumMismatch, WrongObjectException,
     MissingCommitError, ObjectMissing, HangupException,
     UnexpectedCommandError)
 from dulwich.repo import Repo as DulwichRepo, Tag
 from dulwich.server import update_server_info
 
 from vcsserver import exceptions, settings, subprocessio
 from vcsserver.utils import safe_str
-from vcsserver.base import RepoFactory, obfuscate_qs
+from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
 from vcsserver.hgcompat import (
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler)
-
+from vcsserver.git_lfs.lib import LFSOidStore
 
 DIR_STAT = stat.S_IFDIR
 FILE_MODE = stat.S_IFMT
 GIT_LINK = objects.S_IFGITLINK
 
 log = logging.getLogger(__name__)
 
 
 def reraise_safe_exceptions(func):
     """Converts Dulwich exceptions to something neutral."""
     @wraps(func)
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except (ChecksumMismatch, WrongObjectException, MissingCommitError,
                 ObjectMissing) as e:
             raise exceptions.LookupException(e.message)
         except (HangupException, UnexpectedCommandError) as e:
             raise exceptions.VcsException(e.message)
+        except Exception as e:
+            # NOTE(marcink): becuase of how dulwich handles some exceptions
+            # (KeyError on empty repos), we cannot track this and catch all
+            # exceptions, it's an exceptions from other handlers
+            #if not hasattr(e, '_vcs_kind'):
+                #log.exception("Unhandled exception in git remote call")
+                #raise_from_original(exceptions.UnhandledException)
+            raise
     return wrapper
 
 
 class Repo(DulwichRepo):
     """
     A wrapper for dulwich Repo class.
 
     Since dulwich is sometimes keeping .idx file descriptors open, it leads to
     "Too many open files" error. We need to close all opened file descriptors
     once the repo object is destroyed.
 
     TODO: mikhail: please check if we need this wrapper after updating dulwich
     to 0.12.0 +
     """
     def __del__(self):
         if hasattr(self, 'object_store'):
             self.close()
 
 
 class GitFactory(RepoFactory):
 
     def _create_repo(self, wire, create):
         repo_path = str_to_dulwich(wire['path'])
         return Repo(repo_path)
 
 
 class GitRemote(object):
 
     def __init__(self, factory):
         self._factory = factory
 
         self._bulk_methods = {
             "author": self.commit_attribute,
             "date": self.get_object_attrs,
             "message": self.commit_attribute,
             "parents": self.commit_attribute,
             "_commit": self.revision,
         }
 
+    def _wire_to_config(self, wire):
+        if 'config' in wire:
+            return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']])
+        return {}
+
     def _assign_ref(self, wire, ref, commit_id):
         repo = self._factory.repo(wire)
         repo[ref] = commit_id
 
     @reraise_safe_exceptions
     def add_object(self, wire, content):
         repo = self._factory.repo(wire)
         blob = objects.Blob()
         blob.set_raw_string(content)
         repo.object_store.add_object(blob)
         return blob.id
 
     @reraise_safe_exceptions
     def assert_correct_path(self, wire):
         try:
             self._factory.repo(wire)
         except NotGitRepository as e:
             # Exception can contain unicode which we convert
             raise exceptions.AbortException(repr(e))
 
     @reraise_safe_exceptions
     def bare(self, wire):
         repo = self._factory.repo(wire)
         return repo.bare
 
     @reraise_safe_exceptions
     def blob_as_pretty_string(self, wire, sha):
         repo = self._factory.repo(wire)
         return repo[sha].as_pretty_string()
 
     @reraise_safe_exceptions
     def blob_raw_length(self, wire, sha):
         repo = self._factory.repo(wire)
         blob = repo[sha]
         return blob.raw_length()
 
+    def _parse_lfs_pointer(self, raw_content):
+
+        spec_string = 'version https://git-lfs.github.com/spec'
+        if raw_content and raw_content.startswith(spec_string):
+            pattern = re.compile(r"""
+            (?:\n)?
+            ^version[ ]https://git-lfs\.github\.com/spec/(?P<spec_ver>v\d+)\n
+            ^oid[ ] sha256:(?P<oid_hash>[0-9a-f]{64})\n
+            ^size[ ](?P<oid_size>[0-9]+)\n
+            (?:\n)?
+            """, re.VERBOSE | re.MULTILINE)
+            match = pattern.match(raw_content)
+            if match:
+                return match.groupdict()
+
+        return {}
+
+    @reraise_safe_exceptions
+    def is_large_file(self, wire, sha):
+        repo = self._factory.repo(wire)
+        blob = repo[sha]
+        return self._parse_lfs_pointer(blob.as_raw_string())
+
+    @reraise_safe_exceptions
+    def in_largefiles_store(self, wire, oid):
+        repo = self._factory.repo(wire)
+        conf = self._wire_to_config(wire)
+
+        store_location = conf.get('vcs_git_lfs_store_location')
+        if store_location:
+            repo_name = repo.path
+            store = LFSOidStore(
+                oid=oid, repo=repo_name, store_location=store_location)
+            return store.has_oid()
+
+        return False
+
+    @reraise_safe_exceptions
+    def store_path(self, wire, oid):
+        repo = self._factory.repo(wire)
+        conf = self._wire_to_config(wire)
+
+        store_location = conf.get('vcs_git_lfs_store_location')
+        if store_location:
+            repo_name = repo.path
+            store = LFSOidStore(
+                oid=oid, repo=repo_name, store_location=store_location)
+            return store.oid_path
+        raise ValueError('Unable to fetch oid with path {}'.format(oid))
+
     @reraise_safe_exceptions
     def bulk_request(self, wire, rev, pre_load):
         result = {}
         for attr in pre_load:
             try:
                 method = self._bulk_methods[attr]
                 args = [wire, rev]
                 if attr == "date":
                     args.extend(["commit_time", "commit_timezone"])
                 elif attr in ["author", "message", "parents"]:
                     args.append(attr)
                 result[attr] = method(*args)
             except KeyError:
                 raise exceptions.VcsException(
                     "Unknown bulk attribute: %s" % attr)
         return result
 
     def _build_opener(self, url):
         handlers = []
         url_obj = url_parser(url)
         _, authinfo = url_obj.authinfo()
 
         if authinfo:
             # create a password manager
             passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         return urllib2.build_opener(*handlers)
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
         url_obj = url_parser(url)
         test_uri, _ = url_obj.authinfo()
         url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
         url_obj.query = obfuscate_qs(url_obj.query)
         cleaned_uri = str(url_obj)
         log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
 
         if not test_uri.endswith('info/refs'):
             test_uri = test_uri.rstrip('/') + '/info/refs'
 
         o = self._build_opener(url)
         o.addheaders = [('User-Agent', 'git/1.7.8.0')]  # fake some git
 
         q = {"service": 'git-upload-pack'}
         qs = '?%s' % urllib.urlencode(q)
         cu = "%s%s" % (test_uri, qs)
         req = urllib2.Request(cu, None, {})
 
         try:
             log.debug("Trying to open URL %s", cleaned_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError('Return Code is not 200')
         except Exception as e:
             log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
             # means it cannot be cloned
             raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
 
         # now detect if it's proper git repo
         gitdata = resp.read()
         if 'service=git-upload-pack' in gitdata:
             pass
         elif re.findall(r'[0-9a-fA-F]{40}\s+refs', gitdata):
             # old style git can return some other format !
             pass
         else:
             raise exceptions.URLError(
                 "url [%s] does not look like an git" % (cleaned_uri,))
 
         return True
 
     @reraise_safe_exceptions
     def clone(self, wire, url, deferred, valid_refs, update_after_clone):
         remote_refs = self.fetch(wire, url, apply_refs=False)
         repo = self._factory.repo(wire)
         if isinstance(valid_refs, list):
             valid_refs = tuple(valid_refs)
 
         for k in remote_refs:
             # only parse heads/tags and skip so called deferred tags
             if k.startswith(valid_refs) and not k.endswith(deferred):
                 repo[k] = remote_refs[k]
 
         if update_after_clone:
             # we want to checkout HEAD
             repo["HEAD"] = remote_refs["HEAD"]
             index.build_index_from_tree(repo.path, repo.index_path(),
                                         repo.object_store, repo["HEAD"].tree)
 
     # TODO: this is quite complex, check if that can be simplified
     @reraise_safe_exceptions
     def commit(self, wire, commit_data, branch, commit_tree, updated, removed):
         repo = self._factory.repo(wire)
         object_store = repo.object_store
 
         # Create tree and populates it with blobs
         commit_tree = commit_tree and repo[commit_tree] or objects.Tree()
 
         for node in updated:
             # Compute subdirs if needed
             dirpath, nodename = vcspath.split(node['path'])
             dirnames = map(safe_str, dirpath and dirpath.split('/') or [])
             parent = commit_tree
             ancestors = [('', parent)]
 
             # Tries to dig for the deepest existing tree
             while dirnames:
                 curdir = dirnames.pop(0)
                 try:
                     dir_id = parent[curdir][1]
                 except KeyError:
                     # put curdir back into dirnames and stops
                     dirnames.insert(0, curdir)
                     break
                 else:
                     # If found, updates parent
                     parent = repo[dir_id]
                     ancestors.append((curdir, parent))
             # Now parent is deepest existing tree and we need to create
             # subtrees for dirnames (in reverse order)
             # [this only applies for nodes from added]
             new_trees = []
 
             blob = objects.Blob.from_string(node['content'])
 
             if dirnames:
                 # If there are trees which should be created we need to build
                 # them now (in reverse order)
                 reversed_dirnames = list(reversed(dirnames))
                 curtree = objects.Tree()
                 curtree[node['node_path']] = node['mode'], blob.id
                 new_trees.append(curtree)
                 for dirname in reversed_dirnames[:-1]:
                     newtree = objects.Tree()
                     newtree[dirname] = (DIR_STAT, curtree.id)
                     new_trees.append(newtree)
                     curtree = newtree
                 parent[reversed_dirnames[-1]] = (DIR_STAT, curtree.id)
             else:
                 parent.add(
                     name=node['node_path'], mode=node['mode'], hexsha=blob.id)
 
             new_trees.append(parent)
             # Update ancestors
             reversed_ancestors = reversed(
                 [(a[1], b[1], b[0]) for a, b in zip(ancestors, ancestors[1:])])
             for parent, tree, path in reversed_ancestors:
                 parent[path] = (DIR_STAT, tree.id)
                 object_store.add_object(tree)
 
             object_store.add_object(blob)
             for tree in new_trees:
                 object_store.add_object(tree)
 
         for node_path in removed:
             paths = node_path.split('/')
             tree = commit_tree
             trees = [tree]
             # Traverse deep into the forest...
             for path in paths:
                 try:
                     obj = repo[tree[path][1]]
                     if isinstance(obj, objects.Tree):
                         trees.append(obj)
                         tree = obj
                 except KeyError:
                     break
             # Cut down the blob and all rotten trees on the way back...
             for path, tree in reversed(zip(paths, trees)):
                 del tree[path]
                 if tree:
                     # This tree still has elements - don't remove it or any
                     # of it's parents
                     break
 
         object_store.add_object(commit_tree)
 
         # Create commit
         commit = objects.Commit()
         commit.tree = commit_tree.id
         for k, v in commit_data.iteritems():
             setattr(commit, k, v)
         object_store.add_object(commit)
 
         ref = 'refs/heads/%s' % branch
         repo.refs[ref] = commit.id
 
         return commit.id
 
     @reraise_safe_exceptions
     def fetch(self, wire, url, apply_refs=True, refs=None):
         if url != 'default' and '://' not in url:
             client = LocalGitClient(url)
         else:
             url_obj = url_parser(url)
             o = self._build_opener(url)
             url, _ = url_obj.authinfo()
             client = HttpGitClient(base_url=url, opener=o)
         repo = self._factory.repo(wire)
 
         determine_wants = repo.object_store.determine_wants_all
         if refs:
             def determine_wants_requested(references):
                 return [references[r] for r in references if r in refs]
             determine_wants = determine_wants_requested
 
         try:
             remote_refs = client.fetch(
                 path=url, target=repo, determine_wants=determine_wants)
         except NotGitRepository as e:
             log.warning(
                 'Trying to fetch from "%s" failed, not a Git repository.', url)
             # Exception can contain unicode which we convert
             raise exceptions.AbortException(repr(e))
 
         # mikhail: client.fetch() returns all the remote refs, but fetches only
         # refs filtered by `determine_wants` function. We need to filter result
         # as well
         if refs:
             remote_refs = {k: remote_refs[k] for k in remote_refs if k in refs}
 
         if apply_refs:
             # TODO: johbo: Needs proper test coverage with a git repository
             # that contains a tag object, so that we would end up with
             # a peeled ref at this point.
             PEELED_REF_MARKER = '^{}'
             for k in remote_refs:
                 if k.endswith(PEELED_REF_MARKER):
                     log.info("Skipping peeled reference %s", k)
                     continue
                 repo[k] = remote_refs[k]
 
             if refs:
                 # mikhail: explicitly set the head to the last ref.
                 repo['HEAD'] = remote_refs[refs[-1]]
 
             # TODO: mikhail: should we return remote_refs here to be
             # consistent?
         else:
             return remote_refs
 
     @reraise_safe_exceptions
     def get_remote_refs(self, wire, url):
         repo = Repo(url)
         return repo.get_refs()
 
     @reraise_safe_exceptions
     def get_description(self, wire):
         repo = self._factory.repo(wire)
         return repo.get_description()
 
     @reraise_safe_exceptions
     def get_file_history(self, wire, file_path, commit_id, limit):
         repo = self._factory.repo(wire)
         include = [commit_id]
         paths = [file_path]
 
         walker = repo.get_walker(include, paths=paths, max_entries=limit)
         return [x.commit.id for x in walker]
 
     @reraise_safe_exceptions
     def get_missing_revs(self, wire, rev1, rev2, path2):
         repo = self._factory.repo(wire)
         LocalGitClient(thin_packs=False).fetch(path2, repo)
 
         wire_remote = wire.copy()
         wire_remote['path'] = path2
         repo_remote = self._factory.repo(wire_remote)
         LocalGitClient(thin_packs=False).fetch(wire["path"], repo_remote)
 
         revs = [
             x.commit.id
             for x in repo_remote.get_walker(include=[rev2], exclude=[rev1])]
         return revs
 
     @reraise_safe_exceptions
     def get_object(self, wire, sha):
         repo = self._factory.repo(wire)
         obj = repo.get_object(sha)
         commit_id = obj.id
 
         if isinstance(obj, Tag):
             commit_id = obj.object[1]
 
         return {
             'id': obj.id,
             'type': obj.type_name,
             'commit_id': commit_id
         }
 
     @reraise_safe_exceptions
     def get_object_attrs(self, wire, sha, *attrs):
         repo = self._factory.repo(wire)
         obj = repo.get_object(sha)
         return list(getattr(obj, a) for a in attrs)
 
     @reraise_safe_exceptions
     def get_refs(self, wire):
         repo = self._factory.repo(wire)
         result = {}
         for ref, sha in repo.refs.as_dict().items():
             peeled_sha = repo.get_peeled(ref)
             result[ref] = peeled_sha
         return result
 
     @reraise_safe_exceptions
     def get_refs_path(self, wire):
         repo = self._factory.repo(wire)
         return repo.refs.path
 
     @reraise_safe_exceptions
     def head(self, wire):
         repo = self._factory.repo(wire)
         return repo.head()
 
     @reraise_safe_exceptions
     def init(self, wire):
         repo_path = str_to_dulwich(wire['path'])
         self.repo = Repo.init(repo_path)
 
     @reraise_safe_exceptions
     def init_bare(self, wire):
         repo_path = str_to_dulwich(wire['path'])
         self.repo = Repo.init_bare(repo_path)
 
     @reraise_safe_exceptions
     def revision(self, wire, rev):
         repo = self._factory.repo(wire)
         obj = repo[rev]
         obj_data = {
             'id': obj.id,
         }
         try:
             obj_data['tree'] = obj.tree
         except AttributeError:
             pass
         return obj_data
 
     @reraise_safe_exceptions
     def commit_attribute(self, wire, rev, attr):
         repo = self._factory.repo(wire)
         obj = repo[rev]
         return getattr(obj, attr)
 
     @reraise_safe_exceptions
     def set_refs(self, wire, key, value):
         repo = self._factory.repo(wire)
         repo.refs[key] = value
 
     @reraise_safe_exceptions
     def remove_ref(self, wire, key):
         repo = self._factory.repo(wire)
         del repo.refs[key]
 
     @reraise_safe_exceptions
     def tree_changes(self, wire, source_id, target_id):
         repo = self._factory.repo(wire)
         source = repo[source_id].tree if source_id else None
         target = repo[target_id].tree
         result = repo.object_store.tree_changes(source, target)
         return list(result)
 
     @reraise_safe_exceptions
     def tree_items(self, wire, tree_id):
         repo = self._factory.repo(wire)
         tree = repo[tree_id]
 
         result = []
         for item in tree.iteritems():
             item_sha = item.sha
             item_mode = item.mode
 
             if FILE_MODE(item_mode) == GIT_LINK:
                 item_type = "link"
             else:
                 item_type = repo[item_sha].type_name
 
             result.append((item.path, item_mode, item_sha, item_type))
         return result
 
     @reraise_safe_exceptions
     def update_server_info(self, wire):
         repo = self._factory.repo(wire)
         update_server_info(repo)
 
     @reraise_safe_exceptions
     def discover_git_version(self):
         stdout, _ = self.run_git_command(
             {}, ['--version'], _bare=True, _safe=True)
         prefix = 'git version'
         if stdout.startswith(prefix):
             stdout = stdout[len(prefix):]
         return stdout.strip()
 
     @reraise_safe_exceptions
     def run_git_command(self, wire, cmd, **opts):
         path = wire.get('path', None)
 
         if path and os.path.isdir(path):
             opts['cwd'] = path
 
         if '_bare' in opts:
             _copts = []
             del opts['_bare']
         else:
             _copts = ['-c', 'core.quotepath=false', ]
         safe_call = False
         if '_safe' in opts:
             # no exc on failure
             del opts['_safe']
             safe_call = True
 
         gitenv = os.environ.copy()
         gitenv.update(opts.pop('extra_env', {}))
         # need to clean fix GIT_DIR !
         if 'GIT_DIR' in gitenv:
             del gitenv['GIT_DIR']
         gitenv['GIT_CONFIG_NOGLOBAL'] = '1'
 
         cmd = [settings.GIT_EXECUTABLE] + _copts + cmd
 
         try:
             _opts = {'env': gitenv, 'shell': False}
             _opts.update(opts)
             p = subprocessio.SubprocessIOChunker(cmd, **_opts)
 
             return ''.join(p), ''.join(p.error)
         except (EnvironmentError, OSError) as err:
             tb_err = ("Couldn't run git command (%s).\n"
                       "Original error was:%s\n" % (cmd, err))
             log.exception(tb_err)
             if safe_call:
                 return '', err
             else:
                 raise exceptions.VcsException(tb_err)
 
 
 def str_to_dulwich(value):
     """
     Dulwich 0.10.1a requires `unicode` objects to be passed in.
     """
     return value.decode(settings.WIRE_ENCODING)

vcsserver/hg.py

@@ -1,725 +1,727 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2017 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import io
 import logging
 import stat
-import sys
 import urllib
 import urllib2
 
 from hgext import largefiles, rebase
 from hgext.strip import strip as hgext_strip
 from mercurial import commands
 from mercurial import unionrepo
+from mercurial import verify
 
 from vcsserver import exceptions
-from vcsserver.base import RepoFactory, obfuscate_qs
+from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original
 from vcsserver.hgcompat import (
     archival, bin, clone, config as hgconfig, diffopts, hex,
     hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler,
     httppeer, localrepository, match, memctx, exchange, memfilectx, nullrev,
     patch, peer, revrange, ui, Abort, LookupError, RepoError, RepoLookupError,
     InterventionRequired, RequirementError)
 
 log = logging.getLogger(__name__)
 
 
 def make_ui_from_config(repo_config):
     baseui = ui.ui()
 
     # clean the baseui object
     baseui._ocfg = hgconfig.config()
     baseui._ucfg = hgconfig.config()
     baseui._tcfg = hgconfig.config()
 
     for section, option, value in repo_config:
         baseui.setconfig(section, option, value)
 
     # make our hgweb quiet so it doesn't print output
     baseui.setconfig('ui', 'quiet', 'true')
 
     # force mercurial to only use 1 thread, otherwise it may try to set a
     # signal in a non-main thread, thus generating a ValueError.
     baseui.setconfig('worker', 'numcpus', 1)
 
     # If there is no config for the largefiles extension, we explicitly disable
     # it here. This overrides settings from repositories hgrc file. Recent
     # mercurial versions enable largefiles in hgrc on clone from largefile
     # repo.
     if not baseui.hasconfig('extensions', 'largefiles'):
         log.debug('Explicitly disable largefiles extension for repo.')
         baseui.setconfig('extensions', 'largefiles', '!')
 
     return baseui
 
 
 def reraise_safe_exceptions(func):
     """Decorator for converting mercurial exceptions to something neutral."""
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except (Abort, InterventionRequired):
             raise_from_original(exceptions.AbortException)
         except RepoLookupError:
             raise_from_original(exceptions.LookupException)
         except RequirementError:
             raise_from_original(exceptions.RequirementException)
         except RepoError:
             raise_from_original(exceptions.VcsException)
         except LookupError:
             raise_from_original(exceptions.LookupException)
         except Exception as e:
             if not hasattr(e, '_vcs_kind'):
                 log.exception("Unhandled exception in hg remote call")
                 raise_from_original(exceptions.UnhandledException)
             raise
     return wrapper
 
 
-def raise_from_original(new_type):
-    """
-    Raise a new exception type with original args and traceback.
-    """
-    _, original, traceback = sys.exc_info()
-    try:
-        raise new_type(*original.args), None, traceback
-    finally:
-        del traceback
-
-
 class MercurialFactory(RepoFactory):
 
     def _create_config(self, config, hooks=True):
         if not hooks:
             hooks_to_clean = frozenset((
                 'changegroup.repo_size', 'preoutgoing.pre_pull',
                 'outgoing.pull_logger', 'prechangegroup.pre_push'))
             new_config = []
             for section, option, value in config:
                 if section == 'hooks' and option in hooks_to_clean:
                     continue
                 new_config.append((section, option, value))
             config = new_config
 
         baseui = make_ui_from_config(config)
         return baseui
 
     def _create_repo(self, wire, create):
         baseui = self._create_config(wire["config"])
         return localrepository(baseui, wire["path"], create)
 
 
 class HgRemote(object):
 
     def __init__(self, factory):
         self._factory = factory
 
         self._bulk_methods = {
             "affected_files": self.ctx_files,
             "author": self.ctx_user,
             "branch": self.ctx_branch,
             "children": self.ctx_children,
             "date": self.ctx_date,
             "message": self.ctx_description,
             "parents": self.ctx_parents,
             "status": self.ctx_status,
             "_file_paths": self.ctx_list,
         }
 
     @reraise_safe_exceptions
     def discover_hg_version(self):
         from mercurial import util
         return util.version()
 
     @reraise_safe_exceptions
     def archive_repo(self, archive_path, mtime, file_info, kind):
         if kind == "tgz":
             archiver = archival.tarit(archive_path, mtime, "gz")
         elif kind == "tbz2":
             archiver = archival.tarit(archive_path, mtime, "bz2")
         elif kind == 'zip':
             archiver = archival.zipit(archive_path, mtime)
         else:
             raise exceptions.ArchiveException(
                 'Remote does not support: "%s".' % kind)
 
         for f_path, f_mode, f_is_link, f_content in file_info:
             archiver.addfile(f_path, f_mode, f_is_link, f_content)
         archiver.done()
 
     @reraise_safe_exceptions
     def bookmarks(self, wire):
         repo = self._factory.repo(wire)
         return dict(repo._bookmarks)
 
     @reraise_safe_exceptions
     def branches(self, wire, normal, closed):
         repo = self._factory.repo(wire)
         iter_branches = repo.branchmap().iterbranches()
         bt = {}
         for branch_name, _heads, tip, is_closed in iter_branches:
             if normal and not is_closed:
                 bt[branch_name] = tip
             if closed and is_closed:
                 bt[branch_name] = tip
 
         return bt
 
     @reraise_safe_exceptions
     def bulk_request(self, wire, rev, pre_load):
         result = {}
         for attr in pre_load:
             try:
                 method = self._bulk_methods[attr]
                 result[attr] = method(wire, rev)
             except KeyError:
                 raise exceptions.VcsException(
                     'Unknown bulk attribute: "%s"' % attr)
         return result
 
     @reraise_safe_exceptions
     def clone(self, wire, source, dest, update_after_clone=False, hooks=True):
         baseui = self._factory._create_config(wire["config"], hooks=hooks)
         clone(baseui, source, dest, noupdate=not update_after_clone)
 
     @reraise_safe_exceptions
     def commitctx(
             self, wire, message, parents, commit_time, commit_timezone,
             user, files, extra, removed, updated):
 
         def _filectxfn(_repo, memctx, path):
             """
             Marks given path as added/changed/removed in a given _repo. This is
             for internal mercurial commit function.
             """
 
             # check if this path is removed
             if path in removed:
                 # returning None is a way to mark node for removal
                 return None
 
             # check if this path is added
             for node in updated:
                 if node['path'] == path:
                     return memfilectx(
                         _repo,
                         path=node['path'],
                         data=node['content'],
                         islink=False,
                         isexec=bool(node['mode'] & stat.S_IXUSR),
                         copied=False,
                         memctx=memctx)
 
             raise exceptions.AbortException(
                 "Given path haven't been marked as added, "
                 "changed or removed (%s)" % path)
 
         repo = self._factory.repo(wire)
 
         commit_ctx = memctx(
             repo=repo,
             parents=parents,
             text=message,
             files=files,
             filectxfn=_filectxfn,
             user=user,
             date=(commit_time, commit_timezone),
             extra=extra)
 
         n = repo.commitctx(commit_ctx)
         new_id = hex(n)
 
         return new_id
 
     @reraise_safe_exceptions
     def ctx_branch(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.branch()
 
     @reraise_safe_exceptions
     def ctx_children(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return [child.rev() for child in ctx.children()]
 
     @reraise_safe_exceptions
     def ctx_date(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.date()
 
     @reraise_safe_exceptions
     def ctx_description(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.description()
 
     @reraise_safe_exceptions
     def ctx_diff(
             self, wire, revision, git=True, ignore_whitespace=True, context=3):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         result = ctx.diff(
             git=git, ignore_whitespace=ignore_whitespace, context=context)
         return list(result)
 
     @reraise_safe_exceptions
     def ctx_files(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.files()
 
     @reraise_safe_exceptions
     def ctx_list(self, path, revision):
         repo = self._factory.repo(path)
         ctx = repo[revision]
         return list(ctx)
 
     @reraise_safe_exceptions
     def ctx_parents(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return [parent.rev() for parent in ctx.parents()]
 
     @reraise_safe_exceptions
     def ctx_substate(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.substate
 
     @reraise_safe_exceptions
     def ctx_status(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         status = repo[ctx.p1().node()].status(other=ctx.node())
         # object of status (odd, custom named tuple in mercurial) is not
         # correctly serializable via Pyro, we make it a list, as the underling
         # API expects this to be a list
         return list(status)
 
     @reraise_safe_exceptions
     def ctx_user(self, wire, revision):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         return ctx.user()
 
     @reraise_safe_exceptions
     def check_url(self, url, config):
         _proto = None
         if '+' in url[:url.find('://')]:
             _proto = url[0:url.find('+')]
             url = url[url.find('+') + 1:]
         handlers = []
         url_obj = url_parser(url)
         test_uri, authinfo = url_obj.authinfo()
         url_obj.passwd = '*****' if url_obj.passwd else url_obj.passwd
         url_obj.query = obfuscate_qs(url_obj.query)
 
         cleaned_uri = str(url_obj)
         log.info("Checking URL for remote cloning/import: %s", cleaned_uri)
 
         if authinfo:
             # create a password manager
             passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
             passmgr.add_password(*authinfo)
 
             handlers.extend((httpbasicauthhandler(passmgr),
                              httpdigestauthhandler(passmgr)))
 
         o = urllib2.build_opener(*handlers)
         o.addheaders = [('Content-Type', 'application/mercurial-0.1'),
                         ('Accept', 'application/mercurial-0.1')]
 
         q = {"cmd": 'between'}
         q.update({'pairs': "%s-%s" % ('0' * 40, '0' * 40)})
         qs = '?%s' % urllib.urlencode(q)
         cu = "%s%s" % (test_uri, qs)
         req = urllib2.Request(cu, None, {})
 
         try:
             log.debug("Trying to open URL %s", cleaned_uri)
             resp = o.open(req)
             if resp.code != 200:
                 raise exceptions.URLError('Return Code is not 200')
         except Exception as e:
             log.warning("URL cannot be opened: %s", cleaned_uri, exc_info=True)
             # means it cannot be cloned
             raise exceptions.URLError("[%s] org_exc: %s" % (cleaned_uri, e))
 
         # now check if it's a proper hg repo, but don't do it for svn
         try:
             if _proto == 'svn':
                 pass
             else:
                 # check for pure hg repos
                 log.debug(
                     "Verifying if URL is a Mercurial repository: %s",
                     cleaned_uri)
                 httppeer(make_ui_from_config(config), url).lookup('tip')
         except Exception as e:
             log.warning("URL is not a valid Mercurial repository: %s",
                         cleaned_uri)
             raise exceptions.URLError(
                 "url [%s] does not look like an hg repo org_exc: %s"
                 % (cleaned_uri, e))
 
         log.info("URL is a valid Mercurial repository: %s", cleaned_uri)
         return True
 
     @reraise_safe_exceptions
     def diff(
             self, wire, rev1, rev2, file_filter, opt_git, opt_ignorews,
             context):
         repo = self._factory.repo(wire)
 
         if file_filter:
             match_filter = match(file_filter[0], '', [file_filter[1]])
         else:
             match_filter = file_filter
         opts = diffopts(git=opt_git, ignorews=opt_ignorews, context=context)
 
         try:
             return "".join(patch.diff(
                 repo, node1=rev1, node2=rev2, match=match_filter, opts=opts))
         except RepoLookupError:
             raise exceptions.LookupException()
 
     @reraise_safe_exceptions
     def file_history(self, wire, revision, path, limit):
         repo = self._factory.repo(wire)
 
         ctx = repo[revision]
         fctx = ctx.filectx(path)
 
         def history_iter():
             limit_rev = fctx.rev()
             for obj in reversed(list(fctx.filelog())):
                 obj = fctx.filectx(obj)
                 if limit_rev >= obj.rev():
                     yield obj
 
         history = []
         for cnt, obj in enumerate(history_iter()):
             if limit and cnt >= limit:
                 break
             history.append(hex(obj.node()))
 
         return [x for x in history]
 
     @reraise_safe_exceptions
     def file_history_untill(self, wire, revision, path, limit):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
 
         file_log = list(fctx.filelog())
         if limit:
             # Limit to the last n items
             file_log = file_log[-limit:]
 
         return [hex(fctx.filectx(cs).node()) for cs in reversed(file_log)]
 
     @reraise_safe_exceptions
     def fctx_annotate(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
 
         result = []
         for i, annotate_data in enumerate(fctx.annotate()):
             ln_no = i + 1
             node_info, content = annotate_data
             sha = hex(node_info[0].node())
             result.append((ln_no, sha, content))
         return result
 
     @reraise_safe_exceptions
     def fctx_data(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
         return fctx.data()
 
     @reraise_safe_exceptions
     def fctx_flags(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
         return fctx.flags()
 
     @reraise_safe_exceptions
     def fctx_size(self, wire, revision, path):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         fctx = ctx.filectx(path)
         return fctx.size()
 
     @reraise_safe_exceptions
     def get_all_commit_ids(self, wire, name):
         repo = self._factory.repo(wire)
         revs = repo.filtered(name).changelog.index
         return map(lambda x: hex(x[7]), revs)[:-1]
 
     @reraise_safe_exceptions
     def get_config_value(self, wire, section, name, untrusted=False):
         repo = self._factory.repo(wire)
         return repo.ui.config(section, name, untrusted=untrusted)
 
     @reraise_safe_exceptions
     def get_config_bool(self, wire, section, name, untrusted=False):
         repo = self._factory.repo(wire)
         return repo.ui.configbool(section, name, untrusted=untrusted)
 
     @reraise_safe_exceptions
     def get_config_list(self, wire, section, name, untrusted=False):
         repo = self._factory.repo(wire)
         return repo.ui.configlist(section, name, untrusted=untrusted)
 
     @reraise_safe_exceptions
     def is_large_file(self, wire, path):
         return largefiles.lfutil.isstandin(path)
 
     @reraise_safe_exceptions
-    def in_store(self, wire, sha):
+    def in_largefiles_store(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.instore(repo, sha)
 
     @reraise_safe_exceptions
     def in_user_cache(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.inusercache(repo.ui, sha)
 
     @reraise_safe_exceptions
     def store_path(self, wire, sha):
         repo = self._factory.repo(wire)
         return largefiles.lfutil.storepath(repo, sha)
 
     @reraise_safe_exceptions
     def link(self, wire, sha, path):
         repo = self._factory.repo(wire)
         largefiles.lfutil.link(
             largefiles.lfutil.usercachepath(repo.ui, sha), path)
 
     @reraise_safe_exceptions
     def localrepository(self, wire, create=False):
         self._factory.repo(wire, create=create)
 
     @reraise_safe_exceptions
     def lookup(self, wire, revision, both):
         # TODO Paris: Ugly hack to "deserialize" long for msgpack
         if isinstance(revision, float):
             revision = long(revision)
         repo = self._factory.repo(wire)
         try:
             ctx = repo[revision]
         except RepoLookupError:
             raise exceptions.LookupException(revision)
         except LookupError as e:
             raise exceptions.LookupException(e.name)
 
         if not both:
             return ctx.hex()
 
         ctx = repo[ctx.hex()]
         return ctx.hex(), ctx.rev()
 
     @reraise_safe_exceptions
     def pull(self, wire, url, commit_ids=None):
         repo = self._factory.repo(wire)
         remote = peer(repo, {}, url)
         if commit_ids:
             commit_ids = [bin(commit_id) for commit_id in commit_ids]
 
         return exchange.pull(
             repo, remote, heads=commit_ids, force=None).cgresult
 
     @reraise_safe_exceptions
     def revision(self, wire, rev):
         repo = self._factory.repo(wire)
         ctx = repo[rev]
         return ctx.rev()
 
     @reraise_safe_exceptions
     def rev_range(self, wire, filter):
         repo = self._factory.repo(wire)
         revisions = [rev for rev in revrange(repo, filter)]
         return revisions
 
     @reraise_safe_exceptions
     def rev_range_hash(self, wire, node):
         repo = self._factory.repo(wire)
 
         def get_revs(repo, rev_opt):
             if rev_opt:
                 revs = revrange(repo, rev_opt)
                 if len(revs) == 0:
                     return (nullrev, nullrev)
                 return max(revs), min(revs)
             else:
                 return len(repo) - 1, 0
 
         stop, start = get_revs(repo, [node + ':'])
         revs = [hex(repo[r].node()) for r in xrange(start, stop + 1)]
         return revs
 
     @reraise_safe_exceptions
     def revs_from_revspec(self, wire, rev_spec, *args, **kwargs):
         other_path = kwargs.pop('other_path', None)
 
         # case when we want to compare two independent repositories
         if other_path and other_path != wire["path"]:
             baseui = self._factory._create_config(wire["config"])
             repo = unionrepo.unionrepository(baseui, other_path, wire["path"])
         else:
             repo = self._factory.repo(wire)
         return list(repo.revs(rev_spec, *args))
 
     @reraise_safe_exceptions
     def strip(self, wire, revision, update, backup):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         hgext_strip(
             repo.baseui, repo, ctx.node(), update=update, backup=backup)
 
     @reraise_safe_exceptions
+    def verify(self, wire,):
+        repo = self._factory.repo(wire)
+        baseui = self._factory._create_config(wire['config'])
+        baseui.setconfig('ui', 'quiet', 'false')
+        output = io.BytesIO()
+
+        def write(data, **unused_kwargs):
+            output.write(data)
+        baseui.write = write
+
+        repo.ui = baseui
+        verify.verify(repo)
+        return output.getvalue()
+
+    @reraise_safe_exceptions
     def tag(self, wire, name, revision, message, local, user,
             tag_time, tag_timezone):
         repo = self._factory.repo(wire)
         ctx = repo[revision]
         node = ctx.node()
 
         date = (tag_time, tag_timezone)
         try:
             repo.tag(name, node, message, local, user, date)
         except Abort as e:
             log.exception("Tag operation aborted")
             # Exception can contain unicode which we convert
             raise exceptions.AbortException(repr(e))
 
     @reraise_safe_exceptions
     def tags(self, wire):
         repo = self._factory.repo(wire)
         return repo.tags()
 
     @reraise_safe_exceptions
     def update(self, wire, node=None, clean=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         commands.update(baseui, repo, node=node, clean=clean)
 
     @reraise_safe_exceptions
     def identify(self, wire):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         output = io.BytesIO()
         baseui.write = output.write
         # This is required to get a full node id
         baseui.debugflag = True
         commands.identify(baseui, repo, id=True)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def pull_cmd(self, wire, source, bookmark=None, branch=None, revision=None,
                  hooks=True):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'], hooks=hooks)
 
         # Mercurial internally has a lot of logic that checks ONLY if
         # option is defined, we just pass those if they are defined then
         opts = {}
         if bookmark:
             opts['bookmark'] = bookmark
         if branch:
             opts['branch'] = branch
         if revision:
             opts['rev'] = revision
 
         commands.pull(baseui, repo, source, **opts)
 
     @reraise_safe_exceptions
     def heads(self, wire, branch=None):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         output = io.BytesIO()
 
         def write(data, **unused_kwargs):
             output.write(data)
 
         baseui.write = write
         if branch:
             args = [branch]
         else:
             args = []
         commands.heads(baseui, repo, template='{node} ', *args)
 
         return output.getvalue()
 
     @reraise_safe_exceptions
     def ancestor(self, wire, revision1, revision2):
         repo = self._factory.repo(wire)
-        baseui = self._factory._create_config(wire['config'])
-        output = io.BytesIO()
-        baseui.write = output.write
-        commands.debugancestor(baseui, repo, revision1, revision2)
-
-        return output.getvalue()
+        changelog = repo.changelog
+        lookup = repo.lookup
+        a = changelog.ancestor(lookup(revision1), lookup(revision2))
+        return hex(a)
 
     @reraise_safe_exceptions
     def push(self, wire, revisions, dest_path, hooks=True,
              push_branches=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'], hooks=hooks)
         commands.push(baseui, repo, dest=dest_path, rev=revisions,
                       new_branch=push_branches)
 
     @reraise_safe_exceptions
     def merge(self, wire, revision):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'merge', 'internal:dump')
 
         # In case of sub repositories are used mercurial prompts the user in
         # case of merge conflicts or different sub repository sources. By
         # setting the interactive flag to `False` mercurial doesn't prompt the
         # used but instead uses a default value.
         repo.ui.setconfig('ui', 'interactive', False)
 
         commands.merge(baseui, repo, rev=revision)
 
     @reraise_safe_exceptions
     def commit(self, wire, message, username):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'username', username)
         commands.commit(baseui, repo, message=message)
 
     @reraise_safe_exceptions
     def rebase(self, wire, source=None, dest=None, abort=False):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         repo.ui.setconfig('ui', 'merge', 'internal:dump')
         rebase.rebase(
             baseui, repo, base=source, dest=dest, abort=abort, keep=not abort)
 
     @reraise_safe_exceptions
     def bookmark(self, wire, bookmark, revision=None):
         repo = self._factory.repo(wire)
         baseui = self._factory._create_config(wire['config'])
         commands.bookmark(baseui, repo, bookmark, rev=revision, force=True)

vcsserver/hooks.py

@@ -1,372 +1,404 b''
 # -*- coding: utf-8 -*-
 
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2017 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
+import io
+import sys
+import json
+import logging
 import collections
 import importlib
-import io
-import json
 import subprocess
-import sys
+
 from httplib import HTTPConnection
 
 
 import mercurial.scmutil
 import mercurial.node
 import Pyro4
 import simplejson as json
 
 from vcsserver import exceptions
 
+log = logging.getLogger(__name__)
+
 
 class HooksHttpClient(object):
     connection = None
 
     def __init__(self, hooks_uri):
         self.hooks_uri = hooks_uri
 
     def __call__(self, method, extras):
         connection = HTTPConnection(self.hooks_uri)
         body = self._serialize(method, extras)
         connection.request('POST', '/', body)
         response = connection.getresponse()
         return json.loads(response.read())
 
     def _serialize(self, hook_name, extras):
         data = {
             'method': hook_name,
             'extras': extras
         }
         return json.dumps(data)
 
 
 class HooksDummyClient(object):
     def __init__(self, hooks_module):
         self._hooks_module = importlib.import_module(hooks_module)
 
     def __call__(self, hook_name, extras):
         with self._hooks_module.Hooks() as hooks:
             return getattr(hooks, hook_name)(extras)
 
 
 class HooksPyro4Client(object):
     def __init__(self, hooks_uri):
         self.hooks_uri = hooks_uri
 
     def __call__(self, hook_name, extras):
         with Pyro4.Proxy(self.hooks_uri) as hooks:
             return getattr(hooks, hook_name)(extras)
 
 
 class RemoteMessageWriter(object):
     """Writer base class."""
     def write(message):
         raise NotImplementedError()
 
 
 class HgMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to mercurial clients."""
 
     def __init__(self, ui):
         self.ui = ui
 
     def write(self, message):
         # TODO: Check why the quiet flag is set by default.
         old = self.ui.quiet
         self.ui.quiet = False
         self.ui.status(message.encode('utf-8'))
         self.ui.quiet = old
 
 
 class GitMessageWriter(RemoteMessageWriter):
     """Writer that knows how to send messages to git clients."""
 
     def __init__(self, stdout=None):
         self.stdout = stdout or sys.stdout
 
     def write(self, message):
         self.stdout.write(message.encode('utf-8'))
 
 
 def _handle_exception(result):
     exception_class = result.get('exception')
+    exception_traceback = result.get('exception_traceback')
+
+    if exception_traceback:
+        log.error('Got traceback from remote call:%s', exception_traceback)
+
     if exception_class == 'HTTPLockedRC':
         raise exceptions.RepositoryLockedException(*result['exception_args'])
     elif exception_class == 'RepositoryError':
         raise exceptions.VcsException(*result['exception_args'])
     elif exception_class:
         raise Exception('Got remote exception "%s" with args "%s"' %
                         (exception_class, result['exception_args']))
 
 
 def _get_hooks_client(extras):
     if 'hooks_uri' in extras:
         protocol = extras.get('hooks_protocol')
         return (
             HooksHttpClient(extras['hooks_uri'])
             if protocol == 'http'
             else HooksPyro4Client(extras['hooks_uri'])
         )
     else:
         return HooksDummyClient(extras['hooks_module'])
 
 
 def _call_hook(hook_name, extras, writer):
     hooks = _get_hooks_client(extras)
     result = hooks(hook_name, extras)
     writer.write(result['output'])
     _handle_exception(result)
 
     return result['status']
 
 
 def _extras_from_ui(ui):
     extras = json.loads(ui.config('rhodecode', 'RC_SCM_DATA'))
     return extras
 
 
 def repo_size(ui, repo, **kwargs):
     return _call_hook('repo_size', _extras_from_ui(ui), HgMessageWriter(ui))
 
 
 def pre_pull(ui, repo, **kwargs):
     return _call_hook('pre_pull', _extras_from_ui(ui), HgMessageWriter(ui))
 
 
 def post_pull(ui, repo, **kwargs):
     return _call_hook('post_pull', _extras_from_ui(ui), HgMessageWriter(ui))
 
 
-def pre_push(ui, repo, **kwargs):
-    return _call_hook('pre_push', _extras_from_ui(ui), HgMessageWriter(ui))
+def pre_push(ui, repo, node=None, **kwargs):
+    extras = _extras_from_ui(ui)
+
+    rev_data = []
+    if node and kwargs.get('hooktype') == 'pretxnchangegroup':
+        branches = collections.defaultdict(list)
+        for commit_id, branch in _rev_range_hash(repo, node, with_branch=True):
+            branches[branch].append(commit_id)
+
+        for branch, commits in branches.iteritems():
+            old_rev = kwargs.get('node_last') or commits[0]
+            rev_data.append({
+                'old_rev': old_rev,
+                'new_rev': commits[-1],
+                'ref': '',
+                'type': 'branch',
+                'name': branch,
+            })
+
+    extras['commit_ids'] = rev_data
+    return _call_hook('pre_push', extras, HgMessageWriter(ui))
 
 
-# N.B.(skreft): the two functions below were taken and adapted from
-# rhodecode.lib.vcs.remote.handle_git_pre_receive
-# They are required to compute the commit_ids
-def _get_revs(repo, rev_opt):
-    revs = [rev for rev in mercurial.scmutil.revrange(repo, rev_opt)]
-    if len(revs) == 0:
-        return (mercurial.node.nullrev, mercurial.node.nullrev)
+def _rev_range_hash(repo, node, with_branch=False):
 
-    return max(revs), min(revs)
-
+    commits = []
+    for rev in xrange(repo[node], len(repo)):
+        ctx = repo[rev]
+        commit_id = mercurial.node.hex(ctx.node())
+        branch = ctx.branch()
+        if with_branch:
+            commits.append((commit_id, branch))
+        else:
+            commits.append(commit_id)
 
-def _rev_range_hash(repo, node):
-    stop, start = _get_revs(repo, [node + ':'])
-    revs = [mercurial.node.hex(repo[r].node()) for r in xrange(start, stop + 1)]
-
-    return revs
+    return commits
 
 
 def post_push(ui, repo, node, **kwargs):
     commit_ids = _rev_range_hash(repo, node)
 
     extras = _extras_from_ui(ui)
     extras['commit_ids'] = commit_ids
 
     return _call_hook('post_push', extras, HgMessageWriter(ui))
 
 
 # backward compat
 log_pull_action = post_pull
 
 # backward compat
 log_push_action = post_push
 
 
 def handle_git_pre_receive(unused_repo_path, unused_revs, unused_env):
     """
     Old hook name: keep here for backward compatibility.
 
     This is only required when the installed git hooks are not upgraded.
     """
     pass
 
 
 def handle_git_post_receive(unused_repo_path, unused_revs, unused_env):
     """
     Old hook name: keep here for backward compatibility.
 
     This is only required when the installed git hooks are not upgraded.
     """
     pass
 
 
 HookResponse = collections.namedtuple('HookResponse', ('status', 'output'))
 
 
 def git_pre_pull(extras):
     """
     Pre pull hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
     stdout = io.BytesIO()
     try:
         status = _call_hook('pre_pull', extras, GitMessageWriter(stdout))
     except Exception as error:
         status = 128
         stdout.write('ERROR: %s\n' % str(error))
 
     return HookResponse(status, stdout.getvalue())
 
 
 def git_post_pull(extras):
     """
     Post pull hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     if 'pull' not in extras['hooks']:
         return HookResponse(0, '')
 
     stdout = io.BytesIO()
     try:
         status = _call_hook('post_pull', extras, GitMessageWriter(stdout))
     except Exception as error:
         status = 128
         stdout.write('ERROR: %s\n' % error)
 
     return HookResponse(status, stdout.getvalue())
 
 
-def git_pre_receive(unused_repo_path, unused_revs, env):
+def _parse_git_ref_lines(revision_lines):
+    rev_data = []
+    for revision_line in revision_lines or []:
+        old_rev, new_rev, ref = revision_line.strip().split(' ')
+        ref_data = ref.split('/', 2)
+        if ref_data[1] in ('tags', 'heads'):
+            rev_data.append({
+                'old_rev': old_rev,
+                'new_rev': new_rev,
+                'ref': ref,
+                'type': ref_data[1],
+                'name': ref_data[2],
+            })
+    return rev_data
+
+
+def git_pre_receive(unused_repo_path, revision_lines, env):
     """
     Pre push hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
+    rev_data = _parse_git_ref_lines(revision_lines)
     if 'push' not in extras['hooks']:
         return 0
+    extras['commit_ids'] = rev_data
     return _call_hook('pre_push', extras, GitMessageWriter())
 
 
 def _run_command(arguments):
     """
     Run the specified command and return the stdout.
 
-    :param arguments: sequence of program arugments (including the program name)
+    :param arguments: sequence of program arguments (including the program name)
     :type arguments: list[str]
     """
     # TODO(skreft): refactor this method and all the other similar ones.
     # Probably this should be using subprocessio.
     process = subprocess.Popen(
         arguments, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     stdout, _ = process.communicate()
 
     if process.returncode != 0:
         raise Exception(
             'Command %s exited with exit code %s' % (arguments,
                                                      process.returncode))
 
     return stdout
 
 
 def git_post_receive(unused_repo_path, revision_lines, env):
     """
     Post push hook.
 
     :param extras: dictionary containing the keys defined in simplevcs
     :type extras: dict
 
     :return: status code of the hook. 0 for success.
     :rtype: int
     """
     extras = json.loads(env['RC_SCM_DATA'])
     if 'push' not in extras['hooks']:
         return 0
 
-    rev_data = []
-    for revision_line in revision_lines:
-        old_rev, new_rev, ref = revision_line.strip().split(' ')
-        ref_data = ref.split('/', 2)
-        if ref_data[1] in ('tags', 'heads'):
-            rev_data.append({
-                'old_rev': old_rev,
-                'new_rev': new_rev,
-                'ref': ref,
-                'type': ref_data[1],
-                'name': ref_data[2],
-            })
+    rev_data = _parse_git_ref_lines(revision_lines)
 
     git_revs = []
 
     # N.B.(skreft): it is ok to just call git, as git before calling a
     # subcommand sets the PATH environment variable so that it point to the
     # correct version of the git executable.
     empty_commit_id = '0' * 40
     for push_ref in rev_data:
         type_ = push_ref['type']
         if type_ == 'heads':
             if push_ref['old_rev'] == empty_commit_id:
 
                 # Fix up head revision if needed
                 cmd = ['git', 'show', 'HEAD']
                 try:
                     _run_command(cmd)
                 except Exception:
                     cmd = ['git', 'symbolic-ref', 'HEAD',
                            'refs/heads/%s' % push_ref['name']]
-                    print "Setting default branch to %s" % push_ref['name']
+                    print("Setting default branch to %s" % push_ref['name'])
                     _run_command(cmd)
 
                 cmd = ['git', 'for-each-ref', '--format=%(refname)',
                        'refs/heads/*']
                 heads = _run_command(cmd)
                 heads = heads.replace(push_ref['ref'], '')
                 heads = ' '.join(head for head in heads.splitlines() if head)
                 cmd = ['git', 'log', '--reverse', '--pretty=format:%H',
                         '--', push_ref['new_rev'], '--not', heads]
                 git_revs.extend(_run_command(cmd).splitlines())
             elif push_ref['new_rev'] == empty_commit_id:
                 # delete branch case
                 git_revs.append('delete_branch=>%s' % push_ref['name'])
             else:
                 cmd = ['git', 'log',
                        '{old_rev}..{new_rev}'.format(**push_ref),
                        '--reverse', '--pretty=format:%H']
                 git_revs.extend(_run_command(cmd).splitlines())
         elif type_ == 'tags':
             git_revs.append('tag=>%s' % push_ref['name'])
 
     extras['commit_ids'] = git_revs
 
     if 'repo_size' in extras['hooks']:
         try:
             _call_hook('repo_size', extras, GitMessageWriter())
         except:
             pass
 
     return _call_hook('post_push', extras, GitMessageWriter())

vcsserver/http_main.py

@@ -1,408 +1,434 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2017 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import base64
 import locale
 import logging
 import uuid
 import wsgiref.util
 import traceback
 from itertools import chain
 
 import msgpack
 from beaker.cache import CacheManager
 from beaker.util import parse_cache_config_options
 from pyramid.config import Configurator
 from pyramid.wsgi import wsgiapp
 
 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
+from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
 from vcsserver.echo_stub.echo_app import EchoApp
 from vcsserver.exceptions import HTTPRepoLocked
 from vcsserver.server import VcsServer
 
 try:
     from vcsserver.git import GitFactory, GitRemote
 except ImportError:
     GitFactory = None
     GitRemote = None
+
 try:
     from vcsserver.hg import MercurialFactory, HgRemote
 except ImportError:
     MercurialFactory = None
     HgRemote = None
+
 try:
     from vcsserver.svn import SubversionFactory, SvnRemote
 except ImportError:
     SubversionFactory = None
     SvnRemote = None
 
 log = logging.getLogger(__name__)
 
 
 class VCS(object):
     def __init__(self, locale=None, cache_config=None):
         self.locale = locale
         self.cache_config = cache_config
         self._configure_locale()
         self._initialize_cache()
 
         if GitFactory and GitRemote:
             git_repo_cache = self.cache.get_cache_region(
                 'git', region='repo_object')
             git_factory = GitFactory(git_repo_cache)
             self._git_remote = GitRemote(git_factory)
         else:
             log.info("Git client import failed")
 
         if MercurialFactory and HgRemote:
             hg_repo_cache = self.cache.get_cache_region(
                 'hg', region='repo_object')
             hg_factory = MercurialFactory(hg_repo_cache)
             self._hg_remote = HgRemote(hg_factory)
         else:
             log.info("Mercurial client import failed")
 
         if SubversionFactory and SvnRemote:
             svn_repo_cache = self.cache.get_cache_region(
                 'svn', region='repo_object')
             svn_factory = SubversionFactory(svn_repo_cache)
             self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
         else:
             log.info("Subversion client import failed")
 
         self._vcsserver = VcsServer()
 
     def _initialize_cache(self):
         cache_config = parse_cache_config_options(self.cache_config)
         log.info('Initializing beaker cache: %s' % cache_config)
         self.cache = CacheManager(**cache_config)
 
     def _configure_locale(self):
         if self.locale:
             log.info('Settings locale: `LC_ALL` to %s' % self.locale)
         else:
             log.info(
                 'Configuring locale subsystem based on environment variables')
         try:
             # If self.locale is the empty string, then the locale
             # module will use the environment variables. See the
             # documentation of the package `locale`.
             locale.setlocale(locale.LC_ALL, self.locale)
 
             language_code, encoding = locale.getlocale()
             log.info(
                 'Locale set to language code "%s" with encoding "%s".',
                 language_code, encoding)
         except locale.Error:
             log.exception(
                 'Cannot set locale, not configuring the locale system')
 
 
 class WsgiProxy(object):
     def __init__(self, wsgi):
         self.wsgi = wsgi
 
     def __call__(self, environ, start_response):
         input_data = environ['wsgi.input'].read()
         input_data = msgpack.unpackb(input_data)
 
         error = None
         try:
             data, status, headers = self.wsgi.handle(
                 input_data['environment'], input_data['input_data'],
                 *input_data['args'], **input_data['kwargs'])
         except Exception as e:
             data, status, headers = [], None, None
             error = {
                 'message': str(e),
                 '_vcs_kind': getattr(e, '_vcs_kind', None)
             }
 
         start_response(200, {})
         return self._iterator(error, status, headers, data)
 
     def _iterator(self, error, status, headers, data):
         initial_data = [
             error,
             status,
             headers,
         ]
 
         for d in chain(initial_data, data):
             yield msgpack.packb(d)
 
 
 class HTTPApplication(object):
     ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
 
     remote_wsgi = remote_wsgi
     _use_echo_app = False
 
-    def __init__(self, settings=None):
+    def __init__(self, settings=None, global_config=None):
         self.config = Configurator(settings=settings)
+        self.global_config = global_config
+
         locale = settings.get('locale', '') or 'en_US.UTF-8'
         vcs = VCS(locale=locale, cache_config=settings)
         self._remotes = {
             'hg': vcs._hg_remote,
             'git': vcs._git_remote,
             'svn': vcs._svn_remote,
             'server': vcs._vcsserver,
         }
         if settings.get('dev.use_echo_app', 'false').lower() == 'true':
             self._use_echo_app = True
             log.warning("Using EchoApp for VCS operations.")
             self.remote_wsgi = remote_wsgi_stub
         self._configure_settings(settings)
         self._configure()
 
     def _configure_settings(self, app_settings):
         """
         Configure the settings module.
         """
         git_path = app_settings.get('git_path', None)
         if git_path:
             settings.GIT_EXECUTABLE = git_path
 
     def _configure(self):
         self.config.add_renderer(
             name='msgpack',
             factory=self._msgpack_renderer_factory)
 
         self.config.add_route('service', '/_service')
         self.config.add_route('status', '/status')
         self.config.add_route('hg_proxy', '/proxy/hg')
         self.config.add_route('git_proxy', '/proxy/git')
         self.config.add_route('vcs', '/{backend}')
         self.config.add_route('stream_git', '/stream/git/*repo_name')
         self.config.add_route('stream_hg', '/stream/hg/*repo_name')
 
         self.config.add_view(
             self.status_view, route_name='status', renderer='json')
         self.config.add_view(
             self.service_view, route_name='service', renderer='msgpack')
 
         self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
         self.config.add_view(self.git_proxy(), route_name='git_proxy')
         self.config.add_view(
             self.vcs_view, route_name='vcs', renderer='msgpack',
             custom_predicates=[self.is_vcs_view])
 
         self.config.add_view(self.hg_stream(), route_name='stream_hg')
         self.config.add_view(self.git_stream(), route_name='stream_git')
 
         def notfound(request):
             return {'status': '404 NOT FOUND'}
         self.config.add_notfound_view(notfound, renderer='json')
 
-        self.config.add_view(
-            self.handle_vcs_exception, context=Exception,
-            custom_predicates=[self.is_vcs_exception])
-
-        self.config.add_view(
-            self.general_error_handler, context=Exception)
+        self.config.add_view(self.handle_vcs_exception, context=Exception)
 
         self.config.add_tween(
             'vcsserver.tweens.RequestWrapperTween',
         )
 
     def wsgi_app(self):
         return self.config.make_wsgi_app()
 
     def vcs_view(self, request):
         remote = self._remotes[request.matchdict['backend']]
         payload = msgpack.unpackb(request.body, use_list=True)
         method = payload.get('method')
         params = payload.get('params')
         wire = params.get('wire')
         args = params.get('args')
         kwargs = params.get('kwargs')
         if wire:
             try:
                 wire['context'] = uuid.UUID(wire['context'])
             except KeyError:
                 pass
             args.insert(0, wire)
 
         log.debug('method called:%s with kwargs:%s', method, kwargs)
         try:
             resp = getattr(remote, method)(*args, **kwargs)
         except Exception as e:
             tb_info = traceback.format_exc()
 
             type_ = e.__class__.__name__
             if type_ not in self.ALLOWED_EXCEPTIONS:
                 type_ = None
 
             resp = {
                 'id': payload.get('id'),
                 'error': {
                     'message': e.message,
                     'traceback': tb_info,
                     'type': type_
                 }
             }
             try:
                 resp['error']['_vcs_kind'] = e._vcs_kind
             except AttributeError:
                 pass
         else:
             resp = {
                 'id': payload.get('id'),
                 'result': resp
             }
 
         return resp
 
     def status_view(self, request):
         return {'status': 'OK'}
 
     def service_view(self, request):
         import vcsserver
+        import ConfigParser as configparser
+
         payload = msgpack.unpackb(request.body, use_list=True)
+
+        try:
+            path = self.global_config['__file__']
+            config = configparser.ConfigParser()
+            config.read(path)
+            parsed_ini = config
+            if parsed_ini.has_section('server:main'):
+                parsed_ini = dict(parsed_ini.items('server:main'))
+        except Exception:
+            log.exception('Failed to read .ini file for display')
+            parsed_ini = {}
+
         resp = {
             'id': payload.get('id'),
             'result': dict(
                 version=vcsserver.__version__,
-                config={},
+                config=parsed_ini,
                 payload=payload,
             )
         }
         return resp
 
     def _msgpack_renderer_factory(self, info):
         def _render(value, system):
             value = msgpack.packb(value)
             request = system.get('request')
             if request is not None:
                 response = request.response
                 ct = response.content_type
                 if ct == response.default_content_type:
                     response.content_type = 'application/x-msgpack'
             return value
         return _render
 
     def hg_proxy(self):
         @wsgiapp
         def _hg_proxy(environ, start_response):
             app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
             return app(environ, start_response)
         return _hg_proxy
 
     def git_proxy(self):
         @wsgiapp
         def _git_proxy(environ, start_response):
             app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
             return app(environ, start_response)
         return _git_proxy
 
     def hg_stream(self):
         if self._use_echo_app:
             @wsgiapp
             def _hg_stream(environ, start_response):
                 app = EchoApp('fake_path', 'fake_name', None)
                 return app(environ, start_response)
             return _hg_stream
         else:
             @wsgiapp
             def _hg_stream(environ, start_response):
                 repo_path = environ['HTTP_X_RC_REPO_PATH']
                 repo_name = environ['HTTP_X_RC_REPO_NAME']
                 packed_config = base64.b64decode(
                     environ['HTTP_X_RC_REPO_CONFIG'])
                 config = msgpack.unpackb(packed_config)
                 app = scm_app.create_hg_wsgi_app(
                     repo_path, repo_name, config)
 
                 # Consitent path information for hgweb
                 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
                 environ['REPO_NAME'] = repo_name
                 return app(environ, ResponseFilter(start_response))
             return _hg_stream
 
     def git_stream(self):
         if self._use_echo_app:
             @wsgiapp
             def _git_stream(environ, start_response):
                 app = EchoApp('fake_path', 'fake_name', None)
                 return app(environ, start_response)
             return _git_stream
         else:
             @wsgiapp
             def _git_stream(environ, start_response):
                 repo_path = environ['HTTP_X_RC_REPO_PATH']
                 repo_name = environ['HTTP_X_RC_REPO_NAME']
                 packed_config = base64.b64decode(
                     environ['HTTP_X_RC_REPO_CONFIG'])
                 config = msgpack.unpackb(packed_config)
 
                 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
+                content_type = environ.get('CONTENT_TYPE', '')
+
+                path = environ['PATH_INFO']
+                is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
+                log.debug(
+                    'LFS: Detecting if request `%s` is LFS server path based '
+                    'on content type:`%s`, is_lfs:%s',
+                    path, content_type, is_lfs_request)
+
+                if not is_lfs_request:
+                    # fallback detection by path
+                    if GIT_LFS_PROTO_PAT.match(path):
+                        is_lfs_request = True
+                    log.debug(
+                        'LFS: fallback detection by path of: `%s`, is_lfs:%s',
+                        path, is_lfs_request)
+
+                if is_lfs_request:
+                    app = scm_app.create_git_lfs_wsgi_app(
+                        repo_path, repo_name, config)
+                else:
                 app = scm_app.create_git_wsgi_app(
                     repo_path, repo_name, config)
                 return app(environ, start_response)
+
             return _git_stream
 
     def is_vcs_view(self, context, request):
         """
         View predicate that returns true if given backend is supported by
         defined remotes.
         """
         backend = request.matchdict.get('backend')
         return backend in self._remotes
 
-    def is_vcs_exception(self, context, request):
-        """
-        View predicate that returns true if the context object is a VCS
-        exception.
-        """
-        return hasattr(context, '_vcs_kind')
-
     def handle_vcs_exception(self, exception, request):
-        if exception._vcs_kind == 'repo_locked':
+        _vcs_kind = getattr(exception, '_vcs_kind', '')
+        if _vcs_kind == 'repo_locked':
             # Get custom repo-locked status code if present.
             status_code = request.headers.get('X-RC-Locked-Status-Code')
             return HTTPRepoLocked(
                 title=exception.message, status_code=status_code)
 
         # Re-raise exception if we can not handle it.
-        raise exception
-
-    def general_error_handler(self, exception, request):
         log.exception(
-            'error occurred handling this request for path: %s',
-            request.path)
+            'error occurred handling this request for path: %s', request.path)
         raise exception
 
 
 class ResponseFilter(object):
 
     def __init__(self, start_response):
         self._start_response = start_response
 
     def __call__(self, status, response_headers, exc_info=None):
         headers = tuple(
             (h, v) for h, v in response_headers
             if not wsgiref.util.is_hop_by_hop(h))
         return self._start_response(status, headers, exc_info)
 
 
 def main(global_config, **settings):
     if MercurialFactory:
         hgpatches.patch_largefiles_capabilities()
         hgpatches.patch_subrepo_type_mapping()
-    app = HTTPApplication(settings=settings)
+    app = HTTPApplication(settings=settings, global_config=global_config)
     return app.wsgi_app()

vcsserver/scm_app.py

@@ -1,174 +1,209 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2017 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
+import os
 import logging
-import os
 
 import mercurial
 import mercurial.error
 import mercurial.hgweb.common
 import mercurial.hgweb.hgweb_mod
 import mercurial.hgweb.protocol
 import webob.exc
 
-from vcsserver import pygrack, exceptions, settings
+from vcsserver import pygrack, exceptions, settings, git_lfs
 
 
 log = logging.getLogger(__name__)
 
 
 # propagated from mercurial documentation
 HG_UI_SECTIONS = [
     'alias', 'auth', 'decode/encode', 'defaults', 'diff', 'email', 'extensions',
     'format', 'merge-patterns', 'merge-tools', 'hooks', 'http_proxy', 'smtp',
     'patch', 'paths', 'profiling', 'server', 'trusted', 'ui', 'web',
 ]
 
 
 class HgWeb(mercurial.hgweb.hgweb_mod.hgweb):
     """Extension of hgweb that simplifies some functions."""
 
     def _get_view(self, repo):
         """Views are not supported."""
         return repo
 
     def loadsubweb(self):
         """The result is only used in the templater method which is not used."""
         return None
 
     def run(self):
         """Unused function so raise an exception if accidentally called."""
         raise NotImplementedError
 
     def templater(self, req):
         """Function used in an unreachable code path.
 
         This code is unreachable because we guarantee that the HTTP request,
         corresponds to a Mercurial command. See the is_hg method. So, we are
         never going to get a user-visible url.
         """
         raise NotImplementedError
 
     def archivelist(self, nodeid):
         """Unused function so raise an exception if accidentally called."""
         raise NotImplementedError
 
     def run_wsgi(self, req):
         """Check the request has a valid command, failing fast otherwise."""
         cmd = req.form.get('cmd', [''])[0]
         if not mercurial.hgweb.protocol.iscmd(cmd):
             req.respond(
                 mercurial.hgweb.common.ErrorResponse(
                     mercurial.hgweb.common.HTTP_BAD_REQUEST),
                 mercurial.hgweb.protocol.HGTYPE
             )
             return ['']
 
         return super(HgWeb, self).run_wsgi(req)
 
 
 def make_hg_ui_from_config(repo_config):
     baseui = mercurial.ui.ui()
 
     # clean the baseui object
     baseui._ocfg = mercurial.config.config()
     baseui._ucfg = mercurial.config.config()
     baseui._tcfg = mercurial.config.config()
 
     for section, option, value in repo_config:
         baseui.setconfig(section, option, value)
 
     # make our hgweb quiet so it doesn't print output
     baseui.setconfig('ui', 'quiet', 'true')
 
     return baseui
 
 
 def update_hg_ui_from_hgrc(baseui, repo_path):
     path = os.path.join(repo_path, '.hg', 'hgrc')
 
     if not os.path.isfile(path):
         log.debug('hgrc file is not present at %s, skipping...', path)
         return
     log.debug('reading hgrc from %s', path)
     cfg = mercurial.config.config()
     cfg.read(path)
     for section in HG_UI_SECTIONS:
         for k, v in cfg.items(section):
             log.debug('settings ui from file: [%s] %s=%s', section, k, v)
             baseui.setconfig(section, k, v)
 
 
 def create_hg_wsgi_app(repo_path, repo_name, config):
     """
     Prepares a WSGI application to handle Mercurial requests.
 
     :param config: is a list of 3-item tuples representing a ConfigObject
         (it is the serialized version of the config object).
     """
     log.debug("Creating Mercurial WSGI application")
 
     baseui = make_hg_ui_from_config(config)
     update_hg_ui_from_hgrc(baseui, repo_path)
 
     try:
         return HgWeb(repo_path, name=repo_name, baseui=baseui)
     except mercurial.error.RequirementError as exc:
         raise exceptions.RequirementException(exc)
 
 
 class GitHandler(object):
+    """
+    Handler for Git operations like push/pull etc
+    """
     def __init__(self, repo_location, repo_name, git_path, update_server_info,
                  extras):
         if not os.path.isdir(repo_location):
             raise OSError(repo_location)
         self.content_path = repo_location
         self.repo_name = repo_name
         self.repo_location = repo_location
         self.extras = extras
         self.git_path = git_path
         self.update_server_info = update_server_info
 
     def __call__(self, environ, start_response):
         app = webob.exc.HTTPNotFound()
         candidate_paths = (
             self.content_path, os.path.join(self.content_path, '.git'))
 
         for content_path in candidate_paths:
             try:
                 app = pygrack.GitRepository(
                     self.repo_name, content_path, self.git_path,
                     self.update_server_info, self.extras)
                 break
             except OSError:
                 continue
 
         return app(environ, start_response)
 
 
 def create_git_wsgi_app(repo_path, repo_name, config):
     """
     Creates a WSGI application to handle Git requests.
 
     :param config: is a dictionary holding the extras.
     """
     git_path = settings.GIT_EXECUTABLE
     update_server_info = config.pop('git_update_server_info')
     app = GitHandler(
         repo_path, repo_name, git_path, update_server_info, config)
 
     return app
+
+
+class GitLFSHandler(object):
+    """
+    Handler for Git LFS operations
+    """
+
+    def __init__(self, repo_location, repo_name, git_path, update_server_info,
+                 extras):
+        if not os.path.isdir(repo_location):
+            raise OSError(repo_location)
+        self.content_path = repo_location
+        self.repo_name = repo_name
+        self.repo_location = repo_location
+        self.extras = extras
+        self.git_path = git_path
+        self.update_server_info = update_server_info
+
+    def get_app(self, git_lfs_enabled, git_lfs_store_path):
+        app = git_lfs.create_app(git_lfs_enabled, git_lfs_store_path)
+        return app
+
+
+def create_git_lfs_wsgi_app(repo_path, repo_name, config):
+    git_path = settings.GIT_EXECUTABLE
+    update_server_info = config.pop('git_update_server_info')
+    git_lfs_enabled = config.pop('git_lfs_enabled')
+    git_lfs_store_path = config.pop('git_lfs_store_path')
+    app = GitLFSHandler(
+        repo_path, repo_name, git_path, update_server_info, config)
+
+    return app.get_app(git_lfs_enabled, git_lfs_store_path)

vcsserver/svn.py

@@ -1,651 +1,644 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2017 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 from __future__ import absolute_import
 
 from urllib2 import URLError
 import logging
 import posixpath as vcspath
 import StringIO
 import subprocess
 import urllib
 
 import svn.client
 import svn.core
 import svn.delta
 import svn.diff
 import svn.fs
 import svn.repos
 
 from vcsserver import svn_diff
 from vcsserver import exceptions
-from vcsserver.base import RepoFactory
+from vcsserver.base import RepoFactory, raise_from_original
 
 
 log = logging.getLogger(__name__)
 
 
 # Set of svn compatible version flags.
 # Compare with subversion/svnadmin/svnadmin.c
 svn_compatible_versions = set([
     'pre-1.4-compatible',
     'pre-1.5-compatible',
     'pre-1.6-compatible',
     'pre-1.8-compatible',
 ])
 
 
 def reraise_safe_exceptions(func):
     """Decorator for converting svn exceptions to something neutral."""
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except Exception as e:
             if not hasattr(e, '_vcs_kind'):
                 log.exception("Unhandled exception in hg remote call")
                 raise_from_original(exceptions.UnhandledException)
             raise
     return wrapper
 
 
-def raise_from_original(new_type):
-    """
-    Raise a new exception type with original args and traceback.
-    """
-    _, original, traceback = sys.exc_info()
-    try:
-        raise new_type(*original.args), None, traceback
-    finally:
-        del traceback
-
-
 class SubversionFactory(RepoFactory):
 
     def _create_repo(self, wire, create, compatible_version):
         path = svn.core.svn_path_canonicalize(wire['path'])
         if create:
             fs_config = {}
             if compatible_version:
                 if compatible_version not in svn_compatible_versions:
                     raise Exception('Unknown SVN compatible version "{}"'
                                     .format(compatible_version))
                 log.debug('Create SVN repo with compatible version "%s"',
                           compatible_version)
                 fs_config[compatible_version] = '1'
             repo = svn.repos.create(path, "", "", None, fs_config)
         else:
             repo = svn.repos.open(path)
         return repo
 
     def repo(self, wire, create=False, compatible_version=None):
         def create_new_repo():
             return self._create_repo(wire, create, compatible_version)
 
         return self._repo(wire, create_new_repo)
 
 
 
 NODE_TYPE_MAPPING = {
     svn.core.svn_node_file: 'file',
     svn.core.svn_node_dir: 'dir',
 }
 
 
 class SvnRemote(object):
 
     def __init__(self, factory, hg_factory=None):
         self._factory = factory
         # TODO: Remove once we do not use internal Mercurial objects anymore
         # for subversion
         self._hg_factory = hg_factory
 
     @reraise_safe_exceptions
     def discover_svn_version(self):
         try:
             import svn.core
             svn_ver = svn.core.SVN_VERSION
         except ImportError:
             svn_ver = None
         return svn_ver
 
     def check_url(self, url, config_items):
         # this can throw exception if not installed, but we detect this
         from hgsubversion import svnrepo
 
         baseui = self._hg_factory._create_config(config_items)
         # uuid function get's only valid UUID from proper repo, else
         # throws exception
         try:
             svnrepo.svnremoterepo(baseui, url).svn.uuid
         except:
             log.debug("Invalid svn url: %s", url)
             raise URLError(
                 '"%s" is not a valid Subversion source url.' % (url, ))
         return True
 
     def is_path_valid_repository(self, wire, path):
         try:
             svn.repos.open(path)
         except svn.core.SubversionException:
             log.debug("Invalid Subversion path %s", path)
             return False
         return True
 
     def lookup(self, wire, revision):
         if revision not in [-1, None, 'HEAD']:
             raise NotImplementedError
         repo = self._factory.repo(wire)
         fs_ptr = svn.repos.fs(repo)
         head = svn.fs.youngest_rev(fs_ptr)
         return head
 
     def lookup_interval(self, wire, start_ts, end_ts):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         start_rev = None
         end_rev = None
         if start_ts:
             start_ts_svn = apr_time_t(start_ts)
             start_rev = svn.repos.dated_revision(repo, start_ts_svn) + 1
         else:
             start_rev = 1
         if end_ts:
             end_ts_svn = apr_time_t(end_ts)
             end_rev = svn.repos.dated_revision(repo, end_ts_svn)
         else:
             end_rev = svn.fs.youngest_rev(fsobj)
         return start_rev, end_rev
 
     def revision_properties(self, wire, revision):
         repo = self._factory.repo(wire)
         fs_ptr = svn.repos.fs(repo)
         return svn.fs.revision_proplist(fs_ptr, revision)
 
     def revision_changes(self, wire, revision):
 
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         rev_root = svn.fs.revision_root(fsobj, revision)
 
         editor = svn.repos.ChangeCollector(fsobj, rev_root)
         editor_ptr, editor_baton = svn.delta.make_editor(editor)
         base_dir = ""
         send_deltas = False
         svn.repos.replay2(
             rev_root, base_dir, svn.core.SVN_INVALID_REVNUM, send_deltas,
             editor_ptr, editor_baton, None)
 
         added = []
         changed = []
         removed = []
 
         # TODO: CHANGE_ACTION_REPLACE: Figure out where it belongs
         for path, change in editor.changes.iteritems():
             # TODO: Decide what to do with directory nodes. Subversion can add
             # empty directories.
 
             if change.item_kind == svn.core.svn_node_dir:
                 continue
             if change.action in [svn.repos.CHANGE_ACTION_ADD]:
                 added.append(path)
             elif change.action in [svn.repos.CHANGE_ACTION_MODIFY,
                                    svn.repos.CHANGE_ACTION_REPLACE]:
                 changed.append(path)
             elif change.action in [svn.repos.CHANGE_ACTION_DELETE]:
                 removed.append(path)
             else:
                 raise NotImplementedError(
                     "Action %s not supported on path %s" % (
                         change.action, path))
 
         changes = {
             'added': added,
             'changed': changed,
             'removed': removed,
         }
         return changes
 
     def node_history(self, wire, path, revision, limit):
         cross_copies = False
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         rev_root = svn.fs.revision_root(fsobj, revision)
 
         history_revisions = []
         history = svn.fs.node_history(rev_root, path)
         history = svn.fs.history_prev(history, cross_copies)
         while history:
             __, node_revision = svn.fs.history_location(history)
             history_revisions.append(node_revision)
             if limit and len(history_revisions) >= limit:
                 break
             history = svn.fs.history_prev(history, cross_copies)
         return history_revisions
 
     def node_properties(self, wire, path, revision):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         rev_root = svn.fs.revision_root(fsobj, revision)
         return svn.fs.node_proplist(rev_root, path)
 
     def file_annotate(self, wire, path, revision):
         abs_path = 'file://' + urllib.pathname2url(
             vcspath.join(wire['path'], path))
         file_uri = svn.core.svn_path_canonicalize(abs_path)
 
         start_rev = svn_opt_revision_value_t(0)
         peg_rev = svn_opt_revision_value_t(revision)
         end_rev = peg_rev
 
         annotations = []
 
         def receiver(line_no, revision, author, date, line, pool):
             annotations.append((line_no, revision, line))
 
         # TODO: Cannot use blame5, missing typemap function in the swig code
         try:
             svn.client.blame2(
                 file_uri, peg_rev, start_rev, end_rev,
                 receiver, svn.client.create_context())
         except svn.core.SubversionException as exc:
             log.exception("Error during blame operation.")
             raise Exception(
                 "Blame not supported or file does not exist at path %s. "
                 "Error %s." % (path, exc))
 
         return annotations
 
     def get_node_type(self, wire, path, rev=None):
         repo = self._factory.repo(wire)
         fs_ptr = svn.repos.fs(repo)
         if rev is None:
             rev = svn.fs.youngest_rev(fs_ptr)
         root = svn.fs.revision_root(fs_ptr, rev)
         node = svn.fs.check_path(root, path)
         return NODE_TYPE_MAPPING.get(node, None)
 
     def get_nodes(self, wire, path, revision=None):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         if revision is None:
             revision = svn.fs.youngest_rev(fsobj)
         root = svn.fs.revision_root(fsobj, revision)
         entries = svn.fs.dir_entries(root, path)
         result = []
         for entry_path, entry_info in entries.iteritems():
             result.append(
                 (entry_path, NODE_TYPE_MAPPING.get(entry_info.kind, None)))
         return result
 
     def get_file_content(self, wire, path, rev=None):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         if rev is None:
             rev = svn.fs.youngest_revision(fsobj)
         root = svn.fs.revision_root(fsobj, rev)
         content = svn.core.Stream(svn.fs.file_contents(root, path))
         return content.read()
 
     def get_file_size(self, wire, path, revision=None):
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
         if revision is None:
             revision = svn.fs.youngest_revision(fsobj)
         root = svn.fs.revision_root(fsobj, revision)
         size = svn.fs.file_length(root, path)
         return size
 
     def create_repository(self, wire, compatible_version=None):
         log.info('Creating Subversion repository in path "%s"', wire['path'])
         self._factory.repo(wire, create=True,
                            compatible_version=compatible_version)
 
     def import_remote_repository(self, wire, src_url):
         repo_path = wire['path']
         if not self.is_path_valid_repository(wire, repo_path):
             raise Exception(
                 "Path %s is not a valid Subversion repository." % repo_path)
         # TODO: johbo: URL checks ?
         rdump = subprocess.Popen(
             ['svnrdump', 'dump', '--non-interactive', src_url],
             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         load = subprocess.Popen(
             ['svnadmin', 'load', repo_path], stdin=rdump.stdout)
 
         # TODO: johbo: This can be a very long operation, might be better
         # to track some kind of status and provide an api to check if the
         # import is done.
         rdump.wait()
         load.wait()
 
         if rdump.returncode != 0:
             errors = rdump.stderr.read()
             log.error('svnrdump dump failed: statuscode %s: message: %s',
                       rdump.returncode, errors)
             reason = 'UNKNOWN'
             if 'svnrdump: E230001:' in errors:
                 reason = 'INVALID_CERTIFICATE'
             raise Exception(
                 'Failed to dump the remote repository from %s.' % src_url,
                 reason)
         if load.returncode != 0:
             raise Exception(
                 'Failed to load the dump of remote repository from %s.' %
                 (src_url, ))
 
     def commit(self, wire, message, author, timestamp, updated, removed):
         assert isinstance(message, str)
         assert isinstance(author, str)
 
         repo = self._factory.repo(wire)
         fsobj = svn.repos.fs(repo)
 
         rev = svn.fs.youngest_rev(fsobj)
         txn = svn.repos.fs_begin_txn_for_commit(repo, rev, author, message)
         txn_root = svn.fs.txn_root(txn)
 
         for node in updated:
             TxnNodeProcessor(node, txn_root).update()
         for node in removed:
             TxnNodeProcessor(node, txn_root).remove()
 
         commit_id = svn.repos.fs_commit_txn(repo, txn)
 
         if timestamp:
             apr_time = apr_time_t(timestamp)
             ts_formatted = svn.core.svn_time_to_cstring(apr_time)
             svn.fs.change_rev_prop(fsobj, commit_id, 'svn:date', ts_formatted)
 
         log.debug('Committed revision "%s" to "%s".', commit_id, wire['path'])
         return commit_id
 
     def diff(self, wire, rev1, rev2, path1=None, path2=None,
              ignore_whitespace=False, context=3):
 
         wire.update(cache=False)
         repo = self._factory.repo(wire)
         diff_creator = SvnDiffer(
             repo, rev1, path1, rev2, path2, ignore_whitespace, context)
         try:
             return diff_creator.generate_diff()
         except svn.core.SubversionException as e:
             log.exception(
                 "Error during diff operation operation. "
                 "Path might not exist %s, %s" % (path1, path2))
             return ""
 
+    @reraise_safe_exceptions
+    def is_large_file(self, wire, path):
+        return False
+
 
 class SvnDiffer(object):
     """
     Utility to create diffs based on difflib and the Subversion api
     """
 
     binary_content = False
 
     def __init__(
             self, repo, src_rev, src_path, tgt_rev, tgt_path,
             ignore_whitespace, context):
         self.repo = repo
         self.ignore_whitespace = ignore_whitespace
         self.context = context
 
         fsobj = svn.repos.fs(repo)
 
         self.tgt_rev = tgt_rev
         self.tgt_path = tgt_path or ''
         self.tgt_root = svn.fs.revision_root(fsobj, tgt_rev)
         self.tgt_kind = svn.fs.check_path(self.tgt_root, self.tgt_path)
 
         self.src_rev = src_rev
         self.src_path = src_path or self.tgt_path
         self.src_root = svn.fs.revision_root(fsobj, src_rev)
         self.src_kind = svn.fs.check_path(self.src_root, self.src_path)
 
         self._validate()
 
     def _validate(self):
         if (self.tgt_kind != svn.core.svn_node_none and
                 self.src_kind != svn.core.svn_node_none and
                 self.src_kind != self.tgt_kind):
             # TODO: johbo: proper error handling
             raise Exception(
                 "Source and target are not compatible for diff generation. "
                 "Source type: %s, target type: %s" %
                 (self.src_kind, self.tgt_kind))
 
     def generate_diff(self):
         buf = StringIO.StringIO()
         if self.tgt_kind == svn.core.svn_node_dir:
             self._generate_dir_diff(buf)
         else:
             self._generate_file_diff(buf)
         return buf.getvalue()
 
     def _generate_dir_diff(self, buf):
         editor = DiffChangeEditor()
         editor_ptr, editor_baton = svn.delta.make_editor(editor)
         svn.repos.dir_delta2(
             self.src_root,
             self.src_path,
             '',  # src_entry
             self.tgt_root,
             self.tgt_path,
             editor_ptr, editor_baton,
             authorization_callback_allow_all,
             False,  # text_deltas
             svn.core.svn_depth_infinity,  # depth
             False,  # entry_props
             False,  # ignore_ancestry
         )
 
         for path, __, change in sorted(editor.changes):
             self._generate_node_diff(
                 buf, change, path, self.tgt_path, path, self.src_path)
 
     def _generate_file_diff(self, buf):
         change = None
         if self.src_kind == svn.core.svn_node_none:
             change = "add"
         elif self.tgt_kind == svn.core.svn_node_none:
             change = "delete"
         tgt_base, tgt_path = vcspath.split(self.tgt_path)
         src_base, src_path = vcspath.split(self.src_path)
         self._generate_node_diff(
             buf, change, tgt_path, tgt_base, src_path, src_base)
 
     def _generate_node_diff(
             self, buf, change, tgt_path, tgt_base, src_path, src_base):
 
         if self.src_rev == self.tgt_rev and tgt_base == src_base:
             # makes consistent behaviour with git/hg to return empty diff if
             # we compare same revisions
             return
 
         tgt_full_path = vcspath.join(tgt_base, tgt_path)
         src_full_path = vcspath.join(src_base, src_path)
 
         self.binary_content = False
         mime_type = self._get_mime_type(tgt_full_path)
 
         if mime_type and not mime_type.startswith('text'):
             self.binary_content = True
             buf.write("=" * 67 + '\n')
             buf.write("Cannot display: file marked as a binary type.\n")
             buf.write("svn:mime-type = %s\n" % mime_type)
         buf.write("Index: %s\n" % (tgt_path, ))
         buf.write("=" * 67 + '\n')
         buf.write("diff --git a/%(tgt_path)s b/%(tgt_path)s\n" % {
             'tgt_path': tgt_path})
 
         if change == 'add':
             # TODO: johbo: SVN is missing a zero here compared to git
             buf.write("new file mode 10644\n")
 
             #TODO(marcink): intro to binary detection of svn patches
             # if self.binary_content:
             #     buf.write('GIT binary patch\n')
 
             buf.write("--- /dev/null\t(revision 0)\n")
             src_lines = []
         else:
             if change == 'delete':
                 buf.write("deleted file mode 10644\n")
 
             #TODO(marcink): intro to binary detection of svn patches
             # if self.binary_content:
             #     buf.write('GIT binary patch\n')
 
             buf.write("--- a/%s\t(revision %s)\n" % (
                 src_path, self.src_rev))
             src_lines = self._svn_readlines(self.src_root, src_full_path)
 
         if change == 'delete':
             buf.write("+++ /dev/null\t(revision %s)\n" % (self.tgt_rev, ))
             tgt_lines = []
         else:
             buf.write("+++ b/%s\t(revision %s)\n" % (
                 tgt_path, self.tgt_rev))
             tgt_lines = self._svn_readlines(self.tgt_root, tgt_full_path)
 
         if not self.binary_content:
             udiff = svn_diff.unified_diff(
                 src_lines, tgt_lines, context=self.context,
                 ignore_blank_lines=self.ignore_whitespace,
                 ignore_case=False,
                 ignore_space_changes=self.ignore_whitespace)
             buf.writelines(udiff)
 
     def _get_mime_type(self, path):
         try:
             mime_type = svn.fs.node_prop(
                 self.tgt_root, path, svn.core.SVN_PROP_MIME_TYPE)
         except svn.core.SubversionException:
             mime_type = svn.fs.node_prop(
                 self.src_root, path, svn.core.SVN_PROP_MIME_TYPE)
         return mime_type
 
     def _svn_readlines(self, fs_root, node_path):
         if self.binary_content:
             return []
         node_kind = svn.fs.check_path(fs_root, node_path)
         if node_kind not in (
                 svn.core.svn_node_file, svn.core.svn_node_symlink):
             return []
         content = svn.core.Stream(
             svn.fs.file_contents(fs_root, node_path)).read()
         return content.splitlines(True)
 
 
 class DiffChangeEditor(svn.delta.Editor):
     """
     Records changes between two given revisions
     """
 
     def __init__(self):
         self.changes = []
 
     def delete_entry(self, path, revision, parent_baton, pool=None):
         self.changes.append((path, None, 'delete'))
 
     def add_file(
             self, path, parent_baton, copyfrom_path, copyfrom_revision,
             file_pool=None):
         self.changes.append((path, 'file', 'add'))
 
     def open_file(self, path, parent_baton, base_revision, file_pool=None):
         self.changes.append((path, 'file', 'change'))
 
 
 def authorization_callback_allow_all(root, path, pool):
     return True
 
 
 class TxnNodeProcessor(object):
     """
     Utility to process the change of one node within a transaction root.
 
     It encapsulates the knowledge of how to add, update or remove
     a node for a given transaction root. The purpose is to support the method
     `SvnRemote.commit`.
     """
 
     def __init__(self, node, txn_root):
         assert isinstance(node['path'], str)
 
         self.node = node
         self.txn_root = txn_root
 
     def update(self):
         self._ensure_parent_dirs()
         self._add_file_if_node_does_not_exist()
         self._update_file_content()
         self._update_file_properties()
 
     def remove(self):
         svn.fs.delete(self.txn_root, self.node['path'])
         # TODO: Clean up directory if empty
 
     def _ensure_parent_dirs(self):
         curdir = vcspath.dirname(self.node['path'])
         dirs_to_create = []
         while not self._svn_path_exists(curdir):
             dirs_to_create.append(curdir)
             curdir = vcspath.dirname(curdir)
 
         for curdir in reversed(dirs_to_create):
             log.debug('Creating missing directory "%s"', curdir)
             svn.fs.make_dir(self.txn_root, curdir)
 
     def _svn_path_exists(self, path):
         path_status = svn.fs.check_path(self.txn_root, path)
         return path_status != svn.core.svn_node_none
 
     def _add_file_if_node_does_not_exist(self):
         kind = svn.fs.check_path(self.txn_root, self.node['path'])
         if kind == svn.core.svn_node_none:
             svn.fs.make_file(self.txn_root, self.node['path'])
 
     def _update_file_content(self):
         assert isinstance(self.node['content'], str)
         handler, baton = svn.fs.apply_textdelta(
             self.txn_root, self.node['path'], None, None)
         svn.delta.svn_txdelta_send_string(self.node['content'], handler, baton)
 
     def _update_file_properties(self):
         properties = self.node.get('properties', {})
         for key, value in properties.iteritems():
             svn.fs.change_node_prop(
                 self.txn_root, self.node['path'], key, value)
 
 
 def apr_time_t(timestamp):
     """
     Convert a Python timestamp into APR timestamp type apr_time_t
     """
     return timestamp * 1E6
 
 
 def svn_opt_revision_value_t(num):
     """
     Put `num` into a `svn_opt_revision_value_t` structure.
     """
     value = svn.core.svn_opt_revision_value_t()
     value.number = num
     revision = svn.core.svn_opt_revision_t()
     revision.kind = svn.core.svn_opt_revision_number
     revision.value = value
     return revision

vcsserver/utils.py

@@ -1,57 +1,72 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2017 RodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 
+def safe_int(val, default=None):
+    """
+    Returns int() of val if val is not convertable to int use default
+    instead
 
-# TODO: johbo: That's a copy from rhodecode
+    :param val:
+    :param default:
+    """
+
+    try:
+        val = int(val)
+    except (ValueError, TypeError):
+        val = default
+
+    return val
+
+
 def safe_str(unicode_, to_encoding=['utf8']):
     """
     safe str function. Does few trick to turn unicode_ into string
 
     In case of UnicodeEncodeError, we try to return it with encoding detected
     by chardet library if it fails fallback to string with errors replaced
 
     :param unicode_: unicode to encode
     :rtype: str
     :returns: str object
     """
 
     # if it's not basestr cast to str
     if not isinstance(unicode_, basestring):
         return str(unicode_)
 
     if isinstance(unicode_, str):
         return unicode_
 
     if not isinstance(to_encoding, (list, tuple)):
         to_encoding = [to_encoding]
 
     for enc in to_encoding:
         try:
             return unicode_.encode(enc)
         except UnicodeEncodeError:
             pass
 
     try:
         import chardet
         encoding = chardet.detect(unicode_)['encoding']
         if encoding is None:
             raise UnicodeEncodeError()
 
         return unicode_.encode(encoding)
     except (ImportError, UnicodeEncodeError):
         return unicode_.encode(to_encoding[0], 'replace')