# HG changeset patch # User Marcin Kuzminski # Date 2017-03-22 13:53:45 # Node ID bb8ad7ab25209e52a326b561c5785da102ec001b # Parent d21df11cf1cc2b865892cd9354f2065ffcd6a6dc git-lfs: added vcsserver handling of git-lfs objects. - fixes #4235 diff --git a/vcsserver/git_lfs/__init__.py b/vcsserver/git_lfs/__init__.py new file mode 100644 --- /dev/null +++ b/vcsserver/git_lfs/__init__.py @@ -0,0 +1,19 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2017 RodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + +from app import create_app diff --git a/vcsserver/git_lfs/app.py b/vcsserver/git_lfs/app.py new file mode 100644 --- /dev/null +++ b/vcsserver/git_lfs/app.py @@ -0,0 +1,276 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2017 RodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import re +import logging +from wsgiref.util import FileWrapper + +import simplejson as json +from pyramid.config import Configurator +from pyramid.response import Response, FileIter +from pyramid.httpexceptions import ( + HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden, + HTTPUnprocessableEntity) + +from vcsserver.git_lfs.lib import OidHandler, LFSOidStore +from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator +from vcsserver.utils import safe_int + +log = logging.getLogger(__name__) + + +GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ? +GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))') + + +def write_response_error(http_exception, text=None): + content_type = 'application/json' + _exception = http_exception(content_type=content_type) + _exception.content_type = content_type + if text: + _exception.body = json.dumps({'message': text}) + log.debug('LFS: writing response of type %s to client with text:%s', + http_exception, text) + return _exception + + +class AuthHeaderRequired(object): + """ + Decorator to check if request has proper auth-header + """ + + def __call__(self, func): + return get_cython_compat_decorator(self.__wrapper, func) + + def __wrapper(self, func, *fargs, **fkwargs): + request = fargs[1] + auth = request.authorization + if not auth: + return write_response_error(HTTPForbidden) + return func(*fargs[1:], **fkwargs) + + +# views + +def lfs_objects(request): + # indicate not supported, V1 API + log.warning('LFS: v1 api not supported, reporting it back to client') + return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported') + + +@AuthHeaderRequired() +def lfs_objects_batch(request): + """ + The client sends the following information to the Batch endpoint to transfer some objects: + + operation - Should be download or upload. + transfers - An optional Array of String identifiers for transfer + adapters that the client has configured. If omitted, the basic + transfer adapter MUST be assumed by the server. + objects - An Array of objects to download. + oid - String OID of the LFS object. + size - Integer byte size of the LFS object. Must be at least zero. + """ + auth = request.authorization + + repo = request.matchdict.get('repo') + + data = request.json + operation = data.get('operation') + if operation not in ('download', 'upload'): + log.debug('LFS: unsupported operation:%s', operation) + return write_response_error( + HTTPBadRequest, 'unsupported operation mode: `%s`' % operation) + + if 'objects' not in data: + log.debug('LFS: missing objects data') + return write_response_error( + HTTPBadRequest, 'missing objects data') + + log.debug('LFS: handling operation of type: %s', operation) + + objects = [] + for o in data['objects']: + try: + oid = o['oid'] + obj_size = o['size'] + except KeyError: + log.exception('LFS, failed to extract data') + return write_response_error( + HTTPBadRequest, 'unsupported data in objects') + + obj_data = {'oid': oid} + + obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid) + obj_verify_href = request.route_url('lfs_objects_verify', repo=repo) + store = LFSOidStore( + repo, oid, store_location=request.registry.git_lfs_store_path) + handler = OidHandler( + store, repo, auth, oid, obj_size, obj_data, + obj_href, obj_verify_href) + + # this verifies also OIDs + actions, errors = handler.exec_operation(operation) + if errors: + log.warning('LFS: got following errors: %s', errors) + obj_data['errors'] = errors + + if actions: + obj_data['actions'] = actions + + obj_data['size'] = obj_size + obj_data['authenticated'] = True + objects.append(obj_data) + + result = {'objects': objects, 'transfer': 'basic'} + log.debug('LFS Response %s', safe_result(result)) + + return result + + +def lfs_objects_oid_upload(request): + repo = request.matchdict.get('repo') + oid = request.matchdict.get('oid') + store = LFSOidStore( + repo, oid, store_location=request.registry.git_lfs_store_path) + engine = store.get_engine(mode='wb') + log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid) + with engine as f: + for chunk in FileWrapper(request.body_file_seekable, blksize=64 * 1024): + f.write(chunk) + + return {'upload': 'ok'} + + +def lfs_objects_oid_download(request): + repo = request.matchdict.get('repo') + oid = request.matchdict.get('oid') + + store = LFSOidStore( + repo, oid, store_location=request.registry.git_lfs_store_path) + if not store.has_oid(): + log.debug('LFS: oid %s does not exists in store', oid) + return write_response_error( + HTTPNotFound, 'requested file with oid `%s` not found in store' % oid) + + # TODO(marcink): support range header ? + # Range: bytes=0-, `bytes=(\d+)\-.*` + + f = open(store.oid_path, 'rb') + response = Response( + content_type='application/octet-stream', app_iter=FileIter(f)) + response.headers.add('X-RC-LFS-Response-Oid', str(oid)) + return response + + +def lfs_objects_verify(request): + repo = request.matchdict.get('repo') + + data = request.json + oid = data.get('oid') + size = safe_int(data.get('size')) + + if not (oid and size): + return write_response_error( + HTTPBadRequest, 'missing oid and size in request data') + + store = LFSOidStore(repo, oid, + store_location=request.registry.git_lfs_store_path) + if not store.has_oid(): + log.debug('LFS: oid %s does not exists in store', oid) + return write_response_error( + HTTPNotFound, 'oid `%s` does not exists in store' % oid) + + store_size = store.size_oid() + if store_size != size: + msg = 'requested file size mismatch store size:%s requested:%s' % ( + store_size, size) + return write_response_error( + HTTPUnprocessableEntity, msg) + + return {'message': {'size': 'ok', 'in_store': 'ok'}} + + +def lfs_objects_lock(request): + return write_response_error( + HTTPNotImplemented, 'GIT LFS locking api not supported') + + +def not_found(request): + return write_response_error( + HTTPNotFound, 'request path not found') + + +def lfs_disabled(request): + return write_response_error( + HTTPNotImplemented, 'GIT LFS disabled for this repo') + + +def git_lfs_app(config): + + # v1 API deprecation endpoint + config.add_route('lfs_objects', + '/{repo:.*?[^/]}/info/lfs/objects') + config.add_view(lfs_objects, route_name='lfs_objects', + request_method='POST', renderer='json') + + # locking API + config.add_route('lfs_objects_lock', + '/{repo:.*?[^/]}/info/lfs/locks') + config.add_view(lfs_objects_lock, route_name='lfs_objects_lock', + request_method=('POST', 'GET'), renderer='json') + + config.add_route('lfs_objects_lock_verify', + '/{repo:.*?[^/]}/info/lfs/locks/verify') + config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify', + request_method=('POST', 'GET'), renderer='json') + + # batch API + config.add_route('lfs_objects_batch', + '/{repo:.*?[^/]}/info/lfs/objects/batch') + config.add_view(lfs_objects_batch, route_name='lfs_objects_batch', + request_method='POST', renderer='json') + + # oid upload/download API + config.add_route('lfs_objects_oid', + '/{repo:.*?[^/]}/info/lfs/objects/{oid}') + config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid', + request_method='PUT', renderer='json') + config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid', + request_method='GET', renderer='json') + + # verification API + config.add_route('lfs_objects_verify', + '/{repo:.*?[^/]}/info/lfs/verify') + config.add_view(lfs_objects_verify, route_name='lfs_objects_verify', + request_method='POST', renderer='json') + + # not found handler for API + config.add_notfound_view(not_found, renderer='json') + + +def create_app(git_lfs_enabled, git_lfs_store_path): + config = Configurator() + if git_lfs_enabled: + config.include(git_lfs_app) + config.registry.git_lfs_store_path = git_lfs_store_path + else: + # not found handler for API, reporting disabled LFS support + config.add_notfound_view(lfs_disabled, renderer='json') + + app = config.make_wsgi_app() + return app diff --git a/vcsserver/git_lfs/lib.py b/vcsserver/git_lfs/lib.py new file mode 100644 --- /dev/null +++ b/vcsserver/git_lfs/lib.py @@ -0,0 +1,167 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2017 RodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import shutil +import logging +from collections import OrderedDict + +log = logging.getLogger(__name__) + + +class OidHandler(object): + + def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href, + obj_verify_href=None): + self.current_store = store + self.repo_name = repo_name + self.auth = auth + self.oid = oid + self.obj_size = obj_size + self.obj_data = obj_data + self.obj_href = obj_href + self.obj_verify_href = obj_verify_href + + def get_store(self, mode=None): + return self.current_store + + def get_auth(self): + """returns auth header for re-use in upload/download""" + return " ".join(self.auth) + + def download(self): + + store = self.get_store() + response = None + has_errors = None + + if not store.has_oid(): + # error reply back to client that something is wrong with dl + err_msg = 'object: {} does not exist in store'.format(store.oid) + has_errors = OrderedDict( + error=OrderedDict( + code=404, + message=err_msg + ) + ) + + download_action = OrderedDict( + href=self.obj_href, + header=OrderedDict([("Authorization", self.get_auth())]) + ) + if not has_errors: + response = OrderedDict(download=download_action) + return response, has_errors + + def upload(self, skip_existing=True): + """ + Write upload action for git-lfs server + """ + + store = self.get_store() + response = None + has_errors = None + + # verify if we have the OID before, if we do, reply with empty + if store.has_oid(): + log.debug('LFS: store already has oid %s', store.oid) + if skip_existing: + log.debug('LFS: skipping further action as oid is existing') + return response, has_errors + + upload_action = OrderedDict( + href=self.obj_href, + header=OrderedDict([("Authorization", self.get_auth())]) + ) + if not has_errors: + response = OrderedDict(upload=upload_action) + # if specified in handler, return the verification endpoint + if self.obj_verify_href: + verify_action = OrderedDict( + href=self.obj_verify_href, + header=OrderedDict([("Authorization", self.get_auth())]) + ) + response['verify'] = verify_action + return response, has_errors + + def exec_operation(self, operation, *args, **kwargs): + handler = getattr(self, operation) + log.debug('LFS: handling request using %s handler', handler) + return handler(*args, **kwargs) + + +class LFSOidStore(object): + + def __init__(self, repo, oid, store_location=None): + self._store = store_location or self.get_default_store() + self.oid = oid + self.repo = repo + self.store_path = os.path.join(self._store, repo) + self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp') + self.oid_path = os.path.join(self.store_path, oid) + self.fd = None + + def get_engine(self, mode): + """ + engine = .get_engine(mode='wb') + with engine as f: + f.write('...') + """ + + class StoreEngine(object): + def __init__(self, mode, store_path, oid_path, tmp_oid_path): + self.mode = mode + self.store_path = store_path + self.oid_path = oid_path + self.tmp_oid_path = tmp_oid_path + + def __enter__(self): + if not os.path.isdir(self.store_path): + os.makedirs(self.store_path) + + # TODO(marcink): maybe write metadata here with size/oid ? + fd = open(self.tmp_oid_path, self.mode) + self.fd = fd + return fd + + def __exit__(self, exc_type, exc_value, traceback): + # close tmp file, and rename to final destination + self.fd.close() + shutil.move(self.tmp_oid_path, self.oid_path) + + return StoreEngine( + mode, self.store_path, self.oid_path, self.tmp_oid_path) + + def get_default_store(self): + """ + Default store, consistent with defaults of Mercurial large files store + which is /home/username/.cache/largefiles + """ + user_home = os.path.expanduser("~") + return os.path.join(user_home, '.cache', 'lfs-store') + + def has_oid(self): + return os.path.exists(os.path.join(self.store_path, self.oid)) + + def size_oid(self): + size = -1 + + if self.has_oid(): + oid = os.path.join(self.store_path, self.oid) + size = os.stat(oid).st_size + + return size \ No newline at end of file diff --git a/vcsserver/git_lfs/tests/__init__.py b/vcsserver/git_lfs/tests/__init__.py new file mode 100644 --- /dev/null +++ b/vcsserver/git_lfs/tests/__init__.py @@ -0,0 +1,16 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2017 RodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA diff --git a/vcsserver/git_lfs/tests/test_lfs_app.py b/vcsserver/git_lfs/tests/test_lfs_app.py new file mode 100644 --- /dev/null +++ b/vcsserver/git_lfs/tests/test_lfs_app.py @@ -0,0 +1,233 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2017 RodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import pytest +from webtest.app import TestApp as WebObTestApp + +from vcsserver.git_lfs.app import create_app + + +@pytest.fixture(scope='function') +def git_lfs_app(tmpdir): + custom_app = WebObTestApp(create_app( + git_lfs_enabled=True, git_lfs_store_path=str(tmpdir))) + custom_app._store = str(tmpdir) + return custom_app + + +@pytest.fixture() +def http_auth(): + return {'HTTP_AUTHORIZATION': "Basic XXXXX"} + + +class TestLFSApplication(object): + + def test_app_wrong_path(self, git_lfs_app): + git_lfs_app.get('/repo/info/lfs/xxx', status=404) + + def test_app_deprecated_endpoint(self, git_lfs_app): + response = git_lfs_app.post('/repo/info/lfs/objects', status=501) + assert response.status_code == 501 + assert response.json == {u'message': u'LFS: v1 api not supported'} + + def test_app_lock_verify_api_not_available(self, git_lfs_app): + response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501) + assert response.status_code == 501 + assert response.json == { + u'message': u'GIT LFS locking api not supported'} + + def test_app_lock_api_not_available(self, git_lfs_app): + response = git_lfs_app.post('/repo/info/lfs/locks', status=501) + assert response.status_code == 501 + assert response.json == { + u'message': u'GIT LFS locking api not supported'} + + def test_app_batch_api_missing_auth(self, git_lfs_app,): + git_lfs_app.post_json( + '/repo/info/lfs/objects/batch', params={}, status=403) + + def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth): + response = git_lfs_app.post_json( + '/repo/info/lfs/objects/batch', params={}, status=400, + extra_environ=http_auth) + assert response.json == { + u'message': u'unsupported operation mode: `None`'} + + def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth): + response = git_lfs_app.post_json( + '/repo/info/lfs/objects/batch', params={'operation': 'download'}, + status=400, extra_environ=http_auth) + assert response.json == { + u'message': u'missing objects data'} + + def test_app_batch_api_unsupported_data_in_objects( + self, git_lfs_app, http_auth): + params = {'operation': 'download', + 'objects': [{}]} + response = git_lfs_app.post_json( + '/repo/info/lfs/objects/batch', params=params, status=400, + extra_environ=http_auth) + assert response.json == { + u'message': u'unsupported data in objects'} + + def test_app_batch_api_download_missing_object( + self, git_lfs_app, http_auth): + params = {'operation': 'download', + 'objects': [{'oid': '123', 'size': '1024'}]} + response = git_lfs_app.post_json( + '/repo/info/lfs/objects/batch', params=params, + extra_environ=http_auth) + + expected_objects = [ + {u'authenticated': True, + u'errors': {u'error': { + u'code': 404, + u'message': u'object: 123 does not exist in store'}}, + u'oid': u'123', + u'size': u'1024'} + ] + assert response.json == { + 'objects': expected_objects, 'transfer': 'basic'} + + def test_app_batch_api_download(self, git_lfs_app, http_auth): + oid = '456' + oid_path = os.path.join(git_lfs_app._store, 'repo', oid) + os.makedirs(os.path.dirname(oid_path)) + with open(oid_path, 'wb') as f: + f.write('OID_CONTENT') + + params = {'operation': 'download', + 'objects': [{'oid': oid, 'size': '1024'}]} + response = git_lfs_app.post_json( + '/repo/info/lfs/objects/batch', params=params, + extra_environ=http_auth) + + expected_objects = [ + {u'authenticated': True, + u'actions': { + u'download': { + u'header': {u'Authorization': u'Basic XXXXX'}, + u'href': u'http://localhost/repo/info/lfs/objects/456'}, + }, + u'oid': u'456', + u'size': u'1024'} + ] + assert response.json == { + 'objects': expected_objects, 'transfer': 'basic'} + + def test_app_batch_api_upload(self, git_lfs_app, http_auth): + params = {'operation': 'upload', + 'objects': [{'oid': '123', 'size': '1024'}]} + response = git_lfs_app.post_json( + '/repo/info/lfs/objects/batch', params=params, + extra_environ=http_auth) + expected_objects = [ + {u'authenticated': True, + u'actions': { + u'upload': { + u'header': {u'Authorization': u'Basic XXXXX'}, + u'href': u'http://localhost/repo/info/lfs/objects/123'}, + u'verify': { + u'header': {u'Authorization': u'Basic XXXXX'}, + u'href': u'http://localhost/repo/info/lfs/verify'} + }, + u'oid': u'123', + u'size': u'1024'} + ] + assert response.json == { + 'objects': expected_objects, 'transfer': 'basic'} + + def test_app_verify_api_missing_data(self, git_lfs_app): + params = {'oid': 'missing',} + response = git_lfs_app.post_json( + '/repo/info/lfs/verify', params=params, + status=400) + + assert response.json == { + u'message': u'missing oid and size in request data'} + + def test_app_verify_api_missing_obj(self, git_lfs_app): + params = {'oid': 'missing', 'size': '1024'} + response = git_lfs_app.post_json( + '/repo/info/lfs/verify', params=params, + status=404) + + assert response.json == { + u'message': u'oid `missing` does not exists in store'} + + def test_app_verify_api_size_mismatch(self, git_lfs_app): + oid = 'existing' + oid_path = os.path.join(git_lfs_app._store, 'repo', oid) + os.makedirs(os.path.dirname(oid_path)) + with open(oid_path, 'wb') as f: + f.write('OID_CONTENT') + + params = {'oid': oid, 'size': '1024'} + response = git_lfs_app.post_json( + '/repo/info/lfs/verify', params=params, status=422) + + assert response.json == { + u'message': u'requested file size mismatch ' + u'store size:11 requested:1024'} + + def test_app_verify_api(self, git_lfs_app): + oid = 'existing' + oid_path = os.path.join(git_lfs_app._store, 'repo', oid) + os.makedirs(os.path.dirname(oid_path)) + with open(oid_path, 'wb') as f: + f.write('OID_CONTENT') + + params = {'oid': oid, 'size': 11} + response = git_lfs_app.post_json( + '/repo/info/lfs/verify', params=params) + + assert response.json == { + u'message': {u'size': u'ok', u'in_store': u'ok'}} + + def test_app_download_api_oid_not_existing(self, git_lfs_app): + oid = 'missing' + + response = git_lfs_app.get( + '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404) + + assert response.json == { + u'message': u'requested file with oid `missing` not found in store'} + + def test_app_download_api(self, git_lfs_app): + oid = 'existing' + oid_path = os.path.join(git_lfs_app._store, 'repo', oid) + os.makedirs(os.path.dirname(oid_path)) + with open(oid_path, 'wb') as f: + f.write('OID_CONTENT') + + response = git_lfs_app.get( + '/repo/info/lfs/objects/{oid}'.format(oid=oid)) + assert response + + def test_app_upload(self, git_lfs_app): + oid = 'uploaded' + + response = git_lfs_app.put( + '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT') + + assert response.json == {u'upload': u'ok'} + + # verify that we actually wrote that OID + oid_path = os.path.join(git_lfs_app._store, 'repo', oid) + assert os.path.isfile(oid_path) + assert 'CONTENT' == open(oid_path).read() diff --git a/vcsserver/git_lfs/tests/test_lib.py b/vcsserver/git_lfs/tests/test_lib.py new file mode 100644 --- /dev/null +++ b/vcsserver/git_lfs/tests/test_lib.py @@ -0,0 +1,121 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2017 RodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import pytest +from vcsserver.git_lfs.lib import OidHandler, LFSOidStore + + +@pytest.fixture() +def lfs_store(tmpdir): + repo = 'test' + oid = '123456789' + store = LFSOidStore(repo=repo, oid=oid, store_location=str(tmpdir)) + return store + + +@pytest.fixture() +def oid_handler(lfs_store): + store = lfs_store + repo = store.repo + oid = store.oid + + oid_handler = OidHandler( + store=store, repo_name=repo, auth=('basic', 'xxxx'), + oid=oid, + obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid', + obj_verify_href='http://localhost/verify') + return oid_handler + + +class TestOidHandler(object): + + @pytest.mark.parametrize('exec_action', [ + 'download', + 'upload', + ]) + def test_exec_action(self, exec_action, oid_handler): + handler = oid_handler.exec_operation(exec_action) + assert handler + + def test_exec_action_undefined(self, oid_handler): + with pytest.raises(AttributeError): + oid_handler.exec_operation('wrong') + + def test_download_oid_not_existing(self, oid_handler): + response, has_errors = oid_handler.exec_operation('download') + + assert response is None + assert has_errors['error'] == { + 'code': 404, + 'message': 'object: 123456789 does not exist in store'} + + def test_download_oid(self, oid_handler): + store = oid_handler.get_store() + + os.makedirs(os.path.dirname(store.oid_path)) + with open(store.oid_path, 'wb') as f: + f.write('CONTENT') + + response, has_errors = oid_handler.exec_operation('download') + + assert has_errors is None + assert response['download'] == { + 'header': {'Authorization': 'basic xxxx'}, + 'href': 'http://localhost/handle_oid' + } + + def test_upload_oid_that_exists(self, oid_handler): + store = oid_handler.get_store() + + os.makedirs(os.path.dirname(store.oid_path)) + with open(store.oid_path, 'wb') as f: + f.write('CONTENT') + + response, has_errors = oid_handler.exec_operation('upload') + assert has_errors is None + assert response is None + + def test_upload_oid(self, oid_handler): + response, has_errors = oid_handler.exec_operation('upload') + assert has_errors is None + assert response['upload'] == { + 'header': {'Authorization': 'basic xxxx'}, + 'href': 'http://localhost/handle_oid' + } + + +class TestLFSStore(object): + def test_write_oid(self, lfs_store): + oid_location = lfs_store.oid_path + + assert not os.path.isfile(oid_location) + + engine = lfs_store.get_engine(mode='wb') + with engine as f: + f.write('CONTENT') + + assert os.path.isfile(oid_location) + + def test_detect_has_oid(self, lfs_store): + + assert lfs_store.has_oid() is False + engine = lfs_store.get_engine(mode='wb') + with engine as f: + f.write('CONTENT') + + assert lfs_store.has_oid() is True \ No newline at end of file diff --git a/vcsserver/git_lfs/utils.py b/vcsserver/git_lfs/utils.py new file mode 100644 --- /dev/null +++ b/vcsserver/git_lfs/utils.py @@ -0,0 +1,50 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2017 RodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import copy +from functools import wraps + + +def get_cython_compat_decorator(wrapper, func): + """ + Creates a cython compatible decorator. The previously used + decorator.decorator() function seems to be incompatible with cython. + + :param wrapper: __wrapper method of the decorator class + :param func: decorated function + """ + @wraps(func) + def local_wrapper(*args, **kwds): + return wrapper(func, *args, **kwds) + local_wrapper.__wrapped__ = func + return local_wrapper + + +def safe_result(result): + """clean result for better representation in logs""" + clean_copy = copy.deepcopy(result) + + try: + if 'objects' in clean_copy: + for oid_data in clean_copy['objects']: + if 'actions' in oid_data: + for action_name, data in oid_data['actions'].items(): + if 'header' in data: + data['header'] = {'Authorization': '*****'} + except Exception: + return result + + return clean_copy diff --git a/vcsserver/http_main.py b/vcsserver/http_main.py --- a/vcsserver/http_main.py +++ b/vcsserver/http_main.py @@ -30,6 +30,7 @@ from pyramid.config import Configurator from pyramid.wsgi import wsgiapp from vcsserver import remote_wsgi, scm_app, settings, hgpatches +from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub from vcsserver.echo_stub.echo_app import EchoApp from vcsserver.exceptions import HTTPRepoLocked @@ -40,11 +41,13 @@ try: except ImportError: GitFactory = None GitRemote = None + try: from vcsserver.hg import MercurialFactory, HgRemote except ImportError: MercurialFactory = None HgRemote = None + try: from vcsserver.svn import SubversionFactory, SvnRemote except ImportError: @@ -362,9 +365,31 @@ class HTTPApplication(object): config = msgpack.unpackb(packed_config) environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO'] - app = scm_app.create_git_wsgi_app( - repo_path, repo_name, config) + content_type = environ.get('CONTENT_TYPE', '') + + path = environ['PATH_INFO'] + is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type + log.debug( + 'LFS: Detecting if request `%s` is LFS server path based ' + 'on content type:`%s`, is_lfs:%s', + path, content_type, is_lfs_request) + + if not is_lfs_request: + # fallback detection by path + if GIT_LFS_PROTO_PAT.match(path): + is_lfs_request = True + log.debug( + 'LFS: fallback detection by path of: `%s`, is_lfs:%s', + path, is_lfs_request) + + if is_lfs_request: + app = scm_app.create_git_lfs_wsgi_app( + repo_path, repo_name, config) + else: + app = scm_app.create_git_wsgi_app( + repo_path, repo_name, config) return app(environ, start_response) + return _git_stream def is_vcs_view(self, context, request): diff --git a/vcsserver/scm_app.py b/vcsserver/scm_app.py --- a/vcsserver/scm_app.py +++ b/vcsserver/scm_app.py @@ -15,8 +15,8 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import os import logging -import os import mercurial import mercurial.error @@ -25,7 +25,7 @@ import mercurial.hgweb.hgweb_mod import mercurial.hgweb.protocol import webob.exc -from vcsserver import pygrack, exceptions, settings +from vcsserver import pygrack, exceptions, settings, git_lfs log = logging.getLogger(__name__) @@ -132,6 +132,9 @@ def create_hg_wsgi_app(repo_path, repo_n class GitHandler(object): + """ + Handler for Git operations like push/pull etc + """ def __init__(self, repo_location, repo_name, git_path, update_server_info, extras): if not os.path.isdir(repo_location): @@ -172,3 +175,35 @@ def create_git_wsgi_app(repo_path, repo_ repo_path, repo_name, git_path, update_server_info, config) return app + + +class GitLFSHandler(object): + """ + Handler for Git LFS operations + """ + + def __init__(self, repo_location, repo_name, git_path, update_server_info, + extras): + if not os.path.isdir(repo_location): + raise OSError(repo_location) + self.content_path = repo_location + self.repo_name = repo_name + self.repo_location = repo_location + self.extras = extras + self.git_path = git_path + self.update_server_info = update_server_info + + def get_app(self, git_lfs_enabled, git_lfs_store_path): + app = git_lfs.create_app(git_lfs_enabled, git_lfs_store_path) + return app + + +def create_git_lfs_wsgi_app(repo_path, repo_name, config): + git_path = settings.GIT_EXECUTABLE + update_server_info = config.pop('git_update_server_info') + git_lfs_enabled = config.pop('git_lfs_enabled') + git_lfs_store_path = config.pop('git_lfs_store_path') + app = GitLFSHandler( + repo_path, repo_name, git_path, update_server_info, config) + + return app.get_app(git_lfs_enabled, git_lfs_store_path) diff --git a/vcsserver/utils.py b/vcsserver/utils.py --- a/vcsserver/utils.py +++ b/vcsserver/utils.py @@ -16,8 +16,23 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +def safe_int(val, default=None): + """ + Returns int() of val if val is not convertable to int use default + instead -# TODO: johbo: That's a copy from rhodecode + :param val: + :param default: + """ + + try: + val = int(val) + except (ValueError, TypeError): + val = default + + return val + + def safe_str(unicode_, to_encoding=['utf8']): """ safe str function. Does few trick to turn unicode_ into string