diff --git a/.bumpversion.cfg b/.bumpversion.cfg --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.6.1 +current_version = 4.7.0 message = release: Bump version {current_version} to {new_version} [bumpversion:file:vcsserver/VERSION] diff --git a/.release.cfg b/.release.cfg --- a/.release.cfg +++ b/.release.cfg @@ -5,12 +5,10 @@ done = false done = true [task:fixes_on_stable] -done = true [task:pip2nix_generated] -done = true [release] -state = prepared -version = 4.6.1 +state = in_progress +version = 4.7.0 diff --git a/configs/production_http.ini b/configs/production_http.ini --- a/configs/production_http.ini +++ b/configs/production_http.ini @@ -15,10 +15,8 @@ port = 9900 ########################## ## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini use = egg:gunicorn#main -## Sets the number of process workers. You must set `instance_id = *` -## when this option is set to more than one worker, recommended +## Sets the number of process workers. Recommended ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers -## The `instance_id = *` must be set in the [app:main] section below workers = 2 ## process name proc_name = rhodecode_vcsserver diff --git a/default.nix b/default.nix --- a/default.nix +++ b/default.nix @@ -89,6 +89,7 @@ let name = "rhodecode-vcsserver-${version}"; releaseName = "RhodeCodeVCSServer-${version}"; src = rhodecode-vcsserver-src; + dontStrip = true; # prevent strip, we don't need it. propagatedBuildInputs = attrs.propagatedBuildInputs ++ ([ pkgs.git diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -159,13 +159,13 @@ }; }; decorator = super.buildPythonPackage { - name = "decorator-4.0.10"; + name = "decorator-4.0.11"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/13/8a/4eed41e338e8dcc13ca41c94b142d4d20c0de684ee5065523fee406ce76f/decorator-4.0.10.tar.gz"; - md5 = "434b57fdc3230c500716c5aff8896100"; + url = "https://pypi.python.org/packages/cc/ac/5a16f1fc0506ff72fcc8fd4e858e3a1c231f224ab79bb7c4c9b2094cc570/decorator-4.0.11.tar.gz"; + md5 = "73644c8f0bd4983d1b6a34b49adec0ae"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal { fullName = "new BSD License"; } ]; @@ -315,13 +315,13 @@ }; }; mercurial = super.buildPythonPackage { - name = "mercurial-4.0.2"; + name = "mercurial-4.1.2"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/85/1b/0296aacd697228974a473d2508f013532f987ed6b1bacfe5abd6d5be6332/mercurial-4.0.2.tar.gz"; - md5 = "fa72a08e2723e4fa2a21c4e66437f3fa"; + url = "https://pypi.python.org/packages/88/c1/f0501fd67f5e69346da41ee0bd7b2619ce4bbc9854bb645074c418b9941f/mercurial-4.1.2.tar.gz"; + md5 = "934c99808bdc8385e074b902d59b0d93"; }; meta = { license = [ pkgs.lib.licenses.gpl1 pkgs.lib.licenses.gpl2Plus ]; @@ -445,13 +445,13 @@ }; }; pyramid = super.buildPythonPackage { - name = "pyramid-1.6.1"; + name = "pyramid-1.7.4"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [setuptools WebOb repoze.lru zope.interface zope.deprecation venusian translationstring PasteDeploy]; src = fetchurl { - url = "https://pypi.python.org/packages/30/b3/fcc4a2a4800cbf21989e00454b5828cf1f7fe35c63e0810b350e56d4c475/pyramid-1.6.1.tar.gz"; - md5 = "b18688ff3cc33efdbb098a35b45dd122"; + url = "https://pypi.python.org/packages/33/91/55f5c661f8923902cd1f68d75f2b937c45e7682857356cf18f0be5493899/pyramid-1.7.4.tar.gz"; + md5 = "6ef1dfdcff9136d04490410757c4c446"; }; meta = { license = [ { fullName = "Repoze Public License"; } { fullName = "BSD-derived (http://www.repoze.org/LICENSE.txt)"; } ]; @@ -588,7 +588,7 @@ }; }; rhodecode-vcsserver = super.buildPythonPackage { - name = "rhodecode-vcsserver-4.6.1"; + name = "rhodecode-vcsserver-4.7.0"; buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage configobj]; doCheck = true; propagatedBuildInputs = with self; [Beaker configobj dulwich hgsubversion infrae.cache mercurial msgpack-python pyramid pyramid-jinja2 pyramid-mako repoze.lru simplejson subprocess32 subvertpy six translationstring WebOb wheel zope.deprecation zope.interface ipdb ipython gevent greenlet gunicorn waitress Pyro4 serpent pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage]; diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,16 @@ -# core +## core setuptools==30.1.0 Beaker==1.7.0 configobj==5.0.6 +decorator==4.0.11 dulwich==0.13.0 hgsubversion==1.8.6 infrae.cache==1.0.1 -mercurial==4.0.2 +mercurial==4.1.2 msgpack-python==0.4.8 -pyramid==1.6.1 pyramid-jinja2==2.5 +pyramid==1.7.4 pyramid-mako==1.0.2 repoze.lru==0.6 simplejson==3.7.2 @@ -28,7 +29,6 @@ zope.interface==4.1.3 ## debug ipdb==0.10.1 ipython==5.1.0 - # http servers gevent==1.1.2 greenlet==0.4.10 diff --git a/vcsserver/VERSION b/vcsserver/VERSION --- a/vcsserver/VERSION +++ b/vcsserver/VERSION @@ -1,1 +1,1 @@ -4.6.1 \ No newline at end of file +4.7.0 \ No newline at end of file diff --git a/vcsserver/base.py b/vcsserver/base.py --- a/vcsserver/base.py +++ b/vcsserver/base.py @@ -15,6 +15,8 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import sys +import traceback import logging import urlparse @@ -80,3 +82,17 @@ def obfuscate_qs(query_string): return '&'.join('{}{}'.format( k, '={}'.format(v) if v else '') for k, v in parsed) + + +def raise_from_original(new_type): + """ + Raise a new exception type with original args and traceback. + """ + exc_type, exc_value, exc_traceback = sys.exc_info() + + traceback.format_exception(exc_type, exc_value, exc_traceback) + + try: + raise new_type(*exc_value.args), None, exc_traceback + finally: + del exc_traceback diff --git a/vcsserver/git.py b/vcsserver/git.py --- a/vcsserver/git.py +++ b/vcsserver/git.py @@ -35,10 +35,10 @@ from dulwich.server import update_server from vcsserver import exceptions, settings, subprocessio from vcsserver.utils import safe_str -from vcsserver.base import RepoFactory, obfuscate_qs +from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original from vcsserver.hgcompat import ( hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler) - +from vcsserver.git_lfs.lib import LFSOidStore DIR_STAT = stat.S_IFDIR FILE_MODE = stat.S_IFMT @@ -58,6 +58,14 @@ def reraise_safe_exceptions(func): raise exceptions.LookupException(e.message) except (HangupException, UnexpectedCommandError) as e: raise exceptions.VcsException(e.message) + except Exception as e: + # NOTE(marcink): becuase of how dulwich handles some exceptions + # (KeyError on empty repos), we cannot track this and catch all + # exceptions, it's an exceptions from other handlers + #if not hasattr(e, '_vcs_kind'): + #log.exception("Unhandled exception in git remote call") + #raise_from_original(exceptions.UnhandledException) + raise return wrapper @@ -97,6 +105,11 @@ class GitRemote(object): "_commit": self.revision, } + def _wire_to_config(self, wire): + if 'config' in wire: + return dict([(x[0] + '_' + x[1], x[2]) for x in wire['config']]) + return {} + def _assign_ref(self, wire, ref, commit_id): repo = self._factory.repo(wire) repo[ref] = commit_id @@ -133,6 +146,56 @@ class GitRemote(object): blob = repo[sha] return blob.raw_length() + def _parse_lfs_pointer(self, raw_content): + + spec_string = 'version https://git-lfs.github.com/spec' + if raw_content and raw_content.startswith(spec_string): + pattern = re.compile(r""" + (?:\n)? + ^version[ ]https://git-lfs\.github\.com/spec/(?Pv\d+)\n + ^oid[ ] sha256:(?P[0-9a-f]{64})\n + ^size[ ](?P[0-9]+)\n + (?:\n)? + """, re.VERBOSE | re.MULTILINE) + match = pattern.match(raw_content) + if match: + return match.groupdict() + + return {} + + @reraise_safe_exceptions + def is_large_file(self, wire, sha): + repo = self._factory.repo(wire) + blob = repo[sha] + return self._parse_lfs_pointer(blob.as_raw_string()) + + @reraise_safe_exceptions + def in_largefiles_store(self, wire, oid): + repo = self._factory.repo(wire) + conf = self._wire_to_config(wire) + + store_location = conf.get('vcs_git_lfs_store_location') + if store_location: + repo_name = repo.path + store = LFSOidStore( + oid=oid, repo=repo_name, store_location=store_location) + return store.has_oid() + + return False + + @reraise_safe_exceptions + def store_path(self, wire, oid): + repo = self._factory.repo(wire) + conf = self._wire_to_config(wire) + + store_location = conf.get('vcs_git_lfs_store_location') + if store_location: + repo_name = repo.path + store = LFSOidStore( + oid=oid, repo=repo_name, store_location=store_location) + return store.oid_path + raise ValueError('Unable to fetch oid with path {}'.format(oid)) + @reraise_safe_exceptions def bulk_request(self, wire, rev, pre_load): result = {} diff --git a/vcsserver/git_lfs/__init__.py b/vcsserver/git_lfs/__init__.py new file mode 100644 --- /dev/null +++ b/vcsserver/git_lfs/__init__.py @@ -0,0 +1,19 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2017 RodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + +from app import create_app diff --git a/vcsserver/git_lfs/app.py b/vcsserver/git_lfs/app.py new file mode 100644 --- /dev/null +++ b/vcsserver/git_lfs/app.py @@ -0,0 +1,276 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2017 RodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import re +import logging +from wsgiref.util import FileWrapper + +import simplejson as json +from pyramid.config import Configurator +from pyramid.response import Response, FileIter +from pyramid.httpexceptions import ( + HTTPBadRequest, HTTPNotImplemented, HTTPNotFound, HTTPForbidden, + HTTPUnprocessableEntity) + +from vcsserver.git_lfs.lib import OidHandler, LFSOidStore +from vcsserver.git_lfs.utils import safe_result, get_cython_compat_decorator +from vcsserver.utils import safe_int + +log = logging.getLogger(__name__) + + +GIT_LFS_CONTENT_TYPE = 'application/vnd.git-lfs' #+json ? +GIT_LFS_PROTO_PAT = re.compile(r'^/(.+)/(info/lfs/(.+))') + + +def write_response_error(http_exception, text=None): + content_type = 'application/json' + _exception = http_exception(content_type=content_type) + _exception.content_type = content_type + if text: + _exception.body = json.dumps({'message': text}) + log.debug('LFS: writing response of type %s to client with text:%s', + http_exception, text) + return _exception + + +class AuthHeaderRequired(object): + """ + Decorator to check if request has proper auth-header + """ + + def __call__(self, func): + return get_cython_compat_decorator(self.__wrapper, func) + + def __wrapper(self, func, *fargs, **fkwargs): + request = fargs[1] + auth = request.authorization + if not auth: + return write_response_error(HTTPForbidden) + return func(*fargs[1:], **fkwargs) + + +# views + +def lfs_objects(request): + # indicate not supported, V1 API + log.warning('LFS: v1 api not supported, reporting it back to client') + return write_response_error(HTTPNotImplemented, 'LFS: v1 api not supported') + + +@AuthHeaderRequired() +def lfs_objects_batch(request): + """ + The client sends the following information to the Batch endpoint to transfer some objects: + + operation - Should be download or upload. + transfers - An optional Array of String identifiers for transfer + adapters that the client has configured. If omitted, the basic + transfer adapter MUST be assumed by the server. + objects - An Array of objects to download. + oid - String OID of the LFS object. + size - Integer byte size of the LFS object. Must be at least zero. + """ + auth = request.authorization + + repo = request.matchdict.get('repo') + + data = request.json + operation = data.get('operation') + if operation not in ('download', 'upload'): + log.debug('LFS: unsupported operation:%s', operation) + return write_response_error( + HTTPBadRequest, 'unsupported operation mode: `%s`' % operation) + + if 'objects' not in data: + log.debug('LFS: missing objects data') + return write_response_error( + HTTPBadRequest, 'missing objects data') + + log.debug('LFS: handling operation of type: %s', operation) + + objects = [] + for o in data['objects']: + try: + oid = o['oid'] + obj_size = o['size'] + except KeyError: + log.exception('LFS, failed to extract data') + return write_response_error( + HTTPBadRequest, 'unsupported data in objects') + + obj_data = {'oid': oid} + + obj_href = request.route_url('lfs_objects_oid', repo=repo, oid=oid) + obj_verify_href = request.route_url('lfs_objects_verify', repo=repo) + store = LFSOidStore( + oid, repo, store_location=request.registry.git_lfs_store_path) + handler = OidHandler( + store, repo, auth, oid, obj_size, obj_data, + obj_href, obj_verify_href) + + # this verifies also OIDs + actions, errors = handler.exec_operation(operation) + if errors: + log.warning('LFS: got following errors: %s', errors) + obj_data['errors'] = errors + + if actions: + obj_data['actions'] = actions + + obj_data['size'] = obj_size + obj_data['authenticated'] = True + objects.append(obj_data) + + result = {'objects': objects, 'transfer': 'basic'} + log.debug('LFS Response %s', safe_result(result)) + + return result + + +def lfs_objects_oid_upload(request): + repo = request.matchdict.get('repo') + oid = request.matchdict.get('oid') + store = LFSOidStore( + oid, repo, store_location=request.registry.git_lfs_store_path) + engine = store.get_engine(mode='wb') + log.debug('LFS: starting chunked write of LFS oid: %s to storage', oid) + with engine as f: + for chunk in FileWrapper(request.body_file_seekable, blksize=64 * 1024): + f.write(chunk) + + return {'upload': 'ok'} + + +def lfs_objects_oid_download(request): + repo = request.matchdict.get('repo') + oid = request.matchdict.get('oid') + + store = LFSOidStore( + oid, repo, store_location=request.registry.git_lfs_store_path) + if not store.has_oid(): + log.debug('LFS: oid %s does not exists in store', oid) + return write_response_error( + HTTPNotFound, 'requested file with oid `%s` not found in store' % oid) + + # TODO(marcink): support range header ? + # Range: bytes=0-, `bytes=(\d+)\-.*` + + f = open(store.oid_path, 'rb') + response = Response( + content_type='application/octet-stream', app_iter=FileIter(f)) + response.headers.add('X-RC-LFS-Response-Oid', str(oid)) + return response + + +def lfs_objects_verify(request): + repo = request.matchdict.get('repo') + + data = request.json + oid = data.get('oid') + size = safe_int(data.get('size')) + + if not (oid and size): + return write_response_error( + HTTPBadRequest, 'missing oid and size in request data') + + store = LFSOidStore( + oid, repo, store_location=request.registry.git_lfs_store_path) + if not store.has_oid(): + log.debug('LFS: oid %s does not exists in store', oid) + return write_response_error( + HTTPNotFound, 'oid `%s` does not exists in store' % oid) + + store_size = store.size_oid() + if store_size != size: + msg = 'requested file size mismatch store size:%s requested:%s' % ( + store_size, size) + return write_response_error( + HTTPUnprocessableEntity, msg) + + return {'message': {'size': 'ok', 'in_store': 'ok'}} + + +def lfs_objects_lock(request): + return write_response_error( + HTTPNotImplemented, 'GIT LFS locking api not supported') + + +def not_found(request): + return write_response_error( + HTTPNotFound, 'request path not found') + + +def lfs_disabled(request): + return write_response_error( + HTTPNotImplemented, 'GIT LFS disabled for this repo') + + +def git_lfs_app(config): + + # v1 API deprecation endpoint + config.add_route('lfs_objects', + '/{repo:.*?[^/]}/info/lfs/objects') + config.add_view(lfs_objects, route_name='lfs_objects', + request_method='POST', renderer='json') + + # locking API + config.add_route('lfs_objects_lock', + '/{repo:.*?[^/]}/info/lfs/locks') + config.add_view(lfs_objects_lock, route_name='lfs_objects_lock', + request_method=('POST', 'GET'), renderer='json') + + config.add_route('lfs_objects_lock_verify', + '/{repo:.*?[^/]}/info/lfs/locks/verify') + config.add_view(lfs_objects_lock, route_name='lfs_objects_lock_verify', + request_method=('POST', 'GET'), renderer='json') + + # batch API + config.add_route('lfs_objects_batch', + '/{repo:.*?[^/]}/info/lfs/objects/batch') + config.add_view(lfs_objects_batch, route_name='lfs_objects_batch', + request_method='POST', renderer='json') + + # oid upload/download API + config.add_route('lfs_objects_oid', + '/{repo:.*?[^/]}/info/lfs/objects/{oid}') + config.add_view(lfs_objects_oid_upload, route_name='lfs_objects_oid', + request_method='PUT', renderer='json') + config.add_view(lfs_objects_oid_download, route_name='lfs_objects_oid', + request_method='GET', renderer='json') + + # verification API + config.add_route('lfs_objects_verify', + '/{repo:.*?[^/]}/info/lfs/verify') + config.add_view(lfs_objects_verify, route_name='lfs_objects_verify', + request_method='POST', renderer='json') + + # not found handler for API + config.add_notfound_view(not_found, renderer='json') + + +def create_app(git_lfs_enabled, git_lfs_store_path): + config = Configurator() + if git_lfs_enabled: + config.include(git_lfs_app) + config.registry.git_lfs_store_path = git_lfs_store_path + else: + # not found handler for API, reporting disabled LFS support + config.add_notfound_view(lfs_disabled, renderer='json') + + app = config.make_wsgi_app() + return app diff --git a/vcsserver/git_lfs/lib.py b/vcsserver/git_lfs/lib.py new file mode 100644 --- /dev/null +++ b/vcsserver/git_lfs/lib.py @@ -0,0 +1,166 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2017 RodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import shutil +import logging +from collections import OrderedDict + +log = logging.getLogger(__name__) + + +class OidHandler(object): + + def __init__(self, store, repo_name, auth, oid, obj_size, obj_data, obj_href, + obj_verify_href=None): + self.current_store = store + self.repo_name = repo_name + self.auth = auth + self.oid = oid + self.obj_size = obj_size + self.obj_data = obj_data + self.obj_href = obj_href + self.obj_verify_href = obj_verify_href + + def get_store(self, mode=None): + return self.current_store + + def get_auth(self): + """returns auth header for re-use in upload/download""" + return " ".join(self.auth) + + def download(self): + + store = self.get_store() + response = None + has_errors = None + + if not store.has_oid(): + # error reply back to client that something is wrong with dl + err_msg = 'object: {} does not exist in store'.format(store.oid) + has_errors = OrderedDict( + error=OrderedDict( + code=404, + message=err_msg + ) + ) + + download_action = OrderedDict( + href=self.obj_href, + header=OrderedDict([("Authorization", self.get_auth())]) + ) + if not has_errors: + response = OrderedDict(download=download_action) + return response, has_errors + + def upload(self, skip_existing=True): + """ + Write upload action for git-lfs server + """ + + store = self.get_store() + response = None + has_errors = None + + # verify if we have the OID before, if we do, reply with empty + if store.has_oid(): + log.debug('LFS: store already has oid %s', store.oid) + if skip_existing: + log.debug('LFS: skipping further action as oid is existing') + return response, has_errors + + upload_action = OrderedDict( + href=self.obj_href, + header=OrderedDict([("Authorization", self.get_auth())]) + ) + if not has_errors: + response = OrderedDict(upload=upload_action) + # if specified in handler, return the verification endpoint + if self.obj_verify_href: + verify_action = OrderedDict( + href=self.obj_verify_href, + header=OrderedDict([("Authorization", self.get_auth())]) + ) + response['verify'] = verify_action + return response, has_errors + + def exec_operation(self, operation, *args, **kwargs): + handler = getattr(self, operation) + log.debug('LFS: handling request using %s handler', handler) + return handler(*args, **kwargs) + + +class LFSOidStore(object): + + def __init__(self, oid, repo, store_location=None): + self.oid = oid + self.repo = repo + self.store_path = store_location or self.get_default_store() + self.tmp_oid_path = os.path.join(self.store_path, oid + '.tmp') + self.oid_path = os.path.join(self.store_path, oid) + self.fd = None + + def get_engine(self, mode): + """ + engine = .get_engine(mode='wb') + with engine as f: + f.write('...') + """ + + class StoreEngine(object): + def __init__(self, mode, store_path, oid_path, tmp_oid_path): + self.mode = mode + self.store_path = store_path + self.oid_path = oid_path + self.tmp_oid_path = tmp_oid_path + + def __enter__(self): + if not os.path.isdir(self.store_path): + os.makedirs(self.store_path) + + # TODO(marcink): maybe write metadata here with size/oid ? + fd = open(self.tmp_oid_path, self.mode) + self.fd = fd + return fd + + def __exit__(self, exc_type, exc_value, traceback): + # close tmp file, and rename to final destination + self.fd.close() + shutil.move(self.tmp_oid_path, self.oid_path) + + return StoreEngine( + mode, self.store_path, self.oid_path, self.tmp_oid_path) + + def get_default_store(self): + """ + Default store, consistent with defaults of Mercurial large files store + which is /home/username/.cache/largefiles + """ + user_home = os.path.expanduser("~") + return os.path.join(user_home, '.cache', 'lfs-store') + + def has_oid(self): + return os.path.exists(os.path.join(self.store_path, self.oid)) + + def size_oid(self): + size = -1 + + if self.has_oid(): + oid = os.path.join(self.store_path, self.oid) + size = os.stat(oid).st_size + + return size diff --git a/vcsserver/git_lfs/tests/__init__.py b/vcsserver/git_lfs/tests/__init__.py new file mode 100644 --- /dev/null +++ b/vcsserver/git_lfs/tests/__init__.py @@ -0,0 +1,16 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2017 RodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA diff --git a/vcsserver/git_lfs/tests/test_lfs_app.py b/vcsserver/git_lfs/tests/test_lfs_app.py new file mode 100644 --- /dev/null +++ b/vcsserver/git_lfs/tests/test_lfs_app.py @@ -0,0 +1,237 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2017 RodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import pytest +from webtest.app import TestApp as WebObTestApp + +from vcsserver.git_lfs.app import create_app + + +@pytest.fixture(scope='function') +def git_lfs_app(tmpdir): + custom_app = WebObTestApp(create_app( + git_lfs_enabled=True, git_lfs_store_path=str(tmpdir))) + custom_app._store = str(tmpdir) + return custom_app + + +@pytest.fixture() +def http_auth(): + return {'HTTP_AUTHORIZATION': "Basic XXXXX"} + + +class TestLFSApplication(object): + + def test_app_wrong_path(self, git_lfs_app): + git_lfs_app.get('/repo/info/lfs/xxx', status=404) + + def test_app_deprecated_endpoint(self, git_lfs_app): + response = git_lfs_app.post('/repo/info/lfs/objects', status=501) + assert response.status_code == 501 + assert response.json == {u'message': u'LFS: v1 api not supported'} + + def test_app_lock_verify_api_not_available(self, git_lfs_app): + response = git_lfs_app.post('/repo/info/lfs/locks/verify', status=501) + assert response.status_code == 501 + assert response.json == { + u'message': u'GIT LFS locking api not supported'} + + def test_app_lock_api_not_available(self, git_lfs_app): + response = git_lfs_app.post('/repo/info/lfs/locks', status=501) + assert response.status_code == 501 + assert response.json == { + u'message': u'GIT LFS locking api not supported'} + + def test_app_batch_api_missing_auth(self, git_lfs_app,): + git_lfs_app.post_json( + '/repo/info/lfs/objects/batch', params={}, status=403) + + def test_app_batch_api_unsupported_operation(self, git_lfs_app, http_auth): + response = git_lfs_app.post_json( + '/repo/info/lfs/objects/batch', params={}, status=400, + extra_environ=http_auth) + assert response.json == { + u'message': u'unsupported operation mode: `None`'} + + def test_app_batch_api_missing_objects(self, git_lfs_app, http_auth): + response = git_lfs_app.post_json( + '/repo/info/lfs/objects/batch', params={'operation': 'download'}, + status=400, extra_environ=http_auth) + assert response.json == { + u'message': u'missing objects data'} + + def test_app_batch_api_unsupported_data_in_objects( + self, git_lfs_app, http_auth): + params = {'operation': 'download', + 'objects': [{}]} + response = git_lfs_app.post_json( + '/repo/info/lfs/objects/batch', params=params, status=400, + extra_environ=http_auth) + assert response.json == { + u'message': u'unsupported data in objects'} + + def test_app_batch_api_download_missing_object( + self, git_lfs_app, http_auth): + params = {'operation': 'download', + 'objects': [{'oid': '123', 'size': '1024'}]} + response = git_lfs_app.post_json( + '/repo/info/lfs/objects/batch', params=params, + extra_environ=http_auth) + + expected_objects = [ + {u'authenticated': True, + u'errors': {u'error': { + u'code': 404, + u'message': u'object: 123 does not exist in store'}}, + u'oid': u'123', + u'size': u'1024'} + ] + assert response.json == { + 'objects': expected_objects, 'transfer': 'basic'} + + def test_app_batch_api_download(self, git_lfs_app, http_auth): + oid = '456' + oid_path = os.path.join(git_lfs_app._store, oid) + if not os.path.isdir(os.path.dirname(oid_path)): + os.makedirs(os.path.dirname(oid_path)) + with open(oid_path, 'wb') as f: + f.write('OID_CONTENT') + + params = {'operation': 'download', + 'objects': [{'oid': oid, 'size': '1024'}]} + response = git_lfs_app.post_json( + '/repo/info/lfs/objects/batch', params=params, + extra_environ=http_auth) + + expected_objects = [ + {u'authenticated': True, + u'actions': { + u'download': { + u'header': {u'Authorization': u'Basic XXXXX'}, + u'href': u'http://localhost/repo/info/lfs/objects/456'}, + }, + u'oid': u'456', + u'size': u'1024'} + ] + assert response.json == { + 'objects': expected_objects, 'transfer': 'basic'} + + def test_app_batch_api_upload(self, git_lfs_app, http_auth): + params = {'operation': 'upload', + 'objects': [{'oid': '123', 'size': '1024'}]} + response = git_lfs_app.post_json( + '/repo/info/lfs/objects/batch', params=params, + extra_environ=http_auth) + expected_objects = [ + {u'authenticated': True, + u'actions': { + u'upload': { + u'header': {u'Authorization': u'Basic XXXXX'}, + u'href': u'http://localhost/repo/info/lfs/objects/123'}, + u'verify': { + u'header': {u'Authorization': u'Basic XXXXX'}, + u'href': u'http://localhost/repo/info/lfs/verify'} + }, + u'oid': u'123', + u'size': u'1024'} + ] + assert response.json == { + 'objects': expected_objects, 'transfer': 'basic'} + + def test_app_verify_api_missing_data(self, git_lfs_app): + params = {'oid': 'missing',} + response = git_lfs_app.post_json( + '/repo/info/lfs/verify', params=params, + status=400) + + assert response.json == { + u'message': u'missing oid and size in request data'} + + def test_app_verify_api_missing_obj(self, git_lfs_app): + params = {'oid': 'missing', 'size': '1024'} + response = git_lfs_app.post_json( + '/repo/info/lfs/verify', params=params, + status=404) + + assert response.json == { + u'message': u'oid `missing` does not exists in store'} + + def test_app_verify_api_size_mismatch(self, git_lfs_app): + oid = 'existing' + oid_path = os.path.join(git_lfs_app._store, oid) + if not os.path.isdir(os.path.dirname(oid_path)): + os.makedirs(os.path.dirname(oid_path)) + with open(oid_path, 'wb') as f: + f.write('OID_CONTENT') + + params = {'oid': oid, 'size': '1024'} + response = git_lfs_app.post_json( + '/repo/info/lfs/verify', params=params, status=422) + + assert response.json == { + u'message': u'requested file size mismatch ' + u'store size:11 requested:1024'} + + def test_app_verify_api(self, git_lfs_app): + oid = 'existing' + oid_path = os.path.join(git_lfs_app._store, oid) + if not os.path.isdir(os.path.dirname(oid_path)): + os.makedirs(os.path.dirname(oid_path)) + with open(oid_path, 'wb') as f: + f.write('OID_CONTENT') + + params = {'oid': oid, 'size': 11} + response = git_lfs_app.post_json( + '/repo/info/lfs/verify', params=params) + + assert response.json == { + u'message': {u'size': u'ok', u'in_store': u'ok'}} + + def test_app_download_api_oid_not_existing(self, git_lfs_app): + oid = 'missing' + + response = git_lfs_app.get( + '/repo/info/lfs/objects/{oid}'.format(oid=oid), status=404) + + assert response.json == { + u'message': u'requested file with oid `missing` not found in store'} + + def test_app_download_api(self, git_lfs_app): + oid = 'existing' + oid_path = os.path.join(git_lfs_app._store, oid) + if not os.path.isdir(os.path.dirname(oid_path)): + os.makedirs(os.path.dirname(oid_path)) + with open(oid_path, 'wb') as f: + f.write('OID_CONTENT') + + response = git_lfs_app.get( + '/repo/info/lfs/objects/{oid}'.format(oid=oid)) + assert response + + def test_app_upload(self, git_lfs_app): + oid = 'uploaded' + + response = git_lfs_app.put( + '/repo/info/lfs/objects/{oid}'.format(oid=oid), params='CONTENT') + + assert response.json == {u'upload': u'ok'} + + # verify that we actually wrote that OID + oid_path = os.path.join(git_lfs_app._store, oid) + assert os.path.isfile(oid_path) + assert 'CONTENT' == open(oid_path).read() diff --git a/vcsserver/git_lfs/tests/test_lib.py b/vcsserver/git_lfs/tests/test_lib.py new file mode 100644 --- /dev/null +++ b/vcsserver/git_lfs/tests/test_lib.py @@ -0,0 +1,123 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2017 RodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import pytest +from vcsserver.git_lfs.lib import OidHandler, LFSOidStore + + +@pytest.fixture() +def lfs_store(tmpdir): + repo = 'test' + oid = '123456789' + store = LFSOidStore(oid=oid, repo=repo, store_location=str(tmpdir)) + return store + + +@pytest.fixture() +def oid_handler(lfs_store): + store = lfs_store + repo = store.repo + oid = store.oid + + oid_handler = OidHandler( + store=store, repo_name=repo, auth=('basic', 'xxxx'), + oid=oid, + obj_size='1024', obj_data={}, obj_href='http://localhost/handle_oid', + obj_verify_href='http://localhost/verify') + return oid_handler + + +class TestOidHandler(object): + + @pytest.mark.parametrize('exec_action', [ + 'download', + 'upload', + ]) + def test_exec_action(self, exec_action, oid_handler): + handler = oid_handler.exec_operation(exec_action) + assert handler + + def test_exec_action_undefined(self, oid_handler): + with pytest.raises(AttributeError): + oid_handler.exec_operation('wrong') + + def test_download_oid_not_existing(self, oid_handler): + response, has_errors = oid_handler.exec_operation('download') + + assert response is None + assert has_errors['error'] == { + 'code': 404, + 'message': 'object: 123456789 does not exist in store'} + + def test_download_oid(self, oid_handler): + store = oid_handler.get_store() + if not os.path.isdir(os.path.dirname(store.oid_path)): + os.makedirs(os.path.dirname(store.oid_path)) + + with open(store.oid_path, 'wb') as f: + f.write('CONTENT') + + response, has_errors = oid_handler.exec_operation('download') + + assert has_errors is None + assert response['download'] == { + 'header': {'Authorization': 'basic xxxx'}, + 'href': 'http://localhost/handle_oid' + } + + def test_upload_oid_that_exists(self, oid_handler): + store = oid_handler.get_store() + if not os.path.isdir(os.path.dirname(store.oid_path)): + os.makedirs(os.path.dirname(store.oid_path)) + + with open(store.oid_path, 'wb') as f: + f.write('CONTENT') + + response, has_errors = oid_handler.exec_operation('upload') + assert has_errors is None + assert response is None + + def test_upload_oid(self, oid_handler): + response, has_errors = oid_handler.exec_operation('upload') + assert has_errors is None + assert response['upload'] == { + 'header': {'Authorization': 'basic xxxx'}, + 'href': 'http://localhost/handle_oid' + } + + +class TestLFSStore(object): + def test_write_oid(self, lfs_store): + oid_location = lfs_store.oid_path + + assert not os.path.isfile(oid_location) + + engine = lfs_store.get_engine(mode='wb') + with engine as f: + f.write('CONTENT') + + assert os.path.isfile(oid_location) + + def test_detect_has_oid(self, lfs_store): + + assert lfs_store.has_oid() is False + engine = lfs_store.get_engine(mode='wb') + with engine as f: + f.write('CONTENT') + + assert lfs_store.has_oid() is True \ No newline at end of file diff --git a/vcsserver/git_lfs/utils.py b/vcsserver/git_lfs/utils.py new file mode 100644 --- /dev/null +++ b/vcsserver/git_lfs/utils.py @@ -0,0 +1,50 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2017 RodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import copy +from functools import wraps + + +def get_cython_compat_decorator(wrapper, func): + """ + Creates a cython compatible decorator. The previously used + decorator.decorator() function seems to be incompatible with cython. + + :param wrapper: __wrapper method of the decorator class + :param func: decorated function + """ + @wraps(func) + def local_wrapper(*args, **kwds): + return wrapper(func, *args, **kwds) + local_wrapper.__wrapped__ = func + return local_wrapper + + +def safe_result(result): + """clean result for better representation in logs""" + clean_copy = copy.deepcopy(result) + + try: + if 'objects' in clean_copy: + for oid_data in clean_copy['objects']: + if 'actions' in oid_data: + for action_name, data in oid_data['actions'].items(): + if 'header' in data: + data['header'] = {'Authorization': '*****'} + except Exception: + return result + + return clean_copy diff --git a/vcsserver/hg.py b/vcsserver/hg.py --- a/vcsserver/hg.py +++ b/vcsserver/hg.py @@ -18,7 +18,6 @@ import io import logging import stat -import sys import urllib import urllib2 @@ -26,9 +25,10 @@ from hgext import largefiles, rebase from hgext.strip import strip as hgext_strip from mercurial import commands from mercurial import unionrepo +from mercurial import verify from vcsserver import exceptions -from vcsserver.base import RepoFactory, obfuscate_qs +from vcsserver.base import RepoFactory, obfuscate_qs, raise_from_original from vcsserver.hgcompat import ( archival, bin, clone, config as hgconfig, diffopts, hex, hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler, @@ -91,17 +91,6 @@ def reraise_safe_exceptions(func): return wrapper -def raise_from_original(new_type): - """ - Raise a new exception type with original args and traceback. - """ - _, original, traceback = sys.exc_info() - try: - raise new_type(*original.args), None, traceback - finally: - del traceback - - class MercurialFactory(RepoFactory): def _create_config(self, config, hooks=True): @@ -496,7 +485,7 @@ class HgRemote(object): return largefiles.lfutil.isstandin(path) @reraise_safe_exceptions - def in_store(self, wire, sha): + def in_largefiles_store(self, wire, sha): repo = self._factory.repo(wire) return largefiles.lfutil.instore(repo, sha) @@ -598,6 +587,21 @@ class HgRemote(object): repo.baseui, repo, ctx.node(), update=update, backup=backup) @reraise_safe_exceptions + def verify(self, wire,): + repo = self._factory.repo(wire) + baseui = self._factory._create_config(wire['config']) + baseui.setconfig('ui', 'quiet', 'false') + output = io.BytesIO() + + def write(data, **unused_kwargs): + output.write(data) + baseui.write = write + + repo.ui = baseui + verify.verify(repo) + return output.getvalue() + + @reraise_safe_exceptions def tag(self, wire, name, revision, message, local, user, tag_time, tag_timezone): repo = self._factory.repo(wire) @@ -674,12 +678,10 @@ class HgRemote(object): @reraise_safe_exceptions def ancestor(self, wire, revision1, revision2): repo = self._factory.repo(wire) - baseui = self._factory._create_config(wire['config']) - output = io.BytesIO() - baseui.write = output.write - commands.debugancestor(baseui, repo, revision1, revision2) - - return output.getvalue() + changelog = repo.changelog + lookup = repo.lookup + a = changelog.ancestor(lookup(revision1), lookup(revision2)) + return hex(a) @reraise_safe_exceptions def push(self, wire, revisions, dest_path, hooks=True, diff --git a/vcsserver/hooks.py b/vcsserver/hooks.py --- a/vcsserver/hooks.py +++ b/vcsserver/hooks.py @@ -17,12 +17,14 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import io +import sys +import json +import logging import collections import importlib -import io -import json import subprocess -import sys + from httplib import HTTPConnection @@ -33,6 +35,8 @@ import simplejson as json from vcsserver import exceptions +log = logging.getLogger(__name__) + class HooksHttpClient(object): connection = None @@ -105,6 +109,11 @@ class GitMessageWriter(RemoteMessageWrit def _handle_exception(result): exception_class = result.get('exception') + exception_traceback = result.get('exception_traceback') + + if exception_traceback: + log.error('Got traceback from remote call:%s', exception_traceback) + if exception_class == 'HTTPLockedRC': raise exceptions.RepositoryLockedException(*result['exception_args']) elif exception_class == 'RepositoryError': @@ -152,26 +161,42 @@ def post_pull(ui, repo, **kwargs): return _call_hook('post_pull', _extras_from_ui(ui), HgMessageWriter(ui)) -def pre_push(ui, repo, **kwargs): - return _call_hook('pre_push', _extras_from_ui(ui), HgMessageWriter(ui)) +def pre_push(ui, repo, node=None, **kwargs): + extras = _extras_from_ui(ui) + + rev_data = [] + if node and kwargs.get('hooktype') == 'pretxnchangegroup': + branches = collections.defaultdict(list) + for commit_id, branch in _rev_range_hash(repo, node, with_branch=True): + branches[branch].append(commit_id) + + for branch, commits in branches.iteritems(): + old_rev = kwargs.get('node_last') or commits[0] + rev_data.append({ + 'old_rev': old_rev, + 'new_rev': commits[-1], + 'ref': '', + 'type': 'branch', + 'name': branch, + }) + + extras['commit_ids'] = rev_data + return _call_hook('pre_push', extras, HgMessageWriter(ui)) -# N.B.(skreft): the two functions below were taken and adapted from -# rhodecode.lib.vcs.remote.handle_git_pre_receive -# They are required to compute the commit_ids -def _get_revs(repo, rev_opt): - revs = [rev for rev in mercurial.scmutil.revrange(repo, rev_opt)] - if len(revs) == 0: - return (mercurial.node.nullrev, mercurial.node.nullrev) +def _rev_range_hash(repo, node, with_branch=False): - return max(revs), min(revs) - + commits = [] + for rev in xrange(repo[node], len(repo)): + ctx = repo[rev] + commit_id = mercurial.node.hex(ctx.node()) + branch = ctx.branch() + if with_branch: + commits.append((commit_id, branch)) + else: + commits.append(commit_id) -def _rev_range_hash(repo, node): - stop, start = _get_revs(repo, [node + ':']) - revs = [mercurial.node.hex(repo[r].node()) for r in xrange(start, stop + 1)] - - return revs + return commits def post_push(ui, repo, node, **kwargs): @@ -257,7 +282,23 @@ def git_post_pull(extras): return HookResponse(status, stdout.getvalue()) -def git_pre_receive(unused_repo_path, unused_revs, env): +def _parse_git_ref_lines(revision_lines): + rev_data = [] + for revision_line in revision_lines or []: + old_rev, new_rev, ref = revision_line.strip().split(' ') + ref_data = ref.split('/', 2) + if ref_data[1] in ('tags', 'heads'): + rev_data.append({ + 'old_rev': old_rev, + 'new_rev': new_rev, + 'ref': ref, + 'type': ref_data[1], + 'name': ref_data[2], + }) + return rev_data + + +def git_pre_receive(unused_repo_path, revision_lines, env): """ Pre push hook. @@ -268,8 +309,10 @@ def git_pre_receive(unused_repo_path, un :rtype: int """ extras = json.loads(env['RC_SCM_DATA']) + rev_data = _parse_git_ref_lines(revision_lines) if 'push' not in extras['hooks']: return 0 + extras['commit_ids'] = rev_data return _call_hook('pre_push', extras, GitMessageWriter()) @@ -277,7 +320,7 @@ def _run_command(arguments): """ Run the specified command and return the stdout. - :param arguments: sequence of program arugments (including the program name) + :param arguments: sequence of program arguments (including the program name) :type arguments: list[str] """ # TODO(skreft): refactor this method and all the other similar ones. @@ -308,18 +351,7 @@ def git_post_receive(unused_repo_path, r if 'push' not in extras['hooks']: return 0 - rev_data = [] - for revision_line in revision_lines: - old_rev, new_rev, ref = revision_line.strip().split(' ') - ref_data = ref.split('/', 2) - if ref_data[1] in ('tags', 'heads'): - rev_data.append({ - 'old_rev': old_rev, - 'new_rev': new_rev, - 'ref': ref, - 'type': ref_data[1], - 'name': ref_data[2], - }) + rev_data = _parse_git_ref_lines(revision_lines) git_revs = [] @@ -339,7 +371,7 @@ def git_post_receive(unused_repo_path, r except Exception: cmd = ['git', 'symbolic-ref', 'HEAD', 'refs/heads/%s' % push_ref['name']] - print "Setting default branch to %s" % push_ref['name'] + print("Setting default branch to %s" % push_ref['name']) _run_command(cmd) cmd = ['git', 'for-each-ref', '--format=%(refname)', diff --git a/vcsserver/http_main.py b/vcsserver/http_main.py --- a/vcsserver/http_main.py +++ b/vcsserver/http_main.py @@ -30,6 +30,7 @@ from pyramid.config import Configurator from pyramid.wsgi import wsgiapp from vcsserver import remote_wsgi, scm_app, settings, hgpatches +from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub from vcsserver.echo_stub.echo_app import EchoApp from vcsserver.exceptions import HTTPRepoLocked @@ -40,11 +41,13 @@ try: except ImportError: GitFactory = None GitRemote = None + try: from vcsserver.hg import MercurialFactory, HgRemote except ImportError: MercurialFactory = None HgRemote = None + try: from vcsserver.svn import SubversionFactory, SvnRemote except ImportError: @@ -153,8 +156,10 @@ class HTTPApplication(object): remote_wsgi = remote_wsgi _use_echo_app = False - def __init__(self, settings=None): + def __init__(self, settings=None, global_config=None): self.config = Configurator(settings=settings) + self.global_config = global_config + locale = settings.get('locale', '') or 'en_US.UTF-8' vcs = VCS(locale=locale, cache_config=settings) self._remotes = { @@ -209,12 +214,7 @@ class HTTPApplication(object): return {'status': '404 NOT FOUND'} self.config.add_notfound_view(notfound, renderer='json') - self.config.add_view( - self.handle_vcs_exception, context=Exception, - custom_predicates=[self.is_vcs_exception]) - - self.config.add_view( - self.general_error_handler, context=Exception) + self.config.add_view(self.handle_vcs_exception, context=Exception) self.config.add_tween( 'vcsserver.tweens.RequestWrapperTween', @@ -273,12 +273,26 @@ class HTTPApplication(object): def service_view(self, request): import vcsserver + import ConfigParser as configparser + payload = msgpack.unpackb(request.body, use_list=True) + + try: + path = self.global_config['__file__'] + config = configparser.ConfigParser() + config.read(path) + parsed_ini = config + if parsed_ini.has_section('server:main'): + parsed_ini = dict(parsed_ini.items('server:main')) + except Exception: + log.exception('Failed to read .ini file for display') + parsed_ini = {} + resp = { 'id': payload.get('id'), 'result': dict( version=vcsserver.__version__, - config={}, + config=parsed_ini, payload=payload, ) } @@ -351,9 +365,31 @@ class HTTPApplication(object): config = msgpack.unpackb(packed_config) environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO'] - app = scm_app.create_git_wsgi_app( - repo_path, repo_name, config) + content_type = environ.get('CONTENT_TYPE', '') + + path = environ['PATH_INFO'] + is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type + log.debug( + 'LFS: Detecting if request `%s` is LFS server path based ' + 'on content type:`%s`, is_lfs:%s', + path, content_type, is_lfs_request) + + if not is_lfs_request: + # fallback detection by path + if GIT_LFS_PROTO_PAT.match(path): + is_lfs_request = True + log.debug( + 'LFS: fallback detection by path of: `%s`, is_lfs:%s', + path, is_lfs_request) + + if is_lfs_request: + app = scm_app.create_git_lfs_wsgi_app( + repo_path, repo_name, config) + else: + app = scm_app.create_git_wsgi_app( + repo_path, repo_name, config) return app(environ, start_response) + return _git_stream def is_vcs_view(self, context, request): @@ -364,27 +400,17 @@ class HTTPApplication(object): backend = request.matchdict.get('backend') return backend in self._remotes - def is_vcs_exception(self, context, request): - """ - View predicate that returns true if the context object is a VCS - exception. - """ - return hasattr(context, '_vcs_kind') - def handle_vcs_exception(self, exception, request): - if exception._vcs_kind == 'repo_locked': + _vcs_kind = getattr(exception, '_vcs_kind', '') + if _vcs_kind == 'repo_locked': # Get custom repo-locked status code if present. status_code = request.headers.get('X-RC-Locked-Status-Code') return HTTPRepoLocked( title=exception.message, status_code=status_code) # Re-raise exception if we can not handle it. - raise exception - - def general_error_handler(self, exception, request): log.exception( - 'error occurred handling this request for path: %s', - request.path) + 'error occurred handling this request for path: %s', request.path) raise exception @@ -404,5 +430,5 @@ def main(global_config, **settings): if MercurialFactory: hgpatches.patch_largefiles_capabilities() hgpatches.patch_subrepo_type_mapping() - app = HTTPApplication(settings=settings) + app = HTTPApplication(settings=settings, global_config=global_config) return app.wsgi_app() diff --git a/vcsserver/scm_app.py b/vcsserver/scm_app.py --- a/vcsserver/scm_app.py +++ b/vcsserver/scm_app.py @@ -15,8 +15,8 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import os import logging -import os import mercurial import mercurial.error @@ -25,7 +25,7 @@ import mercurial.hgweb.hgweb_mod import mercurial.hgweb.protocol import webob.exc -from vcsserver import pygrack, exceptions, settings +from vcsserver import pygrack, exceptions, settings, git_lfs log = logging.getLogger(__name__) @@ -132,6 +132,9 @@ def create_hg_wsgi_app(repo_path, repo_n class GitHandler(object): + """ + Handler for Git operations like push/pull etc + """ def __init__(self, repo_location, repo_name, git_path, update_server_info, extras): if not os.path.isdir(repo_location): @@ -172,3 +175,35 @@ def create_git_wsgi_app(repo_path, repo_ repo_path, repo_name, git_path, update_server_info, config) return app + + +class GitLFSHandler(object): + """ + Handler for Git LFS operations + """ + + def __init__(self, repo_location, repo_name, git_path, update_server_info, + extras): + if not os.path.isdir(repo_location): + raise OSError(repo_location) + self.content_path = repo_location + self.repo_name = repo_name + self.repo_location = repo_location + self.extras = extras + self.git_path = git_path + self.update_server_info = update_server_info + + def get_app(self, git_lfs_enabled, git_lfs_store_path): + app = git_lfs.create_app(git_lfs_enabled, git_lfs_store_path) + return app + + +def create_git_lfs_wsgi_app(repo_path, repo_name, config): + git_path = settings.GIT_EXECUTABLE + update_server_info = config.pop('git_update_server_info') + git_lfs_enabled = config.pop('git_lfs_enabled') + git_lfs_store_path = config.pop('git_lfs_store_path') + app = GitLFSHandler( + repo_path, repo_name, git_path, update_server_info, config) + + return app.get_app(git_lfs_enabled, git_lfs_store_path) diff --git a/vcsserver/svn.py b/vcsserver/svn.py --- a/vcsserver/svn.py +++ b/vcsserver/svn.py @@ -33,7 +33,7 @@ import svn.repos from vcsserver import svn_diff from vcsserver import exceptions -from vcsserver.base import RepoFactory +from vcsserver.base import RepoFactory, raise_from_original log = logging.getLogger(__name__) @@ -62,17 +62,6 @@ def reraise_safe_exceptions(func): return wrapper -def raise_from_original(new_type): - """ - Raise a new exception type with original args and traceback. - """ - _, original, traceback = sys.exc_info() - try: - raise new_type(*original.args), None, traceback - finally: - del traceback - - class SubversionFactory(RepoFactory): def _create_repo(self, wire, create, compatible_version): @@ -388,6 +377,10 @@ class SvnRemote(object): "Path might not exist %s, %s" % (path1, path2)) return "" + @reraise_safe_exceptions + def is_large_file(self, wire, path): + return False + class SvnDiffer(object): """ diff --git a/vcsserver/utils.py b/vcsserver/utils.py --- a/vcsserver/utils.py +++ b/vcsserver/utils.py @@ -16,8 +16,23 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +def safe_int(val, default=None): + """ + Returns int() of val if val is not convertable to int use default + instead -# TODO: johbo: That's a copy from rhodecode + :param val: + :param default: + """ + + try: + val = int(val) + except (ValueError, TypeError): + val = default + + return val + + def safe_str(unicode_, to_encoding=['utf8']): """ safe str function. Does few trick to turn unicode_ into string