# HG changeset patch # User Marcin Kuzminski # Date 2018-04-23 10:55:47 # Node ID ce60257b92a603cfed2c165fd3685c0e954d8c21 # Parent 18f67141766ba2ba0f35cec3a4712f7df06c72d7 # Parent 78d5d613c08ae08e8793e3b6f32b32ca19597478 release: Merge default into stable for release preparation diff --git a/.bumpversion.cfg b/.bumpversion.cfg --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.11.6 +current_version = 4.12.0 message = release: Bump version {current_version} to {new_version} [bumpversion:file:vcsserver/VERSION] diff --git a/.release.cfg b/.release.cfg --- a/.release.cfg +++ b/.release.cfg @@ -5,12 +5,10 @@ done = false done = true [task:fixes_on_stable] -done = true [task:pip2nix_generated] -done = true [release] -state = prepared -version = 4.11.6 +state = in_progress +version = 4.12.0 diff --git a/MANIFEST.in b/MANIFEST.in --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,6 +8,9 @@ include vcsserver/VERSION # all config files recursive-include configs * +# hook templates +recursive-include vcsserver/hook_utils/hook_templates * + # skip any tests files recursive-exclude vcsserver/tests * diff --git a/configs/development_http.ini b/configs/development_http.ini --- a/configs/development_http.ini +++ b/configs/development_http.ini @@ -20,6 +20,10 @@ beaker.cache.repo_object.max_items = 100 beaker.cache.repo_object.expire = 300 beaker.cache.repo_object.enabled = true +# path to binaries for vcsserver, it should be set by the installer +# at installation time, e.g /home/user/vcsserver-1/profile/bin +core.binary_dir = "" + [server:main] ## COMMON ## host = 0.0.0.0 diff --git a/configs/production_http.ini b/configs/production_http.ini --- a/configs/production_http.ini +++ b/configs/production_http.ini @@ -39,7 +39,7 @@ use = egg:rhodecode-vcsserver pyramid.default_locale_name = en pyramid.includes = -## default locale used by VCS systems +# default locale used by VCS systems locale = en_US.UTF-8 # cache regions, please don't change @@ -50,6 +50,10 @@ beaker.cache.repo_object.max_items = 100 beaker.cache.repo_object.expire = 300 beaker.cache.repo_object.enabled = true +# path to binaries for vcsserver, it should be set by the installer +# at installation time, e.g /home/user/vcsserver-1/profile/bin +core.binary_dir = "" + ################################ ### LOGGING CONFIGURATION #### diff --git a/pkgs/patch-beaker-lock-func-debug.diff b/pkgs/patch-beaker-lock-func-debug.diff new file mode 100644 --- /dev/null +++ b/pkgs/patch-beaker-lock-func-debug.diff @@ -0,0 +1,20 @@ +diff -rup Beaker-1.9.1-orig/beaker/container.py Beaker-1.9.1/beaker/container.py +--- Beaker-1.9.1-orig/beaker/container.py 2018-04-10 10:23:04.000000000 +0200 ++++ Beaker-1.9.1/beaker/container.py 2018-04-10 10:23:34.000000000 +0200 +@@ -353,13 +353,13 @@ class Value(object): + debug("get_value returning old value while new one is created") + return value + else: +- debug("lock_creatfunc (didnt wait)") ++ debug("lock_creatfunc `%s` (didnt wait)", self.createfunc.__name__) + has_createlock = True + + if not has_createlock: +- debug("lock_createfunc (waiting)") ++ debug("lock_createfunc `%s` (waiting)", self.createfunc.__name__) + creation_lock.acquire() +- debug("lock_createfunc (waited)") ++ debug("lock_createfunc `%s` (waited)", self.createfunc.__name__) + + try: + # see if someone created the value already diff --git a/pkgs/python-packages-overrides.nix b/pkgs/python-packages-overrides.nix --- a/pkgs/python-packages-overrides.nix +++ b/pkgs/python-packages-overrides.nix @@ -12,6 +12,12 @@ in self: super: { + Beaker = super.Beaker.override (attrs: { + patches = [ + ./patch-beaker-lock-func-debug.diff + ]; + }); + subvertpy = super.subvertpy.override (attrs: { # TODO: johbo: Remove the "or" once we drop 16.03 support SVN_PREFIX = "${pkgs.subversion.dev or pkgs.subversion}"; diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -3,26 +3,26 @@ { Beaker = super.buildPythonPackage { - name = "Beaker-1.9.0"; + name = "Beaker-1.9.1"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [funcsigs]; src = fetchurl { - url = "https://pypi.python.org/packages/93/b2/12de6937b06e9615dbb3cb3a1c9af17f133f435bdef59f4ad42032b6eb49/Beaker-1.9.0.tar.gz"; - md5 = "38b3fcdfa24faf97c6cf66991eb54e9c"; + url = "https://pypi.python.org/packages/ca/14/a626188d0d0c7b55dd7cf1902046c2743bd392a7078bb53073e13280eb1e/Beaker-1.9.1.tar.gz"; + md5 = "46fda0a164e2b0d24ccbda51a2310301"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; Jinja2 = super.buildPythonPackage { - name = "Jinja2-2.10"; + name = "Jinja2-2.9.6"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [MarkupSafe]; src = fetchurl { - url = "https://pypi.python.org/packages/56/e6/332789f295cf22308386cf5bbd1f4e00ed11484299c5d7383378cf48ba47/Jinja2-2.10.tar.gz"; - md5 = "61ef1117f945486472850819b8d1eb3d"; + url = "https://pypi.python.org/packages/90/61/f820ff0076a2599dd39406dcb858ecb239438c02ce706c8e91131ab9c7f1/Jinja2-2.9.6.tar.gz"; + md5 = "6411537324b4dba0956aaa8109f3c77b"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -237,13 +237,13 @@ }; }; greenlet = super.buildPythonPackage { - name = "greenlet-0.4.12"; + name = "greenlet-0.4.13"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/be/76/82af375d98724054b7e273b5d9369346937324f9bcc20980b45b068ef0b0/greenlet-0.4.12.tar.gz"; - md5 = "e8637647d58a26c4a1f51ca393e53c00"; + url = "https://pypi.python.org/packages/13/de/ba92335e9e76040ca7274224942282a80d54f85e342a5e33c5277c7f87eb/greenlet-0.4.13.tar.gz"; + md5 = "6e0b9dd5385f81d478451ec8ed1d62b3"; }; meta = { license = [ pkgs.lib.licenses.mit ]; @@ -372,8 +372,8 @@ doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/15/45/30273ee91feb60dabb8fbb2da7868520525f02cf910279b3047182feed80/mock-1.0.1.zip"; - md5 = "869f08d003c289a97c1a6610faf5e913"; + url = "https://pypi.python.org/packages/a2/52/7edcd94f0afb721a2d559a5b9aae8af4f8f2c79bc63fdbe8a8a6c9b23bbe/mock-1.0.1.tar.gz"; + md5 = "c3971991738caa55ec7c356bbc154ee2"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal ]; @@ -406,13 +406,13 @@ }; }; pexpect = super.buildPythonPackage { - name = "pexpect-4.3.0"; + name = "pexpect-4.4.0"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [ptyprocess]; src = fetchurl { - url = "https://pypi.python.org/packages/f8/44/5466c30e49762bb92e442bbdf4472d6904608d211258eb3198a11f0309a4/pexpect-4.3.0.tar.gz"; - md5 = "047a486dcd26134b74f2e67046bb61a0"; + url = "https://pypi.python.org/packages/fa/c3/60c0cbf96f242d0b47a82e9ca634dcd6dcb043832cf05e17540812e1c707/pexpect-4.4.0.tar.gz"; + md5 = "e9b07f0765df8245ac72201d757baaef"; }; meta = { license = [ pkgs.lib.licenses.isc { fullName = "ISC License (ISCL)"; } ]; @@ -653,7 +653,7 @@ }; }; rhodecode-vcsserver = super.buildPythonPackage { - name = "rhodecode-vcsserver-4.11.6"; + name = "rhodecode-vcsserver-4.12.0"; buildInputs = with self; [pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage configobj]; doCheck = true; propagatedBuildInputs = with self; [Beaker configobj decorator dulwich hgsubversion hg-evolve infrae.cache mercurial msgpack-python pyramid pyramid-jinja2 pyramid-mako repoze.lru simplejson subprocess32 subvertpy six translationstring WebOb wheel zope.deprecation zope.interface ipdb ipython gevent greenlet gunicorn waitress pytest py pytest-cov pytest-sugar pytest-runner pytest-catchlog pytest-profiling gprof2dot pytest-timeout mock WebTest cov-core coverage]; @@ -663,13 +663,13 @@ }; }; scandir = super.buildPythonPackage { - name = "scandir-1.6"; + name = "scandir-1.7"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; []; src = fetchurl { - url = "https://pypi.python.org/packages/77/3f/916f524f50ee65e3f465a280d2851bd63685250fddb3020c212b3977664d/scandir-1.6.tar.gz"; - md5 = "0180ddb97c96cbb2d4f25d2ae11c64ac"; + url = "https://pypi.python.org/packages/13/bb/e541b74230bbf7a20a3949a2ee6631be299378a784f5445aa5d0047c192b/scandir-1.7.tar.gz"; + md5 = "037e5f24d1a0e78b17faca72dea9555f"; }; meta = { license = [ pkgs.lib.licenses.bsdOriginal { fullName = "New BSD License"; } ]; @@ -845,26 +845,26 @@ }; }; zope.deprecation = super.buildPythonPackage { - name = "zope.deprecation-4.1.2"; + name = "zope.deprecation-4.3.0"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [setuptools]; src = fetchurl { - url = "https://pypi.python.org/packages/c1/d3/3919492d5e57d8dd01b36f30b34fc8404a30577392b1eb817c303499ad20/zope.deprecation-4.1.2.tar.gz"; - md5 = "e9a663ded58f4f9f7881beb56cae2782"; + url = "https://pypi.python.org/packages/a1/18/2dc5e6bfe64fdc3b79411b67464c55bb0b43b127051a20f7f492ab767758/zope.deprecation-4.3.0.tar.gz"; + md5 = "2166b2cb7e0e96a21104e6f8f9b696bb"; }; meta = { license = [ pkgs.lib.licenses.zpt21 ]; }; }; zope.interface = super.buildPythonPackage { - name = "zope.interface-4.1.3"; + name = "zope.interface-4.4.3"; buildInputs = with self; []; doCheck = false; propagatedBuildInputs = with self; [setuptools]; src = fetchurl { - url = "https://pypi.python.org/packages/9d/81/2509ca3c6f59080123c1a8a97125eb48414022618cec0e64eb1313727bfe/zope.interface-4.1.3.tar.gz"; - md5 = "9ae3d24c0c7415deb249dd1a132f0f79"; + url = "https://pypi.python.org/packages/bd/d2/25349ed41f9dcff7b3baf87bd88a4c82396cf6e02f1f42bb68657a3132af/zope.interface-4.4.3.tar.gz"; + md5 = "8700a4f527c1203b34b10c2b4e7a6912"; }; meta = { license = [ pkgs.lib.licenses.zpt21 ]; diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ ## core setuptools==30.1.0 -Beaker==1.9.0 +Beaker==1.9.1 configobj==5.0.6 decorator==4.1.2 dulwich==0.13.0 @@ -11,6 +11,7 @@ infrae.cache==1.0.1 mercurial==4.4.2 msgpack-python==0.4.8 pyramid-jinja2==2.7 +Jinja2==2.9.6 pyramid==1.9.1 pyramid-mako==1.0.2 repoze.lru==0.7 @@ -23,12 +24,12 @@ six==1.11.0 translationstring==1.3 WebOb==1.7.4 wheel==0.29.0 -zope.deprecation==4.1.2 -zope.interface==4.1.3 +zope.deprecation==4.3.0 +zope.interface==4.4.3 ## http servers gevent==1.2.2 -greenlet==0.4.12 +greenlet==0.4.13 gunicorn==19.7.1 waitress==1.1.0 diff --git a/vcsserver/VERSION b/vcsserver/VERSION --- a/vcsserver/VERSION +++ b/vcsserver/VERSION @@ -1,1 +1,1 @@ -4.11.6 \ No newline at end of file +4.12.0 \ No newline at end of file diff --git a/vcsserver/git.py b/vcsserver/git.py --- a/vcsserver/git.py +++ b/vcsserver/git.py @@ -451,8 +451,9 @@ class GitRemote(object): if self.check_url(url, wire): repo = self._factory.repo(wire) self.run_git_command( - wire, ['push', url, '--mirror'], fail_on_stderr=False) - + wire, ['push', url, '--mirror'], fail_on_stderr=False, + _copts=['-c', 'core.askpass=""'], + extra_env={'GIT_TERMINAL_PROMPT': '0'}) @reraise_safe_exceptions def get_remote_refs(self, wire, url): @@ -625,6 +626,10 @@ class GitRemote(object): del opts['_safe'] safe_call = True + if '_copts' in opts: + _copts.extend(opts['_copts'] or []) + del opts['_copts'] + gitenv = os.environ.copy() gitenv.update(opts.pop('extra_env', {})) # need to clean fix GIT_DIR ! @@ -650,6 +655,12 @@ class GitRemote(object): else: raise exceptions.VcsException(tb_err) + @reraise_safe_exceptions + def install_hooks(self, wire, force=False): + from vcsserver.hook_utils import install_git_hooks + repo = self._factory.repo(wire) + return install_git_hooks(repo.path, repo.bare, force_create=force) + def str_to_dulwich(value): """ diff --git a/vcsserver/hg.py b/vcsserver/hg.py --- a/vcsserver/hg.py +++ b/vcsserver/hg.py @@ -553,7 +553,13 @@ class HgRemote(object): @reraise_safe_exceptions def pull(self, wire, url, commit_ids=None): repo = self._factory.repo(wire) + # Disable any prompts for this repo + repo.ui.setconfig('ui', 'interactive', 'off', '-y') + remote = peer(repo, {}, url) + # Disable any prompts for this remote + remote.ui.setconfig('ui', 'interactive', 'off', '-y') + if commit_ids: commit_ids = [bin(commit_id) for commit_id in commit_ids] @@ -564,8 +570,15 @@ class HgRemote(object): def sync_push(self, wire, url): if self.check_url(url, wire['config']): repo = self._factory.repo(wire) + + # Disable any prompts for this repo + repo.ui.setconfig('ui', 'interactive', 'off', '-y') + bookmarks = dict(repo._bookmarks).keys() remote = peer(repo, {}, url) + # Disable any prompts for this remote + remote.ui.setconfig('ui', 'interactive', 'off', '-y') + return exchange.push( repo, remote, newbranch=True, bookmarks=bookmarks).cgresult @@ -756,3 +769,8 @@ class HgRemote(object): repo = self._factory.repo(wire) baseui = self._factory._create_config(wire['config']) commands.bookmark(baseui, repo, bookmark, rev=revision, force=True) + + @reraise_safe_exceptions + def install_hooks(self, wire, force=False): + # we don't need any special hooks for Mercurial + pass diff --git a/vcsserver/hgpatches.py b/vcsserver/hgpatches.py --- a/vcsserver/hgpatches.py +++ b/vcsserver/hgpatches.py @@ -87,7 +87,7 @@ def patch_subrepo_type_mapping(): """ return True - def dirty(self, ignoreupdate=False): + def dirty(self, ignoreupdate=False, missing=False): """returns true if the dirstate of the subrepo is dirty or does not match current stored state. If ignoreupdate is true, only check whether the subrepo has uncommitted changes in its dirstate. diff --git a/vcsserver/hook_utils/__init__.py b/vcsserver/hook_utils/__init__.py new file mode 100644 --- /dev/null +++ b/vcsserver/hook_utils/__init__.py @@ -0,0 +1,154 @@ +# -*- coding: utf-8 -*- + +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import re +import os +import sys +import datetime +import logging +import pkg_resources + +import vcsserver + +log = logging.getLogger(__name__) + + +def install_git_hooks(repo_path, bare, executable=None, force_create=False): + """ + Creates a RhodeCode hook inside a git repository + + :param repo_path: path to repository + :param executable: binary executable to put in the hooks + :param force_create: Create even if same name hook exists + """ + executable = executable or sys.executable + hooks_path = os.path.join(repo_path, 'hooks') + if not bare: + hooks_path = os.path.join(repo_path, '.git', 'hooks') + if not os.path.isdir(hooks_path): + os.makedirs(hooks_path, mode=0777) + + tmpl_post = pkg_resources.resource_string( + 'vcsserver', '/'.join( + ('hook_utils', 'hook_templates', 'git_post_receive.py.tmpl'))) + tmpl_pre = pkg_resources.resource_string( + 'vcsserver', '/'.join( + ('hook_utils', 'hook_templates', 'git_pre_receive.py.tmpl'))) + + path = '' # not used for now + timestamp = datetime.datetime.utcnow().isoformat() + + for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]: + log.debug('Installing git hook in repo %s', repo_path) + _hook_file = os.path.join(hooks_path, '%s-receive' % h_type) + _rhodecode_hook = check_rhodecode_hook(_hook_file) + + if _rhodecode_hook or force_create: + log.debug('writing git %s hook file at %s !', h_type, _hook_file) + try: + with open(_hook_file, 'wb') as f: + template = template.replace( + '_TMPL_', vcsserver.__version__) + template = template.replace('_DATE_', timestamp) + template = template.replace('_ENV_', executable) + template = template.replace('_PATH_', path) + f.write(template) + os.chmod(_hook_file, 0755) + except IOError: + log.exception('error writing hook file %s', _hook_file) + else: + log.debug('skipping writing hook file') + + return True + + +def install_svn_hooks(repo_path, executable=None, force_create=False): + """ + Creates RhodeCode hooks inside a svn repository + + :param repo_path: path to repository + :param executable: binary executable to put in the hooks + :param force_create: Create even if same name hook exists + """ + executable = executable or sys.executable + hooks_path = os.path.join(repo_path, 'hooks') + if not os.path.isdir(hooks_path): + os.makedirs(hooks_path, mode=0777) + + tmpl_post = pkg_resources.resource_string( + 'vcsserver', '/'.join( + ('hook_utils', 'hook_templates', 'svn_post_commit_hook.py.tmpl'))) + tmpl_pre = pkg_resources.resource_string( + 'vcsserver', '/'.join( + ('hook_utils', 'hook_templates', 'svn_pre_commit_hook.py.tmpl'))) + + path = '' # not used for now + timestamp = datetime.datetime.utcnow().isoformat() + + for h_type, template in [('pre', tmpl_pre), ('post', tmpl_post)]: + log.debug('Installing svn hook in repo %s', repo_path) + _hook_file = os.path.join(hooks_path, '%s-commit' % h_type) + _rhodecode_hook = check_rhodecode_hook(_hook_file) + + if _rhodecode_hook or force_create: + log.debug('writing svn %s hook file at %s !', h_type, _hook_file) + + try: + with open(_hook_file, 'wb') as f: + template = template.replace( + '_TMPL_', vcsserver.__version__) + template = template.replace('_DATE_', timestamp) + template = template.replace('_ENV_', executable) + template = template.replace('_PATH_', path) + + f.write(template) + os.chmod(_hook_file, 0755) + except IOError: + log.exception('error writing hook file %s', _hook_file) + else: + log.debug('skipping writing hook file') + + return True + + +def check_rhodecode_hook(hook_path): + """ + Check if the hook was created by RhodeCode + """ + if not os.path.exists(hook_path): + return True + + log.debug('hook exists, checking if it is from rhodecode') + hook_content = read_hook_content(hook_path) + matches = re.search(r'(?:RC_HOOK_VER)\s*=\s*(.*)', hook_content) + if matches: + try: + version = matches.groups()[0] + log.debug('got version %s from hooks.', version) + return True + except Exception: + log.exception("Exception while reading the hook version.") + + return False + + +def read_hook_content(hook_path): + with open(hook_path, 'rb') as f: + content = f.read() + return content diff --git a/vcsserver/hook_utils/hook_templates/git_post_receive.py.tmpl b/vcsserver/hook_utils/hook_templates/git_post_receive.py.tmpl new file mode 100644 --- /dev/null +++ b/vcsserver/hook_utils/hook_templates/git_post_receive.py.tmpl @@ -0,0 +1,51 @@ +#!_ENV_ +import os +import sys +path_adjust = [_PATH_] + +if path_adjust: + sys.path = path_adjust + +try: + from vcsserver import hooks +except ImportError: + if os.environ.get('RC_DEBUG_GIT_HOOK'): + import traceback + print traceback.format_exc() + hooks = None + + +# TIMESTAMP: _DATE_ +RC_HOOK_VER = '_TMPL_' + + +def main(): + if hooks is None: + # exit with success if we cannot import vcsserver.hooks !! + # this allows simply push to this repo even without rhodecode + sys.exit(0) + + if os.environ.get('RC_SKIP_HOOKS'): + sys.exit(0) + + repo_path = os.getcwd() + push_data = sys.stdin.readlines() + os.environ['RC_HOOK_VER'] = RC_HOOK_VER + # os.environ is modified here by a subprocess call that + # runs git and later git executes this hook. + # Environ gets some additional info from rhodecode system + # like IP or username from basic-auth + try: + result = hooks.git_post_receive(repo_path, push_data, os.environ) + sys.exit(result) + except Exception as error: + # TODO: johbo: Improve handling of this special case + if not getattr(error, '_vcs_kind', None) == 'repo_locked': + raise + print 'ERROR:', error + sys.exit(1) + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/vcsserver/hook_utils/hook_templates/git_pre_receive.py.tmpl b/vcsserver/hook_utils/hook_templates/git_pre_receive.py.tmpl new file mode 100644 --- /dev/null +++ b/vcsserver/hook_utils/hook_templates/git_pre_receive.py.tmpl @@ -0,0 +1,51 @@ +#!_ENV_ +import os +import sys +path_adjust = [_PATH_] + +if path_adjust: + sys.path = path_adjust + +try: + from vcsserver import hooks +except ImportError: + if os.environ.get('RC_DEBUG_GIT_HOOK'): + import traceback + print traceback.format_exc() + hooks = None + + +# TIMESTAMP: _DATE_ +RC_HOOK_VER = '_TMPL_' + + +def main(): + if hooks is None: + # exit with success if we cannot import vcsserver.hooks !! + # this allows simply push to this repo even without rhodecode + sys.exit(0) + + if os.environ.get('RC_SKIP_HOOKS'): + sys.exit(0) + + repo_path = os.getcwd() + push_data = sys.stdin.readlines() + os.environ['RC_HOOK_VER'] = RC_HOOK_VER + # os.environ is modified here by a subprocess call that + # runs git and later git executes this hook. + # Environ gets some additional info from rhodecode system + # like IP or username from basic-auth + try: + result = hooks.git_pre_receive(repo_path, push_data, os.environ) + sys.exit(result) + except Exception as error: + # TODO: johbo: Improve handling of this special case + if not getattr(error, '_vcs_kind', None) == 'repo_locked': + raise + print 'ERROR:', error + sys.exit(1) + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/vcsserver/hook_utils/hook_templates/svn_post_commit_hook.py.tmpl b/vcsserver/hook_utils/hook_templates/svn_post_commit_hook.py.tmpl new file mode 100644 --- /dev/null +++ b/vcsserver/hook_utils/hook_templates/svn_post_commit_hook.py.tmpl @@ -0,0 +1,50 @@ +#!_ENV_ + +import os +import sys +path_adjust = [_PATH_] + +if path_adjust: + sys.path = path_adjust + +try: + from vcsserver import hooks +except ImportError: + if os.environ.get('RC_DEBUG_SVN_HOOK'): + import traceback + print traceback.format_exc() + hooks = None + + +# TIMESTAMP: _DATE_ +RC_HOOK_VER = '_TMPL_' + + +def main(): + if hooks is None: + # exit with success if we cannot import vcsserver.hooks !! + # this allows simply push to this repo even without rhodecode + sys.exit(0) + + if os.environ.get('RC_SKIP_HOOKS'): + sys.exit(0) + repo_path = os.getcwd() + push_data = sys.argv[1:] + + os.environ['RC_HOOK_VER'] = RC_HOOK_VER + + try: + result = hooks.svn_post_commit(repo_path, push_data, os.environ) + sys.exit(result) + except Exception as error: + # TODO: johbo: Improve handling of this special case + if not getattr(error, '_vcs_kind', None) == 'repo_locked': + raise + print 'ERROR:', error + sys.exit(1) + sys.exit(0) + + + +if __name__ == '__main__': + main() diff --git a/vcsserver/hook_utils/hook_templates/svn_pre_commit_hook.py.tmpl b/vcsserver/hook_utils/hook_templates/svn_pre_commit_hook.py.tmpl new file mode 100644 --- /dev/null +++ b/vcsserver/hook_utils/hook_templates/svn_pre_commit_hook.py.tmpl @@ -0,0 +1,52 @@ +#!_ENV_ + +import os +import sys +path_adjust = [_PATH_] + +if path_adjust: + sys.path = path_adjust + +try: + from vcsserver import hooks +except ImportError: + if os.environ.get('RC_DEBUG_SVN_HOOK'): + import traceback + print traceback.format_exc() + hooks = None + + +# TIMESTAMP: _DATE_ +RC_HOOK_VER = '_TMPL_' + + +def main(): + if os.environ.get('SSH_READ_ONLY') == '1': + sys.stderr.write('Only read-only access is allowed') + sys.exit(1) + + if hooks is None: + # exit with success if we cannot import vcsserver.hooks !! + # this allows simply push to this repo even without rhodecode + sys.exit(0) + if os.environ.get('RC_SKIP_HOOKS'): + sys.exit(0) + repo_path = os.getcwd() + push_data = sys.argv[1:] + + os.environ['RC_HOOK_VER'] = RC_HOOK_VER + + try: + result = hooks.svn_pre_commit(repo_path, push_data, os.environ) + sys.exit(result) + except Exception as error: + # TODO: johbo: Improve handling of this special case + if not getattr(error, '_vcs_kind', None) == 'repo_locked': + raise + print 'ERROR:', error + sys.exit(1) + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/vcsserver/hooks.py b/vcsserver/hooks.py --- a/vcsserver/hooks.py +++ b/vcsserver/hooks.py @@ -20,10 +20,10 @@ import io import os import sys -import json import logging import collections import importlib +import base64 from httplib import HTTPConnection @@ -46,7 +46,11 @@ class HooksHttpClient(object): def __call__(self, method, extras): connection = HTTPConnection(self.hooks_uri) body = self._serialize(method, extras) - connection.request('POST', '/', body) + try: + connection.request('POST', '/', body) + except Exception: + log.error('Connection failed on %s', connection) + raise response = connection.getresponse() return json.loads(response.read()) @@ -97,6 +101,17 @@ class GitMessageWriter(RemoteMessageWrit self.stdout.write(message.encode('utf-8')) +class SvnMessageWriter(RemoteMessageWriter): + """Writer that knows how to send messages to svn clients.""" + + def __init__(self, stderr=None): + # SVN needs data sent to stderr for back-to-client messaging + self.stderr = stderr or sys.stderr + + def write(self, message): + self.stderr.write(message.encode('utf-8')) + + def _handle_exception(result): exception_class = result.get('exception') exception_traceback = result.get('exception_traceback') @@ -122,8 +137,9 @@ def _get_hooks_client(extras): def _call_hook(hook_name, extras, writer): - hooks = _get_hooks_client(extras) - result = hooks(hook_name, extras) + hooks_client = _get_hooks_client(extras) + log.debug('Hooks, using client:%s', hooks_client) + result = hooks_client(hook_name, extras) log.debug('Hooks got result: %s', result) writer.write(result['output']) _handle_exception(result) @@ -465,3 +481,61 @@ def git_post_receive(unused_repo_path, r pass return _call_hook('post_push', extras, GitMessageWriter()) + + +def svn_pre_commit(repo_path, commit_data, env): + path, txn_id = commit_data + branches = [] + tags = [] + + cmd = ['svnlook', 'pget', + '-t', txn_id, + '--revprop', path, 'rc-scm-extras'] + stdout, stderr = subprocessio.run_command( + cmd, env=os.environ.copy()) + extras = json.loads(base64.urlsafe_b64decode(stdout)) + + extras['commit_ids'] = [] + extras['txn_id'] = txn_id + extras['new_refs'] = { + 'branches': branches, + 'bookmarks': [], + 'tags': tags, + } + sys.stderr.write(str(extras)) + return _call_hook('pre_push', extras, SvnMessageWriter()) + + +def svn_post_commit(repo_path, commit_data, env): + """ + commit_data is path, rev, txn_id + """ + path, commit_id, txn_id = commit_data + branches = [] + tags = [] + + cmd = ['svnlook', 'pget', + '-r', commit_id, + '--revprop', path, 'rc-scm-extras'] + stdout, stderr = subprocessio.run_command( + cmd, env=os.environ.copy()) + + extras = json.loads(base64.urlsafe_b64decode(stdout)) + + extras['commit_ids'] = [commit_id] + extras['txn_id'] = txn_id + extras['new_refs'] = { + 'branches': branches, + 'bookmarks': [], + 'tags': tags, + } + + if 'repo_size' in extras['hooks']: + try: + _call_hook('repo_size', extras, SvnMessageWriter()) + except: + pass + + return _call_hook('post_push', extras, SvnMessageWriter()) + + diff --git a/vcsserver/http_main.py b/vcsserver/http_main.py --- a/vcsserver/http_main.py +++ b/vcsserver/http_main.py @@ -15,6 +15,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import os import base64 import locale import logging @@ -29,6 +30,7 @@ from beaker.cache import CacheManager from beaker.util import parse_cache_config_options from pyramid.config import Configurator from pyramid.wsgi import wsgiapp +from pyramid.compat import configparser from vcsserver import remote_wsgi, scm_app, settings, hgpatches from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT @@ -90,6 +92,10 @@ class VCS(object): svn_repo_cache = self.cache.get_cache_region( 'svn', region='repo_object') svn_factory = SubversionFactory(svn_repo_cache) + # hg factory is used for svn url validation + hg_repo_cache = self.cache.get_cache_region( + 'hg', region='repo_object') + hg_factory = MercurialFactory(hg_repo_cache) self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory) else: log.info("Subversion client import failed") @@ -188,6 +194,9 @@ class HTTPApplication(object): git_path = app_settings.get('git_path', None) if git_path: settings.GIT_EXECUTABLE = git_path + binary_dir = app_settings.get('core.binary_dir', None) + if binary_dir: + settings.BINARY_DIR = binary_dir def _configure(self): self.config.add_renderer( @@ -276,11 +285,11 @@ class HTTPApplication(object): def status_view(self, request): import vcsserver - return {'status': 'OK', 'vcsserver_version': vcsserver.__version__} + return {'status': 'OK', 'vcsserver_version': vcsserver.__version__, + 'pid': os.getpid()} def service_view(self, request): import vcsserver - import ConfigParser as configparser payload = msgpack.unpackb(request.body, use_list=True) diff --git a/vcsserver/settings.py b/vcsserver/settings.py --- a/vcsserver/settings.py +++ b/vcsserver/settings.py @@ -17,3 +17,4 @@ WIRE_ENCODING = 'UTF-8' GIT_EXECUTABLE = 'git' +BINARY_DIR = '' diff --git a/vcsserver/svn.py b/vcsserver/svn.py --- a/vcsserver/svn.py +++ b/vcsserver/svn.py @@ -32,7 +32,7 @@ import svn.diff import svn.fs import svn.repos -from vcsserver import svn_diff, exceptions, subprocessio +from vcsserver import svn_diff, exceptions, subprocessio, settings from vcsserver.base import RepoFactory, raise_from_original log = logging.getLogger(__name__) @@ -414,6 +414,17 @@ class SvnRemote(object): def is_large_file(self, wire, path): return False + @reraise_safe_exceptions + def install_hooks(self, wire, force=False): + from vcsserver.hook_utils import install_svn_hooks + repo_path = wire['path'] + binary_dir = settings.BINARY_DIR + executable = None + if binary_dir: + executable = os.path.join(binary_dir, 'python') + return install_svn_hooks( + repo_path, executable=executable, force_create=force) + class SvnDiffer(object): """ @@ -576,6 +587,7 @@ class SvnDiffer(object): return content.splitlines(True) + class DiffChangeEditor(svn.delta.Editor): """ Records changes between two given revisions diff --git a/vcsserver/tests/__init__.py b/vcsserver/tests/__init__.py new file mode 100644 --- /dev/null +++ b/vcsserver/tests/__init__.py @@ -0,0 +1,16 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA diff --git a/vcsserver/tests/conftest.py b/vcsserver/tests/conftest.py --- a/vcsserver/tests/conftest.py +++ b/vcsserver/tests/conftest.py @@ -55,3 +55,4 @@ def get_available_port(): mysocket.close() del mysocket return port + diff --git a/vcsserver/tests/fixture.py b/vcsserver/tests/fixture.py --- a/vcsserver/tests/fixture.py +++ b/vcsserver/tests/fixture.py @@ -69,3 +69,18 @@ class ContextINI(object): def __exit__(self, exc_type, exc_val, exc_tb): if self.destroy: os.remove(self.new_path) + + +def no_newline_id_generator(test_name): + """ + Generates a test name without spaces or newlines characters. Used for + nicer output of progress of test + """ + org_name = test_name + test_name = test_name\ + .replace('\n', '_N') \ + .replace('\r', '_N') \ + .replace('\t', '_T') \ + .replace(' ', '_S') + + return test_name or 'test-with-empty-name' diff --git a/vcsserver/tests/test_install_hooks.py b/vcsserver/tests/test_install_hooks.py new file mode 100644 --- /dev/null +++ b/vcsserver/tests/test_install_hooks.py @@ -0,0 +1,206 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import sys +import stat +import pytest +import vcsserver +import tempfile +from vcsserver import hook_utils +from vcsserver.tests.fixture import no_newline_id_generator +from vcsserver.utils import AttributeDict + + +class TestCheckRhodecodeHook(object): + + def test_returns_false_when_hook_file_is_wrong_found(self, tmpdir): + hook = os.path.join(str(tmpdir), 'fake_hook_file.py') + with open(hook, 'wb') as f: + f.write('dummy test') + result = hook_utils.check_rhodecode_hook(hook) + assert result is False + + def test_returns_true_when_no_hook_file_found(self, tmpdir): + hook = os.path.join(str(tmpdir), 'fake_hook_file_not_existing.py') + result = hook_utils.check_rhodecode_hook(hook) + assert result + + @pytest.mark.parametrize("file_content, expected_result", [ + ("RC_HOOK_VER = '3.3.3'\n", True), + ("RC_HOOK = '3.3.3'\n", False), + ], ids=no_newline_id_generator) + def test_signatures(self, file_content, expected_result, tmpdir): + hook = os.path.join(str(tmpdir), 'fake_hook_file_1.py') + with open(hook, 'wb') as f: + f.write(file_content) + + result = hook_utils.check_rhodecode_hook(hook) + + assert result is expected_result + + +class BaseInstallHooks(object): + HOOK_FILES = () + + def _check_hook_file_mode(self, file_path): + assert os.path.exists(file_path), 'path %s missing' % file_path + stat_info = os.stat(file_path) + + file_mode = stat.S_IMODE(stat_info.st_mode) + expected_mode = int('755', 8) + assert expected_mode == file_mode + + def _check_hook_file_content(self, file_path, executable): + executable = executable or sys.executable + with open(file_path, 'rt') as hook_file: + content = hook_file.read() + + expected_env = '#!{}'.format(executable) + expected_rc_version = "\nRC_HOOK_VER = '{}'\n".format( + vcsserver.__version__) + assert content.strip().startswith(expected_env) + assert expected_rc_version in content + + def _create_fake_hook(self, file_path, content): + with open(file_path, 'w') as hook_file: + hook_file.write(content) + + def create_dummy_repo(self, repo_type): + tmpdir = tempfile.mkdtemp() + repo = AttributeDict() + if repo_type == 'git': + repo.path = os.path.join(tmpdir, 'test_git_hooks_installation_repo') + os.makedirs(repo.path) + os.makedirs(os.path.join(repo.path, 'hooks')) + repo.bare = True + + elif repo_type == 'svn': + repo.path = os.path.join(tmpdir, 'test_svn_hooks_installation_repo') + os.makedirs(repo.path) + os.makedirs(os.path.join(repo.path, 'hooks')) + + return repo + + def check_hooks(self, repo_path, repo_bare=True): + for file_name in self.HOOK_FILES: + if repo_bare: + file_path = os.path.join(repo_path, 'hooks', file_name) + else: + file_path = os.path.join(repo_path, '.git', 'hooks', file_name) + self._check_hook_file_mode(file_path) + self._check_hook_file_content(file_path, sys.executable) + + +class TestInstallGitHooks(BaseInstallHooks): + HOOK_FILES = ('pre-receive', 'post-receive') + + def test_hooks_are_installed(self): + repo = self.create_dummy_repo('git') + result = hook_utils.install_git_hooks(repo.path, repo.bare) + assert result + self.check_hooks(repo.path, repo.bare) + + def test_hooks_are_replaced(self): + repo = self.create_dummy_repo('git') + hooks_path = os.path.join(repo.path, 'hooks') + for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]: + self._create_fake_hook( + file_path, content="RC_HOOK_VER = 'abcde'\n") + + result = hook_utils.install_git_hooks(repo.path, repo.bare) + assert result + self.check_hooks(repo.path, repo.bare) + + def test_non_rc_hooks_are_not_replaced(self): + repo = self.create_dummy_repo('git') + hooks_path = os.path.join(repo.path, 'hooks') + non_rc_content = 'echo "non rc hook"\n' + for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]: + self._create_fake_hook( + file_path, content=non_rc_content) + + result = hook_utils.install_git_hooks(repo.path, repo.bare) + assert result + + for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]: + with open(file_path, 'rt') as hook_file: + content = hook_file.read() + assert content == non_rc_content + + def test_non_rc_hooks_are_replaced_with_force_flag(self): + repo = self.create_dummy_repo('git') + hooks_path = os.path.join(repo.path, 'hooks') + non_rc_content = 'echo "non rc hook"\n' + for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]: + self._create_fake_hook( + file_path, content=non_rc_content) + + result = hook_utils.install_git_hooks( + repo.path, repo.bare, force_create=True) + assert result + self.check_hooks(repo.path, repo.bare) + + +class TestInstallSvnHooks(BaseInstallHooks): + HOOK_FILES = ('pre-commit', 'post-commit') + + def test_hooks_are_installed(self): + repo = self.create_dummy_repo('svn') + result = hook_utils.install_svn_hooks(repo.path) + assert result + self.check_hooks(repo.path) + + def test_hooks_are_replaced(self): + repo = self.create_dummy_repo('svn') + hooks_path = os.path.join(repo.path, 'hooks') + for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]: + self._create_fake_hook( + file_path, content="RC_HOOK_VER = 'abcde'\n") + + result = hook_utils.install_svn_hooks(repo.path) + assert result + self.check_hooks(repo.path) + + def test_non_rc_hooks_are_not_replaced(self): + repo = self.create_dummy_repo('svn') + hooks_path = os.path.join(repo.path, 'hooks') + non_rc_content = 'echo "non rc hook"\n' + for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]: + self._create_fake_hook( + file_path, content=non_rc_content) + + result = hook_utils.install_svn_hooks(repo.path) + assert result + + for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]: + with open(file_path, 'rt') as hook_file: + content = hook_file.read() + assert content == non_rc_content + + def test_non_rc_hooks_are_replaced_with_force_flag(self): + repo = self.create_dummy_repo('svn') + hooks_path = os.path.join(repo.path, 'hooks') + non_rc_content = 'echo "non rc hook"\n' + for file_path in [os.path.join(hooks_path, f) for f in self.HOOK_FILES]: + self._create_fake_hook( + file_path, content=non_rc_content) + + result = hook_utils.install_svn_hooks( + repo.path, force_create=True) + assert result + self.check_hooks(repo.path, ) diff --git a/vcsserver/utils.py b/vcsserver/utils.py --- a/vcsserver/utils.py +++ b/vcsserver/utils.py @@ -14,6 +14,9 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import logging + +log = logging.getLogger(__name__) def safe_int(val, default=None): @@ -70,3 +73,10 @@ def safe_str(unicode_, to_encoding=['utf return unicode_.encode(encoding) except (ImportError, UnicodeEncodeError): return unicode_.encode(to_encoding[0], 'replace') + + +class AttributeDict(dict): + def __getattr__(self, attr): + return self.get(attr, None) + __setattr__ = dict.__setitem__ + __delattr__ = dict.__delitem__