# HG changeset patch # User RhodeCode Admin # Date 2024-05-20 07:01:45 # Node ID fe30068d078021275f96b43af830896b920922f6 # Parent 6b191913846894aea2651599fe6be07285f86b2a fix(svn): fixed problems with svn hooks binary dir not beeing propagates in mod_dav_svn diff --git a/vcsserver/config/settings_maker.py b/vcsserver/config/settings_maker.py --- a/vcsserver/config/settings_maker.py +++ b/vcsserver/config/settings_maker.py @@ -28,6 +28,7 @@ from vcsserver.type_utils import str2boo log = logging.getLogger(__name__) + # skip keys, that are set here, so we don't double process those set_keys = { '__file__': '' @@ -51,6 +52,10 @@ class SettingsMaker: return int(input_val) @classmethod + def _float_func(cls, input_val): + return float(input_val) + + @classmethod def _list_func(cls, input_val, sep=','): return aslist(input_val, sep=sep) @@ -61,8 +66,18 @@ class SettingsMaker: return input_val @classmethod - def _float_func(cls, input_val): - return float(input_val) + def _string_no_quote_func(cls, input_val, lower=True): + """ + Special case string function that detects if value is set to empty quote string + e.g. + + core.binary_dir = "" + """ + + input_val = cls._string_func(input_val, lower=lower) + if input_val in ['""', "''"]: + return '' + return input_val @classmethod def _dir_func(cls, input_val, ensure_dir=False, mode=0o755): @@ -148,10 +163,12 @@ class SettingsMaker: parser_func = { 'bool': self._bool_func, 'int': self._int_func, + 'float': self._float_func, 'list': self._list_func, 'list:newline': functools.partial(self._list_func, sep='/n'), 'list:spacesep': functools.partial(self._list_func, sep=' '), 'string': functools.partial(self._string_func, lower=lower), + 'string:noquote': functools.partial(self._string_no_quote_func, lower=lower), 'dir': self._dir_func, 'dir:ensured': functools.partial(self._dir_func, ensure_dir=True), 'file': self._file_path_func, diff --git a/vcsserver/hook_utils/__init__.py b/vcsserver/hook_utils/__init__.py --- a/vcsserver/hook_utils/__init__.py +++ b/vcsserver/hook_utils/__init__.py @@ -142,9 +142,9 @@ def install_svn_hooks(repo_path, executa env_expand = str([ ('RC_CORE_BINARY_DIR', vcsserver.settings.BINARY_DIR), - ('RC_GIT_EXECUTABLE', vcsserver.settings.GIT_EXECUTABLE), - ('RC_SVN_EXECUTABLE', vcsserver.settings.SVN_EXECUTABLE), - ('RC_SVNLOOK_EXECUTABLE', vcsserver.settings.SVNLOOK_EXECUTABLE), + ('RC_GIT_EXECUTABLE', vcsserver.settings.GIT_EXECUTABLE()), + ('RC_SVN_EXECUTABLE', vcsserver.settings.SVN_EXECUTABLE()), + ('RC_SVNLOOK_EXECUTABLE', vcsserver.settings.SVNLOOK_EXECUTABLE()), ]) try: diff --git a/vcsserver/hooks.py b/vcsserver/hooks.py --- a/vcsserver/hooks.py +++ b/vcsserver/hooks.py @@ -31,7 +31,6 @@ from celery import Celery import mercurial.scmutil import mercurial.node -import vcsserver.settings from vcsserver.lib.rc_json import json from vcsserver import exceptions, subprocessio, settings from vcsserver.str_utils import ascii_str, safe_str @@ -294,20 +293,23 @@ def _get_hg_env(old_rev, new_rev, txnid, return [(k, v) for k, v in env.items()] -def _fix_hooks_executables(): +def _fix_hooks_executables(ini_path=''): """ This is a trick to set proper settings.EXECUTABLE paths for certain execution patterns especially for subversion where hooks strip entire env, and calling just 'svn' command will most likely fail because svn is not on PATH """ - vcsserver.settings.BINARY_DIR = ( - os.environ.get('RC_BINARY_DIR') or vcsserver.settings.BINARY_DIR) - vcsserver.settings.GIT_EXECUTABLE = ( - os.environ.get('RC_GIT_EXECUTABLE') or vcsserver.settings.GIT_EXECUTABLE) - vcsserver.settings.SVN_EXECUTABLE = ( - os.environ.get('RC_SVN_EXECUTABLE') or vcsserver.settings.SVN_EXECUTABLE) - vcsserver.settings.SVNLOOK_EXECUTABLE = ( - os.environ.get('RC_SVNLOOK_EXECUTABLE') or vcsserver.settings.SVNLOOK_EXECUTABLE) + from vcsserver.http_main import sanitize_settings_and_apply_defaults + from vcsserver.lib.config_utils import get_app_config_lightweight + + core_binary_dir = settings.BINARY_DIR or '/usr/local/bin/rhodecode_bin/vcs_bin' + if ini_path: + + ini_settings = get_app_config_lightweight(ini_path) + ini_settings = sanitize_settings_and_apply_defaults({'__file__': ini_path}, ini_settings) + core_binary_dir = ini_settings['core.binary_dir'] + + settings.BINARY_DIR = core_binary_dir def repo_size(ui, repo, **kwargs): @@ -568,10 +570,12 @@ def git_pre_receive(unused_repo_path, re rev_data = _parse_git_ref_lines(revision_lines) if 'push' not in extras['hooks']: return 0 + _fix_hooks_executables() + empty_commit_id = '0' * 40 detect_force_push = extras.get('detect_force_push') - _fix_hooks_executables() + for push_ref in rev_data: # store our git-env which holds the temp store push_ref['git_env'] = _get_git_env() @@ -586,7 +590,7 @@ def git_pre_receive(unused_repo_path, re if type_ == 'heads' and not (new_branch or delete_branch): old_rev = push_ref['old_rev'] new_rev = push_ref['new_rev'] - cmd = [settings.GIT_EXECUTABLE, 'rev-list', old_rev, f'^{new_rev}'] + cmd = [settings.GIT_EXECUTABLE(), 'rev-list', old_rev, f'^{new_rev}'] stdout, stderr = subprocessio.run_command( cmd, env=os.environ.copy()) # means we're having some non-reachable objects, this forced push was used @@ -611,6 +615,7 @@ def git_post_receive(unused_repo_path, r extras = json.loads(env['RC_SCM_DATA']) if 'push' not in extras['hooks']: return 0 + _fix_hooks_executables() rev_data = _parse_git_ref_lines(revision_lines) @@ -645,14 +650,14 @@ def git_post_receive(unused_repo_path, r repo.set_head(need_head_set) print(f"Setting default branch to {push_ref_name}") - cmd = [settings.GIT_EXECUTABLE, 'for-each-ref', '--format=%(refname)', 'refs/heads/*'] + cmd = [settings.GIT_EXECUTABLE(), 'for-each-ref', '--format=%(refname)', 'refs/heads/*'] stdout, stderr = subprocessio.run_command( cmd, env=os.environ.copy()) heads = safe_str(stdout) heads = heads.replace(push_ref['ref'], '') heads = ' '.join(head for head in heads.splitlines() if head) or '.' - cmd = [settings.GIT_EXECUTABLE, 'log', '--reverse', + cmd = [settings.GIT_EXECUTABLE(), 'log', '--reverse', '--pretty=format:%H', '--', push_ref['new_rev'], '--not', heads] stdout, stderr = subprocessio.run_command( @@ -666,7 +671,7 @@ def git_post_receive(unused_repo_path, r if push_ref['name'] not in branches: branches.append(push_ref['name']) - cmd = [settings.GIT_EXECUTABLE, 'log', + cmd = [settings.GIT_EXECUTABLE(), 'log', f'{push_ref["old_rev"]}..{push_ref["new_rev"]}', '--reverse', '--pretty=format:%H'] stdout, stderr = subprocessio.run_command( @@ -716,9 +721,11 @@ def git_post_receive(unused_repo_path, r def _get_extras_from_txn_id(path, txn_id): + _fix_hooks_executables() + extras = {} try: - cmd = [settings.SVNLOOK_EXECUTABLE, 'pget', + cmd = [settings.SVNLOOK_EXECUTABLE(), 'pget', '-t', txn_id, '--revprop', path, 'rc-scm-extras'] stdout, stderr = subprocessio.run_command( @@ -731,9 +738,11 @@ def _get_extras_from_txn_id(path, txn_id def _get_extras_from_commit_id(commit_id, path): + _fix_hooks_executables() + extras = {} try: - cmd = [settings.SVNLOOK_EXECUTABLE, 'pget', + cmd = [settings.SVNLOOK_EXECUTABLE(), 'pget', '-r', commit_id, '--revprop', path, 'rc-scm-extras'] stdout, stderr = subprocessio.run_command( @@ -746,11 +755,11 @@ def _get_extras_from_commit_id(commit_id def svn_pre_commit(repo_path, commit_data, env): + path, txn_id = commit_data branches = [] tags = [] - _fix_hooks_executables() if env.get('RC_SCM_DATA'): extras = json.loads(env['RC_SCM_DATA']) else: @@ -790,7 +799,6 @@ def svn_post_commit(repo_path, commit_da branches = [] tags = [] - _fix_hooks_executables() if env.get('RC_SCM_DATA'): extras = json.loads(env['RC_SCM_DATA']) else: diff --git a/vcsserver/http_main.py b/vcsserver/http_main.py --- a/vcsserver/http_main.py +++ b/vcsserver/http_main.py @@ -263,11 +263,6 @@ class HTTPApplication: settings.BINARY_DIR = binary_dir - # from core.binary dir we set executable paths - settings.GIT_EXECUTABLE = os.path.join(binary_dir, settings.GIT_EXECUTABLE) - settings.SVN_EXECUTABLE = os.path.join(binary_dir, settings.SVN_EXECUTABLE) - settings.SVNLOOK_EXECUTABLE = os.path.join(binary_dir, settings.SVNLOOK_EXECUTABLE) - # Store the settings to make them available to other modules. vcsserver.PYRAMID_SETTINGS = settings_merged vcsserver.CONFIG = settings_merged @@ -716,7 +711,9 @@ def sanitize_settings_and_apply_defaults settings_maker.make_setting('pyramid.default_locale_name', 'en') settings_maker.make_setting('locale', 'en_US.UTF-8') - settings_maker.make_setting('core.binary_dir', '/usr/local/bin/rhodecode_bin/vcs_bin') + settings_maker.make_setting( + 'core.binary_dir', '/usr/local/bin/rhodecode_bin/vcs_bin', + default_when_empty=True, parser='string:noquote') temp_store = tempfile.gettempdir() default_cache_dir = os.path.join(temp_store, 'rc_cache') diff --git a/vcsserver/lib/config_utils.py b/vcsserver/lib/config_utils.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/config_utils.py @@ -0,0 +1,40 @@ +# Copyright (C) 2010-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import os + + +def get_config(ini_path, **kwargs): + import configparser + parser = configparser.ConfigParser(**kwargs) + parser.read(ini_path) + return parser + + +def get_app_config_lightweight(ini_path): + parser = get_config(ini_path) + parser.set('app:main', 'here', os.getcwd()) + parser.set('app:main', '__file__', ini_path) + return dict(parser.items('app:main')) + + +def get_app_config(ini_path): + """ + This loads the app context and provides a heavy type iniliaziation of config + """ + from paste.deploy.loadwsgi import appconfig + return appconfig(f'config:{ini_path}', relative_to=os.getcwd()) diff --git a/vcsserver/remote/git_remote.py b/vcsserver/remote/git_remote.py --- a/vcsserver/remote/git_remote.py +++ b/vcsserver/remote/git_remote.py @@ -40,7 +40,7 @@ from dulwich.repo import Repo as Dulwich import rhodecode from vcsserver import exceptions, settings, subprocessio -from vcsserver.str_utils import safe_str, safe_int, safe_bytes, ascii_bytes, convert_to_str +from vcsserver.str_utils import safe_str, safe_int, safe_bytes, ascii_bytes, convert_to_str, splitnewlines from vcsserver.base import RepoFactory, obfuscate_qs, ArchiveNode, store_archive_in_cache, BytesEnvelope, BinaryEnvelope from vcsserver.hgcompat import ( hg_url as url_parser, httpbasicauthhandler, httpdigestauthhandler) @@ -1347,7 +1347,8 @@ class GitRemote(RemoteBase): with repo_init as repo: commit = repo[commit_id] blame_obj = repo.blame(path, newest_commit=commit_id) - for i, line in enumerate(commit.tree[path].data.splitlines()): + file_content = commit.tree[path].data + for i, line in enumerate(splitnewlines(file_content)): line_no = i + 1 hunk = blame_obj.for_line(line_no) blame_commit_id = hunk.final_commit_id.hex @@ -1423,7 +1424,7 @@ class GitRemote(RemoteBase): gitenv['GIT_CONFIG_NOGLOBAL'] = '1' gitenv['GIT_DISCOVERY_ACROSS_FILESYSTEM'] = '1' - cmd = [settings.GIT_EXECUTABLE] + _copts + cmd + cmd = [settings.GIT_EXECUTABLE()] + _copts + cmd _opts = {'env': gitenv, 'shell': False} proc = None diff --git a/vcsserver/scm_app.py b/vcsserver/scm_app.py --- a/vcsserver/scm_app.py +++ b/vcsserver/scm_app.py @@ -214,7 +214,7 @@ def create_git_wsgi_app(repo_path, repo_ :param config: is a dictionary holding the extras. """ - git_path = settings.GIT_EXECUTABLE + git_path = settings.GIT_EXECUTABLE() update_server_info = config.pop('git_update_server_info') app = GitHandler( repo_path, repo_name, git_path, update_server_info, config) @@ -244,7 +244,7 @@ class GitLFSHandler: def create_git_lfs_wsgi_app(repo_path, repo_name, config): - git_path = settings.GIT_EXECUTABLE + git_path = settings.GIT_EXECUTABLE() update_server_info = config.pop('git_update_server_info') git_lfs_enabled = config.pop('git_lfs_enabled') git_lfs_store_path = config.pop('git_lfs_store_path') diff --git a/vcsserver/settings.py b/vcsserver/settings.py --- a/vcsserver/settings.py +++ b/vcsserver/settings.py @@ -14,9 +14,18 @@ # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import os WIRE_ENCODING = 'UTF-8' -GIT_EXECUTABLE = 'git' -SVN_EXECUTABLE = 'svn' -SVNLOOK_EXECUTABLE = 'svnlook' + +# Path where we can find binary dir BINARY_DIR = '' + +def GIT_EXECUTABLE() -> str: + return os.environ.get('RC_GIT_EXECUTABLE') or os.path.join(BINARY_DIR, 'git') + +def SVN_EXECUTABLE() -> str: + return os.environ.get('RC_SVN_EXECUTABLE') or os.path.join(BINARY_DIR, 'svn') + +def SVNLOOK_EXECUTABLE() -> str: + return os.environ.get('RC_SVNLOOK_EXECUTABLE') or os.path.join(BINARY_DIR, 'svnlook') diff --git a/vcsserver/str_utils.py b/vcsserver/str_utils.py --- a/vcsserver/str_utils.py +++ b/vcsserver/str_utils.py @@ -142,3 +142,17 @@ def convert_to_str(data): return list(convert_to_str(item) for item in data) else: return data + + +def splitnewlines(text: bytes): + """ + like splitlines, but only split on newlines. + """ + + lines = [_l + b'\n' for _l in text.split(b'\n')] + if lines: + if lines[-1] == b'\n': + lines.pop() + else: + lines[-1] = lines[-1][:-1] + return lines \ No newline at end of file