diff --git a/configs/development.ini b/configs/development.ini --- a/configs/development.ini +++ b/configs/development.ini @@ -66,6 +66,10 @@ debugtoolbar.exclude_prefixes = ; or /usr/local/bin/rhodecode_bin/vcs_bin core.binary_dir = +; Redis connection settings for svn integrations logic +; This connection string needs to be the same on ce and vcsserver +vcs.svn.redis_conn = redis://redis:6379/0 + ; Custom exception store path, defaults to TMPDIR ; This is used to store exception from RhodeCode in shared directory #exception_tracker.store_path = diff --git a/configs/production.ini b/configs/production.ini --- a/configs/production.ini +++ b/configs/production.ini @@ -46,6 +46,10 @@ use = egg:rhodecode-vcsserver ; or /usr/local/bin/rhodecode_bin/vcs_bin core.binary_dir = +; Redis connection settings for svn integrations logic +; This connection string needs to be the same on ce and vcsserver +vcs.svn.redis_conn = redis://redis:6379/0 + ; Custom exception store path, defaults to TMPDIR ; This is used to store exception from RhodeCode in shared directory #exception_tracker.store_path = diff --git a/vcsserver/hook_utils/__init__.py b/vcsserver/hook_utils/__init__.py --- a/vcsserver/hook_utils/__init__.py +++ b/vcsserver/hook_utils/__init__.py @@ -87,8 +87,16 @@ def install_git_hooks(repo_path, bare, e if _rhodecode_hook or force_create: log.debug('writing git %s hook file at %s !', h_type, _hook_file) + env_expand = str([ + ('RC_INI_FILE', vcsserver.CONFIG['__file__']), + ('RC_CORE_BINARY_DIR', vcsserver.settings.BINARY_DIR), + ('RC_GIT_EXECUTABLE', vcsserver.settings.GIT_EXECUTABLE()), + ('RC_SVN_EXECUTABLE', vcsserver.settings.SVN_EXECUTABLE()), + ('RC_SVNLOOK_EXECUTABLE', vcsserver.settings.SVNLOOK_EXECUTABLE()), + ]) try: with open(_hook_file, 'wb') as f: + template = template.replace(b'_OS_EXPAND_', safe_bytes(env_expand)) template = template.replace(b'_TMPL_', safe_bytes(vcsserver.get_version())) template = template.replace(b'_DATE_', safe_bytes(timestamp)) template = template.replace(b'_ENV_', safe_bytes(executable)) @@ -141,17 +149,17 @@ def install_svn_hooks(repo_path, executa log.debug('writing svn %s hook file at %s !', h_type, _hook_file) env_expand = str([ + ('RC_INI_FILE', vcsserver.CONFIG['__file__']), ('RC_CORE_BINARY_DIR', vcsserver.settings.BINARY_DIR), ('RC_GIT_EXECUTABLE', vcsserver.settings.GIT_EXECUTABLE()), ('RC_SVN_EXECUTABLE', vcsserver.settings.SVN_EXECUTABLE()), ('RC_SVNLOOK_EXECUTABLE', vcsserver.settings.SVNLOOK_EXECUTABLE()), - ]) try: with open(_hook_file, 'wb') as f: + template = template.replace(b'_OS_EXPAND_', safe_bytes(env_expand)) template = template.replace(b'_TMPL_', safe_bytes(vcsserver.get_version())) template = template.replace(b'_DATE_', safe_bytes(timestamp)) - template = template.replace(b'_OS_EXPAND_', safe_bytes(env_expand)) template = template.replace(b'_ENV_', safe_bytes(executable)) template = template.replace(b'_PATH_', safe_bytes(path)) diff --git a/vcsserver/hook_utils/hook_templates/git_post_receive.py.tmpl b/vcsserver/hook_utils/hook_templates/git_post_receive.py.tmpl --- a/vcsserver/hook_utils/hook_templates/git_post_receive.py.tmpl +++ b/vcsserver/hook_utils/hook_templates/git_post_receive.py.tmpl @@ -1,4 +1,5 @@ #!_ENV_ + import os import sys path_adjust = [_PATH_] @@ -6,6 +7,11 @@ path_adjust = [_PATH_] if path_adjust: sys.path = path_adjust +# special trick to pass in some information from rc to hooks +# mod_dav strips ALL env vars and we can't even access things like PATH +for env_k, env_v in _OS_EXPAND_: + os.environ[env_k] = env_v + try: from vcsserver import hooks except ImportError: @@ -30,11 +36,13 @@ def main(): repo_path = os.getcwd() push_data = sys.stdin.readlines() - os.environ['RC_HOOK_VER'] = RC_HOOK_VER + # os.environ is modified here by a subprocess call that # runs git and later git executes this hook. # Environ gets some additional info from rhodecode system # like IP or username from basic-auth + + os.environ['RC_HOOK_VER'] = RC_HOOK_VER try: result = hooks.git_post_receive(repo_path, push_data, os.environ) sys.exit(result) diff --git a/vcsserver/hook_utils/hook_templates/git_pre_receive.py.tmpl b/vcsserver/hook_utils/hook_templates/git_pre_receive.py.tmpl --- a/vcsserver/hook_utils/hook_templates/git_pre_receive.py.tmpl +++ b/vcsserver/hook_utils/hook_templates/git_pre_receive.py.tmpl @@ -1,4 +1,5 @@ #!_ENV_ + import os import sys path_adjust = [_PATH_] @@ -6,6 +7,11 @@ path_adjust = [_PATH_] if path_adjust: sys.path = path_adjust +# special trick to pass in some information from rc to hooks +# mod_dav strips ALL env vars and we can't even access things like PATH +for env_k, env_v in _OS_EXPAND_: + os.environ[env_k] = env_v + try: from vcsserver import hooks except ImportError: @@ -30,11 +36,13 @@ def main(): repo_path = os.getcwd() push_data = sys.stdin.readlines() - os.environ['RC_HOOK_VER'] = RC_HOOK_VER + # os.environ is modified here by a subprocess call that # runs git and later git executes this hook. # Environ gets some additional info from rhodecode system # like IP or username from basic-auth + + os.environ['RC_HOOK_VER'] = RC_HOOK_VER try: result = hooks.git_pre_receive(repo_path, push_data, os.environ) sys.exit(result) diff --git a/vcsserver/hook_utils/hook_templates/svn_post_commit_hook.py.tmpl b/vcsserver/hook_utils/hook_templates/svn_post_commit_hook.py.tmpl --- a/vcsserver/hook_utils/hook_templates/svn_post_commit_hook.py.tmpl +++ b/vcsserver/hook_utils/hook_templates/svn_post_commit_hook.py.tmpl @@ -7,6 +7,11 @@ path_adjust = [_PATH_] if path_adjust: sys.path = path_adjust +# special trick to pass in some information from rc to hooks +# mod_dav strips ALL env vars and we can't even access things like PATH +for env_k, env_v in _OS_EXPAND_: + os.environ[env_k] = env_v + try: from vcsserver import hooks except ImportError: @@ -20,11 +25,6 @@ except ImportError: RC_HOOK_VER = '_TMPL_' -# special trick to pass in some information from rc to hooks -# mod_dav strips ALL env vars and we can't even access things like PATH -for env_k, env_v in _OS_EXPAND_: - os.environ[env_k] = env_v - def main(): if hooks is None: # exit with success if we cannot import vcsserver.hooks !! @@ -33,13 +33,13 @@ def main(): if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'): sys.exit(0) - repo_path = os.getcwd() + cwd_repo_path = os.getcwd() push_data = sys.argv[1:] os.environ['RC_HOOK_VER'] = RC_HOOK_VER try: - result = hooks.svn_post_commit(repo_path, push_data, os.environ) + result = hooks.svn_post_commit(cwd_repo_path, push_data, os.environ) sys.exit(result) except Exception as error: # TODO: johbo: Improve handling of this special case diff --git a/vcsserver/hook_utils/hook_templates/svn_pre_commit_hook.py.tmpl b/vcsserver/hook_utils/hook_templates/svn_pre_commit_hook.py.tmpl --- a/vcsserver/hook_utils/hook_templates/svn_pre_commit_hook.py.tmpl +++ b/vcsserver/hook_utils/hook_templates/svn_pre_commit_hook.py.tmpl @@ -7,6 +7,11 @@ path_adjust = [_PATH_] if path_adjust: sys.path = path_adjust +# special trick to pass in some information from rc to hooks +# mod_dav strips ALL env vars and we can't even access things like PATH +for env_k, env_v in _OS_EXPAND_: + os.environ[env_k] = env_v + try: from vcsserver import hooks except ImportError: @@ -20,11 +25,6 @@ except ImportError: RC_HOOK_VER = '_TMPL_' -# special trick to pass in some information from rc to hooks -# mod_dav strips ALL env vars and we can't even access things like PATH -for env_k, env_v in _OS_EXPAND_: - os.environ[env_k] = env_v - def main(): if os.environ.get('SSH_READ_ONLY') == '1': sys.stderr.write('Only read-only access is allowed') @@ -37,13 +37,12 @@ def main(): if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'): sys.exit(0) - repo_path = os.getcwd() + cwd_repo_path = os.getcwd() push_data = sys.argv[1:] os.environ['RC_HOOK_VER'] = RC_HOOK_VER - try: - result = hooks.svn_pre_commit(repo_path, push_data, os.environ) + result = hooks.svn_pre_commit(cwd_repo_path, push_data, os.environ) sys.exit(result) except Exception as error: # TODO: johbo: Improve handling of this special case diff --git a/vcsserver/hooks.py b/vcsserver/hooks.py --- a/vcsserver/hooks.py +++ b/vcsserver/hooks.py @@ -31,9 +31,10 @@ from celery import Celery import mercurial.scmutil import mercurial.node +from vcsserver import exceptions, subprocessio, settings from vcsserver.lib.ext_json import json -from vcsserver import exceptions, subprocessio, settings from vcsserver.lib.str_utils import ascii_str, safe_str +from vcsserver.lib.svn_txn_utils import get_txn_id_from_store from vcsserver.remote.git_remote import Repository celery_app = Celery('__vcsserver__') @@ -293,20 +294,28 @@ def _get_hg_env(old_rev, new_rev, txnid, return [(k, v) for k, v in env.items()] +def _get_ini_settings(ini_file): + from vcsserver.http_main import sanitize_settings_and_apply_defaults + from vcsserver.lib.config_utils import get_app_config_lightweight, configure_and_store_settings + + global_config = {'__file__': ini_file} + ini_settings = get_app_config_lightweight(ini_file) + sanitize_settings_and_apply_defaults(global_config, ini_settings) + configure_and_store_settings(global_config, ini_settings) + + return ini_settings + + def _fix_hooks_executables(ini_path=''): """ This is a trick to set proper settings.EXECUTABLE paths for certain execution patterns especially for subversion where hooks strip entire env, and calling just 'svn' command will most likely fail because svn is not on PATH """ - from vcsserver.http_main import sanitize_settings_and_apply_defaults - from vcsserver.lib.config_utils import get_app_config_lightweight - + # set defaults, in case we can't read from ini_file core_binary_dir = settings.BINARY_DIR or '/usr/local/bin/rhodecode_bin/vcs_bin' if ini_path: - - ini_settings = get_app_config_lightweight(ini_path) - ini_settings = sanitize_settings_and_apply_defaults({'__file__': ini_path}, ini_settings) + ini_settings = _get_ini_settings(ini_path) core_binary_dir = ini_settings['core.binary_dir'] settings.BINARY_DIR = core_binary_dir @@ -570,7 +579,7 @@ def git_pre_receive(unused_repo_path, re rev_data = _parse_git_ref_lines(revision_lines) if 'push' not in extras['hooks']: return 0 - _fix_hooks_executables() + _fix_hooks_executables(env.get('RC_INI_FILE')) empty_commit_id = '0' * 40 @@ -616,7 +625,7 @@ def git_post_receive(unused_repo_path, r if 'push' not in extras['hooks']: return 0 - _fix_hooks_executables() + _fix_hooks_executables(env.get('RC_INI_FILE')) rev_data = _parse_git_ref_lines(revision_lines) @@ -720,37 +729,8 @@ def git_post_receive(unused_repo_path, r return status_code -def _get_extras_from_txn_id(path, txn_id): - _fix_hooks_executables() - - extras = {} - try: - cmd = [settings.SVNLOOK_EXECUTABLE(), 'pget', - '-t', txn_id, - '--revprop', path, 'rc-scm-extras'] - stdout, stderr = subprocessio.run_command( - cmd, env=os.environ.copy()) - extras = json.loads(base64.urlsafe_b64decode(stdout)) - except Exception: - log.exception('Failed to extract extras info from txn_id') - - return extras - - -def _get_extras_from_commit_id(commit_id, path): - _fix_hooks_executables() - - extras = {} - try: - cmd = [settings.SVNLOOK_EXECUTABLE(), 'pget', - '-r', commit_id, - '--revprop', path, 'rc-scm-extras'] - stdout, stderr = subprocessio.run_command( - cmd, env=os.environ.copy()) - extras = json.loads(base64.urlsafe_b64decode(stdout)) - except Exception: - log.exception('Failed to extract extras info from commit_id') - +def get_extras_from_txn_id(repo_path, txn_id): + extras = get_txn_id_from_store(repo_path, txn_id) return extras @@ -763,13 +743,18 @@ def svn_pre_commit(repo_path, commit_dat if env.get('RC_SCM_DATA'): extras = json.loads(env['RC_SCM_DATA']) else: + ini_path = env.get('RC_INI_FILE') + if ini_path: + _get_ini_settings(ini_path) # fallback method to read from TXN-ID stored data - extras = _get_extras_from_txn_id(path, txn_id) + extras = get_extras_from_txn_id(path, txn_id) if not extras: - #TODO: temporary fix until svn txn-id changes are merged + raise ValueError('SVN-PRE-COMMIT: Failed to extract context data in called extras for hook execution') + + if extras.get('rc_internal_commit'): + # special marker for internal commit, we don't call hooks client return 0 - raise ValueError('Failed to extract context data called extras for hook execution') extras['hook_type'] = 'pre_commit' extras['commit_ids'] = [txn_id] @@ -805,13 +790,18 @@ def svn_post_commit(repo_path, commit_da if env.get('RC_SCM_DATA'): extras = json.loads(env['RC_SCM_DATA']) else: + ini_path = env.get('RC_INI_FILE') + if ini_path: + _get_ini_settings(ini_path) # fallback method to read from TXN-ID stored data - extras = _get_extras_from_commit_id(commit_id, path) + extras = get_extras_from_txn_id(path, txn_id) - if not extras: - #TODO: temporary fix until svn txn-id changes are merged + if not extras and txn_id: + raise ValueError('SVN-POST-COMMIT: Failed to extract context data in called extras for hook execution') + + if extras.get('rc_internal_commit'): + # special marker for internal commit, we don't call hooks client return 0 - raise ValueError('Failed to extract context data called extras for hook execution') extras['hook_type'] = 'post_commit' extras['commit_ids'] = [commit_id] diff --git a/vcsserver/http_main.py b/vcsserver/http_main.py --- a/vcsserver/http_main.py +++ b/vcsserver/http_main.py @@ -37,20 +37,23 @@ from pyramid.wsgi import wsgiapp from pyramid.response import Response from vcsserver.base import BytesEnvelope, BinaryEnvelope -from vcsserver.lib.ext_json import json + from vcsserver.config.settings_maker import SettingsMaker -from vcsserver.lib.str_utils import safe_int -from vcsserver.lib.statsd_client import StatsdClient + from vcsserver.tweens.request_wrapper import get_headers_call_context -import vcsserver -from vcsserver import remote_wsgi, scm_app, settings, hgpatches +from vcsserver import remote_wsgi, scm_app, hgpatches +from vcsserver.server import VcsServer from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub from vcsserver.echo_stub.echo_app import EchoApp from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected from vcsserver.lib.exc_tracking import store_exception, format_exc -from vcsserver.server import VcsServer +from vcsserver.lib.str_utils import safe_int +from vcsserver.lib.statsd_client import StatsdClient +from vcsserver.lib.ext_json import json +from vcsserver.lib.config_utils import configure_and_store_settings + strict_vcs = True @@ -94,8 +97,7 @@ log = logging.getLogger(__name__) try: locale.setlocale(locale.LC_ALL, '') except locale.Error as e: - log.error( - 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e) + log.error('LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e) os.environ['LC_ALL'] = 'C' @@ -248,25 +250,10 @@ class HTTPApplication: log.warning("Using EchoApp for VCS operations.") self.remote_wsgi = remote_wsgi_stub - self._configure_settings(global_config, settings) + configure_and_store_settings(global_config, settings) self._configure() - def _configure_settings(self, global_config, app_settings): - """ - Configure the settings module. - """ - settings_merged = global_config.copy() - settings_merged.update(app_settings) - - binary_dir = app_settings['core.binary_dir'] - - settings.BINARY_DIR = binary_dir - - # Store the settings to make them available to other modules. - vcsserver.PYRAMID_SETTINGS = settings_merged - vcsserver.CONFIG = settings_merged - def _configure(self): self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory) @@ -715,6 +702,8 @@ def sanitize_settings_and_apply_defaults 'core.binary_dir', '/usr/local/bin/rhodecode_bin/vcs_bin', default_when_empty=True, parser='string:noquote') + settings_maker.make_setting('vcs.svn.redis_conn', 'redis://redis:6379/0') + temp_store = tempfile.gettempdir() default_cache_dir = os.path.join(temp_store, 'rc_cache') # save default, cache dir, and use it for all backends later. diff --git a/vcsserver/lib/config_utils.py b/vcsserver/lib/config_utils.py --- a/vcsserver/lib/config_utils.py +++ b/vcsserver/lib/config_utils.py @@ -16,6 +16,8 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ import os +import vcsserver +import vcsserver.settings def get_config(ini_path, **kwargs): @@ -38,3 +40,19 @@ def get_app_config(ini_path): """ from paste.deploy.loadwsgi import appconfig return appconfig(f'config:{ini_path}', relative_to=os.getcwd()) + + +def configure_and_store_settings(global_config, app_settings): + """ + Configure the settings module. + """ + settings_merged = global_config.copy() + settings_merged.update(app_settings) + + binary_dir = app_settings['core.binary_dir'] + + vcsserver.settings.BINARY_DIR = binary_dir + + # Store the settings to make them available to other modules. + vcsserver.PYRAMID_SETTINGS = settings_merged + vcsserver.CONFIG = settings_merged diff --git a/vcsserver/lib/svn_txn_utils.py b/vcsserver/lib/svn_txn_utils.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/svn_txn_utils.py @@ -0,0 +1,111 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2023 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import logging +import redis + +from ..lib import rc_cache +from ..lib.ext_json import json + + +log = logging.getLogger(__name__) + +redis_client = None + + +class RedisTxnClient: + + def __init__(self, url): + self.url = url + self._create_client(url) + + def _create_client(self, url): + connection_pool = redis.ConnectionPool.from_url(url) + self.writer_client = redis.StrictRedis( + connection_pool=connection_pool + ) + self.reader_client = self.writer_client + + def set(self, key, value): + self.writer_client.set(key, value) + + def get(self, key): + return self.reader_client.get(key) + + def delete(self, key): + self.writer_client.delete(key) + + +def get_redis_client(url=''): + + global redis_client + if redis_client is not None: + return redis_client + if not url: + from vcsserver import CONFIG + url = CONFIG['vcs.svn.redis_conn'] + redis_client = RedisTxnClient(url) + return redis_client + + +def get_txn_id_data_key(repo_path, svn_txn_id): + log.debug('svn-txn-id: %s, obtaining data path', svn_txn_id) + repo_key = rc_cache.utils.compute_key_from_params(repo_path) + final_key = f'{repo_key}.{svn_txn_id}.svn_txn_id' + log.debug('computed final key: %s', final_key) + + return final_key + + +def store_txn_id_data(repo_path, svn_txn_id, data_dict): + log.debug('svn-txn-id: %s, storing data', svn_txn_id) + + if not svn_txn_id: + log.warning('Cannot store txn_id because it is empty') + return + + redis_conn = get_redis_client() + + store_key = get_txn_id_data_key(repo_path, svn_txn_id) + store_data = json.dumps(data_dict) + redis_conn.set(store_key, store_data) + + +def get_txn_id_from_store(repo_path, svn_txn_id, rm_on_read=False): + """ + Reads txn_id from store and if present returns the data for callback manager + """ + log.debug('svn-txn-id: %s, retrieving data', svn_txn_id) + redis_conn = get_redis_client() + + store_key = get_txn_id_data_key(repo_path, svn_txn_id) + data = {} + redis_conn.get(store_key) + raw_data = 'not-set' + try: + raw_data = redis_conn.get(store_key) + if not raw_data: + raise ValueError(f'Failed to get txn_id metadata, from store: {store_key}') + data = json.loads(raw_data) + except Exception: + log.exception('Failed to get txn_id metadata: %s', raw_data) + + if rm_on_read: + log.debug('Cleaning up txn_id at %s', store_key) + redis_conn.delete(store_key) + + return data diff --git a/vcsserver/remote/svn_remote.py b/vcsserver/remote/svn_remote.py --- a/vcsserver/remote/svn_remote.py +++ b/vcsserver/remote/svn_remote.py @@ -28,7 +28,6 @@ import urllib.parse import urllib.error import traceback - import svn.client # noqa import svn.core # noqa import svn.delta # noqa @@ -47,10 +46,11 @@ from vcsserver.base import ( BinaryEnvelope, ) from vcsserver.exceptions import NoContentException +from vcsserver.vcs_base import RemoteBase from vcsserver.lib.str_utils import safe_str, safe_bytes from vcsserver.lib.type_utils import assert_bytes -from vcsserver.vcs_base import RemoteBase from vcsserver.lib.svnremoterepo import svnremoterepo +from vcsserver.lib.svn_txn_utils import store_txn_id_data log = logging.getLogger(__name__) @@ -503,6 +503,11 @@ class SvnRemote(RemoteBase): for node in removed: TxnNodeProcessor(node, txn_root).remove() + svn_txn_id = safe_str(svn.fs.svn_fs_txn_name(txn)) + full_repo_path = wire['path'] + txn_id_data = {'svn_txn_id': svn_txn_id, 'rc_internal_commit': True} + + store_txn_id_data(full_repo_path, svn_txn_id, txn_id_data) commit_id = svn.repos.fs_commit_txn(repo, txn) if timestamp: