fix(svn): svn events fixes

vcsserver/lib/svn_txn_utils.py

@@ -0,0 +1,111 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2023 RhodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import logging
+import redis
+
+from ..lib import rc_cache
+from ..lib.ext_json import json
+
+
+log = logging.getLogger(__name__)
+
+redis_client = None
+
+
+class RedisTxnClient:
+
+    def __init__(self, url):
+        self.url = url
+        self._create_client(url)
+
+    def _create_client(self, url):
+        connection_pool = redis.ConnectionPool.from_url(url)
+        self.writer_client = redis.StrictRedis(
+            connection_pool=connection_pool
+        )
+        self.reader_client = self.writer_client
+
+    def set(self, key, value):
+        self.writer_client.set(key, value)
+
+    def get(self, key):
+        return self.reader_client.get(key)
+
+    def delete(self, key):
+        self.writer_client.delete(key)
+
+
+def get_redis_client(url=''):
+
+    global redis_client
+    if redis_client is not None:
+        return redis_client
+    if not url:
+        from vcsserver import CONFIG
+        url = CONFIG['vcs.svn.redis_conn']
+    redis_client = RedisTxnClient(url)
+    return redis_client
+
+
+def get_txn_id_data_key(repo_path, svn_txn_id):
+    log.debug('svn-txn-id: %s, obtaining data path', svn_txn_id)
+    repo_key = rc_cache.utils.compute_key_from_params(repo_path)
+    final_key = f'{repo_key}.{svn_txn_id}.svn_txn_id'
+    log.debug('computed final key: %s', final_key)
+
+    return final_key
+
+
+def store_txn_id_data(repo_path, svn_txn_id, data_dict):
+    log.debug('svn-txn-id: %s, storing data', svn_txn_id)
+
+    if not svn_txn_id:
+        log.warning('Cannot store txn_id because it is empty')
+        return
+
+    redis_conn = get_redis_client()
+
+    store_key = get_txn_id_data_key(repo_path, svn_txn_id)
+    store_data = json.dumps(data_dict)
+    redis_conn.set(store_key, store_data)
+
+
+def get_txn_id_from_store(repo_path, svn_txn_id, rm_on_read=False):
+    """
+    Reads txn_id from store and if present returns the data for callback manager
+    """
+    log.debug('svn-txn-id: %s, retrieving data', svn_txn_id)
+    redis_conn = get_redis_client()
+
+    store_key = get_txn_id_data_key(repo_path, svn_txn_id)
+    data = {}
+    redis_conn.get(store_key)
+    raw_data = 'not-set'
+    try:
+        raw_data = redis_conn.get(store_key)
+        if not raw_data:
+            raise ValueError(f'Failed to get txn_id metadata, from store: {store_key}')
+        data = json.loads(raw_data)
+    except Exception:
+        log.exception('Failed to get txn_id metadata: %s', raw_data)
+
+    if rm_on_read:
+        log.debug('Cleaning up txn_id at %s', store_key)
+        redis_conn.delete(store_key)
+
+    return data

configs/development.ini

@@ -66,6 +66,10 b' debugtoolbar.exclude_prefixes ='
 ; or /usr/local/bin/rhodecode_bin/vcs_bin
 core.binary_dir =
 
+; Redis connection settings for svn integrations logic
+; This connection string needs to be the same on ce and vcsserver
+vcs.svn.redis_conn = redis://redis:6379/0
+
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =

configs/production.ini

@@ -46,6 +46,10 b' use = egg:rhodecode-vcsserver'
 ; or /usr/local/bin/rhodecode_bin/vcs_bin
 core.binary_dir =
 
+; Redis connection settings for svn integrations logic
+; This connection string needs to be the same on ce and vcsserver
+vcs.svn.redis_conn = redis://redis:6379/0
+
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =

vcsserver/hook_utils/__init__.py

@@ -87,8 +87,16 b' def install_git_hooks(repo_path, bare, e'
 
         if _rhodecode_hook or force_create:
             log.debug('writing git %s hook file at %s !', h_type, _hook_file)
+            env_expand = str([
+                ('RC_INI_FILE',  vcsserver.CONFIG['__file__']),
+                ('RC_CORE_BINARY_DIR', vcsserver.settings.BINARY_DIR),
+                ('RC_GIT_EXECUTABLE', vcsserver.settings.GIT_EXECUTABLE()),
+                ('RC_SVN_EXECUTABLE', vcsserver.settings.SVN_EXECUTABLE()),
+                ('RC_SVNLOOK_EXECUTABLE', vcsserver.settings.SVNLOOK_EXECUTABLE()),
+            ])
             try:
                 with open(_hook_file, 'wb') as f:
+                    template = template.replace(b'_OS_EXPAND_', safe_bytes(env_expand))
                     template = template.replace(b'_TMPL_', safe_bytes(vcsserver.get_version()))
                     template = template.replace(b'_DATE_', safe_bytes(timestamp))
                     template = template.replace(b'_ENV_', safe_bytes(executable))
@@ -141,17 +149,17 b' def install_svn_hooks(repo_path, executa'
             log.debug('writing svn %s hook file at %s !', h_type, _hook_file)
 
             env_expand = str([
+                ('RC_INI_FILE',  vcsserver.CONFIG['__file__']),
                 ('RC_CORE_BINARY_DIR', vcsserver.settings.BINARY_DIR),
                 ('RC_GIT_EXECUTABLE', vcsserver.settings.GIT_EXECUTABLE()),
                 ('RC_SVN_EXECUTABLE', vcsserver.settings.SVN_EXECUTABLE()),
                 ('RC_SVNLOOK_EXECUTABLE', vcsserver.settings.SVNLOOK_EXECUTABLE()),
-
             ])
             try:
                 with open(_hook_file, 'wb') as f:
+                    template = template.replace(b'_OS_EXPAND_', safe_bytes(env_expand))
                     template = template.replace(b'_TMPL_', safe_bytes(vcsserver.get_version()))
                     template = template.replace(b'_DATE_', safe_bytes(timestamp))
-                    template = template.replace(b'_OS_EXPAND_', safe_bytes(env_expand))
                     template = template.replace(b'_ENV_', safe_bytes(executable))
                     template = template.replace(b'_PATH_', safe_bytes(path))
 

vcsserver/hook_utils/hook_templates/git_post_receive.py.tmpl

@@ -1,4 +1,5 b''
 #!_ENV_
+
 import os
 import sys
 path_adjust = [_PATH_]
@@ -6,6 +7,11 b' path_adjust = [_PATH_]'
 if path_adjust:
     sys.path = path_adjust
 
+# special trick to pass in some information from rc to hooks
+# mod_dav strips ALL env vars and we can't even access things like PATH
+for env_k, env_v in _OS_EXPAND_:
+    os.environ[env_k] = env_v
+
 try:
     from vcsserver import hooks
 except ImportError:
@@ -30,11 +36,13 b' def main():'
 
     repo_path = os.getcwd()
     push_data = sys.stdin.readlines()
-    os.environ['RC_HOOK_VER'] = RC_HOOK_VER
+
     # os.environ is modified here by a subprocess call that
     # runs git and later git executes this hook.
     # Environ gets some additional info from rhodecode system
     # like IP or username from basic-auth
+
+    os.environ['RC_HOOK_VER'] = RC_HOOK_VER
     try:
         result = hooks.git_post_receive(repo_path, push_data, os.environ)
         sys.exit(result)

vcsserver/hook_utils/hook_templates/git_pre_receive.py.tmpl

@@ -1,4 +1,5 b''
 #!_ENV_
+
 import os
 import sys
 path_adjust = [_PATH_]
@@ -6,6 +7,11 b' path_adjust = [_PATH_]'
 if path_adjust:
     sys.path = path_adjust
 
+# special trick to pass in some information from rc to hooks
+# mod_dav strips ALL env vars and we can't even access things like PATH
+for env_k, env_v in _OS_EXPAND_:
+    os.environ[env_k] = env_v
+
 try:
     from vcsserver import hooks
 except ImportError:
@@ -30,11 +36,13 b' def main():'
 
     repo_path = os.getcwd()
     push_data = sys.stdin.readlines()
-    os.environ['RC_HOOK_VER'] = RC_HOOK_VER
+
     # os.environ is modified here by a subprocess call that
     # runs git and later git executes this hook.
     # Environ gets some additional info from rhodecode system
     # like IP or username from basic-auth
+
+    os.environ['RC_HOOK_VER'] = RC_HOOK_VER
     try:
         result = hooks.git_pre_receive(repo_path, push_data, os.environ)
         sys.exit(result)

vcsserver/hook_utils/hook_templates/svn_post_commit_hook.py.tmpl

@@ -7,6 +7,11 b' path_adjust = [_PATH_]'
 if path_adjust:
     sys.path = path_adjust
 
+# special trick to pass in some information from rc to hooks
+# mod_dav strips ALL env vars and we can't even access things like PATH
+for env_k, env_v in _OS_EXPAND_:
+    os.environ[env_k] = env_v
+
 try:
     from vcsserver import hooks
 except ImportError:
@@ -20,11 +25,6 b' except ImportError:'
 RC_HOOK_VER = '_TMPL_'
 
 
-# special trick to pass in some information from rc to hooks
-# mod_dav strips ALL env vars and we can't even access things like PATH
-for env_k, env_v in _OS_EXPAND_:
-    os.environ[env_k] = env_v
-
 def main():
     if hooks is None:
         # exit with success if we cannot import vcsserver.hooks !!
@@ -33,13 +33,13 b' def main():'
 
     if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'):
         sys.exit(0)
-    repo_path = os.getcwd()
+    cwd_repo_path = os.getcwd()
     push_data = sys.argv[1:]
 
     os.environ['RC_HOOK_VER'] = RC_HOOK_VER
 
     try:
-        result = hooks.svn_post_commit(repo_path, push_data, os.environ)
+        result = hooks.svn_post_commit(cwd_repo_path, push_data, os.environ)
         sys.exit(result)
     except Exception as error:
         # TODO: johbo: Improve handling of this special case

vcsserver/hook_utils/hook_templates/svn_pre_commit_hook.py.tmpl

@@ -7,6 +7,11 b' path_adjust = [_PATH_]'
 if path_adjust:
     sys.path = path_adjust
 
+# special trick to pass in some information from rc to hooks
+# mod_dav strips ALL env vars and we can't even access things like PATH
+for env_k, env_v in _OS_EXPAND_:
+    os.environ[env_k] = env_v
+
 try:
     from vcsserver import hooks
 except ImportError:
@@ -20,11 +25,6 b' except ImportError:'
 RC_HOOK_VER = '_TMPL_'
 
 
-# special trick to pass in some information from rc to hooks
-# mod_dav strips ALL env vars and we can't even access things like PATH
-for env_k, env_v in _OS_EXPAND_:
-    os.environ[env_k] = env_v
-
 def main():
     if os.environ.get('SSH_READ_ONLY') == '1':
         sys.stderr.write('Only read-only access is allowed')
@@ -37,13 +37,12 b' def main():'
 
     if os.environ.get('RC_SKIP_HOOKS') or os.environ.get('RC_SKIP_SVN_HOOKS'):
         sys.exit(0)
-    repo_path = os.getcwd()
+    cwd_repo_path = os.getcwd()
     push_data = sys.argv[1:]
 
     os.environ['RC_HOOK_VER'] = RC_HOOK_VER
-
     try:
-        result = hooks.svn_pre_commit(repo_path, push_data, os.environ)
+        result = hooks.svn_pre_commit(cwd_repo_path, push_data, os.environ)
         sys.exit(result)
     except Exception as error:
         # TODO: johbo: Improve handling of this special case

vcsserver/hooks.py

@@ -31,9 +31,10 b' from celery import Celery'
 import mercurial.scmutil
 import mercurial.node
 
+from vcsserver import exceptions, subprocessio, settings
 from vcsserver.lib.ext_json import json
-from vcsserver import exceptions, subprocessio, settings
 from vcsserver.lib.str_utils import ascii_str, safe_str
+from vcsserver.lib.svn_txn_utils import get_txn_id_from_store
 from vcsserver.remote.git_remote import Repository
 
 celery_app = Celery('__vcsserver__')
@@ -293,20 +294,28 b' def _get_hg_env(old_rev, new_rev, txnid,'
     return [(k, v) for k, v in env.items()]
 
 
+def _get_ini_settings(ini_file):
+    from vcsserver.http_main import sanitize_settings_and_apply_defaults
+    from vcsserver.lib.config_utils import get_app_config_lightweight, configure_and_store_settings
+
+    global_config = {'__file__': ini_file}
+    ini_settings = get_app_config_lightweight(ini_file)
+    sanitize_settings_and_apply_defaults(global_config, ini_settings)
+    configure_and_store_settings(global_config, ini_settings)
+
+    return ini_settings
+
+
 def _fix_hooks_executables(ini_path=''):
     """
     This is a trick to set proper settings.EXECUTABLE paths for certain execution patterns
     especially for subversion where hooks strip entire env, and calling just 'svn' command will most likely fail
     because svn is not on PATH
     """
-    from vcsserver.http_main import sanitize_settings_and_apply_defaults
-    from vcsserver.lib.config_utils import get_app_config_lightweight
-
+    # set defaults, in case we can't read from ini_file
     core_binary_dir = settings.BINARY_DIR or '/usr/local/bin/rhodecode_bin/vcs_bin'
     if ini_path:
-
-        ini_settings = get_app_config_lightweight(ini_path)
-        ini_settings = sanitize_settings_and_apply_defaults({'__file__': ini_path}, ini_settings)
+        ini_settings = _get_ini_settings(ini_path)
         core_binary_dir = ini_settings['core.binary_dir']
 
     settings.BINARY_DIR = core_binary_dir
@@ -570,7 +579,7 b' def git_pre_receive(unused_repo_path, re'
     rev_data = _parse_git_ref_lines(revision_lines)
     if 'push' not in extras['hooks']:
         return 0
-    _fix_hooks_executables()
+    _fix_hooks_executables(env.get('RC_INI_FILE'))
 
     empty_commit_id = '0' * 40
 
@@ -616,7 +625,7 b' def git_post_receive(unused_repo_path, r'
     if 'push' not in extras['hooks']:
         return 0
 
-    _fix_hooks_executables()
+    _fix_hooks_executables(env.get('RC_INI_FILE'))
 
     rev_data = _parse_git_ref_lines(revision_lines)
 
@@ -720,37 +729,8 b' def git_post_receive(unused_repo_path, r'
     return status_code
 
 
-def _get_extras_from_txn_id(path, txn_id):
-    _fix_hooks_executables()
-
-    extras = {}
-    try:
-        cmd = [settings.SVNLOOK_EXECUTABLE(), 'pget',
-               '-t', txn_id,
-               '--revprop', path, 'rc-scm-extras']
-        stdout, stderr = subprocessio.run_command(
-            cmd, env=os.environ.copy())
-        extras = json.loads(base64.urlsafe_b64decode(stdout))
-    except Exception:
-        log.exception('Failed to extract extras info from txn_id')
-
-    return extras
-
-
-def _get_extras_from_commit_id(commit_id, path):
-    _fix_hooks_executables()
-
-    extras = {}
-    try:
-        cmd = [settings.SVNLOOK_EXECUTABLE(), 'pget',
-               '-r', commit_id,
-               '--revprop', path, 'rc-scm-extras']
-        stdout, stderr = subprocessio.run_command(
-            cmd, env=os.environ.copy())
-        extras = json.loads(base64.urlsafe_b64decode(stdout))
-    except Exception:
-        log.exception('Failed to extract extras info from commit_id')
-
+def get_extras_from_txn_id(repo_path, txn_id):
+    extras = get_txn_id_from_store(repo_path, txn_id)
     return extras
 
 
@@ -763,13 +743,18 b' def svn_pre_commit(repo_path, commit_dat'
     if env.get('RC_SCM_DATA'):
         extras = json.loads(env['RC_SCM_DATA'])
     else:
+        ini_path = env.get('RC_INI_FILE')
+        if ini_path:
+            _get_ini_settings(ini_path)
         # fallback method to read from TXN-ID stored data
-        extras = _get_extras_from_txn_id(path, txn_id)
+        extras = get_extras_from_txn_id(path, txn_id)
 
     if not extras:
-        #TODO: temporary fix until svn txn-id changes are merged
+        raise ValueError('SVN-PRE-COMMIT: Failed to extract context data in called extras for hook execution')
+
+    if extras.get('rc_internal_commit'):
+        # special marker for internal commit, we don't call hooks client
         return 0
-        raise ValueError('Failed to extract context data called extras for hook execution')
 
     extras['hook_type'] = 'pre_commit'
     extras['commit_ids'] = [txn_id]
@@ -805,13 +790,18 b' def svn_post_commit(repo_path, commit_da'
     if env.get('RC_SCM_DATA'):
         extras = json.loads(env['RC_SCM_DATA'])
     else:
+        ini_path = env.get('RC_INI_FILE')
+        if ini_path:
+            _get_ini_settings(ini_path)
         # fallback method to read from TXN-ID stored data
-        extras = _get_extras_from_commit_id(commit_id, path)
+        extras = get_extras_from_txn_id(path, txn_id)
 
-    if not extras:
-        #TODO: temporary fix until svn txn-id changes are merged
+    if not extras and txn_id:
+        raise ValueError('SVN-POST-COMMIT: Failed to extract context data in called extras for hook execution')
+
+    if extras.get('rc_internal_commit'):
+        # special marker for internal commit, we don't call hooks client
         return 0
-        raise ValueError('Failed to extract context data called extras for hook execution')
 
     extras['hook_type'] = 'post_commit'
     extras['commit_ids'] = [commit_id]

vcsserver/http_main.py

@@ -37,20 +37,23 b' from pyramid.wsgi import wsgiapp'
 from pyramid.response import Response
 
 from vcsserver.base import BytesEnvelope, BinaryEnvelope
-from vcsserver.lib.ext_json import json
+
 from vcsserver.config.settings_maker import SettingsMaker
-from vcsserver.lib.str_utils import safe_int
-from vcsserver.lib.statsd_client import StatsdClient
+
 from vcsserver.tweens.request_wrapper import get_headers_call_context
 
-import vcsserver
-from vcsserver import remote_wsgi, scm_app, settings, hgpatches
+from vcsserver import remote_wsgi, scm_app, hgpatches
+from vcsserver.server import VcsServer
 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
 from vcsserver.echo_stub.echo_app import EchoApp
 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
 from vcsserver.lib.exc_tracking import store_exception, format_exc
-from vcsserver.server import VcsServer
+from vcsserver.lib.str_utils import safe_int
+from vcsserver.lib.statsd_client import StatsdClient
+from vcsserver.lib.ext_json import json
+from vcsserver.lib.config_utils import configure_and_store_settings
+
 
 strict_vcs = True
 
@@ -94,8 +97,7 b' log = logging.getLogger(__name__)'
 try:
     locale.setlocale(locale.LC_ALL, '')
 except locale.Error as e:
-    log.error(
-        'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
+    log.error('LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
     os.environ['LC_ALL'] = 'C'
 
 
@@ -248,25 +250,10 b' class HTTPApplication:'
             log.warning("Using EchoApp for VCS operations.")
             self.remote_wsgi = remote_wsgi_stub
 
-        self._configure_settings(global_config, settings)
+        configure_and_store_settings(global_config, settings)
 
         self._configure()
 
-    def _configure_settings(self, global_config, app_settings):
-        """
-        Configure the settings module.
-        """
-        settings_merged = global_config.copy()
-        settings_merged.update(app_settings)
-
-        binary_dir = app_settings['core.binary_dir']
-
-        settings.BINARY_DIR = binary_dir
-
-        # Store the settings to make them available to other modules.
-        vcsserver.PYRAMID_SETTINGS = settings_merged
-        vcsserver.CONFIG = settings_merged
-
     def _configure(self):
         self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
 
@@ -715,6 +702,8 b' def sanitize_settings_and_apply_defaults'
         'core.binary_dir', '/usr/local/bin/rhodecode_bin/vcs_bin',
         default_when_empty=True, parser='string:noquote')
 
+    settings_maker.make_setting('vcs.svn.redis_conn', 'redis://redis:6379/0')
+
     temp_store = tempfile.gettempdir()
     default_cache_dir = os.path.join(temp_store, 'rc_cache')
     # save default, cache dir, and use it for all backends later.

vcsserver/lib/config_utils.py

@@ -16,6 +16,8 b''
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import os
+import vcsserver
+import vcsserver.settings
 
 
 def get_config(ini_path, **kwargs):
@@ -38,3 +40,19 b' def get_app_config(ini_path):'
     """
     from paste.deploy.loadwsgi import appconfig
     return appconfig(f'config:{ini_path}', relative_to=os.getcwd())
+
+
+def configure_and_store_settings(global_config, app_settings):
+    """
+    Configure the settings module.
+    """
+    settings_merged = global_config.copy()
+    settings_merged.update(app_settings)
+
+    binary_dir = app_settings['core.binary_dir']
+
+    vcsserver.settings.BINARY_DIR = binary_dir
+
+    # Store the settings to make them available to other modules.
+    vcsserver.PYRAMID_SETTINGS = settings_merged
+    vcsserver.CONFIG = settings_merged

vcsserver/remote/svn_remote.py

@@ -28,7 +28,6 b' import urllib.parse'
 import urllib.error
 import traceback
 
-
 import svn.client  # noqa
 import svn.core  # noqa
 import svn.delta  # noqa
@@ -47,10 +46,11 b' from vcsserver.base import ('
     BinaryEnvelope,
 )
 from vcsserver.exceptions import NoContentException
+from vcsserver.vcs_base import RemoteBase
 from vcsserver.lib.str_utils import safe_str, safe_bytes
 from vcsserver.lib.type_utils import assert_bytes
-from vcsserver.vcs_base import RemoteBase
 from vcsserver.lib.svnremoterepo import svnremoterepo
+from vcsserver.lib.svn_txn_utils import store_txn_id_data
 
 log = logging.getLogger(__name__)
 
@@ -503,6 +503,11 b' class SvnRemote(RemoteBase):'
         for node in removed:
             TxnNodeProcessor(node, txn_root).remove()
 
+        svn_txn_id = safe_str(svn.fs.svn_fs_txn_name(txn))
+        full_repo_path = wire['path']
+        txn_id_data = {'svn_txn_id': svn_txn_id, 'rc_internal_commit': True}
+
+        store_txn_id_data(full_repo_path, svn_txn_id, txn_id_data)
         commit_id = svn.repos.fs_commit_txn(repo, txn)
 
         if timestamp: