# HG changeset patch # User Serhii Ilin # Date 2024-02-12 08:18:30 # Node ID 79967b24a7d49fbe052ba3cd91895e65eacd3918 # Parent 945790c2bccbd4e2a1ed590cf8f0c8c3961f6d7f feat(celery-hooks): added HooksCeleryClient, removed support od HooksDummyClient, updated tests. Fixes: RCCE-55 diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,26 @@ async-timeout==4.0.3 atomicwrites==1.4.1 +celery==5.3.4 + billiard==4.1.0 + click==8.1.3 + click-didyoumean==0.3.0 + click==8.1.3 + click-plugins==1.1.1 + click==8.1.3 + click-repl==0.2.0 + click==8.1.3 + prompt-toolkit==3.0.38 + wcwidth==0.2.6 + six==1.16.0 + kombu==5.3.2 + amqp==5.1.1 + vine==5.1.0 + vine==5.1.0 + python-dateutil==2.8.2 + six==1.16.0 + tzdata==2023.4 + vine==5.1.0 contextlib2==21.6.0 cov-core==1.15.0 coverage==7.2.3 diff --git a/vcsserver/hooks.py b/vcsserver/hooks.py --- a/vcsserver/hooks.py +++ b/vcsserver/hooks.py @@ -27,6 +27,7 @@ import dataclasses import pygit2 import http.client +from celery import Celery import mercurial.scmutil @@ -37,6 +38,7 @@ from vcsserver import exceptions, subpro from vcsserver.str_utils import ascii_str, safe_str from vcsserver.remote.git_remote import Repository +celery_app = Celery() log = logging.getLogger(__name__) @@ -88,14 +90,24 @@ class HooksHttpClient: return headers, msgpack.packb(data) -class HooksDummyClient: - def __init__(self, hooks_module): - log.debug('HooksDummyClient import: %s', hooks_module) - self._hooks_module = importlib.import_module(hooks_module) +class HooksCeleryClient: + TASK_TIMEOUT = 60 # time in seconds - def __call__(self, hook_name, extras): - with self._hooks_module.Hooks() as hooks: - return getattr(hooks, hook_name)(extras) + def __init__(self, queue, backend): + celery_app.config_from_object({'broker_url': queue, 'result_backend': backend, + 'broker_connection_retry_on_startup': True, + 'task_serializer': 'msgpack', + 'accept_content': ['json', 'msgpack'], + 'result_serializer': 'msgpack', + 'result_accept_content': ['json', 'msgpack'] + }) + self.celery_app = celery_app + + def __call__(self, method, extras): + inquired_task = self.celery_app.signature( + f'rhodecode.lib.celerylib.tasks.{method}' + ) + return inquired_task.delay(extras).get(timeout=self.TASK_TIMEOUT) class HooksShadowRepoClient: @@ -167,19 +179,18 @@ def _handle_exception(result): def _get_hooks_client(extras): hooks_uri = extras.get('hooks_uri') + task_queue = extras.get('task_queue') + task_backend = extras.get('task_backend') is_shadow_repo = extras.get('is_shadow_repo') if hooks_uri: - return HooksHttpClient(extras['hooks_uri']) + return HooksHttpClient(hooks_uri) + elif task_queue and task_backend: + return HooksCeleryClient(task_queue, task_backend) elif is_shadow_repo: return HooksShadowRepoClient() else: - try: - import_module = extras['hooks_module'] - except KeyError: - log.error('Failed to get "hooks_module" from extras: %s', extras) - raise - return HooksDummyClient(import_module) + raise Exception("Hooks client not found!") def _call_hook(hook_name, extras, writer): diff --git a/vcsserver/tests/test_hooks.py b/vcsserver/tests/test_hooks.py --- a/vcsserver/tests/test_hooks.py +++ b/vcsserver/tests/test_hooks.py @@ -132,18 +132,14 @@ class TestGetHooksClient: assert isinstance(result, hooks.HooksHttpClient) assert result.hooks_uri == hooks_uri - def test_returns_dummy_client_when_hooks_uri_not_specified(self): - fake_module = mock.Mock() - import_patcher = mock.patch.object( - hooks.importlib, 'import_module', return_value=fake_module) - fake_module_name = 'fake.module' - with import_patcher as import_mock: - result = hooks._get_hooks_client( - {'hooks_module': fake_module_name}) - - import_mock.assert_called_once_with(fake_module_name) - assert isinstance(result, hooks.HooksDummyClient) - assert result._hooks_module == fake_module + def test_return_celery_client_when_queue_and_backend_provided(self): + task_queue = 'redis://task_queue:0' + task_backend = task_queue + result = hooks._get_hooks_client({ + 'task_queue': task_queue, + 'task_backend': task_backend + }) + assert isinstance(result, hooks.HooksCeleryClient) class TestHooksHttpClient: @@ -182,31 +178,6 @@ class TestHooksHttpClient: assert result == expected_result -class TestHooksDummyClient: - def test_init_imports_hooks_module(self): - hooks_module_name = 'rhodecode.fake.module' - hooks_module = mock.MagicMock() - - import_patcher = mock.patch.object( - hooks.importlib, 'import_module', return_value=hooks_module) - with import_patcher as import_mock: - client = hooks.HooksDummyClient(hooks_module_name) - import_mock.assert_called_once_with(hooks_module_name) - assert client._hooks_module == hooks_module - - def test_call_returns_hook_result(self): - hooks_module_name = 'rhodecode.fake.module' - hooks_module = mock.MagicMock() - import_patcher = mock.patch.object( - hooks.importlib, 'import_module', return_value=hooks_module) - with import_patcher: - client = hooks.HooksDummyClient(hooks_module_name) - - result = client('post_push', {}) - hooks_module.Hooks.assert_called_once_with() - assert result == hooks_module.Hooks().__enter__().post_push() - - @pytest.fixture def http_mirror(request): server = MirrorHttpServer()