fix(tests): fixed potential freeze on vcs_operations tests

rhodecode/tests/fixture_mods/fixture_pyramid.py

@@ -1,226 +1,226 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
 
 from rhodecode.lib.config_utils import get_app_config
 from rhodecode.tests.fixture import TestINI
 from rhodecode.tests import TESTS_TMP_PATH
 from rhodecode.tests.server_utils import RcVCSServer
 
 
 @pytest.fixture(scope='session')
 def vcsserver(request, vcsserver_port, vcsserver_factory):
     """
     Session scope VCSServer.
 
     Tests which need the VCSServer have to rely on this fixture in order
     to ensure it will be running.
 
     For specific needs, the fixture vcsserver_factory can be used. It allows to
     adjust the configuration file for the test run.
 
     Command line args:
 
     --without-vcsserver: Allows to switch this fixture off. You have to
     manually start the server.
 
     --vcsserver-port: Will expect the VCSServer to listen on this port.
     """
 
     if not request.config.getoption('with_vcsserver'):
         return None
 
     return vcsserver_factory(
         request, vcsserver_port=vcsserver_port)
 
 
 @pytest.fixture(scope='session')
 def vcsserver_factory(tmpdir_factory):
     """
     Use this if you need a running vcsserver with a special configuration.
     """
 
     def factory(request, overrides=(), vcsserver_port=None,
-                log_file=None, workers='2'):
+                log_file=None, workers='3'):
 
         if vcsserver_port is None:
             vcsserver_port = get_available_port()
 
         overrides = list(overrides)
         overrides.append({'server:main': {'port': vcsserver_port}})
 
         option_name = 'vcsserver_config_http'
         override_option_name = 'vcsserver_config_override'
         config_file = get_config(
             request.config, option_name=option_name,
             override_option_name=override_option_name, overrides=overrides,
             basetemp=tmpdir_factory.getbasetemp().strpath,
             prefix='test_vcs_')
 
         server = RcVCSServer(config_file, log_file, workers)
         server.start()
 
         @request.addfinalizer
         def cleanup():
             server.shutdown()
 
         server.wait_until_ready()
         return server
 
     return factory
 
 
 def _use_log_level(config):
     level = config.getoption('test_loglevel') or 'critical'
     return level.upper()
 
 
 @pytest.fixture(scope='session')
 def ini_config(request, tmpdir_factory, rcserver_port, vcsserver_port):
     option_name = 'pyramid_config'
     log_level = _use_log_level(request.config)
 
     overrides = [
         {'server:main': {'port': rcserver_port}},
         {'app:main': {
             'cache_dir': '%(here)s/rc-tests/rc_data',
             'vcs.server': f'localhost:{vcsserver_port}',
             # johbo: We will always start the VCSServer on our own based on the
             # fixtures of the test cases. For the test run it must always be
             # off in the INI file.
             'vcs.start_server': 'false',
 
             'vcs.server.protocol': 'http',
             'vcs.scm_app_implementation': 'http',
             'vcs.svn.proxy.enabled': 'true',
             'vcs.hooks.protocol': 'http',
             'vcs.hooks.host': '*',
             'repo_store.path': TESTS_TMP_PATH,
             'app.service_api.token': 'service_secret_token',
         }},
 
         {'handler_console': {
             'class': 'StreamHandler',
             'args': '(sys.stderr,)',
             'level': log_level,
         }},
 
     ]
 
     filename = get_config(
         request.config, option_name=option_name,
         override_option_name='{}_override'.format(option_name),
         overrides=overrides,
         basetemp=tmpdir_factory.getbasetemp().strpath,
         prefix='test_rce_')
     return filename
 
 
 @pytest.fixture(scope='session')
 def ini_settings(ini_config):
     ini_path = ini_config
     return get_app_config(ini_path)
 
 
 def get_available_port(min_port=40000, max_port=55555):
     from rhodecode.lib.utils2 import get_available_port as _get_port
     return _get_port(min_port, max_port)
 
 
 @pytest.fixture(scope='session')
 def rcserver_port(request):
     port = get_available_port()
     print(f'Using rhodecode port {port}')
     return port
 
 
 @pytest.fixture(scope='session')
 def vcsserver_port(request):
     port = request.config.getoption('--vcsserver-port')
     if port is None:
         port = get_available_port()
         print(f'Using vcsserver port {port}')
     return port
 
 
 @pytest.fixture(scope='session')
 def available_port_factory() -> get_available_port:
     """
     Returns a callable which returns free port numbers.
     """
     return get_available_port
 
 
 @pytest.fixture()
 def available_port(available_port_factory):
     """
     Gives you one free port for the current test.
 
     Uses "available_port_factory" to retrieve the port.
     """
     return available_port_factory()
 
 
 @pytest.fixture(scope='session')
 def testini_factory(tmpdir_factory, ini_config):
     """
     Factory to create an INI file based on TestINI.
 
     It will make sure to place the INI file in the correct directory.
     """
     basetemp = tmpdir_factory.getbasetemp().strpath
     return TestIniFactory(basetemp, ini_config)
 
 
 class TestIniFactory(object):
 
     def __init__(self, basetemp, template_ini):
         self._basetemp = basetemp
         self._template_ini = template_ini
 
     def __call__(self, ini_params, new_file_prefix='test'):
         ini_file = TestINI(
             self._template_ini, ini_params=ini_params,
             new_file_prefix=new_file_prefix, dir=self._basetemp)
         result = ini_file.create()
         return result
 
 
 def get_config(
         config, option_name, override_option_name, overrides=None,
         basetemp=None, prefix='test'):
     """
     Find a configuration file and apply overrides for the given `prefix`.
     """
     config_file = (
         config.getoption(option_name) or config.getini(option_name))
     if not config_file:
         pytest.exit(
             "Configuration error, could not extract {}.".format(option_name))
 
     overrides = overrides or []
     config_override = config.getoption(override_option_name)
     if config_override:
         overrides.append(config_override)
     temp_ini_file = TestINI(
         config_file, ini_params=overrides, new_file_prefix=prefix,
         dir=basetemp)
 
     return temp_ini_file.create()

rhodecode/tests/server_utils.py

@@ -1,229 +1,231 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import os
 import time
 import tempfile
 import pytest
 import subprocess
 import logging
 from urllib.request import urlopen
 from urllib.error import URLError
 import configparser
 
 
 from rhodecode.tests import TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS
 from rhodecode.tests.utils import is_url_reachable
 
 log = logging.getLogger(__name__)
 
 
 def get_port(pyramid_config):
     config = configparser.ConfigParser()
     config.read(pyramid_config)
     return config.get('server:main', 'port')
 
 
 def get_host_url(pyramid_config):
     """Construct the host url using the port in the test configuration."""
     port = get_port(pyramid_config)
     return f'127.0.0.1:{port}'
 
 
 def assert_no_running_instance(url):
     if is_url_reachable(url):
         print(f"Hint: Usually this means another instance of server "
               f"is running in the background at {url}.")
         pytest.fail(f"Port is not free at {url}, cannot start server at")
 
 
 class ServerBase(object):
     _args = []
     log_file_name = 'NOT_DEFINED.log'
     status_url_tmpl = 'http://{host}:{port}/_admin/ops/ping'
 
     def __init__(self, config_file, log_file):
         self.config_file = config_file
         config = configparser.ConfigParser()
         config.read(config_file)
 
         self._config = {k: v for k, v in config['server:main'].items()}
 
         self._args = []
         self.log_file = log_file or os.path.join(
             tempfile.gettempdir(), self.log_file_name)
         self.process = None
         self.server_out = None
         log.info("Using the {} configuration:{}".format(
             self.__class__.__name__, config_file))
 
         if not os.path.isfile(config_file):
             raise RuntimeError(f'Failed to get config at {config_file}')
 
     @property
     def command(self):
         return ' '.join(self._args)
 
     @property
     def bind_addr(self):
         return '{host}:{port}'.format(**self._config)
 
     @property
     def http_url(self):
         template = 'http://{host}:{port}/'
         return template.format(**self._config)
 
     def host_url(self):
         host = get_host_url(self.config_file)
         return f'http://{host}'
 
     def get_rc_log(self):
         with open(self.log_file) as f:
             return f.read()
 
     def assert_message_in_server_logs(self, message):
         server_logs = self.get_rc_log()
         assert message in server_logs
 
     def wait_until_ready(self, timeout=30):
         host = self._config['host']
         port = self._config['port']
         status_url = self.status_url_tmpl.format(host=host, port=port)
         start = time.time()
 
         while time.time() - start < timeout:
             try:
                 urlopen(status_url)
                 break
             except URLError:
                 time.sleep(0.2)
         else:
             pytest.fail(
                 "Starting the {} failed or took more than {} "
                 "seconds. cmd: `{}`".format(
                     self.__class__.__name__, timeout, self.command))
 
         log.info('Server of {} ready at url {}'.format(
             self.__class__.__name__, status_url))
 
     def shutdown(self):
         self.process.kill()
         self.server_out.flush()
         self.server_out.close()
 
     def get_log_file_with_port(self):
         log_file = list(self.log_file.partition('.log'))
         log_file.insert(1, get_port(self.config_file))
         log_file = ''.join(log_file)
         return log_file
 
 
 class RcVCSServer(ServerBase):
     """
     Represents a running VCSServer instance.
     """
 
     log_file_name = 'rc-vcsserver.log'
     status_url_tmpl = 'http://{host}:{port}/status'
 
-    def __init__(self, config_file, log_file=None, workers='2'):
+    def __init__(self, config_file, log_file=None, workers='3'):
         super(RcVCSServer, self).__init__(config_file, log_file)
         self._args = [
             'gunicorn',
             '--bind', self.bind_addr,
-            '--worker-class', 'gthread',
-            '--backlog', '16',
+            '--worker-class', 'sync',
+            '--threads', '1',
+            '--backlog', '8',
             '--timeout', '300',
             '--workers', workers,
             '--paste', self.config_file]
 
     def start(self):
         env = os.environ.copy()
 
         self.log_file = self.get_log_file_with_port()
         self.server_out = open(self.log_file, 'w')
 
         host_url = self.host_url()
         assert_no_running_instance(host_url)
 
         print(f'rhodecode-vcsserver starting at: {host_url}')
         print(f'rhodecode-vcsserver command: {self.command}')
         print(f'rhodecode-vcsserver logfile: {self.log_file}')
 
         self.process = subprocess.Popen(
             self._args, bufsize=0, env=env,
             stdout=self.server_out, stderr=self.server_out)
 
 
 class RcWebServer(ServerBase):
     """
     Represents a running RCE web server used as a test fixture.
     """
 
     log_file_name = 'rc-web.log'
     status_url_tmpl = 'http://{host}:{port}/_admin/ops/ping'
 
-    def __init__(self, config_file, log_file=None, workers='1'):
+    def __init__(self, config_file, log_file=None, workers='2'):
         super(RcWebServer, self).__init__(config_file, log_file)
         self._args = [
             'gunicorn',
             '--bind', self.bind_addr,
             '--worker-class', 'gthread',
-            '--backlog', '16',
+            '--threads', '4',
+            '--backlog', '8',
             '--timeout', '300',
             '--workers', workers,
             '--paste', self.config_file]
 
     def start(self):
         env = os.environ.copy()
         env['RC_NO_TMP_PATH'] = '1'
 
         self.log_file = self.get_log_file_with_port()
         self.server_out = open(self.log_file, 'w')
 
         host_url = self.host_url()
         assert_no_running_instance(host_url)
 
         print(f'rhodecode-web starting at: {host_url}')
         print(f'rhodecode-web command: {self.command}')
         print(f'rhodecode-web logfile: {self.log_file}')
 
         self.process = subprocess.Popen(
             self._args, bufsize=0, env=env,
             stdout=self.server_out, stderr=self.server_out)
 
     def repo_clone_url(self, repo_name, **kwargs):
         params = {
             'user': TEST_USER_ADMIN_LOGIN,
             'passwd': TEST_USER_ADMIN_PASS,
             'host': get_host_url(self.config_file),
             'cloned_repo': repo_name,
         }
         params.update(**kwargs)
         _url = f"http://{params['user']}:{params['passwd']}@{params['host']}/{params['cloned_repo']}"
         return _url
 
     def repo_clone_credentials(self, **kwargs):
         params = {
             'user': TEST_USER_ADMIN_LOGIN,
             'passwd': TEST_USER_ADMIN_PASS,
         }
         params.update(**kwargs)
         return params['user'], params['passwd']