tests: fixed starting of customized gunicorn with a explicit --bind call

rhodecode/tests/pylons_plugin.py

@@ -1,285 +1,285 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import json
 import platform
 import socket
 import random
 import pytest
 
 from rhodecode.lib.pyramid_utils import get_app_config
 from rhodecode.tests.fixture import TestINI
 from rhodecode.tests.server_utils import RcVCSServer
 
 
 def _parse_json(value):
     return json.loads(value) if value else None
 
 
 def pytest_addoption(parser):
     parser.addoption(
         '--test-loglevel', dest='test_loglevel',
         help="Set default Logging level for tests, critical(default), error, warn , info, debug")
     group = parser.getgroup('pylons')
     group.addoption(
         '--with-pylons', dest='pyramid_config',
         help="Set up a Pylons environment with the specified config file.")
     group.addoption(
         '--ini-config-override', action='store', type=_parse_json,
         default=None, dest='pyramid_config_override', help=(
             "Overrides the .ini file settings. Should be specified in JSON"
             " format, e.g. '{\"section\": {\"parameter\": \"value\", ...}}'"
         )
     )
     parser.addini(
         'pyramid_config',
         "Set up a Pyramid environment with the specified config file.")
 
     vcsgroup = parser.getgroup('vcs')
     vcsgroup.addoption(
         '--without-vcsserver', dest='with_vcsserver', action='store_false',
         help="Do not start the VCSServer in a background process.")
     vcsgroup.addoption(
         '--with-vcsserver-http', dest='vcsserver_config_http',
         help="Start the HTTP VCSServer with the specified config file.")
     vcsgroup.addoption(
         '--vcsserver-protocol', dest='vcsserver_protocol',
         help="Start the VCSServer with HTTP protocol support.")
     vcsgroup.addoption(
         '--vcsserver-config-override', action='store', type=_parse_json,
         default=None, dest='vcsserver_config_override', help=(
             "Overrides the .ini file settings for the VCSServer. "
             "Should be specified in JSON "
             "format, e.g. '{\"section\": {\"parameter\": \"value\", ...}}'"
         )
     )
     vcsgroup.addoption(
         '--vcsserver-port', action='store', type=int,
         default=None, help=(
             "Allows to set the port of the vcsserver. Useful when testing "
             "against an already running server and random ports cause "
             "trouble."))
     parser.addini(
         'vcsserver_config_http',
         "Start the HTTP VCSServer with the specified config file.")
     parser.addini(
         'vcsserver_protocol',
         "Start the VCSServer with HTTP protocol support.")
 
 
 @pytest.fixture(scope='session')
 def vcsserver(request, vcsserver_port, vcsserver_factory):
     """
     Session scope VCSServer.
 
     Tests wich need the VCSServer have to rely on this fixture in order
     to ensure it will be running.
 
     For specific needs, the fixture vcsserver_factory can be used. It allows to
     adjust the configuration file for the test run.
 
     Command line args:
 
     --without-vcsserver: Allows to switch this fixture off. You have to
     manually start the server.
 
     --vcsserver-port: Will expect the VCSServer to listen on this port.
     """
 
     if not request.config.getoption('with_vcsserver'):
         return None
 
     return vcsserver_factory(
         request, vcsserver_port=vcsserver_port)
 
 
 @pytest.fixture(scope='session')
 def vcsserver_factory(tmpdir_factory):
     """
     Use this if you need a running vcsserver with a special configuration.
     """
 
     def factory(request, overrides=(), vcsserver_port=None,
                 log_file=None):
 
         if vcsserver_port is None:
             vcsserver_port = get_available_port()
 
         overrides = list(overrides)
         overrides.append({'server:main': {'port': vcsserver_port}})
 
         option_name = 'vcsserver_config_http'
         override_option_name = 'vcsserver_config_override'
         config_file = get_config(
             request.config, option_name=option_name,
             override_option_name=override_option_name, overrides=overrides,
             basetemp=tmpdir_factory.getbasetemp().strpath,
             prefix='test_vcs_')
 
         server = RcVCSServer(config_file, log_file)
         server.start()
 
         @request.addfinalizer
         def cleanup():
             server.shutdown()
 
         server.wait_until_ready()
         return server
 
     return factory
 
 
 def is_cygwin():
     return 'cygwin' in platform.system().lower()
 
 
 def _use_log_level(config):
     level = config.getoption('test_loglevel') or 'critical'
     return level.upper()
 
 
 @pytest.fixture(scope='session')
 def ini_config(request, tmpdir_factory, rcserver_port, vcsserver_port):
     option_name = 'pyramid_config'
     log_level = _use_log_level(request.config)
 
     overrides = [
         {'server:main': {'port': rcserver_port}},
         {'app:main': {
             'vcs.server': 'localhost:%s' % vcsserver_port,
             # johbo: We will always start the VCSServer on our own based on the
             # fixtures of the test cases. For the test run it must always be
             # off in the INI file.
             'vcs.start_server': 'false',
 
             'vcs.server.protocol': 'http',
             'vcs.scm_app_implementation': 'http',
             'vcs.hooks.protocol': 'http',
             'vcs.hooks.host': '127.0.0.1',
         }},
 
         {'handler_console': {
-            'class ': 'StreamHandler',
-            'args ': '(sys.stderr,)',
+            'class': 'StreamHandler',
+            'args': '(sys.stderr,)',
             'level': log_level,
         }},
 
     ]
 
     filename = get_config(
         request.config, option_name=option_name,
         override_option_name='{}_override'.format(option_name),
         overrides=overrides,
         basetemp=tmpdir_factory.getbasetemp().strpath,
         prefix='test_rce_')
     return filename
 
 
 @pytest.fixture(scope='session')
 def ini_settings(ini_config):
     ini_path = ini_config
     return get_app_config(ini_path)
 
 
 def get_available_port(min_port=40000, max_port=55555):
     from rhodecode.lib.utils2 import get_available_port as _get_port
     return _get_port(min_port, max_port)
 
 
 @pytest.fixture(scope='session')
 def rcserver_port(request):
     port = get_available_port()
     print('Using rhodecode port {}'.format(port))
     return port
 
 
 @pytest.fixture(scope='session')
 def vcsserver_port(request):
     port = request.config.getoption('--vcsserver-port')
     if port is None:
         port = get_available_port()
         print('Using vcsserver port {}'.format(port))
     return port
 
 
 @pytest.fixture(scope='session')
 def available_port_factory():
     """
     Returns a callable which returns free port numbers.
     """
     return get_available_port
 
 
 @pytest.fixture()
 def available_port(available_port_factory):
     """
     Gives you one free port for the current test.
 
     Uses "available_port_factory" to retrieve the port.
     """
     return available_port_factory()
 
 
 @pytest.fixture(scope='session')
 def testini_factory(tmpdir_factory, ini_config):
     """
     Factory to create an INI file based on TestINI.
 
     It will make sure to place the INI file in the correct directory.
     """
     basetemp = tmpdir_factory.getbasetemp().strpath
     return TestIniFactory(basetemp, ini_config)
 
 
 class TestIniFactory(object):
 
     def __init__(self, basetemp, template_ini):
         self._basetemp = basetemp
         self._template_ini = template_ini
 
     def __call__(self, ini_params, new_file_prefix='test'):
         ini_file = TestINI(
             self._template_ini, ini_params=ini_params,
             new_file_prefix=new_file_prefix, dir=self._basetemp)
         result = ini_file.create()
         return result
 
 
 def get_config(
         config, option_name, override_option_name, overrides=None,
         basetemp=None, prefix='test'):
     """
     Find a configuration file and apply overrides for the given `prefix`.
     """
     config_file = (
         config.getoption(option_name) or config.getini(option_name))
     if not config_file:
         pytest.exit(
             "Configuration error, could not extract {}.".format(option_name))
 
     overrides = overrides or []
     config_override = config.getoption(override_option_name)
     if config_override:
         overrides.append(config_override)
     temp_ini_file = TestINI(
         config_file, ini_params=overrides, new_file_prefix=prefix,
         dir=basetemp)
 
     return temp_ini_file.create()

rhodecode/tests/server_utils.py

@@ -1,201 +1,208 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import os
 import time
 import tempfile
 import pytest
 import subprocess
 import logging
 from urllib.request import urlopen
 from urllib.error import URLError
 import configparser
 
 
 from rhodecode.tests import TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS
 from rhodecode.tests.utils import is_url_reachable
 
 log = logging.getLogger(__name__)
 
 
 def get_port(pyramid_config):
     config = configparser.ConfigParser()
     config.read(pyramid_config)
     return config.get('server:main', 'port')
 
 
 def get_host_url(pyramid_config):
     """Construct the host url using the port in the test configuration."""
     port = get_port(pyramid_config)
     return f'127.0.0.1:{port}'
 
 
 def assert_no_running_instance(url):
     if is_url_reachable(url):
         print(f"Hint: Usually this means another instance of server "
               f"is running in the background at {url}.")
         pytest.fail(f"Port is not free at {url}, cannot start server at")
 
 
 class ServerBase(object):
     _args = []
     log_file_name = 'NOT_DEFINED.log'
     status_url_tmpl = 'http://{host}:{port}'
 
     def __init__(self, config_file, log_file):
         self.config_file = config_file
         config = configparser.ConfigParser()
         config.read(config_file)
 
         self._config = {k: v for k, v in config['server:main'].items()}
 
         self._args = []
         self.log_file = log_file or os.path.join(
             tempfile.gettempdir(), self.log_file_name)
         self.process = None
         self.server_out = None
         log.info("Using the {} configuration:{}".format(
             self.__class__.__name__, config_file))
 
         if not os.path.isfile(config_file):
             raise RuntimeError(f'Failed to get config at {config_file}')
 
     @property
     def command(self):
         return ' '.join(self._args)
 
     @property
+    def bind_addr(self):
+        return '{host}:{port}'.format(**self._config)
+
+    @property
     def http_url(self):
         template = 'http://{host}:{port}/'
         return template.format(**self._config)
 
     def host_url(self):
-        return 'http://' + get_host_url(self.config_file)
+        host = get_host_url(self.config_file)
+        return f'http://{host}'
 
     def get_rc_log(self):
         with open(self.log_file) as f:
             return f.read()
 
     def wait_until_ready(self, timeout=30):
         host = self._config['host']
         port = self._config['port']
         status_url = self.status_url_tmpl.format(host=host, port=port)
         start = time.time()
 
         while time.time() - start < timeout:
             try:
                 urlopen(status_url)
                 break
             except URLError:
                 time.sleep(0.2)
         else:
             pytest.fail(
                 "Starting the {} failed or took more than {} "
                 "seconds. cmd: `{}`".format(
                     self.__class__.__name__, timeout, self.command))
 
         log.info('Server of {} ready at url {}'.format(
             self.__class__.__name__, status_url))
 
     def shutdown(self):
         self.process.kill()
         self.server_out.flush()
         self.server_out.close()
 
     def get_log_file_with_port(self):
         log_file = list(self.log_file.partition('.log'))
         log_file.insert(1, get_port(self.config_file))
         log_file = ''.join(log_file)
         return log_file
 
 
 class RcVCSServer(ServerBase):
     """
     Represents a running VCSServer instance.
     """
 
     log_file_name = 'rc-vcsserver.log'
     status_url_tmpl = 'http://{host}:{port}/status'
 
     def __init__(self, config_file, log_file=None):
         super(RcVCSServer, self).__init__(config_file, log_file)
-        self._args = ['gunicorn', '--paste', self.config_file]
+        self._args = [
+            'gunicorn', '--bind', self.bind_addr,
+            '--paste', self.config_file]
 
     def start(self):
         env = os.environ.copy()
 
         self.log_file = self.get_log_file_with_port()
         self.server_out = open(self.log_file, 'w')
 
         host_url = self.host_url()
         assert_no_running_instance(host_url)
 
-        log.info('rhodecode-vcsserver start command: {}'.format(' '.join(self._args)))
-        log.info('rhodecode-vcsserver starting at: {}'.format(host_url))
-        log.info('rhodecode-vcsserver command: {}'.format(self.command))
-        log.info('rhodecode-vcsserver logfile: {}'.format(self.log_file))
+        print(f'rhodecode-vcsserver starting at: {host_url}')
+        print(f'rhodecode-vcsserver command: {self.command}')
+        print(f'rhodecode-vcsserver logfile: {self.log_file}')
 
         self.process = subprocess.Popen(
             self._args, bufsize=0, env=env,
             stdout=self.server_out, stderr=self.server_out)
 
 
 class RcWebServer(ServerBase):
     """
     Represents a running RCE web server used as a test fixture.
     """
 
     log_file_name = 'rc-web.log'
     status_url_tmpl = 'http://{host}:{port}/_admin/ops/ping'
 
     def __init__(self, config_file, log_file=None):
         super(RcWebServer, self).__init__(config_file, log_file)
         self._args = [
-            'gunicorn', '--worker-class', 'gevent', '--paste', config_file]
+            'gunicorn', '--bind', self.bind_addr, '--worker-class', 'gevent',
+            '--paste', self.config_file]
 
     def start(self):
         env = os.environ.copy()
         env['RC_NO_TMP_PATH'] = '1'
 
         self.log_file = self.get_log_file_with_port()
         self.server_out = open(self.log_file, 'w')
 
         host_url = self.host_url()
         assert_no_running_instance(host_url)
 
-        log.info('rhodecode-web starting at: {}'.format(host_url))
-        log.info('rhodecode-web command: {}'.format(self.command))
-        log.info('rhodecode-web logfile: {}'.format(self.log_file))
+        print(f'rhodecode-web starting at: {host_url}')
+        print(f'rhodecode-web command: {self.command}')
+        print(f'rhodecode-web logfile: {self.log_file}')
 
         self.process = subprocess.Popen(
             self._args, bufsize=0, env=env,
             stdout=self.server_out, stderr=self.server_out)
 
     def repo_clone_url(self, repo_name, **kwargs):
         params = {
             'user': TEST_USER_ADMIN_LOGIN,
             'passwd': TEST_USER_ADMIN_PASS,
             'host': get_host_url(self.config_file),
             'cloned_repo': repo_name,
         }
         params.update(**kwargs)
         _url = 'http://%(user)s:%(passwd)s@%(host)s/%(cloned_repo)s' % params
         return _url