tests: Fix --without-vcsserver parameter

Ensuring now that we always disable the vcsserver auto start for the test suite,
since the test suite bootstraps its own vcsserver process.

Also enhanced TestINI.

rhodecode/tests/fixture.py

@@ -1,331 +1,331 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Helpers for fixture generation
 """
 
 import os
 import time
 import tempfile
 import shutil
 
 import configobj
 
 from rhodecode.tests import *
 from rhodecode.model.db import Repository, User, RepoGroup, UserGroup, Gist
 from rhodecode.model.meta import Session
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.model.gist import GistModel
 
 dn = os.path.dirname
 FIXTURES = os.path.join(dn(dn(os.path.abspath(__file__))), 'tests', 'fixtures')
 
 
 def error_function(*args, **kwargs):
     raise Exception('Total Crash !')
 
 
 class TestINI(object):
     """
     Allows to create a new test.ini file as a copy of existing one with edited
     data. Example usage::
 
         with TestINI('test.ini', [{'section':{'key':val'}]) as new_test_ini_path:
             print 'paster server %s' % new_test_ini
     """
 
     def __init__(self, ini_file_path, ini_params, new_file_prefix='DEFAULT',
                  destroy=True, dir=None):
         self.ini_file_path = ini_file_path
         self.ini_params = ini_params
         self.new_path = None
         self.new_path_prefix = new_file_prefix
         self._destroy = destroy
         self._dir = dir
 
     def __enter__(self):
         return self.create()
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.destroy()
 
     def create(self):
         config = configobj.ConfigObj(
             self.ini_file_path, file_error=True, write_empty_values=True)
 
         for data in self.ini_params:
             section, ini_params = data.items()[0]
-            key, val = ini_params.items()[0]
-            config[section][key] = val
+            for key, val in ini_params.items():
+                config[section][key] = val
         with tempfile.NamedTemporaryFile(
                 prefix=self.new_path_prefix, suffix='.ini', dir=self._dir,
                 delete=False) as new_ini_file:
             config.write(new_ini_file)
             self.new_path = new_ini_file.name
 
         return self.new_path
 
     def destroy(self):
         if self._destroy:
             os.remove(self.new_path)
 
 
 class Fixture(object):
 
     def anon_access(self, status):
         """
         Context process for disabling anonymous access. use like:
         fixture = Fixture()
         with fixture.anon_access(False):
             #tests
 
         after this block anon access will be set to `not status`
         """
 
         class context(object):
             def __enter__(self):
                 anon = User.get_default_user()
                 anon.active = status
                 Session().add(anon)
                 Session().commit()
                 time.sleep(1.5)  # must sleep for cache (1s to expire)
 
             def __exit__(self, exc_type, exc_val, exc_tb):
                 anon = User.get_default_user()
                 anon.active = not status
                 Session().add(anon)
                 Session().commit()
 
         return context()
 
     def _get_repo_create_params(self, **custom):
         defs = {
             'repo_name': None,
             'repo_type': 'hg',
             'clone_uri': '',
             'repo_group': '-1',
             'repo_description': 'DESC',
             'repo_private': False,
             'repo_landing_rev': 'rev:tip',
             'repo_copy_permissions': False,
             'repo_state': Repository.STATE_CREATED,
         }
         defs.update(custom)
         if 'repo_name_full' not in custom:
             defs.update({'repo_name_full': defs['repo_name']})
 
         # fix the repo name if passed as repo_name_full
         if defs['repo_name']:
             defs['repo_name'] = defs['repo_name'].split('/')[-1]
 
         return defs
 
     def _get_group_create_params(self, **custom):
         defs = {
             'group_name': None,
             'group_description': 'DESC',
             'perm_updates': [],
             'perm_additions': [],
             'perm_deletions': [],
             'group_parent_id': -1,
             'enable_locking': False,
             'recursive': False,
         }
         defs.update(custom)
 
         return defs
 
     def _get_user_create_params(self, name, **custom):
         defs = {
             'username': name,
             'password': 'qweqwe',
             'email': '[e-mail]' % name,
             'firstname': 'TestUser',
             'lastname': 'Test',
             'active': True,
             'admin': False,
             'extern_type': 'rhodecode',
             'extern_name': None,
         }
         defs.update(custom)
 
         return defs
 
     def _get_user_group_create_params(self, name, **custom):
         defs = {
             'users_group_name': name,
             'user_group_description': 'DESC',
             'users_group_active': True,
             'user_group_data': {},
         }
         defs.update(custom)
 
         return defs
 
     def create_repo(self, name, **kwargs):
         repo_group = kwargs.get('repo_group')
         if isinstance(repo_group, RepoGroup):
             kwargs['repo_group'] = repo_group.group_id
             name = name.split(Repository.NAME_SEP)[-1]
             name = Repository.NAME_SEP.join((repo_group.group_name, name))
 
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             r = Repository.get_by_repo_name(name)
             if r:
                 return r
 
         form_data = self._get_repo_create_params(repo_name=name, **kwargs)
         cur_user = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         RepoModel().create(form_data, cur_user)
         Session().commit()
         repo = Repository.get_by_repo_name(name)
         assert repo
         return repo
 
     def create_fork(self, repo_to_fork, fork_name, **kwargs):
         repo_to_fork = Repository.get_by_repo_name(repo_to_fork)
 
         form_data = self._get_repo_create_params(repo_name=fork_name,
                                             fork_parent_id=repo_to_fork.repo_id,
                                             repo_type=repo_to_fork.repo_type,
                                             **kwargs)
         #TODO: fix it !!
         form_data['description'] = form_data['repo_description']
         form_data['private'] = form_data['repo_private']
         form_data['landing_rev'] = form_data['repo_landing_rev']
 
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         RepoModel().create_fork(form_data, cur_user=owner)
         Session().commit()
         r = Repository.get_by_repo_name(fork_name)
         assert r
         return r
 
     def destroy_repo(self, repo_name, **kwargs):
         RepoModel().delete(repo_name, **kwargs)
         Session().commit()
 
     def destroy_repo_on_filesystem(self, repo_name):
         rm_path = os.path.join(RepoModel().repos_path, repo_name)
         if os.path.isdir(rm_path):
             shutil.rmtree(rm_path)
 
     def create_repo_group(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             gr = RepoGroup.get_by_group_name(group_name=name)
             if gr:
                 return gr
         form_data = self._get_group_create_params(group_name=name, **kwargs)
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         gr = RepoGroupModel().create(
             group_name=form_data['group_name'],
             group_description=form_data['group_name'],
             owner=owner)
         Session().commit()
         gr = RepoGroup.get_by_group_name(gr.group_name)
         return gr
 
     def destroy_repo_group(self, repogroupid):
         RepoGroupModel().delete(repogroupid)
         Session().commit()
 
     def create_user(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             user = User.get_by_username(name)
             if user:
                 return user
         form_data = self._get_user_create_params(name, **kwargs)
         user = UserModel().create(form_data)
         Session().commit()
         user = User.get_by_username(user.username)
         return user
 
     def destroy_user(self, userid):
         UserModel().delete(userid)
         Session().commit()
 
     def destroy_users(self, userid_iter):
         for user_id in userid_iter:
             if User.get_by_username(user_id):
                 UserModel().delete(user_id)
                 Session().commit()
 
     def create_user_group(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             gr = UserGroup.get_by_group_name(group_name=name)
             if gr:
                 return gr
         form_data = self._get_user_group_create_params(name, **kwargs)
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         user_group = UserGroupModel().create(
             name=form_data['users_group_name'],
             description=form_data['user_group_description'],
             owner=owner, active=form_data['users_group_active'],
             group_data=form_data['user_group_data'])
         Session().commit()
         user_group = UserGroup.get_by_group_name(user_group.users_group_name)
         return user_group
 
     def destroy_user_group(self, usergroupid):
         UserGroupModel().delete(user_group=usergroupid, force=True)
         Session().commit()
 
     def create_gist(self, **kwargs):
         form_data = {
             'description': 'new-gist',
             'owner': TEST_USER_ADMIN_LOGIN,
             'gist_type': GistModel.cls.GIST_PUBLIC,
             'lifetime': -1,
             'acl_level': Gist.ACL_LEVEL_PUBLIC,
             'gist_mapping': {'filename1.txt': {'content': 'hello world'},}
         }
         form_data.update(kwargs)
         gist = GistModel().create(
             description=form_data['description'], owner=form_data['owner'],
             gist_mapping=form_data['gist_mapping'], gist_type=form_data['gist_type'],
             lifetime=form_data['lifetime'], gist_acl_level=form_data['acl_level']
         )
         Session().commit()
         return gist
 
     def destroy_gists(self, gistid=None):
         for g in GistModel.cls.get_all():
             if gistid:
                 if gistid == g.gist_access_id:
                     GistModel().delete(g)
             else:
                 GistModel().delete(g)
         Session().commit()
 
     def load_resource(self, resource_name, strip=False):
         with open(os.path.join(FIXTURES, resource_name)) as f:
             source = f.read()
             if strip:
                 source = source.strip()
 
         return source

rhodecode/tests/pylons_plugin.py

@@ -1,432 +1,439 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2016  RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import json
 import logging.config
 import os
 import platform
 import socket
 import subprocess
 import time
 from urllib2 import urlopen, URLError
 
 import configobj
 import pylons
 import pytest
 import webob
 from beaker.session import SessionObject
 from paste.deploy import loadapp
 from pylons.i18n.translation import _get_translator
 from pylons.util import ContextObj
 from Pyro4.errors import CommunicationError
 from routes.util import URLGenerator
 
 from rhodecode.lib import vcs
 from rhodecode.tests.fixture import TestINI
 import rhodecode
 
 
 def _parse_json(value):
     return json.loads(value) if value else None
 
 
 def pytest_addoption(parser):
     group = parser.getgroup('pylons')
     group.addoption(
         '--with-pylons', dest='pylons_config',
         help="Set up a Pylons environment with the specified config file.")
     group.addoption(
         '--pylons-config-override', action='store', type=_parse_json,
         default=None, dest='pylons_config_override', help=(
             "Overrides the .ini file settings. Should be specified in JSON"
             " format, e.g. '{\"section\": {\"parameter\": \"value\", ...}}'"
         )
     )
     parser.addini(
         'pylons_config',
         "Set up a Pylons environment with the specified config file.")
 
     vcsgroup = parser.getgroup('vcs')
     vcsgroup.addoption(
         '--without-vcsserver', dest='with_vcsserver', action='store_false',
         help="Do not start the VCSServer in a background process.")
     vcsgroup.addoption(
         '--with-vcsserver', dest='vcsserver_config',
         help="Start the VCSServer with the specified config file.")
     vcsgroup.addoption(
         '--with-vcsserver-http', dest='vcsserver_config_http',
         help="Start the HTTP VCSServer with the specified config file.")
     vcsgroup.addoption(
         '--vcsserver-protocol', dest='vcsserver_protocol',
         help="Start the VCSServer with HTTP / Pyro4 protocol support.")
     vcsgroup.addoption(
         '--vcsserver-config-override', action='store', type=_parse_json,
         default=None, dest='vcsserver_config_override', help=(
             "Overrides the .ini file settings for the VCSServer. "
             "Should be specified in JSON "
             "format, e.g. '{\"section\": {\"parameter\": \"value\", ...}}'"
         )
     )
     vcsgroup.addoption(
         '--vcsserver-port', action='store', type=int,
         default=None, help=(
             "Allows to set the port of the vcsserver. Useful when testing "
             "against an already running server and random ports cause "
             "trouble."))
     parser.addini(
         'vcsserver_config',
         "Start the VCSServer with the specified config file.")
     parser.addini(
         'vcsserver_config_http',
         "Start the HTTP VCSServer with the specified config file.")
     parser.addini(
         'vcsserver_protocol',
         "Start the VCSServer with HTTP / Pyro4 protocol support.")
 
 
 @pytest.fixture(scope='session')
 def vcsserver(request, vcsserver_port, vcsserver_factory):
     """
     Session scope VCSServer.
 
     Tests wich need the VCSServer have to rely on this fixture in order
     to ensure it will be running.
 
     For specific needs, the fixture vcsserver_factory can be used. It allows to
     adjust the configuration file for the test run.
 
     Command line args:
 
     --without-vcsserver: Allows to switch this fixture off. You have to
     manually start the server.
 
     --vcsserver-port: Will expect the VCSServer to listen on this port.
     """
 
     if not request.config.getoption('with_vcsserver'):
         return None
 
     use_http = _use_vcs_http_server(request.config)
     return vcsserver_factory(
         request, use_http=use_http, vcsserver_port=vcsserver_port)
 
 
 @pytest.fixture(scope='session')
 def vcsserver_factory(tmpdir_factory):
     """
     Use this if you need a running vcsserver with a special configuration.
     """
 
     def factory(request, use_http=False, overrides=(), vcsserver_port=None):
 
         if vcsserver_port is None:
             vcsserver_port = get_available_port()
 
         overrides = list(overrides)
         if use_http:
             overrides.append({'server:main': {'port': vcsserver_port}})
         else:
             overrides.append({'DEFAULT': {'port': vcsserver_port}})
 
         if is_cygwin():
             platform_override = {'DEFAULT': {
                 'beaker.cache.repo_object.type': 'nocache'}}
             overrides.append(platform_override)
 
         option_name = (
             'vcsserver_config_http' if use_http else 'vcsserver_config')
         override_option_name = 'vcsserver_config_override'
         config_file = get_config(
             request.config, option_name=option_name,
             override_option_name=override_option_name, overrides=overrides,
             basetemp=tmpdir_factory.getbasetemp().strpath,
             prefix='test_vcs_')
 
         print "Using the VCSServer configuration", config_file
         ServerClass = HttpVCSServer if use_http else Pyro4VCSServer
         server = ServerClass(config_file)
         server.start()
 
         @request.addfinalizer
         def cleanup():
             server.shutdown()
 
         server.wait_until_ready()
         return server
 
     return factory
 
 
 def is_cygwin():
     return 'cygwin' in platform.system().lower()
 
 
 def _use_vcs_http_server(config):
     protocol_option = 'vcsserver_protocol'
     protocol = (
         config.getoption(protocol_option) or
         config.getini(protocol_option) or
         'pyro4')
     return protocol == 'http'
 
 
 class VCSServer(object):
     """
     Represents a running VCSServer instance.
     """
 
     _args = []
 
     def start(self):
         print("Starting the VCSServer: {}".format(self._args))
         self.process = subprocess.Popen(self._args)
 
     def wait_until_ready(self, timeout=30):
         raise NotImplementedError()
 
     def shutdown(self):
         self.process.kill()
 
 
 class Pyro4VCSServer(VCSServer):
     def __init__(self, config_file):
         """
         :param config_file: The config file to start the server with
         """
 
         config_data = configobj.ConfigObj(config_file)
         self._config = config_data['DEFAULT']
 
         args = ['vcsserver', '--config', config_file]
         self._args = args
 
     def wait_until_ready(self, timeout=30):
         remote_server = vcs.create_vcsserver_proxy(self.server_and_port)
         start = time.time()
         with remote_server:
             while time.time() - start < timeout:
                 try:
                     remote_server.ping()
                     break
                 except CommunicationError:
                     time.sleep(0.2)
             else:
                 pytest.exit(
                     "Starting the VCSServer failed or took more than {} "
                     "seconds.".format(timeout))
 
     @property
     def server_and_port(self):
         return '{host}:{port}'.format(**self._config)
 
 
 class HttpVCSServer(VCSServer):
     """
     Represents a running VCSServer instance.
     """
     def __init__(self, config_file):
         config_data = configobj.ConfigObj(config_file)
         self._config = config_data['server:main']
 
         args = ['pserve', config_file, 'http_host=0.0.0.0']
         self._args = args
 
     @property
     def http_url(self):
         template = 'http://{host}:{port}/'
         return template.format(**self._config)
 
     def start(self):
         self.process = subprocess.Popen(self._args)
 
     def wait_until_ready(self, timeout=30):
         host = self._config['host']
         port = self._config['port']
         status_url = 'http://{host}:{port}/status'.format(host=host, port=port)
         start = time.time()
 
         while time.time() - start < timeout:
             try:
                 urlopen(status_url)
                 break
             except URLError:
                 time.sleep(0.2)
         else:
             pytest.exit(
                 "Starting the VCSServer failed or took more than {} "
                 "seconds.".format(timeout))
 
     def shutdown(self):
         self.process.kill()
 
 
 @pytest.fixture(scope='session')
 def pylons_config(request, tmpdir_factory, rcserver_port, vcsserver_port):
     option_name = 'pylons_config'
 
     overrides = [
         {'server:main': {'port': rcserver_port}},
-        {'app:main': {'vcs.server': 'localhost:%s' % vcsserver_port}}]
+        {'app:main': {
+            'vcs.server': 'localhost:%s' % vcsserver_port,
+            # johbo: We will always start the VCSServer on our own based on the
+            # fixtures of the test cases. For the test run it must always be
+            # off in the INI file.
+            'vcs.start_server': 'false',
+        }},
+    ]
     if _use_vcs_http_server(request.config):
         overrides.append({'app:main': {'vcs.server.protocol': 'http'}})
 
     filename = get_config(
         request.config, option_name=option_name,
         override_option_name='{}_override'.format(option_name),
         overrides=overrides,
         basetemp=tmpdir_factory.getbasetemp().strpath,
         prefix='test_rce_')
     return filename
 
 
 @pytest.fixture(scope='session')
 def rcserver_port(request):
     port = get_available_port()
     print 'Using rcserver port %s' % (port, )
     return port
 
 
 @pytest.fixture(scope='session')
 def vcsserver_port(request):
     port = request.config.getoption('--vcsserver-port')
     if port is None:
         port = get_available_port()
         print 'Using vcsserver port %s' % (port, )
     return port
 
 
 def get_available_port():
     family = socket.AF_INET
     socktype = socket.SOCK_STREAM
     host = '127.0.0.1'
 
     mysocket = socket.socket(family, socktype)
     mysocket.bind((host, 0))
     port = mysocket.getsockname()[1]
     mysocket.close()
     del mysocket
     return port
 
 
 @pytest.fixture(scope='session')
 def available_port_factory():
     """
     Returns a callable which returns free port numbers.
     """
     return get_available_port
 
 
 @pytest.fixture
 def available_port(available_port_factory):
     """
     Gives you one free port for the current test.
 
     Uses "available_port_factory" to retrieve the port.
     """
     return available_port_factory()
 
 
 @pytest.fixture(scope='session')
 def pylonsapp(pylons_config, vcsserver, http_environ_session):
     logging.config.fileConfig(
         pylons_config, disable_existing_loggers=False)
     app = _setup_pylons_environment(pylons_config, http_environ_session)
     return app
 
 
 @pytest.fixture(scope='session')
 def testini_factory(tmpdir_factory, pylons_config):
     """
     Factory to create an INI file based on TestINI.
 
     It will make sure to place the INI file in the correct directory.
     """
     basetemp = tmpdir_factory.getbasetemp().strpath
     return TestIniFactory(basetemp, pylons_config)
 
 
 class TestIniFactory(object):
 
     def __init__(self, basetemp, template_ini):
         self._basetemp = basetemp
         self._template_ini = template_ini
 
     def __call__(self, ini_params, new_file_prefix='test'):
         ini_file = TestINI(
             self._template_ini, ini_params=ini_params,
             new_file_prefix=new_file_prefix, dir=self._basetemp)
         result = ini_file.create()
         return result
 
 
 def get_config(
         config, option_name, override_option_name, overrides=None,
         basetemp=None, prefix='test'):
     """
     Find a configuration file and apply overrides for the given `prefix`.
     """
     config_file = (
         config.getoption(option_name) or config.getini(option_name))
     if not config_file:
         pytest.exit(
             "Configuration error, could not extract {}.".format(option_name))
 
     overrides = overrides or []
     config_override = config.getoption(override_option_name)
     if config_override:
         overrides.append(config_override)
     temp_ini_file = TestINI(
         config_file, ini_params=overrides, new_file_prefix=prefix,
         dir=basetemp)
 
     return temp_ini_file.create()
 
 
 def _setup_pylons_environment(pylons_config, http_environ):
     current_path = os.getcwd()
     pylonsapp = loadapp(
         'config:' + pylons_config, relative_to=current_path)
 
     # Using rhodecode.CONFIG which is assigned during "load_environment".
     # The indirect approach is used, because "pylonsapp" may actually be
     # the Pyramid application.
     pylonsapp_config = rhodecode.CONFIG
     _init_stack(pylonsapp_config, environ=http_environ)
 
     # For compatibility add the attribute "config" which would be
     # present on the Pylons application.
     pylonsapp.config = pylonsapp_config
     return pylonsapp
 
 
 def _init_stack(config=None, environ=None):
     if not config:
         config = pylons.test.pylonsapp.config
     if not environ:
         environ = {}
     pylons.url._push_object(URLGenerator(config['routes.map'], environ or {}))
     pylons.app_globals._push_object(config['pylons.app_globals'])
     pylons.config._push_object(config)
     pylons.tmpl_context._push_object(ContextObj())
     # Initialize a translator for tests that utilize i18n
     translator = _get_translator(pylons.config.get('lang'))
     pylons.translator._push_object(translator)
     pylons.session._push_object(SessionObject(environ or {}))
     pylons.request._push_object(webob.Request.blank('', environ=environ))