Authentication: cache plugins for auth and their settings in the auth_registry.

- Before that change on each requests 4x we loaded plugins for authentication, this hit
many caches, db invalidation context and loaded the plugins logic each time. This was a heavy performance hit
for SVN and other backends as they needed to load that plugins many many times
- Since Authentication plugins almost never change, we'll not store the plugins listed for authentication into
the authnregistry for each process
- For AuthPlugins settings we now also flush plugins settings, and authnregistry cached plugins

rhodecode/apps/login/tests/test_login.py

@@ -1,578 +1,580 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import urlparse
 
 import mock
 import pytest
 
 from rhodecode.tests import (
     assert_session_flash, HG_REPO, TEST_USER_ADMIN_LOGIN,
     no_newline_id_generator)
 from rhodecode.tests.fixture import Fixture
 from rhodecode.lib.auth import check_password
 from rhodecode.lib import helpers as h
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.db import User, Notification, UserApiKeys
 from rhodecode.model.meta import Session
 
 fixture = Fixture()
 
 whitelist_view = ['RepoCommitsView:repo_commit_raw']
 
 
 def route_path(name, params=None, **kwargs):
     import urllib
     from rhodecode.apps._base import ADMIN_PREFIX
 
     base_url = {
         'login': ADMIN_PREFIX + '/login',
         'logout': ADMIN_PREFIX + '/logout',
         'register': ADMIN_PREFIX + '/register',
         'reset_password':
             ADMIN_PREFIX + '/password_reset',
         'reset_password_confirmation':
             ADMIN_PREFIX + '/password_reset_confirmation',
 
         'admin_permissions_application':
             ADMIN_PREFIX + '/permissions/application',
         'admin_permissions_application_update':
             ADMIN_PREFIX + '/permissions/application/update',
 
         'repo_commit_raw': '/{repo_name}/raw-changeset/{commit_id}'
 
     }[name].format(**kwargs)
 
     if params:
         base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
     return base_url
 
 
 @pytest.mark.usefixtures('app')
 class TestLoginController(object):
     destroy_users = set()
 
     @classmethod
     def teardown_class(cls):
         fixture.destroy_users(cls.destroy_users)
 
     def teardown_method(self, method):
         for n in Notification.query().all():
             Session().delete(n)
 
         Session().commit()
         assert Notification.query().all() == []
 
     def test_index(self):
         response = self.app.get(route_path('login'))
         assert response.status == '200 OK'
         # Test response...
 
     def test_login_admin_ok(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'test_admin',
                                   'password': 'test12'}, status=302)
         response = response.follow()
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == 'test_admin'
         response.mustcontain('logout')
 
     def test_login_regular_ok(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'test_regular',
                                   'password': 'test12'}, status=302)
 
         response = response.follow()
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == 'test_regular'
         response.mustcontain('logout')
 
     def test_login_regular_forbidden_when_super_admin_restriction(self):
         from rhodecode.authentication.plugins.auth_rhodecode import RhodeCodeAuthPlugin
-        with fixture.auth_restriction(RhodeCodeAuthPlugin.AUTH_RESTRICTION_SUPER_ADMIN):
+        with fixture.auth_restriction(self.app._pyramid_registry,
+                                      RhodeCodeAuthPlugin.AUTH_RESTRICTION_SUPER_ADMIN):
             response = self.app.post(route_path('login'),
                                      {'username': 'test_regular',
                                       'password': 'test12'})
 
             response.mustcontain('invalid user name')
             response.mustcontain('invalid password')
 
     def test_login_regular_forbidden_when_scope_restriction(self):
         from rhodecode.authentication.plugins.auth_rhodecode import RhodeCodeAuthPlugin
-        with fixture.scope_restriction(RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_VCS):
+        with fixture.scope_restriction(self.app._pyramid_registry,
+                                       RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_VCS):
             response = self.app.post(route_path('login'),
                                      {'username': 'test_regular',
                                       'password': 'test12'})
 
             response.mustcontain('invalid user name')
             response.mustcontain('invalid password')
 
     def test_login_ok_came_from(self):
         test_came_from = '/_admin/users?branch=stable'
         _url = '{}?came_from={}'.format(route_path('login'), test_came_from)
         response = self.app.post(
             _url, {'username': 'test_admin', 'password': 'test12'}, status=302)
 
         assert 'branch=stable' in response.location
         response = response.follow()
 
         assert response.status == '200 OK'
         response.mustcontain('Users administration')
 
     def test_redirect_to_login_with_get_args(self):
         with fixture.anon_access(False):
             kwargs = {'branch': 'stable'}
             response = self.app.get(
                 h.route_path('repo_summary', repo_name=HG_REPO, _query=kwargs),
                 status=302)
 
             response_query = urlparse.parse_qsl(response.location)
             assert 'branch=stable' in response_query[0][1]
 
     def test_login_form_with_get_args(self):
         _url = '{}?came_from=/_admin/users,branch=stable'.format(route_path('login'))
         response = self.app.get(_url)
         assert 'branch%3Dstable' in response.form.action
 
     @pytest.mark.parametrize("url_came_from", [
         'data:text/html,<script>window.alert("xss")</script>',
         'mailto:[e-mail]',
         'file:///etc/passwd',
         'ftp://some.ftp.server',
         'http://other.domain',
         '/\r\nX-Forwarded-Host: http://example.org',
     ], ids=no_newline_id_generator)
     def test_login_bad_came_froms(self, url_came_from):
         _url = '{}?came_from={}'.format(route_path('login'), url_came_from)
         response = self.app.post(
             _url,
             {'username': 'test_admin', 'password': 'test12'})
         assert response.status == '302 Found'
         response = response.follow()
         assert response.status == '200 OK'
         assert response.request.path == '/'
 
     def test_login_short_password(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'test_admin',
                                   'password': 'as'})
         assert response.status == '200 OK'
 
         response.mustcontain('Enter 3 characters or more')
 
     def test_login_wrong_non_ascii_password(self, user_regular):
         response = self.app.post(
             route_path('login'),
             {'username': user_regular.username,
              'password': u'invalid-non-asci\xe4'.encode('utf8')})
 
         response.mustcontain('invalid user name')
         response.mustcontain('invalid password')
 
     def test_login_with_non_ascii_password(self, user_util):
         password = u'valid-non-ascii\xe4'
         user = user_util.create_user(password=password)
         response = self.app.post(
             route_path('login'),
             {'username': user.username,
              'password': password.encode('utf-8')})
         assert response.status_code == 302
 
     def test_login_wrong_username_password(self):
         response = self.app.post(route_path('login'),
                                  {'username': 'error',
                                   'password': 'test12'})
 
         response.mustcontain('invalid user name')
         response.mustcontain('invalid password')
 
     def test_login_admin_ok_password_migration(self, real_crypto_backend):
         from rhodecode.lib import auth
 
         # create new user, with sha256 password
         temp_user = 'test_admin_sha256'
         user = fixture.create_user(temp_user)
         user.password = auth._RhodeCodeCryptoSha256().hash_create(
             b'test123')
         Session().add(user)
         Session().commit()
         self.destroy_users.add(temp_user)
         response = self.app.post(route_path('login'),
                                  {'username': temp_user,
                                   'password': 'test123'}, status=302)
 
         response = response.follow()
         session = response.get_session_from_response()
         username = session['rhodecode_user'].get('username')
         assert username == temp_user
         response.mustcontain('logout')
 
         # new password should be bcrypted, after log-in and transfer
         user = User.get_by_username(temp_user)
         assert user.password.startswith('$')
 
     # REGISTRATIONS
     def test_register(self):
         response = self.app.get(route_path('register'))
         response.mustcontain('Create an Account')
 
     def test_register_err_same_username(self):
         uname = 'test_admin'
         response = self.app.post(
             route_path('register'),
             {
                 'username': uname,
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         assertr = response.assert_response()
         msg = 'Username "%(username)s" already exists'
         msg = msg % {'username': uname}
         assertr.element_contains('#username+.error-message', msg)
 
     def test_register_err_same_email(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'test_admin_0',
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         assertr = response.assert_response()
         msg = u'This e-mail address is already taken'
         assertr.element_contains('#email+.error-message', msg)
 
     def test_register_err_same_email_case_sensitive(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'test_admin_1',
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
         assertr = response.assert_response()
         msg = u'This e-mail address is already taken'
         assertr.element_contains('#email+.error-message', msg)
 
     def test_register_err_wrong_data(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'xs',
                 'password': 'test',
                 'password_confirmation': 'test',
                 'email': 'goodmailm',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
         assert response.status == '200 OK'
         response.mustcontain('An email address must contain a single @')
         response.mustcontain('Enter a value 6 characters long or more')
 
     def test_register_err_username(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'error user',
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': 'goodmailm',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         response.mustcontain('An email address must contain a single @')
         response.mustcontain(
             'Username may only contain '
             'alphanumeric characters underscores, '
             'periods or dashes and must begin with '
             'alphanumeric character')
 
     def test_register_err_case_sensitive(self):
         usr = 'Test_Admin'
         response = self.app.post(
             route_path('register'),
             {
                 'username': usr,
                 'password': 'test12',
                 'password_confirmation': 'test12',
                 'email': 'goodmailm',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         assertr = response.assert_response()
         msg = u'Username "%(username)s" already exists'
         msg = msg % {'username': usr}
         assertr.element_contains('#username+.error-message', msg)
 
     def test_register_special_chars(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'xxxaxn',
                 'password': 'ąćźżąśśśś',
                 'password_confirmation': 'ąćźżąśśśś',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
 
         msg = u'Invalid characters (non-ascii) in password'
         response.mustcontain(msg)
 
     def test_register_password_mismatch(self):
         response = self.app.post(
             route_path('register'),
             {
                 'username': 'xs',
                 'password': '123qwe',
                 'password_confirmation': 'qwe123',
                 'email': '[e-mail]',
                 'firstname': 'test',
                 'lastname': 'test'
             }
         )
         msg = u'Passwords do not match'
         response.mustcontain(msg)
 
     def test_register_ok(self):
         username = 'test_regular4'
         password = 'qweqwe'
         email = '[e-mail]'
         name = 'testname'
         lastname = 'testlastname'
 
         # this initializes a session
         response = self.app.get(route_path('register'))
         response.mustcontain('Create an Account')
 
 
         response = self.app.post(
             route_path('register'),
             {
                 'username': username,
                 'password': password,
                 'password_confirmation': password,
                 'email': email,
                 'firstname': name,
                 'lastname': lastname,
                 'admin': True
             },
             status=302
         )  # This should be overridden
 
         assert_session_flash(
             response, 'You have successfully registered with RhodeCode. You can log-in now.')
 
         ret = Session().query(User).filter(
             User.username == 'test_regular4').one()
         assert ret.username == username
         assert check_password(password, ret.password)
         assert ret.email == email
         assert ret.name == name
         assert ret.lastname == lastname
         assert ret.auth_tokens is not None
         assert not ret.admin
 
     def test_forgot_password_wrong_mail(self):
         bad_email = '[e-mail]'
         # this initializes a session
         self.app.get(route_path('reset_password'))
 
         response = self.app.post(
             route_path('reset_password'), {'email': bad_email, }
         )
         assert_session_flash(response,
             'If such email exists, a password reset link was sent to it.')
 
     def test_forgot_password(self, user_util):
         # this initializes a session
         self.app.get(route_path('reset_password'))
 
         user = user_util.create_user()
         user_id = user.user_id
         email = user.email
 
         response = self.app.post(route_path('reset_password'), {'email': email, })
 
         assert_session_flash(response,
             'If such email exists, a password reset link was sent to it.')
 
         # BAD KEY
         confirm_url = '{}?key={}'.format(route_path('reset_password_confirmation'), 'badkey')
         response = self.app.get(confirm_url, status=302)
         assert response.location.endswith(route_path('reset_password'))
         assert_session_flash(response, 'Given reset token is invalid')
 
         response.follow()  # cleanup flash
 
         # GOOD KEY
         key = UserApiKeys.query()\
             .filter(UserApiKeys.user_id == user_id)\
             .filter(UserApiKeys.role == UserApiKeys.ROLE_PASSWORD_RESET)\
             .first()
 
         assert key
 
         confirm_url = '{}?key={}'.format(route_path('reset_password_confirmation'), key.api_key)
         response = self.app.get(confirm_url)
         assert response.status == '302 Found'
         assert response.location.endswith(route_path('login'))
 
         assert_session_flash(
             response,
             'Your password reset was successful, '
             'a new password has been sent to your email')
 
         response.follow()
 
     def _get_api_whitelist(self, values=None):
         config = {'api_access_controllers_whitelist': values or []}
         return config
 
     @pytest.mark.parametrize("test_name, auth_token", [
         ('none', None),
         ('empty_string', ''),
         ('fake_number', '123456'),
         ('proper_auth_token', None)
     ])
     def test_access_not_whitelisted_page_via_auth_token(
             self, test_name, auth_token, user_admin):
 
         whitelist = self._get_api_whitelist([])
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert [] == whitelist['api_access_controllers_whitelist']
             if test_name == 'proper_auth_token':
                 # use builtin if api_key is None
                 auth_token = user_admin.api_key
 
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=auth_token)),
                     status=302)
 
     @pytest.mark.parametrize("test_name, auth_token, code", [
         ('none', None, 302),
         ('empty_string', '', 302),
         ('fake_number', '123456', 302),
         ('proper_auth_token', None, 200)
     ])
     def test_access_whitelisted_page_via_auth_token(
             self, test_name, auth_token, code, user_admin):
 
         whitelist = self._get_api_whitelist(whitelist_view)
 
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert whitelist_view == whitelist['api_access_controllers_whitelist']
 
             if test_name == 'proper_auth_token':
                 auth_token = user_admin.api_key
                 assert auth_token
 
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=auth_token)),
                     status=code)
 
     @pytest.mark.parametrize("test_name, auth_token, code", [
         ('proper_auth_token', None, 200),
         ('wrong_auth_token', '123456', 302),
     ])
     def test_access_whitelisted_page_via_auth_token_bound_to_token(
             self, test_name, auth_token, code, user_admin):
 
         expected_token = auth_token
         if test_name == 'proper_auth_token':
             auth_token = user_admin.api_key
             expected_token = auth_token
             assert auth_token
 
         whitelist = self._get_api_whitelist([
             'RepoCommitsView:repo_commit_raw@{}'.format(expected_token)])
 
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
 
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=auth_token)),
                     status=code)
 
     def test_access_page_via_extra_auth_token(self):
         whitelist = self._get_api_whitelist(whitelist_view)
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert whitelist_view == \
                 whitelist['api_access_controllers_whitelist']
 
             new_auth_token = AuthTokenModel().create(
                 TEST_USER_ADMIN_LOGIN, 'test')
             Session().commit()
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=new_auth_token.api_key)),
                     status=200)
 
     def test_access_page_via_expired_auth_token(self):
         whitelist = self._get_api_whitelist(whitelist_view)
         with mock.patch.dict('rhodecode.CONFIG', whitelist):
             assert whitelist_view == \
                 whitelist['api_access_controllers_whitelist']
 
             new_auth_token = AuthTokenModel().create(
                 TEST_USER_ADMIN_LOGIN, 'test')
             Session().commit()
             # patch the api key and make it expired
             new_auth_token.expires = 0
             Session().add(new_auth_token)
             Session().commit()
             with fixture.anon_access(False):
                 self.app.get(
                     route_path('repo_commit_raw',
                                repo_name=HG_REPO, commit_id='tip',
                                params=dict(api_key=new_auth_token.api_key)),
                     status=302)

rhodecode/authentication/base.py

@@ -1,797 +1,808 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Authentication modules
 """
 import socket
 import string
 import colander
 import copy
 import logging
 import time
 import traceback
 import warnings
 import functools
 
 from pyramid.threadlocal import get_current_registry
 
 from rhodecode.authentication.interface import IAuthnPluginRegistry
 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
 from rhodecode.lib import rc_cache
 from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt
 from rhodecode.lib.utils2 import safe_int, safe_str
 from rhodecode.lib.exceptions import LdapConnectionError, LdapUsernameError, \
     LdapPasswordError
 from rhodecode.model.db import User
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import SettingsModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 
 
 log = logging.getLogger(__name__)
 
 # auth types that authenticate() function can receive
 VCS_TYPE = 'vcs'
 HTTP_TYPE = 'http'
 
 external_auth_session_key = 'rhodecode.external_auth'
 
 
 class hybrid_property(object):
     """
     a property decorator that works both for instance and class
     """
     def __init__(self, fget, fset=None, fdel=None, expr=None):
         self.fget = fget
         self.fset = fset
         self.fdel = fdel
         self.expr = expr or fget
         functools.update_wrapper(self, fget)
 
     def __get__(self, instance, owner):
         if instance is None:
             return self.expr(owner)
         else:
             return self.fget(instance)
 
     def __set__(self, instance, value):
         self.fset(instance, value)
 
     def __delete__(self, instance):
         self.fdel(instance)
 
 
 class LazyFormencode(object):
     def __init__(self, formencode_obj, *args, **kwargs):
         self.formencode_obj = formencode_obj
         self.args = args
         self.kwargs = kwargs
 
     def __call__(self, *args, **kwargs):
         from inspect import isfunction
         formencode_obj = self.formencode_obj
         if isfunction(formencode_obj):
             # case we wrap validators into functions
             formencode_obj = self.formencode_obj(*args, **kwargs)
         return formencode_obj(*self.args, **self.kwargs)
 
 
 class RhodeCodeAuthPluginBase(object):
     # UID is used to register plugin to the registry
     uid = None
 
     # cache the authentication request for N amount of seconds. Some kind
     # of authentication methods are very heavy and it's very efficient to cache
     # the result of a call. If it's set to None (default) cache is off
     AUTH_CACHE_TTL = None
     AUTH_CACHE = {}
 
     auth_func_attrs = {
         "username": "unique username",
         "firstname": "first name",
         "lastname": "last name",
         "email": "email address",
         "groups": '["list", "of", "groups"]',
         "user_group_sync":
             'True|False defines if returned user groups should be synced',
         "extern_name": "name in external source of record",
         "extern_type": "type of external source of record",
         "admin": 'True|False defines if user should be RhodeCode super admin',
         "active":
             'True|False defines active state of user internally for RhodeCode',
         "active_from_extern":
             "True|False|None, active state from the external auth, "
             "None means use definition from RhodeCode extern_type active value"
 
     }
     # set on authenticate() method and via set_auth_type func.
     auth_type = None
 
     # set on authenticate() method and via set_calling_scope_repo, this is a
     # calling scope repository when doing authentication most likely on VCS
     # operations
     acl_repo_name = None
 
     # List of setting names to store encrypted. Plugins may override this list
     # to store settings encrypted.
     _settings_encrypted = []
 
     # Mapping of python to DB settings model types. Plugins may override or
     # extend this mapping.
     _settings_type_map = {
         colander.String: 'unicode',
         colander.Integer: 'int',
         colander.Boolean: 'bool',
         colander.List: 'list',
     }
 
     # list of keys in settings that are unsafe to be logged, should be passwords
     # or other crucial credentials
     _settings_unsafe_keys = []
 
     def __init__(self, plugin_id):
         self._plugin_id = plugin_id
+        self._settings = {}
 
     def __str__(self):
         return self.get_id()
 
     def _get_setting_full_name(self, name):
         """
         Return the full setting name used for storing values in the database.
         """
         # TODO: johbo: Using the name here is problematic. It would be good to
         # introduce either new models in the database to hold Plugin and
         # PluginSetting or to use the plugin id here.
         return 'auth_{}_{}'.format(self.name, name)
 
     def _get_setting_type(self, name):
         """
         Return the type of a setting. This type is defined by the SettingsModel
         and determines how the setting is stored in DB. Optionally the suffix
         `.encrypted` is appended to instruct SettingsModel to store it
         encrypted.
         """
         schema_node = self.get_settings_schema().get(name)
         db_type = self._settings_type_map.get(
             type(schema_node.typ), 'unicode')
         if name in self._settings_encrypted:
             db_type = '{}.encrypted'.format(db_type)
         return db_type
 
     @classmethod
     def docs(cls):
         """
         Defines documentation url which helps with plugin setup
         """
         return ''
 
     @classmethod
     def icon(cls):
         """
         Defines ICON in SVG format for authentication method
         """
         return ''
 
     def is_enabled(self):
         """
         Returns true if this plugin is enabled. An enabled plugin can be
         configured in the admin interface but it is not consulted during
         authentication.
         """
         auth_plugins = SettingsModel().get_auth_plugins()
         return self.get_id() in auth_plugins
 
     def is_active(self, plugin_cached_settings=None):
         """
         Returns true if the plugin is activated. An activated plugin is
         consulted during authentication, assumed it is also enabled.
         """
         return self.get_setting_by_name(
             'enabled', plugin_cached_settings=plugin_cached_settings)
 
     def get_id(self):
         """
         Returns the plugin id.
         """
         return self._plugin_id
 
     def get_display_name(self):
         """
         Returns a translation string for displaying purposes.
         """
         raise NotImplementedError('Not implemented in base class')
 
     def get_settings_schema(self):
         """
         Returns a colander schema, representing the plugin settings.
         """
         return AuthnPluginSettingsSchemaBase()
 
-    def get_settings(self):
-        """
-        Returns the plugin settings as dictionary.
-        """
+    def _propagate_settings(self, raw_settings):
         settings = {}
-        raw_settings = SettingsModel().get_all_settings()
         for node in self.get_settings_schema():
             settings[node.name] = self.get_setting_by_name(
                 node.name, plugin_cached_settings=raw_settings)
         return settings
 
+    def get_settings(self, use_cache=True):
+        """
+        Returns the plugin settings as dictionary.
+        """
+        if self._settings != {} and use_cache:
+            return self._settings
+
+        raw_settings = SettingsModel().get_all_settings()
+        settings = self._propagate_settings(raw_settings)
+
+        self._settings = settings
+        return self._settings
+
     def get_setting_by_name(self, name, default=None, plugin_cached_settings=None):
         """
         Returns a plugin setting by name.
         """
         full_name = 'rhodecode_{}'.format(self._get_setting_full_name(name))
         if plugin_cached_settings:
             plugin_settings = plugin_cached_settings
         else:
             plugin_settings = SettingsModel().get_all_settings()
 
         if full_name in plugin_settings:
             return plugin_settings[full_name]
         else:
             return default
 
     def create_or_update_setting(self, name, value):
         """
         Create or update a setting for this plugin in the persistent storage.
         """
         full_name = self._get_setting_full_name(name)
         type_ = self._get_setting_type(name)
         db_setting = SettingsModel().create_or_update_setting(
             full_name, value, type_)
         return db_setting.app_settings_value
 
     def log_safe_settings(self, settings):
         """
         returns a log safe representation of settings, without any secrets
         """
         settings_copy = copy.deepcopy(settings)
         for k in self._settings_unsafe_keys:
             if k in settings_copy:
                 del settings_copy[k]
         return settings_copy
 
     @hybrid_property
     def name(self):
         """
         Returns the name of this authentication plugin.
 
         :returns: string
         """
         raise NotImplementedError("Not implemented in base class")
 
     def get_url_slug(self):
         """
         Returns a slug which should be used when constructing URLs which refer
         to this plugin. By default it returns the plugin name. If the name is
         not suitable for using it in an URL the plugin should override this
         method.
         """
         return self.name
 
     @property
     def is_headers_auth(self):
         """
         Returns True if this authentication plugin uses HTTP headers as
         authentication method.
         """
         return False
 
     @hybrid_property
     def is_container_auth(self):
         """
         Deprecated method that indicates if this authentication plugin uses
         HTTP headers as authentication method.
         """
         warnings.warn(
             'Use is_headers_auth instead.', category=DeprecationWarning)
         return self.is_headers_auth
 
     @hybrid_property
     def allows_creating_users(self):
         """
         Defines if Plugin allows users to be created on-the-fly when
         authentication is called. Controls how external plugins should behave
         in terms if they are allowed to create new users, or not. Base plugins
         should not be allowed to, but External ones should be !
 
         :return: bool
         """
         return False
 
     def set_auth_type(self, auth_type):
         self.auth_type = auth_type
 
     def set_calling_scope_repo(self, acl_repo_name):
         self.acl_repo_name = acl_repo_name
 
     def allows_authentication_from(
             self, user, allows_non_existing_user=True,
             allowed_auth_plugins=None, allowed_auth_sources=None):
         """
         Checks if this authentication module should accept a request for
         the current user.
 
         :param user: user object fetched using plugin's get_user() method.
         :param allows_non_existing_user: if True, don't allow the
             user to be empty, meaning not existing in our database
         :param allowed_auth_plugins: if provided, users extern_type will be
             checked against a list of provided extern types, which are plugin
             auth_names in the end
         :param allowed_auth_sources: authentication type allowed,
             `http` or `vcs` default is both.
             defines if plugin will accept only http authentication vcs
             authentication(git/hg) or both
         :returns: boolean
         """
         if not user and not allows_non_existing_user:
             log.debug('User is empty but plugin does not allow empty users,'
                       'not allowed to authenticate')
             return False
 
         expected_auth_plugins = allowed_auth_plugins or [self.name]
         if user and (user.extern_type and
                      user.extern_type not in expected_auth_plugins):
             log.debug(
                 'User `%s` is bound to `%s` auth type. Plugin allows only '
                 '%s, skipping', user, user.extern_type, expected_auth_plugins)
 
             return False
 
         # by default accept both
         expected_auth_from = allowed_auth_sources or [HTTP_TYPE, VCS_TYPE]
         if self.auth_type not in expected_auth_from:
             log.debug('Current auth source is %s but plugin only allows %s',
                       self.auth_type, expected_auth_from)
             return False
 
         return True
 
     def get_user(self, username=None, **kwargs):
         """
         Helper method for user fetching in plugins, by default it's using
         simple fetch by username, but this method can be custimized in plugins
         eg. headers auth plugin to fetch user by environ params
 
         :param username: username if given to fetch from database
         :param kwargs: extra arguments needed for user fetching.
         """
         user = None
         log.debug(
             'Trying to fetch user `%s` from RhodeCode database', username)
         if username:
             user = User.get_by_username(username)
             if not user:
                 log.debug('User not found, fallback to fetch user in '
                           'case insensitive mode')
                 user = User.get_by_username(username, case_insensitive=True)
         else:
             log.debug('provided username:`%s` is empty skipping...', username)
         if not user:
             log.debug('User `%s` not found in database', username)
         else:
             log.debug('Got DB user:%s', user)
         return user
 
     def user_activation_state(self):
         """
         Defines user activation state when creating new users
 
         :returns: boolean
         """
         raise NotImplementedError("Not implemented in base class")
 
     def auth(self, userobj, username, passwd, settings, **kwargs):
         """
         Given a user object (which may be null), username, a plaintext
         password, and a settings object (containing all the keys needed as
         listed in settings()), authenticate this user's login attempt.
 
         Return None on failure. On success, return a dictionary of the form:
 
             see: RhodeCodeAuthPluginBase.auth_func_attrs
         This is later validated for correctness
         """
         raise NotImplementedError("not implemented in base class")
 
     def _authenticate(self, userobj, username, passwd, settings, **kwargs):
         """
         Wrapper to call self.auth() that validates call on it
 
         :param userobj: userobj
         :param username: username
         :param passwd: plaintext password
         :param settings: plugin settings
         """
         auth = self.auth(userobj, username, passwd, settings, **kwargs)
         if auth:
             auth['_plugin'] = self.name
             auth['_ttl_cache'] = self.get_ttl_cache(settings)
             # check if hash should be migrated ?
             new_hash = auth.get('_hash_migrate')
             if new_hash:
                 self._migrate_hash_to_bcrypt(username, passwd, new_hash)
             if 'user_group_sync' not in auth:
                 auth['user_group_sync'] = False
             return self._validate_auth_return(auth)
         return auth
 
     def _migrate_hash_to_bcrypt(self, username, password, new_hash):
         new_hash_cypher = _RhodeCodeCryptoBCrypt()
         # extra checks, so make sure new hash is correct.
         password_encoded = safe_str(password)
         if new_hash and new_hash_cypher.hash_check(
                 password_encoded, new_hash):
             cur_user = User.get_by_username(username)
             cur_user.password = new_hash
             Session().add(cur_user)
             Session().flush()
             log.info('Migrated user %s hash to bcrypt', cur_user)
 
     def _validate_auth_return(self, ret):
         if not isinstance(ret, dict):
             raise Exception('returned value from auth must be a dict')
         for k in self.auth_func_attrs:
             if k not in ret:
                 raise Exception('Missing %s attribute from returned data' % k)
         return ret
 
     def get_ttl_cache(self, settings=None):
         plugin_settings = settings or self.get_settings()
         # we set default to 30, we make a compromise here,
         # performance > security, mostly due to LDAP/SVN, majority
         # of users pick cache_ttl to be enabled
         from rhodecode.authentication import plugin_default_auth_ttl
         cache_ttl = plugin_default_auth_ttl
 
         if isinstance(self.AUTH_CACHE_TTL, (int, long)):
             # plugin cache set inside is more important than the settings value
             cache_ttl = self.AUTH_CACHE_TTL
         elif plugin_settings.get('cache_ttl'):
             cache_ttl = safe_int(plugin_settings.get('cache_ttl'), 0)
 
         plugin_cache_active = bool(cache_ttl and cache_ttl > 0)
         return plugin_cache_active, cache_ttl
 
 
 class RhodeCodeExternalAuthPlugin(RhodeCodeAuthPluginBase):
 
     @hybrid_property
     def allows_creating_users(self):
         return True
 
     def use_fake_password(self):
         """
         Return a boolean that indicates whether or not we should set the user's
         password to a random value when it is authenticated by this plugin.
         If your plugin provides authentication, then you will generally
         want this.
 
         :returns: boolean
         """
         raise NotImplementedError("Not implemented in base class")
 
     def _authenticate(self, userobj, username, passwd, settings, **kwargs):
         # at this point _authenticate calls plugin's `auth()` function
         auth = super(RhodeCodeExternalAuthPlugin, self)._authenticate(
             userobj, username, passwd, settings, **kwargs)
 
         if auth:
             # maybe plugin will clean the username ?
             # we should use the return value
             username = auth['username']
 
             # if external source tells us that user is not active, we should
             # skip rest of the process. This can prevent from creating users in
             # RhodeCode when using external authentication, but if it's
             # inactive user we shouldn't create that user anyway
             if auth['active_from_extern'] is False:
                 log.warning(
                     "User %s authenticated against %s, but is inactive",
                     username, self.__module__)
                 return None
 
             cur_user = User.get_by_username(username, case_insensitive=True)
             is_user_existing = cur_user is not None
 
             if is_user_existing:
                 log.debug('Syncing user `%s` from '
                           '`%s` plugin', username, self.name)
             else:
                 log.debug('Creating non existing user `%s` from '
                           '`%s` plugin', username, self.name)
 
             if self.allows_creating_users:
                 log.debug('Plugin `%s` allows to '
                           'create new users', self.name)
             else:
                 log.debug('Plugin `%s` does not allow to '
                           'create new users', self.name)
 
             user_parameters = {
                 'username': username,
                 'email': auth["email"],
                 'firstname': auth["firstname"],
                 'lastname': auth["lastname"],
                 'active': auth["active"],
                 'admin': auth["admin"],
                 'extern_name': auth["extern_name"],
                 'extern_type': self.name,
                 'plugin': self,
                 'allow_to_create_user': self.allows_creating_users,
             }
 
             if not is_user_existing:
                 if self.use_fake_password():
                     # Randomize the PW because we don't need it, but don't want
                     # them blank either
                     passwd = PasswordGenerator().gen_password(length=16)
                 user_parameters['password'] = passwd
             else:
                 # Since the password is required by create_or_update method of
                 # UserModel, we need to set it explicitly.
                 # The create_or_update method is smart and recognises the
                 # password hashes as well.
                 user_parameters['password'] = cur_user.password
 
             # we either create or update users, we also pass the flag
             # that controls if this method can actually do that.
             # raises NotAllowedToCreateUserError if it cannot, and we try to.
             user = UserModel().create_or_update(**user_parameters)
             Session().flush()
             # enforce user is just in given groups, all of them has to be ones
             # created from plugins. We store this info in _group_data JSON
             # field
 
             if auth['user_group_sync']:
                 try:
                     groups = auth['groups'] or []
                     log.debug(
                         'Performing user_group sync based on set `%s` '
                         'returned by `%s` plugin', groups, self.name)
                     UserGroupModel().enforce_groups(user, groups, self.name)
                 except Exception:
                     # for any reason group syncing fails, we should
                     # proceed with login
                     log.error(traceback.format_exc())
 
             Session().commit()
         return auth
 
 
 class AuthLdapBase(object):
 
     @classmethod
     def _build_servers(cls, ldap_server_type, ldap_server, port, use_resolver=True):
 
         def host_resolver(host, port, full_resolve=True):
             """
             Main work for this function is to prevent ldap connection issues,
             and detect them early using a "greenified" sockets
             """
             host = host.strip()
             if not full_resolve:
                 return '{}:{}'.format(host, port)
 
             log.debug('LDAP: Resolving IP for LDAP host %s', host)
             try:
                 ip = socket.gethostbyname(host)
                 log.debug('Got LDAP server %s ip %s', host, ip)
             except Exception:
                 raise LdapConnectionError(
                     'Failed to resolve host: `{}`'.format(host))
 
             log.debug('LDAP: Checking if IP %s is accessible', ip)
             s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             try:
                 s.connect((ip, int(port)))
                 s.shutdown(socket.SHUT_RD)
             except Exception:
                 raise LdapConnectionError(
                     'Failed to connect to host: `{}:{}`'.format(host, port))
 
             return '{}:{}'.format(host, port)
 
         if len(ldap_server) == 1:
             # in case of single server use resolver to detect potential
             # connection issues
             full_resolve = True
         else:
             full_resolve = False
 
         return ', '.join(
             ["{}://{}".format(
                 ldap_server_type,
                 host_resolver(host, port, full_resolve=use_resolver and full_resolve))
              for host in ldap_server])
 
     @classmethod
     def _get_server_list(cls, servers):
         return map(string.strip, servers.split(','))
 
     @classmethod
     def get_uid(cls, username, server_addresses):
         uid = username
         for server_addr in server_addresses:
             uid = chop_at(username, "@%s" % server_addr)
         return uid
 
     @classmethod
     def validate_username(cls, username):
         if "," in username:
             raise LdapUsernameError(
                 "invalid character `,` in username: `{}`".format(username))
 
     @classmethod
     def validate_password(cls, username, password):
         if not password:
             msg = "Authenticating user %s with blank password not allowed"
             log.warning(msg, username)
             raise LdapPasswordError(msg)
 
 
 def loadplugin(plugin_id):
     """
     Loads and returns an instantiated authentication plugin.
     Returns the RhodeCodeAuthPluginBase subclass on success,
     or None on failure.
     """
     # TODO: Disusing pyramids thread locals to retrieve the registry.
     authn_registry = get_authn_registry()
     plugin = authn_registry.get_plugin(plugin_id)
     if plugin is None:
         log.error('Authentication plugin not found: "%s"', plugin_id)
     return plugin
 
 
 def get_authn_registry(registry=None):
     registry = registry or get_current_registry()
-    authn_registry = registry.getUtility(IAuthnPluginRegistry)
+    authn_registry = registry.queryUtility(IAuthnPluginRegistry)
     return authn_registry
 
 
 def authenticate(username, password, environ=None, auth_type=None,
                  skip_missing=False, registry=None, acl_repo_name=None):
     """
     Authentication function used for access control,
     It tries to authenticate based on enabled authentication modules.
 
     :param username: username can be empty for headers auth
     :param password: password can be empty for headers auth
     :param environ: environ headers passed for headers auth
     :param auth_type: type of authentication, either `HTTP_TYPE` or `VCS_TYPE`
     :param skip_missing: ignores plugins that are in db but not in environment
     :returns: None if auth failed, plugin_user dict if auth is correct
     """
     if not auth_type or auth_type not in [HTTP_TYPE, VCS_TYPE]:
         raise ValueError('auth type must be on of http, vcs got "%s" instead'
                          % auth_type)
     headers_only = environ and not (username and password)
 
     authn_registry = get_authn_registry(registry)
+
     plugins_to_check = authn_registry.get_plugins_for_authentication()
     log.debug('Starting ordered authentication chain using %s plugins',
               [x.name for x in plugins_to_check])
     for plugin in plugins_to_check:
         plugin.set_auth_type(auth_type)
         plugin.set_calling_scope_repo(acl_repo_name)
 
         if headers_only and not plugin.is_headers_auth:
             log.debug('Auth type is for headers only and plugin `%s` is not '
                       'headers plugin, skipping...', plugin.get_id())
             continue
 
         log.debug('Trying authentication using ** %s **', plugin.get_id())
 
         # load plugin settings from RhodeCode database
         plugin_settings = plugin.get_settings()
         plugin_sanitized_settings = plugin.log_safe_settings(plugin_settings)
         log.debug('Plugin `%s` settings:%s', plugin.get_id(), plugin_sanitized_settings)
 
         # use plugin's method of user extraction.
         user = plugin.get_user(username, environ=environ,
                                settings=plugin_settings)
         display_user = user.username if user else username
         log.debug(
             'Plugin %s extracted user is `%s`', plugin.get_id(), display_user)
 
         if not plugin.allows_authentication_from(user):
             log.debug('Plugin %s does not accept user `%s` for authentication',
                       plugin.get_id(), display_user)
             continue
         else:
             log.debug('Plugin %s accepted user `%s` for authentication',
                       plugin.get_id(), display_user)
 
         log.info('Authenticating user `%s` using %s plugin',
                  display_user, plugin.get_id())
 
         plugin_cache_active, cache_ttl = plugin.get_ttl_cache(plugin_settings)
 
         log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)',
                   plugin.get_id(), plugin_cache_active, cache_ttl)
 
         user_id = user.user_id if user else None
         # don't cache for empty users
         plugin_cache_active = plugin_cache_active and user_id
         cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                                expiration_time=cache_ttl,
                                                condition=plugin_cache_active)
         def compute_auth(
                 cache_name, plugin_name, username, password):
 
             # _authenticate is a wrapper for .auth() method of plugin.
             # it checks if .auth() sends proper data.
             # For RhodeCodeExternalAuthPlugin it also maps users to
             # Database and maps the attributes returned from .auth()
             # to RhodeCode database. If this function returns data
             # then auth is correct.
             log.debug('Running plugin `%s` _authenticate method '
                       'using username and password', plugin.get_id())
             return plugin._authenticate(
                 user, username, password, plugin_settings,
                 environ=environ or {})
 
         start = time.time()
         # for environ based auth, password can be empty, but then the validation is
         # on the server that fills in the env data needed for authentication
         plugin_user = compute_auth('auth', plugin.name, username, (password or ''))
 
         auth_time = time.time() - start
         log.debug('Authentication for plugin `%s` completed in %.4fs, '
                   'expiration time of fetched cache %.1fs.',
                   plugin.get_id(), auth_time, cache_ttl)
 
         log.debug('PLUGIN USER DATA: %s', plugin_user)
 
         if plugin_user:
             log.debug('Plugin returned proper authentication data')
             return plugin_user
         # we failed to Auth because .auth() method didn't return proper user
         log.debug("User `%s` failed to authenticate against %s",
                   display_user, plugin.get_id())
 
     # case when we failed to authenticate against all defined plugins
     return None
 
 
 def chop_at(s, sub, inclusive=False):
     """Truncate string ``s`` at the first occurrence of ``sub``.
 
     If ``inclusive`` is true, truncate just after ``sub`` rather than at it.
 
     >>> chop_at("plutocratic brats", "rat")
     'plutoc'
     >>> chop_at("plutocratic brats", "rat", True)
     'plutocrat'
     """
     pos = s.find(sub)
     if pos == -1:
         return s
     if inclusive:
         return s[:pos+len(sub)]
     return s[:pos]

rhodecode/authentication/registry.py

@@ -1,94 +1,107 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 from pyramid.exceptions import ConfigurationError
 from zope.interface import implementer
 
 from rhodecode.authentication.interface import IAuthnPluginRegistry
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.model.settings import SettingsModel
 
 log = logging.getLogger(__name__)
 
 
 @implementer(IAuthnPluginRegistry)
 class AuthenticationPluginRegistry(object):
 
     # INI settings key to set a fallback authentication plugin.
     fallback_plugin_key = 'rhodecode.auth_plugin_fallback'
 
     def __init__(self, settings):
         self._plugins = {}
+        self._plugins_for_auth = None
         self._fallback_plugin = settings.get(self.fallback_plugin_key, None)
 
     def add_authn_plugin(self, config, plugin):
         plugin_id = plugin.get_id()
         if plugin_id in self._plugins.keys():
             raise ConfigurationError(
                 'Cannot register authentication plugin twice: "%s"', plugin_id)
         else:
             log.debug('Register authentication plugin: "%s"', plugin_id)
             self._plugins[plugin_id] = plugin
 
     def get_plugins(self):
         def sort_key(plugin):
             return str.lower(safe_str(plugin.get_display_name()))
 
         return sorted(self._plugins.values(), key=sort_key)
 
     def get_plugin(self, plugin_id):
         return self._plugins.get(plugin_id, None)
 
     def get_plugin_by_uid(self, plugin_uid):
         for plugin in self._plugins.values():
             if plugin.uid == plugin_uid:
                 return plugin
 
+    def invalidate_plugins_for_auth(self):
+        log.debug('Invalidating cached plugins for authentication')
+        self._plugins_for_auth = None
+
     def get_plugins_for_authentication(self):
         """
         Returns a list of plugins which should be consulted when authenticating
         a user. It only returns plugins which are enabled and active.
         Additionally it includes the fallback plugin from the INI file, if
         `rhodecode.auth_plugin_fallback` is set to a plugin ID.
         """
+        if self._plugins_for_auth is not None:
+            return self._plugins_for_auth
+
         plugins = []
 
         # Add all enabled and active plugins to the list. We iterate over the
         # auth_plugins setting from DB because it also represents the ordering.
         enabled_plugins = SettingsModel().get_auth_plugins()
         raw_settings = SettingsModel().get_all_settings()
         for plugin_id in enabled_plugins:
             plugin = self.get_plugin(plugin_id)
             if plugin is not None and plugin.is_active(
                     plugin_cached_settings=raw_settings):
+
+                # inject settings into plugin, we can re-use the DB fetched settings here
+                plugin._settings = plugin._propagate_settings(raw_settings)
                 plugins.append(plugin)
 
         # Add the fallback plugin from ini file.
         if self._fallback_plugin:
             log.warn(
                 'Using fallback authentication plugin from INI file: "%s"',
                 self._fallback_plugin)
             plugin = self.get_plugin(self._fallback_plugin)
             if plugin is not None and plugin not in plugins:
+                plugin._settings = plugin._propagate_settings(raw_settings)
                 plugins.append(plugin)
 
-        return plugins
+        self._plugins_for_auth = plugins
+        return self._plugins_for_auth

rhodecode/authentication/views.py

@@ -1,179 +1,180 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import colander
 import formencode.htmlfill
 import logging
 
 from pyramid.httpexceptions import HTTPFound
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.apps._base import BaseAppView
 from rhodecode.authentication.base import get_authn_registry
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.model.forms import AuthSettingsForm
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import SettingsModel
 
 log = logging.getLogger(__name__)
 
 
 class AuthnPluginViewBase(BaseAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         self.plugin = self.context.plugin
         return c
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_get(self, defaults=None, errors=None):
         """
         View that displays the plugin settings as a form.
         """
         c = self.load_default_context()
         defaults = defaults or {}
         errors = errors or {}
         schema = self.plugin.get_settings_schema()
 
         # Compute default values for the form. Priority is:
         # 1. Passed to this method 2. DB value 3. Schema default
         for node in schema:
             if node.name not in defaults:
                 defaults[node.name] = self.plugin.get_setting_by_name(
                     node.name, node.default)
 
         template_context = {
             'defaults': defaults,
             'errors': errors,
             'plugin': self.context.plugin,
             'resource': self.context,
         }
 
         return self._get_template_context(c, **template_context)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_post(self):
         """
         View that validates and stores the plugin settings.
         """
         _ = self.request.translate
         self.load_default_context()
         schema = self.plugin.get_settings_schema()
         data = self.request.params
 
         try:
             valid_data = schema.deserialize(data)
         except colander.Invalid as e:
             # Display error message and display form again.
             h.flash(
                 _('Errors exist when saving plugin settings. '
                   'Please check the form inputs.'),
                 category='error')
             defaults = {key: data[key] for key in data if key in schema}
             return self.settings_get(errors=e.asdict(), defaults=defaults)
 
         # Store validated data.
         for name, value in valid_data.items():
             self.plugin.create_or_update_setting(name, value)
         Session().commit()
+        SettingsModel().invalidate_settings_cache()
 
         # Display success message and redirect.
         h.flash(_('Auth settings updated successfully.'), category='success')
-        redirect_to = self.request.resource_path(
-            self.context, route_name='auth_home')
+        redirect_to = self.request.resource_path(self.context, route_name='auth_home')
+
         return HTTPFound(redirect_to)
 
 
 class AuthSettingsView(BaseAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         return c
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def index(self, defaults=None, errors=None, prefix_error=False):
         c = self.load_default_context()
 
         defaults = defaults or {}
         authn_registry = get_authn_registry(self.request.registry)
         enabled_plugins = SettingsModel().get_auth_plugins()
 
         # Create template context and render it.
         template_context = {
             'resource': self.context,
             'available_plugins': authn_registry.get_plugins(),
             'enabled_plugins': enabled_plugins,
         }
         html = render('rhodecode:templates/admin/auth/auth_settings.mako',
                       self._get_template_context(c, **template_context),
                       self.request)
 
         # Create form default values and fill the form.
         form_defaults = {
             'auth_plugins': ',\n'.join(enabled_plugins)
         }
         form_defaults.update(defaults)
         html = formencode.htmlfill.render(
             html,
             defaults=form_defaults,
             errors=errors,
             prefix_error=prefix_error,
             encoding="UTF-8",
             force_defaults=False)
 
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def auth_settings(self):
         _ = self.request.translate
         try:
             form = AuthSettingsForm(self.request.translate)()
             form_result = form.to_python(self.request.POST)
             plugins = ','.join(form_result['auth_plugins'])
             setting = SettingsModel().create_or_update_setting(
                 'auth_plugins', plugins)
             Session().add(setting)
             Session().commit()
-
+            SettingsModel().invalidate_settings_cache()
             h.flash(_('Auth settings updated successfully.'), category='success')
         except formencode.Invalid as errors:
             e = errors.error_dict or {}
             h.flash(_('Errors exist when saving plugin setting. '
                       'Please check the form inputs.'), category='error')
             return self.index(
                 defaults=errors.value,
                 errors=e,
                 prefix_error=False)
         except Exception:
             log.exception('Exception in auth_settings')
             h.flash(_('Error occurred during update of auth settings.'),
                     category='error')
 
-        redirect_to = self.request.resource_path(
-            self.context, route_name='auth_home')
+        redirect_to = self.request.resource_path(self.context, route_name='auth_home')
+
         return HTTPFound(redirect_to)

rhodecode/lib/base.py

@@ -1,615 +1,616 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 The base Controller API
 Provides the BaseController class for subclassing. And usage in different
 controllers
 """
 
 import logging
 import socket
 
 import markupsafe
 import ipaddress
 
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpexceptions import HTTPUnauthorized, HTTPForbidden, get_exception
 from paste.httpheaders import WWW_AUTHENTICATE, AUTHORIZATION
 
 import rhodecode
 from rhodecode.apps._base import TemplateArgs
 from rhodecode.authentication.base import VCS_TYPE
 from rhodecode.lib import auth, utils2
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import AuthUser, CookieStoreWrapper
 from rhodecode.lib.exceptions import UserCreationError
 from rhodecode.lib.utils import (password_changed, get_enabled_hook_classes)
 from rhodecode.lib.utils2 import (
     str2bool, safe_unicode, AttributeDict, safe_int, sha1, aslist, safe_str)
 from rhodecode.model.db import Repository, User, ChangesetComment, UserBookmark
 from rhodecode.model.notification import NotificationModel
 from rhodecode.model.settings import VcsSettingsModel, SettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def _filter_proxy(ip):
     """
     Passed in IP addresses in HEADERS can be in a special format of multiple
     ips. Those comma separated IPs are passed from various proxies in the
     chain of request processing. The left-most being the original client.
     We only care about the first IP which came from the org. client.
 
     :param ip: ip string from headers
     """
     if ',' in ip:
         _ips = ip.split(',')
         _first_ip = _ips[0].strip()
         log.debug('Got multiple IPs %s, using %s', ','.join(_ips), _first_ip)
         return _first_ip
     return ip
 
 
 def _filter_port(ip):
     """
     Removes a port from ip, there are 4 main cases to handle here.
     - ipv4 eg. 127.0.0.1
     - ipv6 eg. ::1
     - ipv4+port eg. 127.0.0.1:8080
     - ipv6+port eg. [::1]:8080
 
     :param ip:
     """
     def is_ipv6(ip_addr):
         if hasattr(socket, 'inet_pton'):
             try:
                 socket.inet_pton(socket.AF_INET6, ip_addr)
             except socket.error:
                 return False
         else:
             # fallback to ipaddress
             try:
                 ipaddress.IPv6Address(safe_unicode(ip_addr))
             except Exception:
                 return False
         return True
 
     if ':' not in ip:  # must be ipv4 pure ip
         return ip
 
     if '[' in ip and ']' in ip:  # ipv6 with port
         return ip.split(']')[0][1:].lower()
 
     # must be ipv6 or ipv4 with port
     if is_ipv6(ip):
         return ip
     else:
         ip, _port = ip.split(':')[:2]  # means ipv4+port
         return ip
 
 
 def get_ip_addr(environ):
     proxy_key = 'HTTP_X_REAL_IP'
     proxy_key2 = 'HTTP_X_FORWARDED_FOR'
     def_key = 'REMOTE_ADDR'
     _filters = lambda x: _filter_port(_filter_proxy(x))
 
     ip = environ.get(proxy_key)
     if ip:
         return _filters(ip)
 
     ip = environ.get(proxy_key2)
     if ip:
         return _filters(ip)
 
     ip = environ.get(def_key, '0.0.0.0')
     return _filters(ip)
 
 
 def get_server_ip_addr(environ, log_errors=True):
     hostname = environ.get('SERVER_NAME')
     try:
         return socket.gethostbyname(hostname)
     except Exception as e:
         if log_errors:
             # in some cases this lookup is not possible, and we don't want to
             # make it an exception in logs
             log.exception('Could not retrieve server ip address: %s', e)
         return hostname
 
 
 def get_server_port(environ):
     return environ.get('SERVER_PORT')
 
 
 def get_access_path(environ):
     path = environ.get('PATH_INFO')
     org_req = environ.get('pylons.original_request')
     if org_req:
         path = org_req.environ.get('PATH_INFO')
     return path
 
 
 def get_user_agent(environ):
     return environ.get('HTTP_USER_AGENT')
 
 
 def vcs_operation_context(
         environ, repo_name, username, action, scm, check_locking=True,
         is_shadow_repo=False, check_branch_perms=False, detect_force_push=False):
     """
     Generate the context for a vcs operation, e.g. push or pull.
 
     This context is passed over the layers so that hooks triggered by the
     vcs operation know details like the user, the user's IP address etc.
 
     :param check_locking: Allows to switch of the computation of the locking
         data. This serves mainly the need of the simplevcs middleware to be
         able to disable this for certain operations.
 
     """
     # Tri-state value: False: unlock, None: nothing, True: lock
     make_lock = None
     locked_by = [None, None, None]
     is_anonymous = username == User.DEFAULT_USER
     user = User.get_by_username(username)
     if not is_anonymous and check_locking:
         log.debug('Checking locking on repository "%s"', repo_name)
         repo = Repository.get_by_repo_name(repo_name)
         make_lock, __, locked_by = repo.get_locking_state(
             action, user.user_id)
     user_id = user.user_id
     settings_model = VcsSettingsModel(repo=repo_name)
     ui_settings = settings_model.get_ui_settings()
 
     # NOTE(marcink): This should be also in sync with
     # rhodecode/apps/ssh_support/lib/backends/base.py:update_environment scm_data
     store = [x for x in ui_settings if x.key == '/']
     repo_store = ''
     if store:
         repo_store = store[0].value
 
     scm_data = {
         'ip': get_ip_addr(environ),
         'username': username,
         'user_id': user_id,
         'action': action,
         'repository': repo_name,
         'scm': scm,
         'config': rhodecode.CONFIG['__file__'],
         'repo_store': repo_store,
         'make_lock': make_lock,
         'locked_by': locked_by,
         'server_url': utils2.get_server_url(environ),
         'user_agent': get_user_agent(environ),
         'hooks': get_enabled_hook_classes(ui_settings),
         'is_shadow_repo': is_shadow_repo,
         'detect_force_push': detect_force_push,
         'check_branch_perms': check_branch_perms,
     }
     return scm_data
 
 
 class BasicAuth(AuthBasicAuthenticator):
 
     def __init__(self, realm, authfunc, registry, auth_http_code=None,
-                 initial_call_detection=False, acl_repo_name=None):
+                 initial_call_detection=False, acl_repo_name=None, rc_realm=''):
         self.realm = realm
+        self.rc_realm = rc_realm
         self.initial_call = initial_call_detection
         self.authfunc = authfunc
         self.registry = registry
         self.acl_repo_name = acl_repo_name
         self._rc_auth_http_code = auth_http_code
 
     def _get_response_from_code(self, http_code):
         try:
             return get_exception(safe_int(http_code))
         except Exception:
             log.exception('Failed to fetch response for code %s', http_code)
             return HTTPForbidden
 
     def get_rc_realm(self):
-        return safe_str(self.registry.rhodecode_settings.get('rhodecode_realm'))
+        return safe_str(self.rc_realm)
 
     def build_authentication(self):
         head = WWW_AUTHENTICATE.tuples('Basic realm="%s"' % self.realm)
         if self._rc_auth_http_code and not self.initial_call:
             # return alternative HTTP code if alternative http return code
             # is specified in RhodeCode config, but ONLY if it's not the
             # FIRST call
             custom_response_klass = self._get_response_from_code(
                 self._rc_auth_http_code)
             return custom_response_klass(headers=head)
         return HTTPUnauthorized(headers=head)
 
     def authenticate(self, environ):
         authorization = AUTHORIZATION(environ)
         if not authorization:
             return self.build_authentication()
         (authmeth, auth) = authorization.split(' ', 1)
         if 'basic' != authmeth.lower():
             return self.build_authentication()
         auth = auth.strip().decode('base64')
         _parts = auth.split(':', 1)
         if len(_parts) == 2:
             username, password = _parts
             auth_data = self.authfunc(
                     username, password, environ, VCS_TYPE,
                     registry=self.registry, acl_repo_name=self.acl_repo_name)
             if auth_data:
                 return {'username': username, 'auth_data': auth_data}
             if username and password:
                 # we mark that we actually executed authentication once, at
                 # that point we can use the alternative auth code
                 self.initial_call = False
 
         return self.build_authentication()
 
     __call__ = authenticate
 
 
 def calculate_version_hash(config):
     return sha1(
         config.get('beaker.session.secret', '') +
         rhodecode.__version__)[:8]
 
 
 def get_current_lang(request):
     # NOTE(marcink): remove after pyramid move
     try:
         return translation.get_lang()[0]
     except:
         pass
 
     return getattr(request, '_LOCALE_', request.locale_name)
 
 
 def attach_context_attributes(context, request, user_id=None):
     """
     Attach variables into template context called `c`.
     """
     config = request.registry.settings
 
     rc_config = SettingsModel().get_all_settings(cache=True, from_request=False)
     context.rc_config = rc_config
     context.rhodecode_version = rhodecode.__version__
     context.rhodecode_edition = config.get('rhodecode.edition')
     # unique secret + version does not leak the version but keep consistency
     context.rhodecode_version_hash = calculate_version_hash(config)
 
     # Default language set for the incoming request
     context.language = get_current_lang(request)
 
     # Visual options
     context.visual = AttributeDict({})
 
     # DB stored Visual Items
     context.visual.show_public_icon = str2bool(
         rc_config.get('rhodecode_show_public_icon'))
     context.visual.show_private_icon = str2bool(
         rc_config.get('rhodecode_show_private_icon'))
     context.visual.stylify_metatags = str2bool(
         rc_config.get('rhodecode_stylify_metatags'))
     context.visual.dashboard_items = safe_int(
         rc_config.get('rhodecode_dashboard_items', 100))
     context.visual.admin_grid_items = safe_int(
         rc_config.get('rhodecode_admin_grid_items', 100))
     context.visual.show_revision_number = str2bool(
         rc_config.get('rhodecode_show_revision_number', True))
     context.visual.show_sha_length = safe_int(
         rc_config.get('rhodecode_show_sha_length', 100))
     context.visual.repository_fields = str2bool(
         rc_config.get('rhodecode_repository_fields'))
     context.visual.show_version = str2bool(
         rc_config.get('rhodecode_show_version'))
     context.visual.use_gravatar = str2bool(
         rc_config.get('rhodecode_use_gravatar'))
     context.visual.gravatar_url = rc_config.get('rhodecode_gravatar_url')
     context.visual.default_renderer = rc_config.get(
         'rhodecode_markup_renderer', 'rst')
     context.visual.comment_types = ChangesetComment.COMMENT_TYPES
     context.visual.rhodecode_support_url = \
         rc_config.get('rhodecode_support_url') or h.route_url('rhodecode_support')
 
     context.visual.affected_files_cut_off = 60
 
     context.pre_code = rc_config.get('rhodecode_pre_code')
     context.post_code = rc_config.get('rhodecode_post_code')
     context.rhodecode_name = rc_config.get('rhodecode_title')
     context.default_encodings = aslist(config.get('default_encoding'), sep=',')
     # if we have specified default_encoding in the request, it has more
     # priority
     if request.GET.get('default_encoding'):
         context.default_encodings.insert(0, request.GET.get('default_encoding'))
     context.clone_uri_tmpl = rc_config.get('rhodecode_clone_uri_tmpl')
     context.clone_uri_ssh_tmpl = rc_config.get('rhodecode_clone_uri_ssh_tmpl')
 
     # INI stored
     context.labs_active = str2bool(
         config.get('labs_settings_active', 'false'))
     context.ssh_enabled = str2bool(
         config.get('ssh.generate_authorized_keyfile', 'false'))
     context.ssh_key_generator_enabled = str2bool(
         config.get('ssh.enable_ui_key_generator', 'true'))
 
     context.visual.allow_repo_location_change = str2bool(
         config.get('allow_repo_location_change', True))
     context.visual.allow_custom_hooks_settings = str2bool(
         config.get('allow_custom_hooks_settings', True))
     context.debug_style = str2bool(config.get('debug_style', False))
 
     context.rhodecode_instanceid = config.get('instance_id')
 
     context.visual.cut_off_limit_diff = safe_int(
         config.get('cut_off_limit_diff'))
     context.visual.cut_off_limit_file = safe_int(
         config.get('cut_off_limit_file'))
 
     context.license = AttributeDict({})
     context.license.hide_license_info = str2bool(
         config.get('license.hide_license_info', False))
 
     # AppEnlight
     context.appenlight_enabled = str2bool(config.get('appenlight', 'false'))
     context.appenlight_api_public_key = config.get(
         'appenlight.api_public_key', '')
     context.appenlight_server_url = config.get('appenlight.server_url', '')
 
     diffmode = {
         "unified": "unified",
         "sideside": "sideside"
     }.get(request.GET.get('diffmode'))
 
     is_api = hasattr(request, 'rpc_user')
     session_attrs = {
         # defaults
         "clone_url_format": "http",
         "diffmode": "sideside"
     }
 
     if not is_api:
         # don't access pyramid session for API calls
         if diffmode and diffmode != request.session.get('rc_user_session_attr.diffmode'):
             request.session['rc_user_session_attr.diffmode'] = diffmode
 
         # session settings per user
 
         for k, v in request.session.items():
             pref = 'rc_user_session_attr.'
             if k and k.startswith(pref):
                 k = k[len(pref):]
                 session_attrs[k] = v
 
     context.user_session_attrs = session_attrs
 
     # JS template context
     context.template_context = {
         'repo_name': None,
         'repo_type': None,
         'repo_landing_commit': None,
         'rhodecode_user': {
             'username': None,
             'email': None,
             'notification_status': False
         },
         'session_attrs': session_attrs,
         'visual': {
             'default_renderer': None
         },
         'commit_data': {
             'commit_id': None
         },
         'pull_request_data': {'pull_request_id': None},
         'timeago': {
             'refresh_time': 120 * 1000,
             'cutoff_limit': 1000 * 60 * 60 * 24 * 7
         },
         'pyramid_dispatch': {
 
         },
         'extra': {'plugins': {}}
     }
     # END CONFIG VARS
     if is_api:
         csrf_token = None
     else:
         csrf_token = auth.get_csrf_token(session=request.session)
 
     context.csrf_token = csrf_token
     context.backends = rhodecode.BACKENDS.keys()
     context.backends.sort()
     unread_count = 0
     user_bookmark_list = []
     if user_id:
         unread_count = NotificationModel().get_unread_cnt_for_user(user_id)
         user_bookmark_list = UserBookmark.get_bookmarks_for_user(user_id)
     context.unread_notifications = unread_count
     context.bookmark_items = user_bookmark_list
 
     # web case
     if hasattr(request, 'user'):
         context.auth_user = request.user
         context.rhodecode_user = request.user
 
     # api case
     if hasattr(request, 'rpc_user'):
         context.auth_user = request.rpc_user
         context.rhodecode_user = request.rpc_user
 
     # attach the whole call context to the request
     request.call_context = context
 
 
 def get_auth_user(request):
     environ = request.environ
     session = request.session
 
     ip_addr = get_ip_addr(environ)
 
     # make sure that we update permissions each time we call controller
     _auth_token = (request.GET.get('auth_token', '') or request.GET.get('api_key', ''))
     if not _auth_token and request.matchdict:
         url_auth_token = request.matchdict.get('_auth_token')
         _auth_token = url_auth_token
         if _auth_token:
             log.debug('Using URL extracted auth token `...%s`', _auth_token[-4:])
 
     if _auth_token:
         # when using API_KEY we assume user exists, and
         # doesn't need auth based on cookies.
         auth_user = AuthUser(api_key=_auth_token, ip_addr=ip_addr)
         authenticated = False
     else:
         cookie_store = CookieStoreWrapper(session.get('rhodecode_user'))
         try:
             auth_user = AuthUser(user_id=cookie_store.get('user_id', None),
                                  ip_addr=ip_addr)
         except UserCreationError as e:
             h.flash(e, 'error')
             # container auth or other auth functions that create users
             # on the fly can throw this exception signaling that there's
             # issue with user creation, explanation should be provided
             # in Exception itself. We then create a simple blank
             # AuthUser
             auth_user = AuthUser(ip_addr=ip_addr)
 
         # in case someone changes a password for user it triggers session
         # flush and forces a re-login
         if password_changed(auth_user, session):
             session.invalidate()
             cookie_store = CookieStoreWrapper(session.get('rhodecode_user'))
             auth_user = AuthUser(ip_addr=ip_addr)
 
         authenticated = cookie_store.get('is_authenticated')
 
     if not auth_user.is_authenticated and auth_user.is_user_object:
         # user is not authenticated and not empty
         auth_user.set_authenticated(authenticated)
 
     return auth_user, _auth_token
 
 
 def h_filter(s):
     """
     Custom filter for Mako templates. Mako by standard uses `markupsafe.escape`
     we wrap this with additional functionality that converts None to empty
     strings
     """
     if s is None:
         return markupsafe.Markup()
     return markupsafe.escape(s)
 
 
 def add_events_routes(config):
     """
     Adds routing that can be used in events. Because some events are triggered
     outside of pyramid context, we need to bootstrap request with some
     routing registered
     """
 
     from rhodecode.apps._base import ADMIN_PREFIX
 
     config.add_route(name='home', pattern='/')
     config.add_route(name='main_page_repos_data', pattern='/_home_repos')
     config.add_route(name='main_page_repo_groups_data', pattern='/_home_repo_groups')
 
     config.add_route(name='login', pattern=ADMIN_PREFIX + '/login')
     config.add_route(name='logout', pattern=ADMIN_PREFIX + '/logout')
     config.add_route(name='repo_summary', pattern='/{repo_name}')
     config.add_route(name='repo_summary_explicit', pattern='/{repo_name}/summary')
     config.add_route(name='repo_group_home', pattern='/{repo_group_name}')
 
     config.add_route(name='pullrequest_show',
                      pattern='/{repo_name}/pull-request/{pull_request_id}')
     config.add_route(name='pull_requests_global',
                      pattern='/pull-request/{pull_request_id}')
 
     config.add_route(name='repo_commit',
                      pattern='/{repo_name}/changeset/{commit_id}')
     config.add_route(name='repo_files',
                      pattern='/{repo_name}/files/{commit_id}/{f_path}')
 
     config.add_route(name='hovercard_user',
                      pattern='/_hovercard/user/{user_id}')
 
     config.add_route(name='hovercard_user_group',
                      pattern='/_hovercard/user_group/{user_group_id}')
 
     config.add_route(name='hovercard_pull_request',
                      pattern='/_hovercard/pull_request/{pull_request_id}')
 
     config.add_route(name='hovercard_repo_commit',
                      pattern='/_hovercard/commit/{repo_name}/{commit_id}')
 
 
 def bootstrap_config(request):
     import pyramid.testing
     registry = pyramid.testing.Registry('RcTestRegistry')
 
     config = pyramid.testing.setUp(registry=registry, request=request)
 
     # allow pyramid lookup in testing
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
 
     add_events_routes(config)
 
     return config
 
 
 def bootstrap_request(**kwargs):
     import pyramid.testing
 
     class TestRequest(pyramid.testing.DummyRequest):
         application_url = kwargs.pop('application_url', 'http://example.com')
         host = kwargs.pop('host', 'example.com:80')
         domain = kwargs.pop('domain', 'example.com')
 
         def translate(self, msg):
             return msg
 
         def plularize(self, singular, plural, n):
             return singular
 
         def get_partial_renderer(self, tmpl_name):
 
             from rhodecode.lib.partial_renderer import get_partial_renderer
             return get_partial_renderer(request=self, tmpl_name=tmpl_name)
 
         _call_context = TemplateArgs()
         _call_context.visual = TemplateArgs()
         _call_context.visual.show_sha_length = 12
         _call_context.visual.show_revision_number = True
 
         @property
         def call_context(self):
             return self._call_context
 
     class TestDummySession(pyramid.testing.DummySession):
         def save(*arg, **kw):
             pass
 
     request = TestRequest(**kwargs)
     request.session = TestDummySession()
 
     return request
 

rhodecode/lib/middleware/simplevcs.py

@@ -1,678 +1,679 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleVCS middleware for handling protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 
 import os
 import re
 import logging
 import importlib
 from functools import wraps
 from StringIO import StringIO
 from lxml import etree
 
 import time
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
 from pyramid.httpexceptions import (
     HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 import rhodecode
 from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
 from rhodecode.lib import rc_cache
 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from rhodecode.lib.base import (
     BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
 from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.middleware import appenlight
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.lib.utils import is_valid_repo, SLUG_RE
 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.backends import base
 
 from rhodecode.model import meta
 from rhodecode.model.db import User, Repository, PullRequest
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.settings import SettingsModel, VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def extract_svn_txn_id(acl_repo_name, data):
     """
     Helper method for extraction of svn txn_id from submitted XML data during
     POST operations
     """
     try:
         root = etree.fromstring(data)
         pat = re.compile(r'/txn/(?P<txn_id>.*)')
         for el in root:
             if el.tag == '{DAV:}source':
                 for sub_el in el:
                     if sub_el.tag == '{DAV:}href':
                         match = pat.search(sub_el.text)
                         if match:
                             svn_tx_id = match.groupdict()['txn_id']
                             txn_id = rc_cache.utils.compute_key_from_params(
                                 acl_repo_name, svn_tx_id)
                             return txn_id
     except Exception:
         log.exception('Failed to extract txn_id')
 
 
 def initialize_generator(factory):
     """
     Initializes the returned generator by draining its first element.
 
     This can be used to give a generator an initializer, which is the code
     up to the first yield statement. This decorator enforces that the first
     produced element has the value ``"__init__"`` to make its special
     purpose very explicit in the using code.
     """
 
     @wraps(factory)
     def wrapper(*args, **kwargs):
         gen = factory(*args, **kwargs)
         try:
             init = gen.next()
         except StopIteration:
             raise ValueError('Generator must yield at least one element.')
         if init != "__init__":
             raise ValueError('First yielded element must be "__init__".')
         return gen
     return wrapper
 
 
 class SimpleVCS(object):
     """Common functionality for SCM HTTP handlers."""
 
     SCM = 'unknown'
 
     acl_repo_name = None
     url_repo_name = None
     vcs_repo_name = None
     rc_extras = {}
 
     # We have to handle requests to shadow repositories different than requests
     # to normal repositories. Therefore we have to distinguish them. To do this
     # we use this regex which will match only on URLs pointing to shadow
     # repositories.
     shadow_repo_re = re.compile(
         '(?P<groups>(?:{slug_pat}/)*)'  # repo groups
         '(?P<target>{slug_pat})/'       # target repo
         'pull-request/(?P<pr_id>\d+)/'  # pull request
         'repository$'                   # shadow repo
         .format(slug_pat=SLUG_RE.pattern))
 
     def __init__(self, config, registry):
         self.registry = registry
         self.config = config
         # re-populated by specialized middleware
         self.repo_vcs_config = base.Config()
-        self.rhodecode_settings = SettingsModel().get_all_settings(cache=True)
 
-        registry.rhodecode_settings = self.rhodecode_settings
+        rc_settings = SettingsModel().get_all_settings(cache=True, from_request=False)
+        realm = rc_settings.get('rhodecode_realm') or 'RhodeCode AUTH'
+
         # authenticate this VCS request using authfunc
         auth_ret_code_detection = \
             str2bool(self.config.get('auth_ret_code_detection', False))
         self.authenticate = BasicAuth(
             '', authenticate, registry, config.get('auth_ret_code'),
-            auth_ret_code_detection)
+            auth_ret_code_detection, rc_realm=realm)
         self.ip_addr = '0.0.0.0'
 
     @LazyProperty
     def global_vcs_config(self):
         try:
             return VcsSettingsModel().get_ui_settings_as_config_obj()
         except Exception:
             return base.Config()
 
     @property
     def base_path(self):
         settings_path = self.repo_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
 
         if not settings_path:
             settings_path = self.global_vcs_config.get(*VcsSettingsModel.PATH_SETTING)
 
         if not settings_path:
             # try, maybe we passed in explicitly as config option
             settings_path = self.config.get('base_path')
 
         if not settings_path:
             raise ValueError('FATAL: base_path is empty')
         return settings_path
 
     def set_repo_names(self, environ):
         """
         This will populate the attributes acl_repo_name, url_repo_name,
         vcs_repo_name and is_shadow_repo. In case of requests to normal (non
         shadow) repositories all names are equal. In case of requests to a
         shadow repository the acl-name points to the target repo of the pull
         request and the vcs-name points to the shadow repo file system path.
         The url-name is always the URL used by the vcs client program.
 
         Example in case of a shadow repo:
             acl_repo_name = RepoGroup/MyRepo
             url_repo_name = RepoGroup/MyRepo/pull-request/3/repository
             vcs_repo_name = /repo/base/path/RepoGroup/.__shadow_MyRepo_pr-3'
         """
         # First we set the repo name from URL for all attributes. This is the
         # default if handling normal (non shadow) repo requests.
         self.url_repo_name = self._get_repository_name(environ)
         self.acl_repo_name = self.vcs_repo_name = self.url_repo_name
         self.is_shadow_repo = False
 
         # Check if this is a request to a shadow repository.
         match = self.shadow_repo_re.match(self.url_repo_name)
         if match:
             match_dict = match.groupdict()
 
             # Build acl repo name from regex match.
             acl_repo_name = safe_unicode('{groups}{target}'.format(
                 groups=match_dict['groups'] or '',
                 target=match_dict['target']))
 
             # Retrieve pull request instance by ID from regex match.
             pull_request = PullRequest.get(match_dict['pr_id'])
 
             # Only proceed if we got a pull request and if acl repo name from
             # URL equals the target repo name of the pull request.
             if pull_request and (acl_repo_name == pull_request.target_repo.repo_name):
 
                 # Get file system path to shadow repository.
                 workspace_id = PullRequestModel()._workspace_id(pull_request)
                 vcs_repo_name = pull_request.target_repo.get_shadow_repository_path(workspace_id)
 
                 # Store names for later usage.
                 self.vcs_repo_name = vcs_repo_name
                 self.acl_repo_name = acl_repo_name
                 self.is_shadow_repo = True
 
         log.debug('Setting all VCS repository names: %s', {
             'acl_repo_name': self.acl_repo_name,
             'url_repo_name': self.url_repo_name,
             'vcs_repo_name': self.vcs_repo_name,
         })
 
     @property
     def scm_app(self):
         custom_implementation = self.config['vcs.scm_app_implementation']
         if custom_implementation == 'http':
             log.debug('Using HTTP implementation of scm app.')
             scm_app_impl = scm_app_http
         else:
             log.debug('Using custom implementation of scm_app: "{}"'.format(
                 custom_implementation))
             scm_app_impl = importlib.import_module(custom_implementation)
         return scm_app_impl
 
     def _get_by_id(self, repo_name):
         """
         Gets a special pattern _<ID> from clone url and tries to replace it
         with a repository_name for support of _<ID> non changeable urls
         """
 
         data = repo_name.split('/')
         if len(data) >= 2:
             from rhodecode.model.repo import RepoModel
             by_id_match = RepoModel().get_repo_by_id(repo_name)
             if by_id_match:
                 data[1] = by_id_match.repo_name
 
         return safe_str('/'.join(data))
 
     def _invalidate_cache(self, repo_name):
         """
         Set's cache for this repository for invalidation on next access
 
         :param repo_name: full repo name, also a cache key
         """
         ScmModel().mark_for_invalidation(repo_name)
 
     def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
         db_repo = Repository.get_by_repo_name(repo_name)
         if not db_repo:
             log.debug('Repository `%s` not found inside the database.',
                       repo_name)
             return False
 
         if db_repo.repo_type != scm_type:
             log.warning(
                 'Repository `%s` have incorrect scm_type, expected %s got %s',
                 repo_name, db_repo.repo_type, scm_type)
             return False
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         return is_valid_repo(
             repo_name, base_path,
             explicit_scm=scm_type, expect_scm=scm_type, config=config)
 
     def valid_and_active_user(self, user):
         """
         Checks if that user is not empty, and if it's actually object it checks
         if he's active.
 
         :param user: user object or None
         :return: boolean
         """
         if user is None:
             return False
 
         elif user.active:
             return True
 
         return False
 
     @property
     def is_shadow_repo_dir(self):
         return os.path.isdir(self.vcs_repo_name)
 
     def _check_permission(self, action, user, auth_user, repo_name, ip_addr=None,
                           plugin_id='', plugin_cache_active=False, cache_ttl=0):
         """
         Checks permissions using action (push/pull) user and repository
         name. If plugin_cache and ttl is set it will use the plugin which
         authenticated the user to store the cached permissions result for N
         amount of seconds as in cache_ttl
 
         :param action: push or pull action
         :param user: user instance
         :param repo_name: repository name
         """
 
         log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
                   plugin_id, plugin_cache_active, cache_ttl)
 
         user_id = user.user_id
         cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                                expiration_time=cache_ttl,
                                                condition=plugin_cache_active)
         def compute_perm_vcs(
                 cache_name, plugin_id, action, user_id, repo_name, ip_addr):
 
             log.debug('auth: calculating permission access now...')
             # check IP
             inherit = user.inherit_default_permissions
             ip_allowed = AuthUser.check_ip_allowed(
                 user_id, ip_addr, inherit_from_default=inherit)
             if ip_allowed:
                 log.info('Access for IP:%s allowed', ip_addr)
             else:
                 return False
 
             if action == 'push':
                 perms = ('repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             else:
                 # any other action need at least read permission
                 perms = (
                     'repository.read', 'repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             return True
 
         start = time.time()
         log.debug('Running plugin `%s` permissions check', plugin_id)
 
         # for environ based auth, password can be empty, but then the validation is
         # on the server that fills in the env data needed for authentication
         perm_result = compute_perm_vcs(
             'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr)
 
         auth_time = time.time() - start
         log.debug('Permissions for plugin `%s` completed in %.4fs, '
                   'expiration time of fetched cache %.1fs.',
                   plugin_id, auth_time, cache_ttl)
 
         return perm_result
 
     def _get_http_scheme(self, environ):
         try:
             return environ['wsgi.url_scheme']
         except Exception:
             log.exception('Failed to read http scheme')
             return 'http'
 
     def _check_ssl(self, environ, start_response):
         """
         Checks the SSL check flag and returns False if SSL is not present
         and required True otherwise
         """
         org_proto = environ['wsgi._org_proto']
         # check if we have SSL required  ! if not it's a bad request !
         require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
         if require_ssl and org_proto == 'http':
             log.debug(
                 'Bad request: detected protocol is `%s` and '
                 'SSL/HTTPS is required.', org_proto)
             return False
         return True
 
     def _get_default_cache_ttl(self):
         # take AUTH_CACHE_TTL from the `rhodecode` auth plugin
         plugin = loadplugin('egg:rhodecode-enterprise-ce#rhodecode')
         plugin_settings = plugin.get_settings()
         plugin_cache_active, cache_ttl = plugin.get_ttl_cache(
             plugin_settings) or (False, 0)
         return plugin_cache_active, cache_ttl
 
     def __call__(self, environ, start_response):
         try:
             return self._handle_request(environ, start_response)
         except Exception:
             log.exception("Exception while handling request")
             appenlight.track_exception(environ)
             return HTTPInternalServerError()(environ, start_response)
         finally:
             meta.Session.remove()
 
     def _handle_request(self, environ, start_response):
         if not self._check_ssl(environ, start_response):
             reason = ('SSL required, while RhodeCode was unable '
                       'to detect this as SSL request')
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         if not self.url_repo_name:
             log.warning('Repository name is empty: %s', self.url_repo_name)
             # failed to get repo name, we fail now
             return HTTPNotFound()(environ, start_response)
         log.debug('Extracted repo name is %s', self.url_repo_name)
 
         ip_addr = get_ip_addr(environ)
         user_agent = get_user_agent(environ)
         username = None
 
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
 
         # ======================================================================
         # GET ACTION PULL or PUSH
         # ======================================================================
         action = self._get_action(environ)
 
         # ======================================================================
         # Check if this is a request to a shadow repository of a pull request.
         # In this case only pull action is allowed.
         # ======================================================================
         if self.is_shadow_repo and action != 'pull':
             reason = 'Only pull action is allowed for shadow repositories.'
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         # Check if the shadow repo actually exists, in case someone refers
         # to it, and it has been deleted because of successful merge.
         if self.is_shadow_repo and not self.is_shadow_repo_dir:
             log.debug(
                 'Shadow repo detected, and shadow repo dir `%s` is missing',
                 self.is_shadow_repo_dir)
             return HTTPNotFound()(environ, start_response)
 
         # ======================================================================
         # CHECK ANONYMOUS PERMISSION
         # ======================================================================
         detect_force_push = False
         check_branch_perms = False
         if action in ['pull', 'push']:
             user_obj = anonymous_user = User.get_default_user()
             auth_user = user_obj.AuthUser()
             username = anonymous_user.username
             if anonymous_user.active:
                 plugin_cache_active, cache_ttl = self._get_default_cache_ttl()
                 # ONLY check permissions if the user is activated
                 anonymous_perm = self._check_permission(
                     action, anonymous_user, auth_user, self.acl_repo_name, ip_addr,
                     plugin_id='anonymous_access',
                     plugin_cache_active=plugin_cache_active,
                     cache_ttl=cache_ttl,
                 )
             else:
                 anonymous_perm = False
 
             if not anonymous_user.active or not anonymous_perm:
                 if not anonymous_user.active:
                     log.debug('Anonymous access is disabled, running '
                               'authentication')
 
                 if not anonymous_perm:
                     log.debug('Not enough credentials to access this '
                               'repository as anonymous user')
 
                 username = None
                 # ==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
                 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
                 # ==============================================================
 
                 # try to auth based on environ, container auth methods
                 log.debug('Running PRE-AUTH for container based authentication')
                 pre_auth = authenticate(
                     '', '', environ, VCS_TYPE, registry=self.registry,
                     acl_repo_name=self.acl_repo_name)
                 if pre_auth and pre_auth.get('username'):
                     username = pre_auth['username']
                 log.debug('PRE-AUTH got %s as username', username)
                 if pre_auth:
                     log.debug('PRE-AUTH successful from %s',
                               pre_auth.get('auth_data', {}).get('_plugin'))
 
                 # If not authenticated by the container, running basic auth
                 # before inject the calling repo_name for special scope checks
                 self.authenticate.acl_repo_name = self.acl_repo_name
 
                 plugin_cache_active, cache_ttl = False, 0
                 plugin = None
                 if not username:
                     self.authenticate.realm = self.authenticate.get_rc_realm()
 
                     try:
                         auth_result = self.authenticate(environ)
                     except (UserCreationError, NotAllowedToCreateUserError) as e:
                         log.error(e)
                         reason = safe_str(e)
                         return HTTPNotAcceptable(reason)(environ, start_response)
 
                     if isinstance(auth_result, dict):
                         AUTH_TYPE.update(environ, 'basic')
                         REMOTE_USER.update(environ, auth_result['username'])
                         username = auth_result['username']
                         plugin = auth_result.get('auth_data', {}).get('_plugin')
                         log.info(
                             'MAIN-AUTH successful for user `%s` from %s plugin',
                             username, plugin)
 
                         plugin_cache_active, cache_ttl = auth_result.get(
                             'auth_data', {}).get('_ttl_cache') or (False, 0)
                     else:
                         return auth_result.wsgi_application(environ, start_response)
 
                 # ==============================================================
                 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
                 # ==============================================================
                 user = User.get_by_username(username)
                 if not self.valid_and_active_user(user):
                     return HTTPForbidden()(environ, start_response)
                 username = user.username
                 user_id = user.user_id
 
                 # check user attributes for password change flag
                 user_obj = user
                 auth_user = user_obj.AuthUser()
                 if user_obj and user_obj.username != User.DEFAULT_USER and \
                         user_obj.user_data.get('force_password_change'):
                     reason = 'password change required'
                     log.debug('User not allowed to authenticate, %s', reason)
                     return HTTPNotAcceptable(reason)(environ, start_response)
 
                 # check permissions for this repository
                 perm = self._check_permission(
                     action, user, auth_user, self.acl_repo_name, ip_addr,
                     plugin, plugin_cache_active, cache_ttl)
                 if not perm:
                     return HTTPForbidden()(environ, start_response)
                 environ['rc_auth_user_id'] = user_id
 
             if action == 'push':
                 perms = auth_user.get_branch_permissions(self.acl_repo_name)
                 if perms:
                     check_branch_perms = True
                     detect_force_push = True
 
         # extras are injected into UI object and later available
         # in hooks executed by RhodeCode
         check_locking = _should_check_locking(environ.get('QUERY_STRING'))
 
         extras = vcs_operation_context(
             environ, repo_name=self.acl_repo_name, username=username,
             action=action, scm=self.SCM, check_locking=check_locking,
             is_shadow_repo=self.is_shadow_repo, check_branch_perms=check_branch_perms,
             detect_force_push=detect_force_push
         )
 
         # ======================================================================
         # REQUEST HANDLING
         # ======================================================================
         repo_path = os.path.join(
             safe_str(self.base_path), safe_str(self.vcs_repo_name))
         log.debug('Repository path is %s', repo_path)
 
         fix_PATH()
 
         log.info(
             '%s action on %s repo "%s" by "%s" from %s %s',
             action, self.SCM, safe_str(self.url_repo_name),
             safe_str(username), ip_addr, user_agent)
 
         return self._generate_vcs_response(
             environ, start_response, repo_path, extras, action)
 
     @initialize_generator
     def _generate_vcs_response(
             self, environ, start_response, repo_path, extras, action):
         """
         Returns a generator for the response content.
 
         This method is implemented as a generator, so that it can trigger
         the cache validation after all content sent back to the client. It
         also handles the locking exceptions which will be triggered when
         the first chunk is produced by the underlying WSGI application.
         """
         txn_id = ''
         if 'CONTENT_LENGTH' in environ and environ['REQUEST_METHOD'] == 'MERGE':
             # case for SVN, we want to re-use the callback daemon port
             # so we use the txn_id, for this we peek the body, and still save
             # it as wsgi.input
             data = environ['wsgi.input'].read()
             environ['wsgi.input'] = StringIO(data)
             txn_id = extract_svn_txn_id(self.acl_repo_name, data)
 
         callback_daemon, extras = self._prepare_callback_daemon(
             extras, environ, action, txn_id=txn_id)
         log.debug('HOOKS extras is %s', extras)
 
         http_scheme = self._get_http_scheme(environ)
 
         config = self._create_config(extras, self.acl_repo_name, scheme=http_scheme)
         app = self._create_wsgi_app(repo_path, self.url_repo_name, config)
         with callback_daemon:
             app.rc_extras = extras
 
             try:
                 response = app(environ, start_response)
             finally:
                 # This statement works together with the decorator
                 # "initialize_generator" above. The decorator ensures that
                 # we hit the first yield statement before the generator is
                 # returned back to the WSGI server. This is needed to
                 # ensure that the call to "app" above triggers the
                 # needed callback to "start_response" before the
                 # generator is actually used.
                 yield "__init__"
 
             # iter content
             for chunk in response:
                 yield chunk
 
             try:
                 # invalidate cache on push
                 if action == 'push':
                     self._invalidate_cache(self.url_repo_name)
             finally:
                 meta.Session.remove()
 
     def _get_repository_name(self, environ):
         """Get repository name out of the environmnent
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _get_action(self, environ):
         """Map request commands into a pull or push command.
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         """Return the WSGI app that will finally handle the request."""
         raise NotImplementedError()
 
     def _create_config(self, extras, repo_name, scheme='http'):
         """Create a safe config representation."""
         raise NotImplementedError()
 
     def _should_use_callback_daemon(self, extras, environ, action):
         if extras.get('is_shadow_repo'):
             # we don't want to execute hooks, and callback daemon for shadow repos
             return False
         return True
 
     def _prepare_callback_daemon(self, extras, environ, action, txn_id=None):
         direct_calls = vcs_settings.HOOKS_DIRECT_CALLS
         if not self._should_use_callback_daemon(extras, environ, action):
             # disable callback daemon for actions that don't require it
             direct_calls = True
 
         return prepare_callback_daemon(
             extras, protocol=vcs_settings.HOOKS_PROTOCOL,
             host=vcs_settings.HOOKS_HOST, use_direct_calls=direct_calls, txn_id=txn_id)
 
 
 def _should_check_locking(query_string):
     # this is kind of hacky, but due to how mercurial handles client-server
     # server see all operation on commit; bookmarks, phases and
     # obsolescence marker in different transaction, we don't want to check
     # locking on those
     return query_string not in ['cmd=listkeys']

rhodecode/model/settings.py

@@ -1,913 +1,920 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import hashlib
 import logging
 import re
 from collections import namedtuple
 from functools import wraps
 import bleach
-from pyramid.threadlocal import get_current_request
+from pyramid.threadlocal import get_current_request, get_current_registry
 
 from rhodecode.lib import rc_cache
 from rhodecode.lib.utils2 import (
     Optional, AttributeDict, safe_str, remove_prefix, str2bool)
 from rhodecode.lib.vcs.backends import base
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     RepoRhodeCodeUi, RepoRhodeCodeSetting, RhodeCodeUi, RhodeCodeSetting, CacheKey)
 from rhodecode.model.meta import Session
 
 
 log = logging.getLogger(__name__)
 
 
 UiSetting = namedtuple(
     'UiSetting', ['section', 'key', 'value', 'active'])
 
 SOCIAL_PLUGINS_LIST = ['github', 'bitbucket', 'twitter', 'google']
 
 
 class SettingNotFound(Exception):
     def __init__(self, setting_id):
         msg = 'Setting `{}` is not found'.format(setting_id)
         super(SettingNotFound, self).__init__(msg)
 
 
 class SettingsModel(BaseModel):
     BUILTIN_HOOKS = (
         RhodeCodeUi.HOOK_REPO_SIZE, RhodeCodeUi.HOOK_PUSH,
         RhodeCodeUi.HOOK_PRE_PUSH, RhodeCodeUi.HOOK_PRETX_PUSH,
         RhodeCodeUi.HOOK_PULL, RhodeCodeUi.HOOK_PRE_PULL,
         RhodeCodeUi.HOOK_PUSH_KEY,)
     HOOKS_SECTION = 'hooks'
 
     def __init__(self, sa=None, repo=None):
         self.repo = repo
         self.UiDbModel = RepoRhodeCodeUi if repo else RhodeCodeUi
         self.SettingsDbModel = (
             RepoRhodeCodeSetting if repo else RhodeCodeSetting)
         super(SettingsModel, self).__init__(sa)
 
     def get_ui_by_key(self, key):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_key == key)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.scalar()
 
     def get_ui_by_section(self, section):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_section == section)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.all()
 
     def get_ui_by_section_and_key(self, section, key):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_section == section)
         q = q.filter(self.UiDbModel.ui_key == key)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.scalar()
 
     def get_ui(self, section=None, key=None):
         q = self.UiDbModel.query()
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
 
         if section:
             q = q.filter(self.UiDbModel.ui_section == section)
         if key:
             q = q.filter(self.UiDbModel.ui_key == key)
 
         # TODO: mikhail: add caching
         result = [
             UiSetting(
                 section=safe_str(r.ui_section), key=safe_str(r.ui_key),
                 value=safe_str(r.ui_value), active=r.ui_active
             )
             for r in q.all()
         ]
         return result
 
     def get_builtin_hooks(self):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_key.in_(self.BUILTIN_HOOKS))
         return self._get_hooks(q)
 
     def get_custom_hooks(self):
         q = self.UiDbModel.query()
         q = q.filter(~self.UiDbModel.ui_key.in_(self.BUILTIN_HOOKS))
         return self._get_hooks(q)
 
     def create_ui_section_value(self, section, val, key=None, active=True):
         new_ui = self.UiDbModel()
         new_ui.ui_section = section
         new_ui.ui_value = val
         new_ui.ui_active = active
 
         repository_id = ''
         if self.repo:
             repo = self._get_repo(self.repo)
             repository_id = repo.repo_id
             new_ui.repository_id = repository_id
 
         if not key:
             # keys are unique so they need appended info
             if self.repo:
                 key = hashlib.sha1(
                     '{}{}{}'.format(section, val, repository_id)).hexdigest()
             else:
                 key = hashlib.sha1('{}{}'.format(section, val)).hexdigest()
 
         new_ui.ui_key = key
 
         Session().add(new_ui)
         return new_ui
 
     def create_or_update_hook(self, key, value):
         ui = (
             self.get_ui_by_section_and_key(self.HOOKS_SECTION, key) or
             self.UiDbModel())
         ui.ui_section = self.HOOKS_SECTION
         ui.ui_active = True
         ui.ui_key = key
         ui.ui_value = value
 
         if self.repo:
             repo = self._get_repo(self.repo)
             repository_id = repo.repo_id
             ui.repository_id = repository_id
 
         Session().add(ui)
         return ui
 
     def delete_ui(self, id_):
         ui = self.UiDbModel.get(id_)
         if not ui:
             raise SettingNotFound(id_)
         Session().delete(ui)
 
     def get_setting_by_name(self, name):
         q = self._get_settings_query()
         q = q.filter(self.SettingsDbModel.app_settings_name == name)
         return q.scalar()
 
     def create_or_update_setting(
             self, name, val=Optional(''), type_=Optional('unicode')):
         """
         Creates or updates RhodeCode setting. If updates is triggered it will
         only update parameters that are explicityl set Optional instance will
         be skipped
 
         :param name:
         :param val:
         :param type_:
         :return:
         """
 
         res = self.get_setting_by_name(name)
         repo = self._get_repo(self.repo) if self.repo else None
 
         if not res:
             val = Optional.extract(val)
             type_ = Optional.extract(type_)
 
             args = (
                 (repo.repo_id, name, val, type_)
                 if repo else (name, val, type_))
             res = self.SettingsDbModel(*args)
 
         else:
             if self.repo:
                 res.repository_id = repo.repo_id
 
             res.app_settings_name = name
             if not isinstance(type_, Optional):
                 # update if set
                 res.app_settings_type = type_
             if not isinstance(val, Optional):
                 # update if set
                 res.app_settings_value = val
 
         Session().add(res)
         return res
 
     def invalidate_settings_cache(self):
         invalidation_namespace = CacheKey.SETTINGS_INVALIDATION_NAMESPACE
         CacheKey.set_invalidate(invalidation_namespace)
 
     def get_all_settings(self, cache=False, from_request=True):
+        from rhodecode.authentication.base import get_authn_registry
+
         # defines if we use GLOBAL, or PER_REPO
         repo = self._get_repo(self.repo) if self.repo else None
         key = "settings_repo.{}".format(repo.repo_id) if repo else "settings_app"
 
         # initially try the requests context, this is the fastest
         # we only fetch global config
         if from_request:
             request = get_current_request()
 
             if request and not repo and hasattr(request, 'call_context') and hasattr(request.call_context, 'rc_config'):
                 rc_config = request.call_context.rc_config
                 if rc_config:
                     return rc_config
 
         region = rc_cache.get_or_create_region('sql_cache_short')
         invalidation_namespace = CacheKey.SETTINGS_INVALIDATION_NAMESPACE
 
         @region.conditional_cache_on_arguments(condition=cache)
         def _get_all_settings(name, key):
             q = self._get_settings_query()
             if not q:
                 raise Exception('Could not get application settings !')
 
             settings = {
                 'rhodecode_' + result.app_settings_name: result.app_settings_value
                 for result in q
             }
             return settings
 
         inv_context_manager = rc_cache.InvalidationContext(
             uid='cache_settings', invalidation_namespace=invalidation_namespace)
         with inv_context_manager as invalidation_context:
             # check for stored invalidation signal, and maybe purge the cache
             # before computing it again
             if invalidation_context.should_invalidate():
                 # NOTE:(marcink) we flush the whole sql_cache_short region, because it
                 # reads different settings etc. It's little too much but those caches
                 # are anyway very short lived and it's a safest way.
                 region = rc_cache.get_or_create_region('sql_cache_short')
                 region.invalidate()
+                registry = get_current_registry()
+                if registry:
+                    authn_registry = get_authn_registry(registry)
+                    if authn_registry:
+                        authn_registry.invalidate_plugins_for_auth()
 
             result = _get_all_settings('rhodecode_settings', key)
             log.debug('Fetching app settings for key: %s took: %.4fs', key,
                       inv_context_manager.compute_time)
 
         return result
 
     def get_auth_settings(self):
         q = self._get_settings_query()
         q = q.filter(
             self.SettingsDbModel.app_settings_name.startswith('auth_'))
         rows = q.all()
         auth_settings = {
             row.app_settings_name: row.app_settings_value for row in rows}
         return auth_settings
 
     def get_auth_plugins(self):
         auth_plugins = self.get_setting_by_name("auth_plugins")
         return auth_plugins.app_settings_value
 
     def get_default_repo_settings(self, strip_prefix=False):
         q = self._get_settings_query()
         q = q.filter(
             self.SettingsDbModel.app_settings_name.startswith('default_'))
         rows = q.all()
 
         result = {}
         for row in rows:
             key = row.app_settings_name
             if strip_prefix:
                 key = remove_prefix(key, prefix='default_')
             result.update({key: row.app_settings_value})
         return result
 
     def get_repo(self):
         repo = self._get_repo(self.repo)
         if not repo:
             raise Exception(
                 'Repository `{}` cannot be found inside the database'.format(
                     self.repo))
         return repo
 
     def _filter_by_repo(self, model, query):
         if self.repo:
             repo = self.get_repo()
             query = query.filter(model.repository_id == repo.repo_id)
         return query
 
     def _get_hooks(self, query):
         query = query.filter(self.UiDbModel.ui_section == self.HOOKS_SECTION)
         query = self._filter_by_repo(RepoRhodeCodeUi, query)
         return query.all()
 
     def _get_settings_query(self):
         q = self.SettingsDbModel.query()
         return self._filter_by_repo(RepoRhodeCodeSetting, q)
 
     def list_enabled_social_plugins(self, settings):
         enabled = []
         for plug in SOCIAL_PLUGINS_LIST:
             if str2bool(settings.get('rhodecode_auth_{}_enabled'.format(plug)
                                      )):
                 enabled.append(plug)
         return enabled
 
 
 def assert_repo_settings(func):
     @wraps(func)
     def _wrapper(self, *args, **kwargs):
         if not self.repo_settings:
             raise Exception('Repository is not specified')
         return func(self, *args, **kwargs)
     return _wrapper
 
 
 class IssueTrackerSettingsModel(object):
     INHERIT_SETTINGS = 'inherit_issue_tracker_settings'
     SETTINGS_PREFIX = 'issuetracker_'
 
     def __init__(self, sa=None, repo=None):
         self.global_settings = SettingsModel(sa=sa)
         self.repo_settings = SettingsModel(sa=sa, repo=repo) if repo else None
 
     @property
     def inherit_global_settings(self):
         if not self.repo_settings:
             return True
         setting = self.repo_settings.get_setting_by_name(self.INHERIT_SETTINGS)
         return setting.app_settings_value if setting else True
 
     @inherit_global_settings.setter
     def inherit_global_settings(self, value):
         if self.repo_settings:
             settings = self.repo_settings.create_or_update_setting(
                 self.INHERIT_SETTINGS, value, type_='bool')
             Session().add(settings)
 
     def _get_keyname(self, key, uid, prefix=''):
         return '{0}{1}{2}_{3}'.format(
             prefix, self.SETTINGS_PREFIX, key, uid)
 
     def _make_dict_for_settings(self, qs):
         prefix_match = self._get_keyname('pat', '', 'rhodecode_')
 
         issuetracker_entries = {}
         # create keys
         for k, v in qs.items():
             if k.startswith(prefix_match):
                 uid = k[len(prefix_match):]
                 issuetracker_entries[uid] = None
 
         def url_cleaner(input_str):
             input_str = input_str.replace('"', '').replace("'", '')
             input_str = bleach.clean(input_str, strip=True)
             return input_str
 
         # populate
         for uid in issuetracker_entries:
             url_data = qs.get(self._get_keyname('url', uid, 'rhodecode_'))
 
             pat = qs.get(self._get_keyname('pat', uid, 'rhodecode_'))
             try:
                 pat_compiled = re.compile(r'%s' % pat)
             except re.error:
                 pat_compiled = None
 
             issuetracker_entries[uid] = AttributeDict({
                 'pat': pat,
                 'pat_compiled': pat_compiled,
                 'url': url_cleaner(
                     qs.get(self._get_keyname('url', uid, 'rhodecode_')) or ''),
                 'pref': bleach.clean(
                     qs.get(self._get_keyname('pref', uid, 'rhodecode_')) or ''),
                 'desc': qs.get(
                     self._get_keyname('desc', uid, 'rhodecode_')),
             })
 
         return issuetracker_entries
 
     def get_global_settings(self, cache=False):
         """
         Returns list of global issue tracker settings
         """
         defaults = self.global_settings.get_all_settings(cache=cache)
         settings = self._make_dict_for_settings(defaults)
         return settings
 
     def get_repo_settings(self, cache=False):
         """
         Returns list of issue tracker settings per repository
         """
         if not self.repo_settings:
             raise Exception('Repository is not specified')
         all_settings = self.repo_settings.get_all_settings(cache=cache)
         settings = self._make_dict_for_settings(all_settings)
         return settings
 
     def get_settings(self, cache=False):
         if self.inherit_global_settings:
             return self.get_global_settings(cache=cache)
         else:
             return self.get_repo_settings(cache=cache)
 
     def delete_entries(self, uid):
         if self.repo_settings:
             all_patterns = self.get_repo_settings()
             settings_model = self.repo_settings
         else:
             all_patterns = self.get_global_settings()
             settings_model = self.global_settings
         entries = all_patterns.get(uid, [])
 
         for del_key in entries:
             setting_name = self._get_keyname(del_key, uid)
             entry = settings_model.get_setting_by_name(setting_name)
             if entry:
                 Session().delete(entry)
 
         Session().commit()
 
     def create_or_update_setting(
             self, name, val=Optional(''), type_=Optional('unicode')):
         if self.repo_settings:
             setting = self.repo_settings.create_or_update_setting(
                 name, val, type_)
         else:
             setting = self.global_settings.create_or_update_setting(
                 name, val, type_)
         return setting
 
 
 class VcsSettingsModel(object):
 
     INHERIT_SETTINGS = 'inherit_vcs_settings'
     GENERAL_SETTINGS = (
         'use_outdated_comments',
         'pr_merge_enabled',
         'hg_use_rebase_for_merging',
         'hg_close_branch_before_merging',
         'git_use_rebase_for_merging',
         'git_close_branch_before_merging',
         'diff_cache',
     )
 
     HOOKS_SETTINGS = (
         ('hooks', 'changegroup.repo_size'),
         ('hooks', 'changegroup.push_logger'),
         ('hooks', 'outgoing.pull_logger'),
     )
     HG_SETTINGS = (
         ('extensions', 'largefiles'),
         ('phases', 'publish'),
         ('extensions', 'evolve'),
         ('extensions', 'topic'),
         ('experimental', 'evolution'),
         ('experimental', 'evolution.exchange'),
     )
     GIT_SETTINGS = (
         ('vcs_git_lfs', 'enabled'),
     )
     GLOBAL_HG_SETTINGS = (
         ('extensions', 'largefiles'),
         ('largefiles', 'usercache'),
         ('phases', 'publish'),
         ('extensions', 'hgsubversion'),
         ('extensions', 'evolve'),
         ('extensions', 'topic'),
         ('experimental', 'evolution'),
         ('experimental', 'evolution.exchange'),
     )
 
     GLOBAL_GIT_SETTINGS = (
         ('vcs_git_lfs', 'enabled'),
         ('vcs_git_lfs', 'store_location')
     )
 
     GLOBAL_SVN_SETTINGS = (
         ('vcs_svn_proxy', 'http_requests_enabled'),
         ('vcs_svn_proxy', 'http_server_url')
     )
 
     SVN_BRANCH_SECTION = 'vcs_svn_branch'
     SVN_TAG_SECTION = 'vcs_svn_tag'
     SSL_SETTING = ('web', 'push_ssl')
     PATH_SETTING = ('paths', '/')
 
     def __init__(self, sa=None, repo=None):
         self.global_settings = SettingsModel(sa=sa)
         self.repo_settings = SettingsModel(sa=sa, repo=repo) if repo else None
         self._ui_settings = (
             self.HG_SETTINGS + self.GIT_SETTINGS + self.HOOKS_SETTINGS)
         self._svn_sections = (self.SVN_BRANCH_SECTION, self.SVN_TAG_SECTION)
 
     @property
     @assert_repo_settings
     def inherit_global_settings(self):
         setting = self.repo_settings.get_setting_by_name(self.INHERIT_SETTINGS)
         return setting.app_settings_value if setting else True
 
     @inherit_global_settings.setter
     @assert_repo_settings
     def inherit_global_settings(self, value):
         self.repo_settings.create_or_update_setting(
             self.INHERIT_SETTINGS, value, type_='bool')
 
     def get_global_svn_branch_patterns(self):
         return self.global_settings.get_ui_by_section(self.SVN_BRANCH_SECTION)
 
     @assert_repo_settings
     def get_repo_svn_branch_patterns(self):
         return self.repo_settings.get_ui_by_section(self.SVN_BRANCH_SECTION)
 
     def get_global_svn_tag_patterns(self):
         return self.global_settings.get_ui_by_section(self.SVN_TAG_SECTION)
 
     @assert_repo_settings
     def get_repo_svn_tag_patterns(self):
         return self.repo_settings.get_ui_by_section(self.SVN_TAG_SECTION)
 
     def get_global_settings(self):
         return self._collect_all_settings(global_=True)
 
     @assert_repo_settings
     def get_repo_settings(self):
         return self._collect_all_settings(global_=False)
 
     @assert_repo_settings
     def get_repo_settings_inherited(self):
         global_settings = self.get_global_settings()
         global_settings.update(self.get_repo_settings())
         return global_settings
 
     @assert_repo_settings
     def create_or_update_repo_settings(
             self, data, inherit_global_settings=False):
         from rhodecode.model.scm import ScmModel
 
         self.inherit_global_settings = inherit_global_settings
 
         repo = self.repo_settings.get_repo()
         if not inherit_global_settings:
             if repo.repo_type == 'svn':
                 self.create_repo_svn_settings(data)
             else:
                 self.create_or_update_repo_hook_settings(data)
                 self.create_or_update_repo_pr_settings(data)
 
             if repo.repo_type == 'hg':
                 self.create_or_update_repo_hg_settings(data)
 
             if repo.repo_type == 'git':
                 self.create_or_update_repo_git_settings(data)
 
         ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
 
     @assert_repo_settings
     def create_or_update_repo_hook_settings(self, data):
         for section, key in self.HOOKS_SETTINGS:
             data_key = self._get_form_ui_key(section, key)
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
 
             active = data.get(data_key)
             repo_setting = self.repo_settings.get_ui_by_section_and_key(
                 section, key)
             if not repo_setting:
                 global_setting = self.global_settings.\
                     get_ui_by_section_and_key(section, key)
                 self.repo_settings.create_ui_section_value(
                     section, global_setting.ui_value, key=key, active=active)
             else:
                 repo_setting.ui_active = active
                 Session().add(repo_setting)
 
     def update_global_hook_settings(self, data):
         for section, key in self.HOOKS_SETTINGS:
             data_key = self._get_form_ui_key(section, key)
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
             active = data.get(data_key)
             repo_setting = self.global_settings.get_ui_by_section_and_key(
                 section, key)
             repo_setting.ui_active = active
             Session().add(repo_setting)
 
     @assert_repo_settings
     def create_or_update_repo_pr_settings(self, data):
         return self._create_or_update_general_settings(
             self.repo_settings, data)
 
     def create_or_update_global_pr_settings(self, data):
         return self._create_or_update_general_settings(
             self.global_settings, data)
 
     @assert_repo_settings
     def create_repo_svn_settings(self, data):
         return self._create_svn_settings(self.repo_settings, data)
 
     def _set_evolution(self, settings, is_enabled):
         if is_enabled:
             # if evolve is active set evolution=all
 
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution'), value='all',
                 active=True)
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution.exchange'), value='yes',
                 active=True)
             # if evolve is active set topics server support
             self._create_or_update_ui(
                 settings, *('extensions', 'topic'), value='',
                 active=True)
 
         else:
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution'), value='',
                 active=False)
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution.exchange'), value='no',
                 active=False)
             self._create_or_update_ui(
                 settings, *('extensions', 'topic'), value='',
                 active=False)
 
     @assert_repo_settings
     def create_or_update_repo_hg_settings(self, data):
         largefiles, phases, evolve = \
             self.HG_SETTINGS[:3]
         largefiles_key, phases_key, evolve_key = \
             self._get_settings_keys(self.HG_SETTINGS[:3], data)
 
         self._create_or_update_ui(
             self.repo_settings, *largefiles, value='',
             active=data[largefiles_key])
         self._create_or_update_ui(
             self.repo_settings, *evolve, value='',
             active=data[evolve_key])
         self._set_evolution(self.repo_settings, is_enabled=data[evolve_key])
 
         self._create_or_update_ui(
             self.repo_settings, *phases, value=safe_str(data[phases_key]))
 
     def create_or_update_global_hg_settings(self, data):
         largefiles, largefiles_store, phases, hgsubversion, evolve \
             = self.GLOBAL_HG_SETTINGS[:5]
         largefiles_key, largefiles_store_key, phases_key, subversion_key, evolve_key \
             = self._get_settings_keys(self.GLOBAL_HG_SETTINGS[:5], data)
 
         self._create_or_update_ui(
             self.global_settings, *largefiles, value='',
             active=data[largefiles_key])
         self._create_or_update_ui(
             self.global_settings, *largefiles_store, value=data[largefiles_store_key])
         self._create_or_update_ui(
             self.global_settings, *phases, value=safe_str(data[phases_key]))
         self._create_or_update_ui(
             self.global_settings, *hgsubversion, active=data[subversion_key])
         self._create_or_update_ui(
             self.global_settings, *evolve, value='',
             active=data[evolve_key])
         self._set_evolution(self.global_settings, is_enabled=data[evolve_key])
 
     def create_or_update_repo_git_settings(self, data):
         # NOTE(marcink): # comma makes unpack work properly
         lfs_enabled, \
             = self.GIT_SETTINGS
 
         lfs_enabled_key, \
             = self._get_settings_keys(self.GIT_SETTINGS, data)
 
         self._create_or_update_ui(
             self.repo_settings, *lfs_enabled, value=data[lfs_enabled_key],
             active=data[lfs_enabled_key])
 
     def create_or_update_global_git_settings(self, data):
         lfs_enabled, lfs_store_location \
             = self.GLOBAL_GIT_SETTINGS
         lfs_enabled_key, lfs_store_location_key \
             = self._get_settings_keys(self.GLOBAL_GIT_SETTINGS, data)
 
         self._create_or_update_ui(
             self.global_settings, *lfs_enabled, value=data[lfs_enabled_key],
             active=data[lfs_enabled_key])
         self._create_or_update_ui(
             self.global_settings, *lfs_store_location,
             value=data[lfs_store_location_key])
 
     def create_or_update_global_svn_settings(self, data):
         # branch/tags patterns
         self._create_svn_settings(self.global_settings, data)
 
         http_requests_enabled, http_server_url = self.GLOBAL_SVN_SETTINGS
         http_requests_enabled_key, http_server_url_key = self._get_settings_keys(
             self.GLOBAL_SVN_SETTINGS, data)
 
         self._create_or_update_ui(
             self.global_settings, *http_requests_enabled,
             value=safe_str(data[http_requests_enabled_key]))
         self._create_or_update_ui(
             self.global_settings, *http_server_url,
             value=data[http_server_url_key])
 
     def update_global_ssl_setting(self, value):
         self._create_or_update_ui(
             self.global_settings, *self.SSL_SETTING, value=value)
 
     def update_global_path_setting(self, value):
         self._create_or_update_ui(
             self.global_settings, *self.PATH_SETTING, value=value)
 
     @assert_repo_settings
     def delete_repo_svn_pattern(self, id_):
         ui = self.repo_settings.UiDbModel.get(id_)
         if ui and ui.repository.repo_name == self.repo_settings.repo:
             # only delete if it's the same repo as initialized settings
             self.repo_settings.delete_ui(id_)
         else:
             # raise error as if we wouldn't find this option
             self.repo_settings.delete_ui(-1)
 
     def delete_global_svn_pattern(self, id_):
         self.global_settings.delete_ui(id_)
 
     @assert_repo_settings
     def get_repo_ui_settings(self, section=None, key=None):
         global_uis = self.global_settings.get_ui(section, key)
         repo_uis = self.repo_settings.get_ui(section, key)
 
         filtered_repo_uis = self._filter_ui_settings(repo_uis)
         filtered_repo_uis_keys = [
             (s.section, s.key) for s in filtered_repo_uis]
 
         def _is_global_ui_filtered(ui):
             return (
                 (ui.section, ui.key) in filtered_repo_uis_keys
                 or ui.section in self._svn_sections)
 
         filtered_global_uis = [
             ui for ui in global_uis if not _is_global_ui_filtered(ui)]
 
         return filtered_global_uis + filtered_repo_uis
 
     def get_global_ui_settings(self, section=None, key=None):
         return self.global_settings.get_ui(section, key)
 
     def get_ui_settings_as_config_obj(self, section=None, key=None):
         config = base.Config()
 
         ui_settings = self.get_ui_settings(section=section, key=key)
 
         for entry in ui_settings:
             config.set(entry.section, entry.key, entry.value)
 
         return config
 
     def get_ui_settings(self, section=None, key=None):
         if not self.repo_settings or self.inherit_global_settings:
             return self.get_global_ui_settings(section, key)
         else:
             return self.get_repo_ui_settings(section, key)
 
     def get_svn_patterns(self, section=None):
         if not self.repo_settings:
             return self.get_global_ui_settings(section)
         else:
             return self.get_repo_ui_settings(section)
 
     @assert_repo_settings
     def get_repo_general_settings(self):
         global_settings = self.global_settings.get_all_settings()
         repo_settings = self.repo_settings.get_all_settings()
         filtered_repo_settings = self._filter_general_settings(repo_settings)
         global_settings.update(filtered_repo_settings)
         return global_settings
 
     def get_global_general_settings(self):
         return self.global_settings.get_all_settings()
 
     def get_general_settings(self):
         if not self.repo_settings or self.inherit_global_settings:
             return self.get_global_general_settings()
         else:
             return self.get_repo_general_settings()
 
     def get_repos_location(self):
         return self.global_settings.get_ui_by_key('/').ui_value
 
     def _filter_ui_settings(self, settings):
         filtered_settings = [
             s for s in settings if self._should_keep_setting(s)]
         return filtered_settings
 
     def _should_keep_setting(self, setting):
         keep = (
             (setting.section, setting.key) in self._ui_settings or
             setting.section in self._svn_sections)
         return keep
 
     def _filter_general_settings(self, settings):
         keys = ['rhodecode_{}'.format(key) for key in self.GENERAL_SETTINGS]
         return {
             k: settings[k]
             for k in settings if k in keys}
 
     def _collect_all_settings(self, global_=False):
         settings = self.global_settings if global_ else self.repo_settings
         result = {}
 
         for section, key in self._ui_settings:
             ui = settings.get_ui_by_section_and_key(section, key)
             result_key = self._get_form_ui_key(section, key)
 
             if ui:
                 if section in ('hooks', 'extensions'):
                     result[result_key] = ui.ui_active
                 elif result_key in ['vcs_git_lfs_enabled']:
                     result[result_key] = ui.ui_active
                 else:
                     result[result_key] = ui.ui_value
 
         for name in self.GENERAL_SETTINGS:
             setting = settings.get_setting_by_name(name)
             if setting:
                 result_key = 'rhodecode_{}'.format(name)
                 result[result_key] = setting.app_settings_value
 
         return result
 
     def _get_form_ui_key(self, section, key):
         return '{section}_{key}'.format(
             section=section, key=key.replace('.', '_'))
 
     def _create_or_update_ui(
             self, settings, section, key, value=None, active=None):
         ui = settings.get_ui_by_section_and_key(section, key)
         if not ui:
             active = True if active is None else active
             settings.create_ui_section_value(
                 section, value, key=key, active=active)
         else:
             if active is not None:
                 ui.ui_active = active
             if value is not None:
                 ui.ui_value = value
             Session().add(ui)
 
     def _create_svn_settings(self, settings, data):
         svn_settings = {
             'new_svn_branch': self.SVN_BRANCH_SECTION,
             'new_svn_tag': self.SVN_TAG_SECTION
         }
         for key in svn_settings:
             if data.get(key):
                 settings.create_ui_section_value(svn_settings[key], data[key])
 
     def _create_or_update_general_settings(self, settings, data):
         for name in self.GENERAL_SETTINGS:
             data_key = 'rhodecode_{}'.format(name)
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
             setting = settings.create_or_update_setting(
                 name, data[data_key], 'bool')
             Session().add(setting)
 
     def _get_settings_keys(self, settings, data):
         data_keys = [self._get_form_ui_key(*s) for s in settings]
         for data_key in data_keys:
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
         return data_keys
 
     def create_largeobjects_dirs_if_needed(self, repo_store_path):
         """
         This is subscribed to the `pyramid.events.ApplicationCreated` event. It
         does a repository scan if enabled in the settings.
         """
 
         from rhodecode.lib.vcs.backends.hg import largefiles_store
         from rhodecode.lib.vcs.backends.git import lfs_store
 
         paths = [
             largefiles_store(repo_store_path),
             lfs_store(repo_store_path)]
 
         for path in paths:
             if os.path.isdir(path):
                 continue
             if os.path.isfile(path):
                 continue
             # not a file nor dir, we try to create it
             try:
                 os.makedirs(path)
             except Exception:
                 log.warning('Failed to create largefiles dir:%s', path)

rhodecode/tests/fixture.py

@@ -1,416 +1,417 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Helpers for fixture generation
 """
 
 import os
 import time
 import tempfile
 import shutil
 
 import configobj
 
+from rhodecode.model.settings import SettingsModel
 from rhodecode.tests import *
 from rhodecode.model.db import Repository, User, RepoGroup, UserGroup, Gist, UserEmailMap
 from rhodecode.model.meta import Session
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.model.gist import GistModel
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.authentication.plugins.auth_rhodecode import \
     RhodeCodeAuthPlugin
 
 dn = os.path.dirname
 FIXTURES = os.path.join(dn(dn(os.path.abspath(__file__))), 'tests', 'fixtures')
 
 
 def error_function(*args, **kwargs):
     raise Exception('Total Crash !')
 
 
 class TestINI(object):
     """
     Allows to create a new test.ini file as a copy of existing one with edited
     data. Example usage::
 
         with TestINI('test.ini', [{'section':{'key':val'}]) as new_test_ini_path:
             print('paster server %s' % new_test_ini)
     """
 
     def __init__(self, ini_file_path, ini_params, new_file_prefix='DEFAULT',
                  destroy=True, dir=None):
         self.ini_file_path = ini_file_path
         self.ini_params = ini_params
         self.new_path = None
         self.new_path_prefix = new_file_prefix
         self._destroy = destroy
         self._dir = dir
 
     def __enter__(self):
         return self.create()
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.destroy()
 
     def create(self):
         config = configobj.ConfigObj(
             self.ini_file_path, file_error=True, write_empty_values=True)
 
         for data in self.ini_params:
             section, ini_params = data.items()[0]
             for key, val in ini_params.items():
                 config[section][key] = val
         with tempfile.NamedTemporaryFile(
                 prefix=self.new_path_prefix, suffix='.ini', dir=self._dir,
                 delete=False) as new_ini_file:
             config.write(new_ini_file)
             self.new_path = new_ini_file.name
 
         return self.new_path
 
     def destroy(self):
         if self._destroy:
             os.remove(self.new_path)
 
 
 class Fixture(object):
 
     def anon_access(self, status):
         """
         Context process for disabling anonymous access. use like:
         fixture = Fixture()
         with fixture.anon_access(False):
             #tests
 
         after this block anon access will be set to `not status`
         """
 
         class context(object):
             def __enter__(self):
                 anon = User.get_default_user()
                 anon.active = status
                 Session().add(anon)
                 Session().commit()
                 time.sleep(1.5)  # must sleep for cache (1s to expire)
 
             def __exit__(self, exc_type, exc_val, exc_tb):
                 anon = User.get_default_user()
                 anon.active = not status
                 Session().add(anon)
                 Session().commit()
 
         return context()
 
-    def auth_restriction(self, auth_restriction):
+    def auth_restriction(self, registry, auth_restriction):
         """
         Context process for changing the builtin rhodecode plugin auth restrictions.
         Use like:
         fixture = Fixture()
         with fixture.auth_restriction('super_admin'):
             #tests
 
         after this block auth restriction will be taken off
         """
 
         class context(object):
             def _get_pluing(self):
-                plugin_id = 'egg:rhodecode-enterprise-ce#{}'.format(
-                    RhodeCodeAuthPlugin.uid)
+                plugin_id = 'egg:rhodecode-enterprise-ce#{}'.format(RhodeCodeAuthPlugin.uid)
                 plugin = RhodeCodeAuthPlugin(plugin_id)
                 return plugin
 
             def __enter__(self):
                 plugin = self._get_pluing()
-                plugin.create_or_update_setting(
-                    'auth_restriction', auth_restriction)
+                plugin.create_or_update_setting('auth_restriction', auth_restriction)
                 Session().commit()
+                SettingsModel().invalidate_settings_cache()
 
             def __exit__(self, exc_type, exc_val, exc_tb):
                 plugin = self._get_pluing()
                 plugin.create_or_update_setting(
                     'auth_restriction', RhodeCodeAuthPlugin.AUTH_RESTRICTION_NONE)
                 Session().commit()
+                SettingsModel().invalidate_settings_cache()
 
         return context()
 
-    def scope_restriction(self, scope_restriction):
+    def scope_restriction(self, registry, scope_restriction):
         """
         Context process for changing the builtin rhodecode plugin scope restrictions.
         Use like:
         fixture = Fixture()
         with fixture.scope_restriction('scope_http'):
             #tests
 
         after this block scope restriction will be taken off
         """
 
         class context(object):
             def _get_pluing(self):
-                plugin_id = 'egg:rhodecode-enterprise-ce#{}'.format(
-                    RhodeCodeAuthPlugin.uid)
+                plugin_id = 'egg:rhodecode-enterprise-ce#{}'.format(RhodeCodeAuthPlugin.uid)
                 plugin = RhodeCodeAuthPlugin(plugin_id)
                 return plugin
 
             def __enter__(self):
                 plugin = self._get_pluing()
-                plugin.create_or_update_setting(
-                    'scope_restriction', scope_restriction)
+                plugin.create_or_update_setting('scope_restriction', scope_restriction)
                 Session().commit()
+                SettingsModel().invalidate_settings_cache()
 
             def __exit__(self, exc_type, exc_val, exc_tb):
                 plugin = self._get_pluing()
                 plugin.create_or_update_setting(
                     'scope_restriction', RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_ALL)
                 Session().commit()
+                SettingsModel().invalidate_settings_cache()
 
         return context()
 
     def _get_repo_create_params(self, **custom):
         defs = {
             'repo_name': None,
             'repo_type': 'hg',
             'clone_uri': '',
             'push_uri': '',
             'repo_group': '-1',
             'repo_description': 'DESC',
             'repo_private': False,
             'repo_landing_rev': 'rev:tip',
             'repo_copy_permissions': False,
             'repo_state': Repository.STATE_CREATED,
         }
         defs.update(custom)
         if 'repo_name_full' not in custom:
             defs.update({'repo_name_full': defs['repo_name']})
 
         # fix the repo name if passed as repo_name_full
         if defs['repo_name']:
             defs['repo_name'] = defs['repo_name'].split('/')[-1]
 
         return defs
 
     def _get_group_create_params(self, **custom):
         defs = {
             'group_name': None,
             'group_description': 'DESC',
             'perm_updates': [],
             'perm_additions': [],
             'perm_deletions': [],
             'group_parent_id': -1,
             'enable_locking': False,
             'recursive': False,
         }
         defs.update(custom)
 
         return defs
 
     def _get_user_create_params(self, name, **custom):
         defs = {
             'username': name,
             'password': 'qweqwe',
             'email': '[e-mail]' % name,
             'firstname': 'TestUser',
             'lastname': 'Test',
             'description': 'test description',
             'active': True,
             'admin': False,
             'extern_type': 'rhodecode',
             'extern_name': None,
         }
         defs.update(custom)
 
         return defs
 
     def _get_user_group_create_params(self, name, **custom):
         defs = {
             'users_group_name': name,
             'user_group_description': 'DESC',
             'users_group_active': True,
             'user_group_data': {},
         }
         defs.update(custom)
 
         return defs
 
     def create_repo(self, name, **kwargs):
         repo_group = kwargs.get('repo_group')
         if isinstance(repo_group, RepoGroup):
             kwargs['repo_group'] = repo_group.group_id
             name = name.split(Repository.NAME_SEP)[-1]
             name = Repository.NAME_SEP.join((repo_group.group_name, name))
 
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             r = Repository.get_by_repo_name(name)
             if r:
                 return r
 
         form_data = self._get_repo_create_params(repo_name=name, **kwargs)
         cur_user = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         RepoModel().create(form_data, cur_user)
         Session().commit()
         repo = Repository.get_by_repo_name(name)
         assert repo
         return repo
 
     def create_fork(self, repo_to_fork, fork_name, **kwargs):
         repo_to_fork = Repository.get_by_repo_name(repo_to_fork)
 
         form_data = self._get_repo_create_params(repo_name=fork_name,
                                             fork_parent_id=repo_to_fork.repo_id,
                                             repo_type=repo_to_fork.repo_type,
                                             **kwargs)
         #TODO: fix it !!
         form_data['description'] = form_data['repo_description']
         form_data['private'] = form_data['repo_private']
         form_data['landing_rev'] = form_data['repo_landing_rev']
 
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         RepoModel().create_fork(form_data, cur_user=owner)
         Session().commit()
         r = Repository.get_by_repo_name(fork_name)
         assert r
         return r
 
     def destroy_repo(self, repo_name, **kwargs):
         RepoModel().delete(repo_name, pull_requests='delete', **kwargs)
         Session().commit()
 
     def destroy_repo_on_filesystem(self, repo_name):
         rm_path = os.path.join(RepoModel().repos_path, repo_name)
         if os.path.isdir(rm_path):
             shutil.rmtree(rm_path)
 
     def create_repo_group(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             gr = RepoGroup.get_by_group_name(group_name=name)
             if gr:
                 return gr
         form_data = self._get_group_create_params(group_name=name, **kwargs)
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         gr = RepoGroupModel().create(
             group_name=form_data['group_name'],
             group_description=form_data['group_name'],
             owner=owner)
         Session().commit()
         gr = RepoGroup.get_by_group_name(gr.group_name)
         return gr
 
     def destroy_repo_group(self, repogroupid):
         RepoGroupModel().delete(repogroupid)
         Session().commit()
 
     def create_user(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             user = User.get_by_username(name)
             if user:
                 return user
         form_data = self._get_user_create_params(name, **kwargs)
         user = UserModel().create(form_data)
 
         # create token for user
         AuthTokenModel().create(
                 user=user, description=u'TEST_USER_TOKEN')
 
         Session().commit()
         user = User.get_by_username(user.username)
         return user
 
     def destroy_user(self, userid):
         UserModel().delete(userid)
         Session().commit()
 
     def create_additional_user_email(self, user, email):
         uem = UserEmailMap()
         uem.user = user
         uem.email = email
         Session().add(uem)
         return uem
 
     def destroy_users(self, userid_iter):
         for user_id in userid_iter:
             if User.get_by_username(user_id):
                 UserModel().delete(user_id)
                 Session().commit()
 
     def create_user_group(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             gr = UserGroup.get_by_group_name(group_name=name)
             if gr:
                 return gr
         # map active flag to the real attribute. For API consistency of fixtures
         if 'active' in kwargs:
             kwargs['users_group_active'] = kwargs['active']
             del kwargs['active']
         form_data = self._get_user_group_create_params(name, **kwargs)
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         user_group = UserGroupModel().create(
             name=form_data['users_group_name'],
             description=form_data['user_group_description'],
             owner=owner, active=form_data['users_group_active'],
             group_data=form_data['user_group_data'])
         Session().commit()
         user_group = UserGroup.get_by_group_name(user_group.users_group_name)
         return user_group
 
     def destroy_user_group(self, usergroupid):
         UserGroupModel().delete(user_group=usergroupid, force=True)
         Session().commit()
 
     def create_gist(self, **kwargs):
         form_data = {
             'description': 'new-gist',
             'owner': TEST_USER_ADMIN_LOGIN,
             'gist_type': GistModel.cls.GIST_PUBLIC,
             'lifetime': -1,
             'acl_level': Gist.ACL_LEVEL_PUBLIC,
             'gist_mapping': {'filename1.txt': {'content': 'hello world'},}
         }
         form_data.update(kwargs)
         gist = GistModel().create(
             description=form_data['description'], owner=form_data['owner'],
             gist_mapping=form_data['gist_mapping'], gist_type=form_data['gist_type'],
             lifetime=form_data['lifetime'], gist_acl_level=form_data['acl_level']
         )
         Session().commit()
         return gist
 
     def destroy_gists(self, gistid=None):
         for g in GistModel.cls.get_all():
             if gistid:
                 if gistid == g.gist_access_id:
                     GistModel().delete(g)
             else:
                 GistModel().delete(g)
         Session().commit()
 
     def load_resource(self, resource_name, strip=False):
         with open(os.path.join(FIXTURES, resource_name)) as f:
             source = f.read()
             if strip:
                 source = source.strip()
 
         return source

rhodecode/tests/vcs_operations/conftest.py

@@ -1,340 +1,342 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 py.test config for test suite for making push/pull operations.
 
 .. important::
 
    You must have git >= 1.8.5 for tests to work fine. With 68b939b git started
    to redirect things to stderr instead of stdout.
 """
 
 import os
 import tempfile
 import textwrap
 import pytest
 
 from rhodecode import events
 from rhodecode.model.db import Integration, UserRepoToPerm, Permission, \
     UserToRepoBranchPermission, User
 from rhodecode.model.integration import IntegrationModel
 from rhodecode.model.db import Repository
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import SettingsModel
 from rhodecode.integrations.types.webhook import WebhookIntegrationType
 
 from rhodecode.tests import GIT_REPO, HG_REPO
 from rhodecode.tests.fixture import Fixture
 from rhodecode.tests.server_utils import RcWebServer
 
 REPO_GROUP = 'a_repo_group'
 HG_REPO_WITH_GROUP = '%s/%s' % (REPO_GROUP, HG_REPO)
 GIT_REPO_WITH_GROUP = '%s/%s' % (REPO_GROUP, GIT_REPO)
 
 
 @pytest.fixture(scope="module")
 def rcextensions(request, db_connection, tmpdir_factory):
     """
     Installs a testing rcextensions pack to ensure they work as expected.
     """
     init_content = textwrap.dedent("""
         # Forward import the example rcextensions to make it
         # active for our tests.
         from rhodecode.tests.other.example_rcextensions import *
     """)
 
     # Note: rcextensions are looked up based on the path of the ini file
     root_path = tmpdir_factory.getbasetemp()
     rcextensions_path = root_path.join('rcextensions')
     init_path = rcextensions_path.join('__init__.py')
 
     if rcextensions_path.check():
         pytest.fail(
             "Path for rcextensions already exists, please clean up before "
             "test run this path: %s" % (rcextensions_path, ))
         return
 
     request.addfinalizer(rcextensions_path.remove)
     init_path.write_binary(init_content, ensure=True)
 
 
 @pytest.fixture(scope="module")
 def repos(request, db_connection):
     """Create a copy of each test repo in a repo group."""
     fixture = Fixture()
     repo_group = fixture.create_repo_group(REPO_GROUP)
     repo_group_id = repo_group.group_id
     fixture.create_fork(HG_REPO, HG_REPO,
                         repo_name_full=HG_REPO_WITH_GROUP,
                         repo_group=repo_group_id)
     fixture.create_fork(GIT_REPO, GIT_REPO,
                         repo_name_full=GIT_REPO_WITH_GROUP,
                         repo_group=repo_group_id)
 
     @request.addfinalizer
     def cleanup():
         fixture.destroy_repo(HG_REPO_WITH_GROUP)
         fixture.destroy_repo(GIT_REPO_WITH_GROUP)
         fixture.destroy_repo_group(repo_group_id)
 
 
 @pytest.fixture(scope="module")
 def rc_web_server_config_modification():
     return []
 
 
 @pytest.fixture(scope="module")
 def rc_web_server_config_factory(testini_factory, rc_web_server_config_modification):
     """
     Configuration file used for the fixture `rc_web_server`.
     """
 
     def factory(rcweb_port, vcsserver_port):
         custom_params = [
             {'handler_console': {'level': 'DEBUG'}},
             {'server:main': {'port': rcweb_port}},
             {'app:main': {'vcs.server': 'localhost:%s' % vcsserver_port}}
         ]
         custom_params.extend(rc_web_server_config_modification)
         return testini_factory(custom_params)
     return factory
 
 
 @pytest.fixture(scope="module")
 def rc_web_server(
         request, vcsserver_factory, available_port_factory,
         rc_web_server_config_factory, repos, rcextensions):
     """
     Run the web server as a subprocess. with it's own instance of vcsserver
     """
     rcweb_port = available_port_factory()
     print('Using rcweb ops test port {}'.format(rcweb_port))
 
     vcsserver_port = available_port_factory()
     print('Using vcsserver ops test port {}'.format(vcsserver_port))
 
     vcs_log = os.path.join(tempfile.gettempdir(), 'rc_op_vcs.log')
     vcsserver_factory(
             request, vcsserver_port=vcsserver_port,
             log_file=vcs_log,
             overrides=(
                 {'server:main': {'workers': 2}},
                 {'server:main': {'graceful_timeout': 10}},
             ))
 
     rc_log = os.path.join(tempfile.gettempdir(), 'rc_op_web.log')
     rc_web_server_config = rc_web_server_config_factory(
         rcweb_port=rcweb_port,
         vcsserver_port=vcsserver_port)
     server = RcWebServer(rc_web_server_config, log_file=rc_log)
     server.start()
 
     @request.addfinalizer
     def cleanup():
         server.shutdown()
 
     server.wait_until_ready()
     return server
 
 
 @pytest.fixture()
 def disable_locking(baseapp):
     r = Repository.get_by_repo_name(GIT_REPO)
     Repository.unlock(r)
     r.enable_locking = False
     Session().add(r)
     Session().commit()
 
     r = Repository.get_by_repo_name(HG_REPO)
     Repository.unlock(r)
     r.enable_locking = False
     Session().add(r)
     Session().commit()
 
 
 @pytest.fixture()
 def enable_auth_plugins(request, baseapp, csrf_token):
     """
     Return a factory object that when called, allows to control which
     authentication plugins are enabled.
     """
     def _enable_plugins(plugins_list, override=None):
         override = override or {}
         params = {
             'auth_plugins': ','.join(plugins_list),
         }
 
         # helper translate some names to others
         name_map = {
             'token': 'authtoken'
         }
 
         for module in plugins_list:
             plugin_name = module.partition('#')[-1]
             if plugin_name in name_map:
                 plugin_name = name_map[plugin_name]
             enabled_plugin = 'auth_%s_enabled' % plugin_name
             cache_ttl = 'auth_%s_cache_ttl' % plugin_name
 
             # default params that are needed for each plugin,
             # `enabled` and `cache_ttl`
             params.update({
                 enabled_plugin: True,
                 cache_ttl: 0
             })
             if override.get:
                 params.update(override.get(module, {}))
 
             validated_params = params
             for k, v in validated_params.items():
                 setting = SettingsModel().create_or_update_setting(k, v)
                 Session().add(setting)
             Session().commit()
 
+        SettingsModel().invalidate_settings_cache()
+
     def cleanup():
         _enable_plugins(['egg:rhodecode-enterprise-ce#rhodecode'])
 
     request.addfinalizer(cleanup)
 
     return _enable_plugins
 
 
 @pytest.fixture()
 def fs_repo_only(request, rhodecode_fixtures):
     def fs_repo_fabric(repo_name, repo_type):
         rhodecode_fixtures.create_repo(repo_name, repo_type=repo_type)
         rhodecode_fixtures.destroy_repo(repo_name, fs_remove=False)
 
         def cleanup():
             rhodecode_fixtures.destroy_repo(repo_name, fs_remove=True)
             rhodecode_fixtures.destroy_repo_on_filesystem(repo_name)
 
         request.addfinalizer(cleanup)
 
     return fs_repo_fabric
 
 
 @pytest.fixture()
 def enable_webhook_push_integration(request):
     integration = Integration()
     integration.integration_type = WebhookIntegrationType.key
     Session().add(integration)
 
     settings = dict(
         url='http://httpbin.org/post',
         secret_token='secret',
         username=None,
         password=None,
         custom_header_key=None,
         custom_header_val=None,
         method_type='post',
         events=[events.RepoPushEvent.name],
         log_data=True
     )
 
     IntegrationModel().update_integration(
         integration,
         name='IntegrationWebhookTest',
         enabled=True,
         settings=settings,
         repo=None,
         repo_group=None,
         child_repos_only=False,
     )
     Session().commit()
     integration_id = integration.integration_id
 
     @request.addfinalizer
     def cleanup():
         integration = Integration.get(integration_id)
         Session().delete(integration)
         Session().commit()
 
 
 @pytest.fixture()
 def branch_permission_setter(request):
     """
 
     def my_test(branch_permission_setter)
         branch_permission_setter(repo_name, username, pattern='*', permission='branch.push')
 
     """
 
     rule_id = None
     write_perm_id = None
     write_perm = None
     rule = None
 
     def _branch_permissions_setter(
             repo_name, username, pattern='*', permission='branch.push_force'):
         global rule_id, write_perm_id
         global rule, write_perm
 
         repo = Repository.get_by_repo_name(repo_name)
         repo_id = repo.repo_id
 
         user = User.get_by_username(username)
         user_id = user.user_id
 
         rule_perm_obj = Permission.get_by_key(permission)
 
         # add new entry, based on existing perm entry
         perm = UserRepoToPerm.query() \
             .filter(UserRepoToPerm.repository_id == repo_id) \
             .filter(UserRepoToPerm.user_id == user_id) \
             .first()
 
         if not perm:
             # such user isn't defined in Permissions for repository
             # we now on-the-fly add new permission
 
             write_perm = UserRepoToPerm()
             write_perm.permission = Permission.get_by_key('repository.write')
             write_perm.repository_id = repo_id
             write_perm.user_id = user_id
             Session().add(write_perm)
             Session().flush()
 
             perm = write_perm
 
         rule = UserToRepoBranchPermission()
         rule.rule_to_perm_id = perm.repo_to_perm_id
         rule.branch_pattern = pattern
         rule.rule_order = 10
         rule.permission = rule_perm_obj
         rule.repository_id = repo_id
         Session().add(rule)
         Session().commit()
 
         return rule
 
     @request.addfinalizer
     def cleanup():
         if rule:
             Session().delete(rule)
             Session().commit()
         if write_perm:
             Session().delete(write_perm)
             Session().commit()
 
     return _branch_permissions_setter