# HG changeset patch # User Marcin Kuzminski # Date 2017-09-28 19:45:19 # Node ID 61a365309898441576507b11d562b72fe7eeda84 # Parent 665d2e3250a1b1e41f42dff2787491039f412655 vcs: reduce sql queries used during pull/push operations. diff --git a/rhodecode/authentication/base.py b/rhodecode/authentication/base.py --- a/rhodecode/authentication/base.py +++ b/rhodecode/authentication/base.py @@ -31,6 +31,7 @@ import warnings import functools from pyramid.threadlocal import get_current_registry +from zope.cachedescriptors.property import Lazy as LazyProperty from rhodecode.authentication.interface import IAuthnPluginRegistry from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase @@ -168,6 +169,11 @@ class RhodeCodeAuthPluginBase(object): db_type = '{}.encrypted'.format(db_type) return db_type + @LazyProperty + def plugin_settings(self): + settings = SettingsModel().get_all_settings() + return settings + def is_enabled(self): """ Returns true if this plugin is enabled. An enabled plugin can be @@ -206,9 +212,10 @@ class RhodeCodeAuthPluginBase(object): """ Returns a plugin setting by name. """ - full_name = self._get_setting_full_name(name) - db_setting = SettingsModel().get_setting_by_name(full_name) - return db_setting.app_settings_value if db_setting else default + full_name = 'rhodecode_{}'.format(self._get_setting_full_name(name)) + plugin_settings = self.plugin_settings + + return plugin_settings.get(full_name) or default def create_or_update_setting(self, name, value): """ diff --git a/rhodecode/authentication/registry.py b/rhodecode/authentication/registry.py --- a/rhodecode/authentication/registry.py +++ b/rhodecode/authentication/registry.py @@ -68,7 +68,7 @@ class AuthenticationPluginRegistry(objec plugins = [] # Add all enabled and active plugins to the list. We iterate over the - # auth_plugins setting from DB beacuse it also represents the ordering. + # auth_plugins setting from DB because it also represents the ordering. enabled_plugins = SettingsModel().get_auth_plugins() for plugin_id in enabled_plugins: plugin = self.get_plugin(plugin_id) diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -45,7 +45,7 @@ from rhodecode.lib.utils import ( get_repo_slug, set_rhodecode_config, password_changed, get_enabled_hook_classes) from rhodecode.lib.utils2 import ( - str2bool, safe_unicode, AttributeDict, safe_int, md5, aslist) + str2bool, safe_unicode, AttributeDict, safe_int, md5, aslist, safe_str) from rhodecode.model import meta from rhodecode.model.db import Repository, User, ChangesetComment from rhodecode.model.notification import NotificationModel @@ -252,6 +252,9 @@ class BasicAuth(AuthBasicAuthenticator): log.exception('Failed to fetch response for code %s' % http_code) return HTTPForbidden + def get_rc_realm(self): + return safe_str(self.registry.rhodecode_settings.get('rhodecode_realm')) + def build_authentication(self): head = WWW_AUTHENTICATE.tuples('Basic realm="%s"' % self.realm) if self._rc_auth_http_code and not self.initial_call: diff --git a/rhodecode/lib/middleware/simplevcs.py b/rhodecode/lib/middleware/simplevcs.py --- a/rhodecode/lib/middleware/simplevcs.py +++ b/rhodecode/lib/middleware/simplevcs.py @@ -45,7 +45,7 @@ from rhodecode.lib.hooks_daemon import p from rhodecode.lib.middleware import appenlight from rhodecode.lib.middleware.utils import scm_app_http from rhodecode.lib.utils import ( - is_valid_repo, get_rhodecode_realm, get_rhodecode_base_path, SLUG_RE) + is_valid_repo, get_rhodecode_base_path, SLUG_RE) from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode from rhodecode.lib.vcs.conf import settings as vcs_settings from rhodecode.lib.vcs.backends import base @@ -53,7 +53,7 @@ from rhodecode.model import meta from rhodecode.model.db import User, Repository, PullRequest from rhodecode.model.scm import ScmModel from rhodecode.model.pull_request import PullRequestModel - +from rhodecode.model.settings import SettingsModel log = logging.getLogger(__name__) @@ -107,9 +107,9 @@ class SimpleVCS(object): self.config = config # re-populated by specialized middleware self.repo_vcs_config = base.Config() - - # base path of repo locations - self.basepath = get_rhodecode_base_path() + self.rhodecode_settings = SettingsModel().get_all_settings(cache=True) + self.basepath = rhodecode.CONFIG['base_path'] + registry.rhodecode_settings = self.rhodecode_settings # authenticate this VCS request using authfunc auth_ret_code_detection = \ str2bool(self.config.get('auth_ret_code_detection', False)) @@ -381,7 +381,7 @@ class SimpleVCS(object): # before inject the calling repo_name for special scope checks self.authenticate.acl_repo_name = self.acl_repo_name if not username: - self.authenticate.realm = get_rhodecode_realm() + self.authenticate.realm = self.authenticate.get_rc_realm() try: result = self.authenticate(environ) diff --git a/rhodecode/tests/lib/middleware/test_simplegit.py b/rhodecode/tests/lib/middleware/test_simplegit.py --- a/rhodecode/tests/lib/middleware/test_simplegit.py +++ b/rhodecode/tests/lib/middleware/test_simplegit.py @@ -73,10 +73,10 @@ def get_environ(url, request_method): ('/info/lfs/info/lfs/objects/batch', 'pull', 'POST'), ]) -def test_get_action(url, expected_action, request_method, pylonsapp): +def test_get_action(url, expected_action, request_method, pylonsapp, request_stub): app = simplegit.SimpleGit(application=None, config={'auth_ret_code': '', 'base_path': ''}, - registry=None) + registry=request_stub.registry) assert expected_action == app._get_action(get_environ(url, request_method)) @@ -102,19 +102,19 @@ def test_get_action(url, expected_action ('/info/lfs/info/lfs/objects/batch', 'info/lfs', 'POST'), ]) -def test_get_repository_name(url, expected_repo_name, request_method, pylonsapp): +def test_get_repository_name(url, expected_repo_name, request_method, pylonsapp, request_stub): app = simplegit.SimpleGit(application=None, config={'auth_ret_code': '', 'base_path': ''}, - registry=None) + registry=request_stub.registry) assert expected_repo_name == app._get_repository_name( get_environ(url, request_method)) -def test_get_config(pylonsapp, user_util): +def test_get_config(user_util, pylonsapp, request_stub): repo = user_util.create_repo(repo_type='git') app = simplegit.SimpleGit(application=None, config={'auth_ret_code': '', 'base_path': ''}, - registry=None) + registry=request_stub.registry) extras = {'foo': 'FOO', 'bar': 'BAR'} # We copy the extras as the method below will change the contents. @@ -130,13 +130,13 @@ def test_get_config(pylonsapp, user_util assert git_config == expected_config -def test_create_wsgi_app_uses_scm_app_from_simplevcs(pylonsapp): +def test_create_wsgi_app_uses_scm_app_from_simplevcs(pylonsapp, request_stub): config = { 'auth_ret_code': '', 'base_path': '', 'vcs.scm_app_implementation': 'rhodecode.tests.lib.middleware.mock_scm_app', } - app = simplegit.SimpleGit(application=None, config=config, registry=None) + app = simplegit.SimpleGit(application=None, config=config, registry=request_stub.registry) wsgi_app = app._create_wsgi_app('/tmp/test', 'test_repo', {}) assert wsgi_app is mock_scm_app.mock_git_wsgi diff --git a/rhodecode/tests/lib/middleware/test_simplehg.py b/rhodecode/tests/lib/middleware/test_simplehg.py --- a/rhodecode/tests/lib/middleware/test_simplehg.py +++ b/rhodecode/tests/lib/middleware/test_simplehg.py @@ -53,10 +53,10 @@ def get_environ(url): # Edge case: not cmd argument ('/foo/bar?key=tip', 'pull'), ]) -def test_get_action(url, expected_action): +def test_get_action(url, expected_action, request_stub): app = simplehg.SimpleHg(application=None, config={'auth_ret_code': '', 'base_path': ''}, - registry=None) + registry=request_stub.registry) assert expected_action == app._get_action(get_environ(url)) @@ -71,18 +71,18 @@ def test_get_action(url, expected_action ('/foo/bar/?cmd=pushkey&key=tip', 'foo/bar'), ('/foo/bar/baz/?cmd=listkeys&key=tip', 'foo/bar/baz'), ]) -def test_get_repository_name(url, expected_repo_name): +def test_get_repository_name(url, expected_repo_name, request_stub): app = simplehg.SimpleHg(application=None, config={'auth_ret_code': '', 'base_path': ''}, - registry=None) + registry=request_stub.registry) assert expected_repo_name == app._get_repository_name(get_environ(url)) -def test_get_config(pylonsapp, user_util): +def test_get_config(user_util, pylonsapp, request_stub): repo = user_util.create_repo(repo_type='git') app = simplehg.SimpleHg(application=None, config={'auth_ret_code': '', 'base_path': ''}, - registry=None) + registry=request_stub.registry) extras = [('foo', 'FOO', 'bar', 'BAR')] hg_config = app._create_config(extras, repo_name=repo.repo_name) @@ -116,13 +116,14 @@ def test_get_config(pylonsapp, user_util assert entry in hg_config -def test_create_wsgi_app_uses_scm_app_from_simplevcs(): +def test_create_wsgi_app_uses_scm_app_from_simplevcs(request_stub): config = { 'auth_ret_code': '', 'base_path': '', 'vcs.scm_app_implementation': 'rhodecode.tests.lib.middleware.mock_scm_app', } - app = simplehg.SimpleHg(application=None, config=config, registry=None) + app = simplehg.SimpleHg( + application=None, config=config, registry=request_stub.registry) wsgi_app = app._create_wsgi_app('/tmp/test', 'test_repo', {}) assert wsgi_app is mock_scm_app.mock_hg_wsgi diff --git a/rhodecode/tests/lib/middleware/test_simplesvn.py b/rhodecode/tests/lib/middleware/test_simplesvn.py --- a/rhodecode/tests/lib/middleware/test_simplesvn.py +++ b/rhodecode/tests/lib/middleware/test_simplesvn.py @@ -29,12 +29,12 @@ from rhodecode.lib.middleware.simplesvn class TestSimpleSvn(object): @pytest.fixture(autouse=True) - def simple_svn(self, pylonsapp): + def simple_svn(self, pylonsapp, request_stub): self.app = SimpleSvn( application='None', config={'auth_ret_code': '', 'base_path': rhodecode.CONFIG['base_path']}, - registry=None) + registry=request_stub.registry) def test_get_config(self): extras = {'foo': 'FOO', 'bar': 'BAR'} diff --git a/rhodecode/tests/lib/middleware/test_simplevcs.py b/rhodecode/tests/lib/middleware/test_simplevcs.py --- a/rhodecode/tests/lib/middleware/test_simplevcs.py +++ b/rhodecode/tests/lib/middleware/test_simplevcs.py @@ -23,6 +23,7 @@ import base64 import mock import pytest +from rhodecode.lib.utils2 import AttributeDict from rhodecode.tests.utils import CustomTestApp from rhodecode.lib.caching_query import FromCache @@ -73,12 +74,12 @@ class StubVCSController(simplevcs.Simple @pytest.fixture -def vcscontroller(pylonsapp, config_stub): +def vcscontroller(pylonsapp, config_stub, request_stub): config_stub.testing_securitypolicy() config_stub.include('rhodecode.authentication') - #set_anonymous_access(True) - controller = StubVCSController(pylonsapp, pylonsapp.config, None) + controller = StubVCSController( + pylonsapp, pylonsapp.config, request_stub.registry) app = HttpsFixup(controller, pylonsapp.config) app = CustomTestApp(app) @@ -134,7 +135,8 @@ class StubFailVCSController(simplevcs.Si @pytest.fixture(scope='module') def fail_controller(pylonsapp): - controller = StubFailVCSController(pylonsapp, pylonsapp.config, None) + controller = StubFailVCSController( + pylonsapp, pylonsapp.config, pylonsapp.config) controller = HttpsFixup(controller, pylonsapp.config) controller = CustomTestApp(controller) return controller @@ -150,16 +152,18 @@ def test_provides_traceback_for_appenlig assert 'appenlight.__traceback' in response.request.environ -def test_provides_utils_scm_app_as_scm_app_by_default(pylonsapp): - controller = StubVCSController(pylonsapp, pylonsapp.config, None) +def test_provides_utils_scm_app_as_scm_app_by_default(pylonsapp, request_stub): + controller = StubVCSController( + pylonsapp, pylonsapp.config, request_stub.registry) assert controller.scm_app is scm_app_http -def test_allows_to_override_scm_app_via_config(pylonsapp): +def test_allows_to_override_scm_app_via_config(pylonsapp, request_stub): config = pylonsapp.config.copy() config['vcs.scm_app_implementation'] = ( 'rhodecode.tests.lib.middleware.mock_scm_app') - controller = StubVCSController(pylonsapp, config, None) + controller = StubVCSController( + pylonsapp, config, request_stub.registry) assert controller.scm_app is mock_scm_app @@ -214,12 +218,14 @@ class TestShadowRepoRegularExpression(ob @pytest.mark.backends('git', 'hg') class TestShadowRepoExposure(object): - def test_pull_on_shadow_repo_propagates_to_wsgi_app(self, pylonsapp): + def test_pull_on_shadow_repo_propagates_to_wsgi_app( + self, pylonsapp, request_stub): """ Check that a pull action to a shadow repo is propagated to the underlying wsgi app. """ - controller = StubVCSController(pylonsapp, pylonsapp.config, None) + controller = StubVCSController( + pylonsapp, pylonsapp.config, request_stub.registry) controller._check_ssl = mock.Mock() controller.is_shadow_repo = True controller._action = 'pull' @@ -238,12 +244,13 @@ class TestShadowRepoExposure(object): # Assert that we got the response from the wsgi app. assert response_body == controller.stub_response_body - def test_pull_on_shadow_repo_that_is_missing(self, pylonsapp): + def test_pull_on_shadow_repo_that_is_missing(self, pylonsapp, request_stub): """ Check that a pull action to a shadow repo is propagated to the underlying wsgi app. """ - controller = StubVCSController(pylonsapp, pylonsapp.config, None) + controller = StubVCSController( + pylonsapp, pylonsapp.config, request_stub.registry) controller._check_ssl = mock.Mock() controller.is_shadow_repo = True controller._action = 'pull' @@ -262,11 +269,12 @@ class TestShadowRepoExposure(object): # Assert that we got the response from the wsgi app. assert '404 Not Found' in response_body - def test_push_on_shadow_repo_raises(self, pylonsapp): + def test_push_on_shadow_repo_raises(self, pylonsapp, request_stub): """ Check that a push action to a shadow repo is aborted. """ - controller = StubVCSController(pylonsapp, pylonsapp.config, None) + controller = StubVCSController( + pylonsapp, pylonsapp.config, request_stub.registry) controller._check_ssl = mock.Mock() controller.is_shadow_repo = True controller._action = 'push' @@ -285,13 +293,14 @@ class TestShadowRepoExposure(object): # Assert that a 406 error is returned. assert '406 Not Acceptable' in response_body - def test_set_repo_names_no_shadow(self, pylonsapp): + def test_set_repo_names_no_shadow(self, pylonsapp, request_stub): """ Check that the set_repo_names method sets all names to the one returned by the _get_repository_name method on a request to a non shadow repo. """ environ_stub = {} - controller = StubVCSController(pylonsapp, pylonsapp.config, None) + controller = StubVCSController( + pylonsapp, pylonsapp.config, request_stub.registry) controller._name = 'RepoGroup/MyRepo' controller.set_repo_names(environ_stub) assert not controller.is_shadow_repo @@ -300,7 +309,8 @@ class TestShadowRepoExposure(object): controller.vcs_repo_name == controller._get_repository_name(environ_stub)) - def test_set_repo_names_with_shadow(self, pylonsapp, pr_util, config_stub): + def test_set_repo_names_with_shadow( + self, pylonsapp, pr_util, config_stub, request_stub): """ Check that the set_repo_names method sets correct names on a request to a shadow repo. @@ -313,7 +323,8 @@ class TestShadowRepoExposure(object): pr_id=pull_request.pull_request_id, pr_segment=TestShadowRepoRegularExpression.pr_segment, shadow_segment=TestShadowRepoRegularExpression.shadow_segment) - controller = StubVCSController(pylonsapp, pylonsapp.config, None) + controller = StubVCSController( + pylonsapp, pylonsapp.config, request_stub.registry) controller._name = shadow_url controller.set_repo_names({}) @@ -329,7 +340,7 @@ class TestShadowRepoExposure(object): assert controller.is_shadow_repo def test_set_repo_names_with_shadow_but_missing_pr( - self, pylonsapp, pr_util, config_stub): + self, pylonsapp, pr_util, config_stub, request_stub): """ Checks that the set_repo_names method enforces matching target repos and pull request IDs. @@ -340,7 +351,8 @@ class TestShadowRepoExposure(object): pr_id=999999999, pr_segment=TestShadowRepoRegularExpression.pr_segment, shadow_segment=TestShadowRepoRegularExpression.shadow_segment) - controller = StubVCSController(pylonsapp, pylonsapp.config, None) + controller = StubVCSController( + pylonsapp, pylonsapp.config, request_stub.registry) controller._name = shadow_url controller.set_repo_names({}) @@ -416,7 +428,8 @@ class TestGenerateVcsResponse(object): 'vcs.hooks.protocol': 'http', 'vcs.hooks.direct_calls': False, } - controller = StubVCSController(None, settings, None) + registry = AttributeDict() + controller = StubVCSController(None, settings, registry) controller._invalidate_cache = mock.Mock() controller.stub_response_body = response_body self.start_response = mock.Mock() @@ -465,11 +478,11 @@ class TestInitializeGenerator(object): class TestPrepareHooksDaemon(object): - def test_calls_imported_prepare_callback_daemon(self, app_settings): + def test_calls_imported_prepare_callback_daemon(self, app_settings, request_stub): expected_extras = {'extra1': 'value1'} daemon = DummyHooksCallbackDaemon() - controller = StubVCSController(None, app_settings, None) + controller = StubVCSController(None, app_settings, request_stub.registry) prepare_patcher = mock.patch.object( simplevcs, 'prepare_callback_daemon', return_value=(daemon, expected_extras))