vcs: reduce sql queries used during pull/push operations.

rhodecode/authentication/base.py

@@ -1,697 +1,704 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Authentication modules
 """
 
 import colander
 import copy
 import logging
 import time
 import traceback
 import warnings
 import functools
 
 from pyramid.threadlocal import get_current_registry
+from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode.authentication.interface import IAuthnPluginRegistry
 from rhodecode.authentication.schema import AuthnPluginSettingsSchemaBase
 from rhodecode.lib import caches
 from rhodecode.lib.auth import PasswordGenerator, _RhodeCodeCryptoBCrypt
 from rhodecode.lib.utils2 import md5_safe, safe_int
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.model.db import User
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import SettingsModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 
 
 log = logging.getLogger(__name__)
 
 # auth types that authenticate() function can receive
 VCS_TYPE = 'vcs'
 HTTP_TYPE = 'http'
 
 
 class hybrid_property(object):
     """
     a property decorator that works both for instance and class
     """
     def __init__(self, fget, fset=None, fdel=None, expr=None):
         self.fget = fget
         self.fset = fset
         self.fdel = fdel
         self.expr = expr or fget
         functools.update_wrapper(self, fget)
 
     def __get__(self, instance, owner):
         if instance is None:
             return self.expr(owner)
         else:
             return self.fget(instance)
 
     def __set__(self, instance, value):
         self.fset(instance, value)
 
     def __delete__(self, instance):
         self.fdel(instance)
 
 
 
 class LazyFormencode(object):
     def __init__(self, formencode_obj, *args, **kwargs):
         self.formencode_obj = formencode_obj
         self.args = args
         self.kwargs = kwargs
 
     def __call__(self, *args, **kwargs):
         from inspect import isfunction
         formencode_obj = self.formencode_obj
         if isfunction(formencode_obj):
             # case we wrap validators into functions
             formencode_obj = self.formencode_obj(*args, **kwargs)
         return formencode_obj(*self.args, **self.kwargs)
 
 
 class RhodeCodeAuthPluginBase(object):
     # cache the authentication request for N amount of seconds. Some kind
     # of authentication methods are very heavy and it's very efficient to cache
     # the result of a call. If it's set to None (default) cache is off
     AUTH_CACHE_TTL = None
     AUTH_CACHE = {}
 
     auth_func_attrs = {
         "username": "unique username",
         "firstname": "first name",
         "lastname": "last name",
         "email": "email address",
         "groups": '["list", "of", "groups"]',
         "extern_name": "name in external source of record",
         "extern_type": "type of external source of record",
         "admin": 'True|False defines if user should be RhodeCode super admin',
         "active":
             'True|False defines active state of user internally for RhodeCode',
         "active_from_extern":
             "True|False\None, active state from the external auth, "
             "None means use definition from RhodeCode extern_type active value"
     }
     # set on authenticate() method and via set_auth_type func.
     auth_type = None
 
     # set on authenticate() method and via set_calling_scope_repo, this is a
     # calling scope repository when doing authentication most likely on VCS
     # operations
     acl_repo_name = None
 
     # List of setting names to store encrypted. Plugins may override this list
     # to store settings encrypted.
     _settings_encrypted = []
 
     # Mapping of python to DB settings model types. Plugins may override or
     # extend this mapping.
     _settings_type_map = {
         colander.String: 'unicode',
         colander.Integer: 'int',
         colander.Boolean: 'bool',
         colander.List: 'list',
     }
 
     # list of keys in settings that are unsafe to be logged, should be passwords
     # or other crucial credentials
     _settings_unsafe_keys = []
 
     def __init__(self, plugin_id):
         self._plugin_id = plugin_id
 
     def __str__(self):
         return self.get_id()
 
     def _get_setting_full_name(self, name):
         """
         Return the full setting name used for storing values in the database.
         """
         # TODO: johbo: Using the name here is problematic. It would be good to
         # introduce either new models in the database to hold Plugin and
         # PluginSetting or to use the plugin id here.
         return 'auth_{}_{}'.format(self.name, name)
 
     def _get_setting_type(self, name):
         """
         Return the type of a setting. This type is defined by the SettingsModel
         and determines how the setting is stored in DB. Optionally the suffix
         `.encrypted` is appended to instruct SettingsModel to store it
         encrypted.
         """
         schema_node = self.get_settings_schema().get(name)
         db_type = self._settings_type_map.get(
             type(schema_node.typ), 'unicode')
         if name in self._settings_encrypted:
             db_type = '{}.encrypted'.format(db_type)
         return db_type
 
+    @LazyProperty
+    def plugin_settings(self):
+        settings = SettingsModel().get_all_settings()
+        return settings
+
     def is_enabled(self):
         """
         Returns true if this plugin is enabled. An enabled plugin can be
         configured in the admin interface but it is not consulted during
         authentication.
         """
         auth_plugins = SettingsModel().get_auth_plugins()
         return self.get_id() in auth_plugins
 
     def is_active(self):
         """
         Returns true if the plugin is activated. An activated plugin is
         consulted during authentication, assumed it is also enabled.
         """
         return self.get_setting_by_name('enabled')
 
     def get_id(self):
         """
         Returns the plugin id.
         """
         return self._plugin_id
 
     def get_display_name(self):
         """
         Returns a translation string for displaying purposes.
         """
         raise NotImplementedError('Not implemented in base class')
 
     def get_settings_schema(self):
         """
         Returns a colander schema, representing the plugin settings.
         """
         return AuthnPluginSettingsSchemaBase()
 
     def get_setting_by_name(self, name, default=None):
         """
         Returns a plugin setting by name.
         """
-        full_name = self._get_setting_full_name(name)
-        db_setting = SettingsModel().get_setting_by_name(full_name)
-        return db_setting.app_settings_value if db_setting else default
+        full_name = 'rhodecode_{}'.format(self._get_setting_full_name(name))
+        plugin_settings = self.plugin_settings
+
+        return plugin_settings.get(full_name) or default
 
     def create_or_update_setting(self, name, value):
         """
         Create or update a setting for this plugin in the persistent storage.
         """
         full_name = self._get_setting_full_name(name)
         type_ = self._get_setting_type(name)
         db_setting = SettingsModel().create_or_update_setting(
             full_name, value, type_)
         return db_setting.app_settings_value
 
     def get_settings(self):
         """
         Returns the plugin settings as dictionary.
         """
         settings = {}
         for node in self.get_settings_schema():
             settings[node.name] = self.get_setting_by_name(node.name)
         return settings
 
     def log_safe_settings(self, settings):
         """
         returns a log safe representation of settings, without any secrets
         """
         settings_copy = copy.deepcopy(settings)
         for k in self._settings_unsafe_keys:
             if k in settings_copy:
                 del settings_copy[k]
         return settings_copy
 
     @property
     def validators(self):
         """
         Exposes RhodeCode validators modules
         """
         # this is a hack to overcome issues with pylons threadlocals and
         # translator object _() not being registered properly.
         class LazyCaller(object):
             def __init__(self, name):
                 self.validator_name = name
 
             def __call__(self, *args, **kwargs):
                 from rhodecode.model import validators as v
                 obj = getattr(v, self.validator_name)
                 # log.debug('Initializing lazy formencode object: %s', obj)
                 return LazyFormencode(obj, *args, **kwargs)
 
         class ProxyGet(object):
             def __getattribute__(self, name):
                 return LazyCaller(name)
 
         return ProxyGet()
 
     @hybrid_property
     def name(self):
         """
         Returns the name of this authentication plugin.
 
         :returns: string
         """
         raise NotImplementedError("Not implemented in base class")
 
     def get_url_slug(self):
         """
         Returns a slug which should be used when constructing URLs which refer
         to this plugin. By default it returns the plugin name. If the name is
         not suitable for using it in an URL the plugin should override this
         method.
         """
         return self.name
 
     @property
     def is_headers_auth(self):
         """
         Returns True if this authentication plugin uses HTTP headers as
         authentication method.
         """
         return False
 
     @hybrid_property
     def is_container_auth(self):
         """
         Deprecated method that indicates if this authentication plugin uses
         HTTP headers as authentication method.
         """
         warnings.warn(
             'Use is_headers_auth instead.', category=DeprecationWarning)
         return self.is_headers_auth
 
     @hybrid_property
     def allows_creating_users(self):
         """
         Defines if Plugin allows users to be created on-the-fly when
         authentication is called. Controls how external plugins should behave
         in terms if they are allowed to create new users, or not. Base plugins
         should not be allowed to, but External ones should be !
 
         :return: bool
         """
         return False
 
     def set_auth_type(self, auth_type):
         self.auth_type = auth_type
 
     def set_calling_scope_repo(self, acl_repo_name):
         self.acl_repo_name = acl_repo_name
 
     def allows_authentication_from(
             self, user, allows_non_existing_user=True,
             allowed_auth_plugins=None, allowed_auth_sources=None):
         """
         Checks if this authentication module should accept a request for
         the current user.
 
         :param user: user object fetched using plugin's get_user() method.
         :param allows_non_existing_user: if True, don't allow the
             user to be empty, meaning not existing in our database
         :param allowed_auth_plugins: if provided, users extern_type will be
             checked against a list of provided extern types, which are plugin
             auth_names in the end
         :param allowed_auth_sources: authentication type allowed,
             `http` or `vcs` default is both.
             defines if plugin will accept only http authentication vcs
             authentication(git/hg) or both
         :returns: boolean
         """
         if not user and not allows_non_existing_user:
             log.debug('User is empty but plugin does not allow empty users,'
                       'not allowed to authenticate')
             return False
 
         expected_auth_plugins = allowed_auth_plugins or [self.name]
         if user and (user.extern_type and
                      user.extern_type not in expected_auth_plugins):
             log.debug(
                 'User `%s` is bound to `%s` auth type. Plugin allows only '
                 '%s, skipping', user, user.extern_type, expected_auth_plugins)
 
             return False
 
         # by default accept both
         expected_auth_from = allowed_auth_sources or [HTTP_TYPE, VCS_TYPE]
         if self.auth_type not in expected_auth_from:
             log.debug('Current auth source is %s but plugin only allows %s',
                       self.auth_type, expected_auth_from)
             return False
 
         return True
 
     def get_user(self, username=None, **kwargs):
         """
         Helper method for user fetching in plugins, by default it's using
         simple fetch by username, but this method can be custimized in plugins
         eg. headers auth plugin to fetch user by environ params
 
         :param username: username if given to fetch from database
         :param kwargs: extra arguments needed for user fetching.
         """
         user = None
         log.debug(
             'Trying to fetch user `%s` from RhodeCode database', username)
         if username:
             user = User.get_by_username(username)
             if not user:
                 log.debug('User not found, fallback to fetch user in '
                           'case insensitive mode')
                 user = User.get_by_username(username, case_insensitive=True)
         else:
             log.debug('provided username:`%s` is empty skipping...', username)
         if not user:
             log.debug('User `%s` not found in database', username)
         else:
             log.debug('Got DB user:%s', user)
         return user
 
     def user_activation_state(self):
         """
         Defines user activation state when creating new users
 
         :returns: boolean
         """
         raise NotImplementedError("Not implemented in base class")
 
     def auth(self, userobj, username, passwd, settings, **kwargs):
         """
         Given a user object (which may be null), username, a plaintext
         password, and a settings object (containing all the keys needed as
         listed in settings()), authenticate this user's login attempt.
 
         Return None on failure. On success, return a dictionary of the form:
 
             see: RhodeCodeAuthPluginBase.auth_func_attrs
         This is later validated for correctness
         """
         raise NotImplementedError("not implemented in base class")
 
     def _authenticate(self, userobj, username, passwd, settings, **kwargs):
         """
         Wrapper to call self.auth() that validates call on it
 
         :param userobj: userobj
         :param username: username
         :param passwd: plaintext password
         :param settings: plugin settings
         """
         auth = self.auth(userobj, username, passwd, settings, **kwargs)
         if auth:
             # check if hash should be migrated ?
             new_hash = auth.get('_hash_migrate')
             if new_hash:
                 self._migrate_hash_to_bcrypt(username, passwd, new_hash)
             return self._validate_auth_return(auth)
         return auth
 
     def _migrate_hash_to_bcrypt(self, username, password, new_hash):
         new_hash_cypher = _RhodeCodeCryptoBCrypt()
         # extra checks, so make sure new hash is correct.
         password_encoded = safe_str(password)
         if new_hash and new_hash_cypher.hash_check(
                 password_encoded, new_hash):
             cur_user = User.get_by_username(username)
             cur_user.password = new_hash
             Session().add(cur_user)
             Session().flush()
             log.info('Migrated user %s hash to bcrypt', cur_user)
 
     def _validate_auth_return(self, ret):
         if not isinstance(ret, dict):
             raise Exception('returned value from auth must be a dict')
         for k in self.auth_func_attrs:
             if k not in ret:
                 raise Exception('Missing %s attribute from returned data' % k)
         return ret
 
 
 class RhodeCodeExternalAuthPlugin(RhodeCodeAuthPluginBase):
 
     @hybrid_property
     def allows_creating_users(self):
         return True
 
     def use_fake_password(self):
         """
         Return a boolean that indicates whether or not we should set the user's
         password to a random value when it is authenticated by this plugin.
         If your plugin provides authentication, then you will generally
         want this.
 
         :returns: boolean
         """
         raise NotImplementedError("Not implemented in base class")
 
     def _authenticate(self, userobj, username, passwd, settings, **kwargs):
         # at this point _authenticate calls plugin's `auth()` function
         auth = super(RhodeCodeExternalAuthPlugin, self)._authenticate(
             userobj, username, passwd, settings, **kwargs)
         if auth:
             # maybe plugin will clean the username ?
             # we should use the return value
             username = auth['username']
 
             # if external source tells us that user is not active, we should
             # skip rest of the process. This can prevent from creating users in
             # RhodeCode when using external authentication, but if it's
             # inactive user we shouldn't create that user anyway
             if auth['active_from_extern'] is False:
                 log.warning(
                     "User %s authenticated against %s, but is inactive",
                     username, self.__module__)
                 return None
 
             cur_user = User.get_by_username(username, case_insensitive=True)
             is_user_existing = cur_user is not None
 
             if is_user_existing:
                 log.debug('Syncing user `%s` from '
                           '`%s` plugin', username, self.name)
             else:
                 log.debug('Creating non existing user `%s` from '
                           '`%s` plugin', username, self.name)
 
             if self.allows_creating_users:
                 log.debug('Plugin `%s` allows to '
                           'create new users', self.name)
             else:
                 log.debug('Plugin `%s` does not allow to '
                           'create new users', self.name)
 
             user_parameters = {
                 'username': username,
                 'email': auth["email"],
                 'firstname': auth["firstname"],
                 'lastname': auth["lastname"],
                 'active': auth["active"],
                 'admin': auth["admin"],
                 'extern_name': auth["extern_name"],
                 'extern_type': self.name,
                 'plugin': self,
                 'allow_to_create_user': self.allows_creating_users,
             }
 
             if not is_user_existing:
                 if self.use_fake_password():
                     # Randomize the PW because we don't need it, but don't want
                     # them blank either
                     passwd = PasswordGenerator().gen_password(length=16)
                 user_parameters['password'] = passwd
             else:
                 # Since the password is required by create_or_update method of
                 # UserModel, we need to set it explicitly.
                 # The create_or_update method is smart and recognises the
                 # password hashes as well.
                 user_parameters['password'] = cur_user.password
 
             # we either create or update users, we also pass the flag
             # that controls if this method can actually do that.
             # raises NotAllowedToCreateUserError if it cannot, and we try to.
             user = UserModel().create_or_update(**user_parameters)
             Session().flush()
             # enforce user is just in given groups, all of them has to be ones
             # created from plugins. We store this info in _group_data JSON
             # field
             try:
                 groups = auth['groups'] or []
                 UserGroupModel().enforce_groups(user, groups, self.name)
             except Exception:
                 # for any reason group syncing fails, we should
                 # proceed with login
                 log.error(traceback.format_exc())
             Session().commit()
         return auth
 
 
 def loadplugin(plugin_id):
     """
     Loads and returns an instantiated authentication plugin.
     Returns the RhodeCodeAuthPluginBase subclass on success,
     or None on failure.
     """
     # TODO: Disusing pyramids thread locals to retrieve the registry.
     authn_registry = get_authn_registry()
     plugin = authn_registry.get_plugin(plugin_id)
     if plugin is None:
         log.error('Authentication plugin not found: "%s"', plugin_id)
     return plugin
 
 
 def get_authn_registry(registry=None):
     registry = registry or get_current_registry()
     authn_registry = registry.getUtility(IAuthnPluginRegistry)
     return authn_registry
 
 
 def get_auth_cache_manager(custom_ttl=None):
     return caches.get_cache_manager(
         'auth_plugins', 'rhodecode.authentication', custom_ttl)
 
 
 def authenticate(username, password, environ=None, auth_type=None,
                  skip_missing=False, registry=None, acl_repo_name=None):
     """
     Authentication function used for access control,
     It tries to authenticate based on enabled authentication modules.
 
     :param username: username can be empty for headers auth
     :param password: password can be empty for headers auth
     :param environ: environ headers passed for headers auth
     :param auth_type: type of authentication, either `HTTP_TYPE` or `VCS_TYPE`
     :param skip_missing: ignores plugins that are in db but not in environment
     :returns: None if auth failed, plugin_user dict if auth is correct
     """
     if not auth_type or auth_type not in [HTTP_TYPE, VCS_TYPE]:
         raise ValueError('auth type must be on of http, vcs got "%s" instead'
                          % auth_type)
     headers_only = environ and not (username and password)
 
     authn_registry = get_authn_registry(registry)
     for plugin in authn_registry.get_plugins_for_authentication():
         plugin.set_auth_type(auth_type)
         plugin.set_calling_scope_repo(acl_repo_name)
 
         if headers_only and not plugin.is_headers_auth:
             log.debug('Auth type is for headers only and plugin `%s` is not '
                       'headers plugin, skipping...', plugin.get_id())
             continue
 
         # load plugin settings from RhodeCode database
         plugin_settings = plugin.get_settings()
         plugin_sanitized_settings = plugin.log_safe_settings(plugin_settings)
         log.debug('Plugin settings:%s', plugin_sanitized_settings)
 
         log.debug('Trying authentication using ** %s **', plugin.get_id())
         # use plugin's method of user extraction.
         user = plugin.get_user(username, environ=environ,
                                settings=plugin_settings)
         display_user = user.username if user else username
         log.debug(
             'Plugin %s extracted user is `%s`', plugin.get_id(), display_user)
 
         if not plugin.allows_authentication_from(user):
             log.debug('Plugin %s does not accept user `%s` for authentication',
                       plugin.get_id(), display_user)
             continue
         else:
             log.debug('Plugin %s accepted user `%s` for authentication',
                       plugin.get_id(), display_user)
 
         log.info('Authenticating user `%s` using %s plugin',
                  display_user, plugin.get_id())
 
         _cache_ttl = 0
 
         if isinstance(plugin.AUTH_CACHE_TTL, (int, long)):
             # plugin cache set inside is more important than the settings value
             _cache_ttl = plugin.AUTH_CACHE_TTL
         elif plugin_settings.get('cache_ttl'):
             _cache_ttl = safe_int(plugin_settings.get('cache_ttl'), 0)
 
         plugin_cache_active = bool(_cache_ttl and _cache_ttl > 0)
 
         # get instance of cache manager configured for a namespace
         cache_manager = get_auth_cache_manager(custom_ttl=_cache_ttl)
 
         log.debug('AUTH_CACHE_TTL for plugin `%s` active: %s (TTL: %s)',
                   plugin.get_id(), plugin_cache_active, _cache_ttl)
 
         # for environ based password can be empty, but then the validation is
         # on the server that fills in the env data needed for authentication
         _password_hash = md5_safe(plugin.name + username + (password or ''))
 
         # _authenticate is a wrapper for .auth() method of plugin.
         # it checks if .auth() sends proper data.
         # For RhodeCodeExternalAuthPlugin it also maps users to
         # Database and maps the attributes returned from .auth()
         # to RhodeCode database. If this function returns data
         # then auth is correct.
         start = time.time()
         log.debug('Running plugin `%s` _authenticate method', plugin.get_id())
 
         def auth_func():
             """
             This function is used internally in Cache of Beaker to calculate
             Results
             """
             return plugin._authenticate(
                 user, username, password, plugin_settings,
                 environ=environ or {})
 
         if plugin_cache_active:
             plugin_user = cache_manager.get(
                 _password_hash, createfunc=auth_func)
         else:
             plugin_user = auth_func()
 
         auth_time = time.time() - start
         log.debug('Authentication for plugin `%s` completed in %.3fs, '
                   'expiration time of fetched cache %.1fs.',
                   plugin.get_id(), auth_time, _cache_ttl)
 
         log.debug('PLUGIN USER DATA: %s', plugin_user)
 
         if plugin_user:
             log.debug('Plugin returned proper authentication data')
             return plugin_user
         # we failed to Auth because .auth() method didn't return proper user
         log.debug("User `%s` failed to authenticate against %s",
                   display_user, plugin.get_id())
     return None
 
 
 def chop_at(s, sub, inclusive=False):
     """Truncate string ``s`` at the first occurrence of ``sub``.
 
     If ``inclusive`` is true, truncate just after ``sub`` rather than at it.
 
     >>> chop_at("plutocratic brats", "rat")
     'plutoc'
     >>> chop_at("plutocratic brats", "rat", True)
     'plutocrat'
     """
     pos = s.find(sub)
     if pos == -1:
         return s
     if inclusive:
         return s[:pos+len(sub)]
     return s[:pos]

rhodecode/authentication/registry.py

@@ -1,87 +1,87 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 from pyramid.exceptions import ConfigurationError
 from zope.interface import implementer
 
 from rhodecode.authentication.interface import IAuthnPluginRegistry
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.model.settings import SettingsModel
 
 log = logging.getLogger(__name__)
 
 
 @implementer(IAuthnPluginRegistry)
 class AuthenticationPluginRegistry(object):
 
     # INI settings key to set a fallback authentication plugin.
     fallback_plugin_key = 'rhodecode.auth_plugin_fallback'
 
     def __init__(self, settings):
         self._plugins = {}
         self._fallback_plugin = settings.get(self.fallback_plugin_key, None)
 
     def add_authn_plugin(self, config, plugin):
         plugin_id = plugin.get_id()
         if plugin_id in self._plugins.keys():
             raise ConfigurationError(
                 'Cannot register authentication plugin twice: "%s"', plugin_id)
         else:
             log.debug('Register authentication plugin: "%s"', plugin_id)
             self._plugins[plugin_id] = plugin
 
     def get_plugins(self):
         def sort_key(plugin):
             return str.lower(safe_str(plugin.get_display_name()))
 
         return sorted(self._plugins.values(), key=sort_key)
 
     def get_plugin(self, plugin_id):
         return self._plugins.get(plugin_id, None)
 
     def get_plugins_for_authentication(self):
         """
         Returns a list of plugins which should be consulted when authenticating
         a user. It only returns plugins which are enabled and active.
         Additionally it includes the fallback plugin from the INI file, if
         `rhodecode.auth_plugin_fallback` is set to a plugin ID.
         """
         plugins = []
 
         # Add all enabled and active plugins to the list. We iterate over the
-        # auth_plugins setting from DB beacuse it also represents the ordering.
+        # auth_plugins setting from DB because it also represents the ordering.
         enabled_plugins = SettingsModel().get_auth_plugins()
         for plugin_id in enabled_plugins:
             plugin = self.get_plugin(plugin_id)
             if plugin is not None and plugin.is_active():
                 plugins.append(plugin)
 
         # Add the fallback plugin from ini file.
         if self._fallback_plugin:
             log.warn(
                 'Using fallback authentication plugin from INI file: "%s"',
                 self._fallback_plugin)
             plugin = self.get_plugin(self._fallback_plugin)
             if plugin is not None and plugin not in plugins:
                 plugins.append(plugin)
 
         return plugins

rhodecode/lib/base.py

@@ -1,627 +1,630 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 The base Controller API
 Provides the BaseController class for subclassing. And usage in different
 controllers
 """
 
 import logging
 import socket
 
 import markupsafe
 import ipaddress
 import pyramid.threadlocal
 
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpexceptions import HTTPUnauthorized, HTTPForbidden, get_exception
 from paste.httpheaders import WWW_AUTHENTICATE, AUTHORIZATION
 
 import rhodecode
 from rhodecode.authentication.base import VCS_TYPE
 from rhodecode.lib import auth, utils2
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import AuthUser, CookieStoreWrapper
 from rhodecode.lib.exceptions import UserCreationError
 from rhodecode.lib.utils import (
     get_repo_slug, set_rhodecode_config, password_changed,
     get_enabled_hook_classes)
 from rhodecode.lib.utils2 import (
-    str2bool, safe_unicode, AttributeDict, safe_int, md5, aslist)
+    str2bool, safe_unicode, AttributeDict, safe_int, md5, aslist, safe_str)
 from rhodecode.model import meta
 from rhodecode.model.db import Repository, User, ChangesetComment
 from rhodecode.model.notification import NotificationModel
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import VcsSettingsModel, SettingsModel
 
 # NOTE(marcink): remove after base controller is no longer required
 from pylons.controllers import WSGIController
 from pylons.i18n import translation
 
 log = logging.getLogger(__name__)
 
 
 # hack to make the migration to pyramid easier
 def render(template_name, extra_vars=None, cache_key=None,
            cache_type=None, cache_expire=None):
     """Render a template with Mako
 
     Accepts the cache options ``cache_key``, ``cache_type``, and
     ``cache_expire``.
 
     """
     from pylons.templating import literal
     from pylons.templating import cached_template, pylons_globals
 
     # Create a render callable for the cache function
     def render_template():
         # Pull in extra vars if needed
         globs = extra_vars or {}
 
         # Second, get the globals
         globs.update(pylons_globals())
 
         globs['_ungettext'] = globs['ungettext']
         # Grab a template reference
         template = globs['app_globals'].mako_lookup.get_template(template_name)
 
         return literal(template.render_unicode(**globs))
 
     return cached_template(template_name, render_template, cache_key=cache_key,
                            cache_type=cache_type, cache_expire=cache_expire)
 
 def _filter_proxy(ip):
     """
     Passed in IP addresses in HEADERS can be in a special format of multiple
     ips. Those comma separated IPs are passed from various proxies in the
     chain of request processing. The left-most being the original client.
     We only care about the first IP which came from the org. client.
 
     :param ip: ip string from headers
     """
     if ',' in ip:
         _ips = ip.split(',')
         _first_ip = _ips[0].strip()
         log.debug('Got multiple IPs %s, using %s', ','.join(_ips), _first_ip)
         return _first_ip
     return ip
 
 
 def _filter_port(ip):
     """
     Removes a port from ip, there are 4 main cases to handle here.
     - ipv4 eg. 127.0.0.1
     - ipv6 eg. ::1
     - ipv4+port eg. 127.0.0.1:8080
     - ipv6+port eg. [::1]:8080
 
     :param ip:
     """
     def is_ipv6(ip_addr):
         if hasattr(socket, 'inet_pton'):
             try:
                 socket.inet_pton(socket.AF_INET6, ip_addr)
             except socket.error:
                 return False
         else:
             # fallback to ipaddress
             try:
                 ipaddress.IPv6Address(safe_unicode(ip_addr))
             except Exception:
                 return False
         return True
 
     if ':' not in ip:  # must be ipv4 pure ip
         return ip
 
     if '[' in ip and ']' in ip:  # ipv6 with port
         return ip.split(']')[0][1:].lower()
 
     # must be ipv6 or ipv4 with port
     if is_ipv6(ip):
         return ip
     else:
         ip, _port = ip.split(':')[:2]  # means ipv4+port
         return ip
 
 
 def get_ip_addr(environ):
     proxy_key = 'HTTP_X_REAL_IP'
     proxy_key2 = 'HTTP_X_FORWARDED_FOR'
     def_key = 'REMOTE_ADDR'
     _filters = lambda x: _filter_port(_filter_proxy(x))
 
     ip = environ.get(proxy_key)
     if ip:
         return _filters(ip)
 
     ip = environ.get(proxy_key2)
     if ip:
         return _filters(ip)
 
     ip = environ.get(def_key, '0.0.0.0')
     return _filters(ip)
 
 
 def get_server_ip_addr(environ, log_errors=True):
     hostname = environ.get('SERVER_NAME')
     try:
         return socket.gethostbyname(hostname)
     except Exception as e:
         if log_errors:
             # in some cases this lookup is not possible, and we don't want to
             # make it an exception in logs
             log.exception('Could not retrieve server ip address: %s', e)
         return hostname
 
 
 def get_server_port(environ):
     return environ.get('SERVER_PORT')
 
 
 def get_access_path(environ):
     path = environ.get('PATH_INFO')
     org_req = environ.get('pylons.original_request')
     if org_req:
         path = org_req.environ.get('PATH_INFO')
     return path
 
 
 def get_user_agent(environ):
     return environ.get('HTTP_USER_AGENT')
 
 
 def vcs_operation_context(
         environ, repo_name, username, action, scm, check_locking=True,
         is_shadow_repo=False):
     """
     Generate the context for a vcs operation, e.g. push or pull.
 
     This context is passed over the layers so that hooks triggered by the
     vcs operation know details like the user, the user's IP address etc.
 
     :param check_locking: Allows to switch of the computation of the locking
         data. This serves mainly the need of the simplevcs middleware to be
         able to disable this for certain operations.
 
     """
     # Tri-state value: False: unlock, None: nothing, True: lock
     make_lock = None
     locked_by = [None, None, None]
     is_anonymous = username == User.DEFAULT_USER
     if not is_anonymous and check_locking:
         log.debug('Checking locking on repository "%s"', repo_name)
         user = User.get_by_username(username)
         repo = Repository.get_by_repo_name(repo_name)
         make_lock, __, locked_by = repo.get_locking_state(
             action, user.user_id)
 
     settings_model = VcsSettingsModel(repo=repo_name)
     ui_settings = settings_model.get_ui_settings()
 
     extras = {
         'ip': get_ip_addr(environ),
         'username': username,
         'action': action,
         'repository': repo_name,
         'scm': scm,
         'config': rhodecode.CONFIG['__file__'],
         'make_lock': make_lock,
         'locked_by': locked_by,
         'server_url': utils2.get_server_url(environ),
         'user_agent': get_user_agent(environ),
         'hooks': get_enabled_hook_classes(ui_settings),
         'is_shadow_repo': is_shadow_repo,
     }
     return extras
 
 
 class BasicAuth(AuthBasicAuthenticator):
 
     def __init__(self, realm, authfunc, registry, auth_http_code=None,
                  initial_call_detection=False, acl_repo_name=None):
         self.realm = realm
         self.initial_call = initial_call_detection
         self.authfunc = authfunc
         self.registry = registry
         self.acl_repo_name = acl_repo_name
         self._rc_auth_http_code = auth_http_code
 
     def _get_response_from_code(self, http_code):
         try:
             return get_exception(safe_int(http_code))
         except Exception:
             log.exception('Failed to fetch response for code %s' % http_code)
             return HTTPForbidden
 
+    def get_rc_realm(self):
+        return safe_str(self.registry.rhodecode_settings.get('rhodecode_realm'))
+
     def build_authentication(self):
         head = WWW_AUTHENTICATE.tuples('Basic realm="%s"' % self.realm)
         if self._rc_auth_http_code and not self.initial_call:
             # return alternative HTTP code if alternative http return code
             # is specified in RhodeCode config, but ONLY if it's not the
             # FIRST call
             custom_response_klass = self._get_response_from_code(
                 self._rc_auth_http_code)
             return custom_response_klass(headers=head)
         return HTTPUnauthorized(headers=head)
 
     def authenticate(self, environ):
         authorization = AUTHORIZATION(environ)
         if not authorization:
             return self.build_authentication()
         (authmeth, auth) = authorization.split(' ', 1)
         if 'basic' != authmeth.lower():
             return self.build_authentication()
         auth = auth.strip().decode('base64')
         _parts = auth.split(':', 1)
         if len(_parts) == 2:
             username, password = _parts
             if self.authfunc(
                     username, password, environ, VCS_TYPE,
                     registry=self.registry, acl_repo_name=self.acl_repo_name):
                 return username
             if username and password:
                 # we mark that we actually executed authentication once, at
                 # that point we can use the alternative auth code
                 self.initial_call = False
 
         return self.build_authentication()
 
     __call__ = authenticate
 
 
 def calculate_version_hash(config):
     return md5(
         config.get('beaker.session.secret', '') +
         rhodecode.__version__)[:8]
 
 
 def get_current_lang(request):
     # NOTE(marcink): remove after pyramid move
     try:
         return translation.get_lang()[0]
     except:
         pass
 
     return getattr(request, '_LOCALE_', request.locale_name)
 
 
 def attach_context_attributes(context, request, user_id):
     """
     Attach variables into template context called `c`, please note that
     request could be pylons or pyramid request in here.
     """
     # NOTE(marcink): remove check after pyramid migration
     if hasattr(request, 'registry'):
         config = request.registry.settings
     else:
         from pylons import config
 
     rc_config = SettingsModel().get_all_settings(cache=True)
 
     context.rhodecode_version = rhodecode.__version__
     context.rhodecode_edition = config.get('rhodecode.edition')
     # unique secret + version does not leak the version but keep consistency
     context.rhodecode_version_hash = calculate_version_hash(config)
 
     # Default language set for the incoming request
     context.language = get_current_lang(request)
 
     # Visual options
     context.visual = AttributeDict({})
 
     # DB stored Visual Items
     context.visual.show_public_icon = str2bool(
         rc_config.get('rhodecode_show_public_icon'))
     context.visual.show_private_icon = str2bool(
         rc_config.get('rhodecode_show_private_icon'))
     context.visual.stylify_metatags = str2bool(
         rc_config.get('rhodecode_stylify_metatags'))
     context.visual.dashboard_items = safe_int(
         rc_config.get('rhodecode_dashboard_items', 100))
     context.visual.admin_grid_items = safe_int(
         rc_config.get('rhodecode_admin_grid_items', 100))
     context.visual.repository_fields = str2bool(
         rc_config.get('rhodecode_repository_fields'))
     context.visual.show_version = str2bool(
         rc_config.get('rhodecode_show_version'))
     context.visual.use_gravatar = str2bool(
         rc_config.get('rhodecode_use_gravatar'))
     context.visual.gravatar_url = rc_config.get('rhodecode_gravatar_url')
     context.visual.default_renderer = rc_config.get(
         'rhodecode_markup_renderer', 'rst')
     context.visual.comment_types = ChangesetComment.COMMENT_TYPES
     context.visual.rhodecode_support_url = \
         rc_config.get('rhodecode_support_url') or h.route_url('rhodecode_support')
 
     context.visual.affected_files_cut_off = 60
 
     context.pre_code = rc_config.get('rhodecode_pre_code')
     context.post_code = rc_config.get('rhodecode_post_code')
     context.rhodecode_name = rc_config.get('rhodecode_title')
     context.default_encodings = aslist(config.get('default_encoding'), sep=',')
     # if we have specified default_encoding in the request, it has more
     # priority
     if request.GET.get('default_encoding'):
         context.default_encodings.insert(0, request.GET.get('default_encoding'))
     context.clone_uri_tmpl = rc_config.get('rhodecode_clone_uri_tmpl')
 
     # INI stored
     context.labs_active = str2bool(
         config.get('labs_settings_active', 'false'))
     context.visual.allow_repo_location_change = str2bool(
         config.get('allow_repo_location_change', True))
     context.visual.allow_custom_hooks_settings = str2bool(
         config.get('allow_custom_hooks_settings', True))
     context.debug_style = str2bool(config.get('debug_style', False))
 
     context.rhodecode_instanceid = config.get('instance_id')
 
     context.visual.cut_off_limit_diff = safe_int(
         config.get('cut_off_limit_diff'))
     context.visual.cut_off_limit_file = safe_int(
         config.get('cut_off_limit_file'))
 
     # AppEnlight
     context.appenlight_enabled = str2bool(config.get('appenlight', 'false'))
     context.appenlight_api_public_key = config.get(
         'appenlight.api_public_key', '')
     context.appenlight_server_url = config.get('appenlight.server_url', '')
 
     # JS template context
     context.template_context = {
         'repo_name': None,
         'repo_type': None,
         'repo_landing_commit': None,
         'rhodecode_user': {
             'username': None,
             'email': None,
             'notification_status': False
         },
         'visual': {
             'default_renderer': None
         },
         'commit_data': {
             'commit_id': None
         },
         'pull_request_data': {'pull_request_id': None},
         'timeago': {
             'refresh_time': 120 * 1000,
             'cutoff_limit': 1000 * 60 * 60 * 24 * 7
         },
         'pyramid_dispatch': {
 
         },
         'extra': {'plugins': {}}
     }
     # END CONFIG VARS
 
     # TODO: This dosn't work when called from pylons compatibility tween.
     # Fix this and remove it from base controller.
     # context.repo_name = get_repo_slug(request)  # can be empty
 
     diffmode = 'sideside'
     if request.GET.get('diffmode'):
         if request.GET['diffmode'] == 'unified':
             diffmode = 'unified'
     elif request.session.get('diffmode'):
         diffmode = request.session['diffmode']
 
     context.diffmode = diffmode
 
     if request.session.get('diffmode') != diffmode:
         request.session['diffmode'] = diffmode
 
     context.csrf_token = auth.get_csrf_token(session=request.session)
     context.backends = rhodecode.BACKENDS.keys()
     context.backends.sort()
     context.unread_notifications = NotificationModel().get_unread_cnt_for_user(user_id)
 
     # NOTE(marcink): when migrated to pyramid we don't need to set this anymore,
     # given request will ALWAYS be pyramid one
     pyramid_request = pyramid.threadlocal.get_current_request()
     context.pyramid_request = pyramid_request
 
     # web case
     if hasattr(pyramid_request, 'user'):
         context.auth_user = pyramid_request.user
         context.rhodecode_user = pyramid_request.user
 
     # api case
     if hasattr(pyramid_request, 'rpc_user'):
         context.auth_user = pyramid_request.rpc_user
         context.rhodecode_user = pyramid_request.rpc_user
 
     # attach the whole call context to the request
     request.call_context = context
 
 
 def get_auth_user(request):
     environ = request.environ
     session = request.session
 
     ip_addr = get_ip_addr(environ)
     # make sure that we update permissions each time we call controller
     _auth_token = (request.GET.get('auth_token', '') or
                    request.GET.get('api_key', ''))
 
     if _auth_token:
         # when using API_KEY we assume user exists, and
         # doesn't need auth based on cookies.
         auth_user = AuthUser(api_key=_auth_token, ip_addr=ip_addr)
         authenticated = False
     else:
         cookie_store = CookieStoreWrapper(session.get('rhodecode_user'))
         try:
             auth_user = AuthUser(user_id=cookie_store.get('user_id', None),
                                  ip_addr=ip_addr)
         except UserCreationError as e:
             h.flash(e, 'error')
             # container auth or other auth functions that create users
             # on the fly can throw this exception signaling that there's
             # issue with user creation, explanation should be provided
             # in Exception itself. We then create a simple blank
             # AuthUser
             auth_user = AuthUser(ip_addr=ip_addr)
 
         if password_changed(auth_user, session):
             session.invalidate()
             cookie_store = CookieStoreWrapper(session.get('rhodecode_user'))
             auth_user = AuthUser(ip_addr=ip_addr)
 
         authenticated = cookie_store.get('is_authenticated')
 
     if not auth_user.is_authenticated and auth_user.is_user_object:
         # user is not authenticated and not empty
         auth_user.set_authenticated(authenticated)
 
     return auth_user
 
 
 class BaseController(WSGIController):
 
     def __before__(self):
         """
         __before__ is called before controller methods and after __call__
         """
         # on each call propagate settings calls into global settings.
         from pylons import config
         from pylons import tmpl_context as c, request, url
         set_rhodecode_config(config)
         attach_context_attributes(c, request, self._rhodecode_user.user_id)
 
         # TODO: Remove this when fixed in attach_context_attributes()
         c.repo_name = get_repo_slug(request)  # can be empty
 
         self.cut_off_limit_diff = safe_int(config.get('cut_off_limit_diff'))
         self.cut_off_limit_file = safe_int(config.get('cut_off_limit_file'))
         self.sa = meta.Session
         self.scm_model = ScmModel(self.sa)
 
         # set user language
         user_lang = getattr(c.pyramid_request, '_LOCALE_', None)
         if user_lang:
             translation.set_lang(user_lang)
             log.debug('set language to %s for user %s',
                       user_lang, self._rhodecode_user)
 
     def _dispatch_redirect(self, with_url, environ, start_response):
         from webob.exc import HTTPFound
         resp = HTTPFound(with_url)
         environ['SCRIPT_NAME'] = ''  # handle prefix middleware
         environ['PATH_INFO'] = with_url
         return resp(environ, start_response)
 
     def __call__(self, environ, start_response):
         """Invoke the Controller"""
         # WSGIController.__call__ dispatches to the Controller method
         # the request is routed to. This routing information is
         # available in environ['pylons.routes_dict']
         from rhodecode.lib import helpers as h
         from pylons import tmpl_context as c, request, url
 
         # Provide the Pylons context to Pyramid's debugtoolbar if it asks
         if environ.get('debugtoolbar.wants_pylons_context', False):
             environ['debugtoolbar.pylons_context'] = c._current_obj()
 
         _route_name = '.'.join([environ['pylons.routes_dict']['controller'],
                                 environ['pylons.routes_dict']['action']])
 
         self.rc_config = SettingsModel().get_all_settings(cache=True)
         self.ip_addr = get_ip_addr(environ)
 
         # The rhodecode auth user is looked up and passed through the
         # environ by the pylons compatibility tween in pyramid.
         # So we can just grab it from there.
         auth_user = environ['rc_auth_user']
 
         # set globals for auth user
         request.user = auth_user
         self._rhodecode_user = auth_user
 
         log.info('IP: %s User: %s accessed %s [%s]' % (
             self.ip_addr, auth_user, safe_unicode(get_access_path(environ)),
             _route_name)
         )
 
         user_obj = auth_user.get_instance()
         if user_obj and user_obj.user_data.get('force_password_change'):
             h.flash('You are required to change your password', 'warning',
                     ignore_duplicate=True)
             return self._dispatch_redirect(
                 url('my_account_password'), environ, start_response)
 
         return WSGIController.__call__(self, environ, start_response)
 
 
 def h_filter(s):
     """
     Custom filter for Mako templates. Mako by standard uses `markupsafe.escape`
     we wrap this with additional functionality that converts None to empty
     strings
     """
     if s is None:
         return markupsafe.Markup()
     return markupsafe.escape(s)
 
 
 def add_events_routes(config):
     """
     Adds routing that can be used in events. Because some events are triggered
     outside of pyramid context, we need to bootstrap request with some
     routing registered
     """
     config.add_route(name='home', pattern='/')
 
     config.add_route(name='repo_summary', pattern='/{repo_name}')
     config.add_route(name='repo_summary_explicit', pattern='/{repo_name}/summary')
     config.add_route(name='repo_group_home', pattern='/{repo_group_name}')
 
     config.add_route(name='pullrequest_show',
                      pattern='/{repo_name}/pull-request/{pull_request_id}')
     config.add_route(name='pull_requests_global',
                      pattern='/pull-request/{pull_request_id}')
 
     config.add_route(name='repo_commit',
                      pattern='/{repo_name}/changeset/{commit_id}')
     config.add_route(name='repo_files',
                      pattern='/{repo_name}/files/{commit_id}/{f_path}')
 
 
 def bootstrap_request(**kwargs):
     import pyramid.testing
 
     class TestRequest(pyramid.testing.DummyRequest):
         application_url = kwargs.pop('application_url', 'http://example.com')
         host = kwargs.pop('host', 'example.com:80')
         domain = kwargs.pop('domain', 'example.com')
 
     class TestDummySession(pyramid.testing.DummySession):
         def save(*arg, **kw):
             pass
 
     request = TestRequest(**kwargs)
     request.session = TestDummySession()
 
     config = pyramid.testing.setUp(request=request)
     add_events_routes(config)
     return request
 

rhodecode/lib/middleware/simplevcs.py

@@ -1,540 +1,540 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleVCS middleware for handling protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 
 import os
 import logging
 import importlib
 import re
 from functools import wraps
 
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 from webob.exc import (
     HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
 
 import rhodecode
 from rhodecode.authentication.base import authenticate, VCS_TYPE
 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from rhodecode.lib.base import (
     BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
 from rhodecode.lib.exceptions import (
     HTTPLockedRC, HTTPRequirementError, UserCreationError,
     NotAllowedToCreateUserError)
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.middleware import appenlight
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.lib.utils import (
-    is_valid_repo, get_rhodecode_realm, get_rhodecode_base_path, SLUG_RE)
+    is_valid_repo, get_rhodecode_base_path, SLUG_RE)
 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool, safe_unicode
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.backends import base
 from rhodecode.model import meta
 from rhodecode.model.db import User, Repository, PullRequest
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.pull_request import PullRequestModel
-
+from rhodecode.model.settings import SettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def initialize_generator(factory):
     """
     Initializes the returned generator by draining its first element.
 
     This can be used to give a generator an initializer, which is the code
     up to the first yield statement. This decorator enforces that the first
     produced element has the value ``"__init__"`` to make its special
     purpose very explicit in the using code.
     """
 
     @wraps(factory)
     def wrapper(*args, **kwargs):
         gen = factory(*args, **kwargs)
         try:
             init = gen.next()
         except StopIteration:
             raise ValueError('Generator must yield at least one element.')
         if init != "__init__":
             raise ValueError('First yielded element must be "__init__".')
         return gen
     return wrapper
 
 
 class SimpleVCS(object):
     """Common functionality for SCM HTTP handlers."""
 
     SCM = 'unknown'
 
     acl_repo_name = None
     url_repo_name = None
     vcs_repo_name = None
 
     # We have to handle requests to shadow repositories different than requests
     # to normal repositories. Therefore we have to distinguish them. To do this
     # we use this regex which will match only on URLs pointing to shadow
     # repositories.
     shadow_repo_re = re.compile(
         '(?P<groups>(?:{slug_pat}/)*)'  # repo groups
         '(?P<target>{slug_pat})/'       # target repo
         'pull-request/(?P<pr_id>\d+)/'  # pull request
         'repository$'                   # shadow repo
         .format(slug_pat=SLUG_RE.pattern))
 
     def __init__(self, application, config, registry):
         self.registry = registry
         self.application = application
         self.config = config
         # re-populated by specialized middleware
         self.repo_vcs_config = base.Config()
-
-        # base path of repo locations
-        self.basepath = get_rhodecode_base_path()
+        self.rhodecode_settings = SettingsModel().get_all_settings(cache=True)
+        self.basepath = rhodecode.CONFIG['base_path']
+        registry.rhodecode_settings = self.rhodecode_settings
         # authenticate this VCS request using authfunc
         auth_ret_code_detection = \
             str2bool(self.config.get('auth_ret_code_detection', False))
         self.authenticate = BasicAuth(
             '', authenticate, registry, config.get('auth_ret_code'),
             auth_ret_code_detection)
         self.ip_addr = '0.0.0.0'
 
     def set_repo_names(self, environ):
         """
         This will populate the attributes acl_repo_name, url_repo_name,
         vcs_repo_name and is_shadow_repo. In case of requests to normal (non
         shadow) repositories all names are equal. In case of requests to a
         shadow repository the acl-name points to the target repo of the pull
         request and the vcs-name points to the shadow repo file system path.
         The url-name is always the URL used by the vcs client program.
 
         Example in case of a shadow repo:
             acl_repo_name = RepoGroup/MyRepo
             url_repo_name = RepoGroup/MyRepo/pull-request/3/repository
             vcs_repo_name = /repo/base/path/RepoGroup/.__shadow_MyRepo_pr-3'
         """
         # First we set the repo name from URL for all attributes. This is the
         # default if handling normal (non shadow) repo requests.
         self.url_repo_name = self._get_repository_name(environ)
         self.acl_repo_name = self.vcs_repo_name = self.url_repo_name
         self.is_shadow_repo = False
 
         # Check if this is a request to a shadow repository.
         match = self.shadow_repo_re.match(self.url_repo_name)
         if match:
             match_dict = match.groupdict()
 
             # Build acl repo name from regex match.
             acl_repo_name = safe_unicode('{groups}{target}'.format(
                 groups=match_dict['groups'] or '',
                 target=match_dict['target']))
 
             # Retrieve pull request instance by ID from regex match.
             pull_request = PullRequest.get(match_dict['pr_id'])
 
             # Only proceed if we got a pull request and if acl repo name from
             # URL equals the target repo name of the pull request.
             if pull_request and (acl_repo_name ==
                                  pull_request.target_repo.repo_name):
                 # Get file system path to shadow repository.
                 workspace_id = PullRequestModel()._workspace_id(pull_request)
                 target_vcs = pull_request.target_repo.scm_instance()
                 vcs_repo_name = target_vcs._get_shadow_repository_path(
                     workspace_id)
 
                 # Store names for later usage.
                 self.vcs_repo_name = vcs_repo_name
                 self.acl_repo_name = acl_repo_name
                 self.is_shadow_repo = True
 
         log.debug('Setting all VCS repository names: %s', {
             'acl_repo_name': self.acl_repo_name,
             'url_repo_name': self.url_repo_name,
             'vcs_repo_name': self.vcs_repo_name,
         })
 
     @property
     def scm_app(self):
         custom_implementation = self.config['vcs.scm_app_implementation']
         if custom_implementation == 'http':
             log.info('Using HTTP implementation of scm app.')
             scm_app_impl = scm_app_http
         else:
             log.info('Using custom implementation of scm_app: "{}"'.format(
                 custom_implementation))
             scm_app_impl = importlib.import_module(custom_implementation)
         return scm_app_impl
 
     def _get_by_id(self, repo_name):
         """
         Gets a special pattern _<ID> from clone url and tries to replace it
         with a repository_name for support of _<ID> non changeable urls
         """
 
         data = repo_name.split('/')
         if len(data) >= 2:
             from rhodecode.model.repo import RepoModel
             by_id_match = RepoModel().get_repo_by_id(repo_name)
             if by_id_match:
                 data[1] = by_id_match.repo_name
 
         return safe_str('/'.join(data))
 
     def _invalidate_cache(self, repo_name):
         """
         Set's cache for this repository for invalidation on next access
 
         :param repo_name: full repo name, also a cache key
         """
         ScmModel().mark_for_invalidation(repo_name)
 
     def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
         db_repo = Repository.get_by_repo_name(repo_name)
         if not db_repo:
             log.debug('Repository `%s` not found inside the database.',
                       repo_name)
             return False
 
         if db_repo.repo_type != scm_type:
             log.warning(
                 'Repository `%s` have incorrect scm_type, expected %s got %s',
                 repo_name, db_repo.repo_type, scm_type)
             return False
 
         return is_valid_repo(repo_name, base_path, explicit_scm=scm_type)
 
     def valid_and_active_user(self, user):
         """
         Checks if that user is not empty, and if it's actually object it checks
         if he's active.
 
         :param user: user object or None
         :return: boolean
         """
         if user is None:
             return False
 
         elif user.active:
             return True
 
         return False
 
     @property
     def is_shadow_repo_dir(self):
         return os.path.isdir(self.vcs_repo_name)
 
     def _check_permission(self, action, user, repo_name, ip_addr=None):
         """
         Checks permissions using action (push/pull) user and repository
         name
 
         :param action: push or pull action
         :param user: user instance
         :param repo_name: repository name
         """
         # check IP
         inherit = user.inherit_default_permissions
         ip_allowed = AuthUser.check_ip_allowed(user.user_id, ip_addr,
                                                inherit_from_default=inherit)
         if ip_allowed:
             log.info('Access for IP:%s allowed', ip_addr)
         else:
             return False
 
         if action == 'push':
             if not HasPermissionAnyMiddleware('repository.write',
                                               'repository.admin')(user,
                                                                   repo_name):
                 return False
 
         else:
             # any other action need at least read permission
             if not HasPermissionAnyMiddleware('repository.read',
                                               'repository.write',
                                               'repository.admin')(user,
                                                                   repo_name):
                 return False
 
         return True
 
     def _check_ssl(self, environ, start_response):
         """
         Checks the SSL check flag and returns False if SSL is not present
         and required True otherwise
         """
         org_proto = environ['wsgi._org_proto']
         # check if we have SSL required  ! if not it's a bad request !
         require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
         if require_ssl and org_proto == 'http':
             log.debug('proto is %s and SSL is required BAD REQUEST !',
                       org_proto)
             return False
         return True
 
     def __call__(self, environ, start_response):
         try:
             return self._handle_request(environ, start_response)
         except Exception:
             log.exception("Exception while handling request")
             appenlight.track_exception(environ)
             return HTTPInternalServerError()(environ, start_response)
         finally:
             meta.Session.remove()
 
     def _handle_request(self, environ, start_response):
 
         if not self._check_ssl(environ, start_response):
             reason = ('SSL required, while RhodeCode was unable '
                       'to detect this as SSL request')
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         if not self.url_repo_name:
             log.warning('Repository name is empty: %s', self.url_repo_name)
             # failed to get repo name, we fail now
             return HTTPNotFound()(environ, start_response)
         log.debug('Extracted repo name is %s', self.url_repo_name)
 
         ip_addr = get_ip_addr(environ)
         user_agent = get_user_agent(environ)
         username = None
 
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
 
         # ======================================================================
         # GET ACTION PULL or PUSH
         # ======================================================================
         action = self._get_action(environ)
 
         # ======================================================================
         # Check if this is a request to a shadow repository of a pull request.
         # In this case only pull action is allowed.
         # ======================================================================
         if self.is_shadow_repo and action != 'pull':
             reason = 'Only pull action is allowed for shadow repositories.'
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         # Check if the shadow repo actually exists, in case someone refers
         # to it, and it has been deleted because of successful merge.
         if self.is_shadow_repo and not self.is_shadow_repo_dir:
             return HTTPNotFound()(environ, start_response)
 
         # ======================================================================
         # CHECK ANONYMOUS PERMISSION
         # ======================================================================
         if action in ['pull', 'push']:
             anonymous_user = User.get_default_user()
             username = anonymous_user.username
             if anonymous_user.active:
                 # ONLY check permissions if the user is activated
                 anonymous_perm = self._check_permission(
                     action, anonymous_user, self.acl_repo_name, ip_addr)
             else:
                 anonymous_perm = False
 
             if not anonymous_user.active or not anonymous_perm:
                 if not anonymous_user.active:
                     log.debug('Anonymous access is disabled, running '
                               'authentication')
 
                 if not anonymous_perm:
                     log.debug('Not enough credentials to access this '
                               'repository as anonymous user')
 
                 username = None
                 # ==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
                 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
                 # ==============================================================
 
                 # try to auth based on environ, container auth methods
                 log.debug('Running PRE-AUTH for container based authentication')
                 pre_auth = authenticate(
                     '', '', environ, VCS_TYPE, registry=self.registry,
                     acl_repo_name=self.acl_repo_name)
                 if pre_auth and pre_auth.get('username'):
                     username = pre_auth['username']
                 log.debug('PRE-AUTH got %s as username', username)
 
                 # If not authenticated by the container, running basic auth
                 # before inject the calling repo_name for special scope checks
                 self.authenticate.acl_repo_name = self.acl_repo_name
                 if not username:
-                    self.authenticate.realm = get_rhodecode_realm()
+                    self.authenticate.realm = self.authenticate.get_rc_realm()
 
                     try:
                         result = self.authenticate(environ)
                     except (UserCreationError, NotAllowedToCreateUserError) as e:
                         log.error(e)
                         reason = safe_str(e)
                         return HTTPNotAcceptable(reason)(environ, start_response)
 
                     if isinstance(result, str):
                         AUTH_TYPE.update(environ, 'basic')
                         REMOTE_USER.update(environ, result)
                         username = result
                     else:
                         return result.wsgi_application(environ, start_response)
 
                 # ==============================================================
                 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
                 # ==============================================================
                 user = User.get_by_username(username)
                 if not self.valid_and_active_user(user):
                     return HTTPForbidden()(environ, start_response)
                 username = user.username
                 user.update_lastactivity()
                 meta.Session().commit()
 
                 # check user attributes for password change flag
                 user_obj = user
                 if user_obj and user_obj.username != User.DEFAULT_USER and \
                         user_obj.user_data.get('force_password_change'):
                     reason = 'password change required'
                     log.debug('User not allowed to authenticate, %s', reason)
                     return HTTPNotAcceptable(reason)(environ, start_response)
 
                 # check permissions for this repository
                 perm = self._check_permission(
                     action, user, self.acl_repo_name, ip_addr)
                 if not perm:
                     return HTTPForbidden()(environ, start_response)
 
         # extras are injected into UI object and later available
         # in hooks executed by rhodecode
         check_locking = _should_check_locking(environ.get('QUERY_STRING'))
         extras = vcs_operation_context(
             environ, repo_name=self.acl_repo_name, username=username,
             action=action, scm=self.SCM, check_locking=check_locking,
             is_shadow_repo=self.is_shadow_repo
         )
 
         # ======================================================================
         # REQUEST HANDLING
         # ======================================================================
         repo_path = os.path.join(
             safe_str(self.basepath), safe_str(self.vcs_repo_name))
         log.debug('Repository path is %s', repo_path)
 
         fix_PATH()
 
         log.info(
             '%s action on %s repo "%s" by "%s" from %s %s',
             action, self.SCM, safe_str(self.url_repo_name),
             safe_str(username), ip_addr, user_agent)
 
         return self._generate_vcs_response(
             environ, start_response, repo_path, extras, action)
 
     @initialize_generator
     def _generate_vcs_response(
             self, environ, start_response, repo_path, extras, action):
         """
         Returns a generator for the response content.
 
         This method is implemented as a generator, so that it can trigger
         the cache validation after all content sent back to the client. It
         also handles the locking exceptions which will be triggered when
         the first chunk is produced by the underlying WSGI application.
         """
         callback_daemon, extras = self._prepare_callback_daemon(extras)
         config = self._create_config(extras, self.acl_repo_name)
         log.debug('HOOKS extras is %s', extras)
         app = self._create_wsgi_app(repo_path, self.url_repo_name, config)
 
         try:
             with callback_daemon:
                 try:
                     response = app(environ, start_response)
                 finally:
                     # This statement works together with the decorator
                     # "initialize_generator" above. The decorator ensures that
                     # we hit the first yield statement before the generator is
                     # returned back to the WSGI server. This is needed to
                     # ensure that the call to "app" above triggers the
                     # needed callback to "start_response" before the
                     # generator is actually used.
                     yield "__init__"
 
                 for chunk in response:
                     yield chunk
         except Exception as exc:
             # TODO: martinb: Exceptions are only raised in case of the Pyro4
             # backend. Refactor this except block after dropping Pyro4 support.
             # TODO: johbo: Improve "translating" back the exception.
             if getattr(exc, '_vcs_kind', None) == 'repo_locked':
                 exc = HTTPLockedRC(*exc.args)
                 _code = rhodecode.CONFIG.get('lock_ret_code')
                 log.debug('Repository LOCKED ret code %s!', (_code,))
             elif getattr(exc, '_vcs_kind', None) == 'requirement':
                 log.debug(
                     'Repository requires features unknown to this Mercurial')
                 exc = HTTPRequirementError(*exc.args)
             else:
                 raise
 
             for chunk in exc(environ, start_response):
                 yield chunk
         finally:
             # invalidate cache on push
             try:
                 if action == 'push':
                     self._invalidate_cache(self.url_repo_name)
             finally:
                 meta.Session.remove()
 
     def _get_repository_name(self, environ):
         """Get repository name out of the environmnent
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _get_action(self, environ):
         """Map request commands into a pull or push command.
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         """Return the WSGI app that will finally handle the request."""
         raise NotImplementedError()
 
     def _create_config(self, extras, repo_name):
         """Create a safe config representation."""
         raise NotImplementedError()
 
     def _prepare_callback_daemon(self, extras):
         return prepare_callback_daemon(
             extras, protocol=vcs_settings.HOOKS_PROTOCOL,
             use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
 
 
 def _should_check_locking(query_string):
     # this is kind of hacky, but due to how mercurial handles client-server
     # server see all operation on commit; bookmarks, phases and
     # obsolescence marker in different transaction, we don't want to check
     # locking on those
     return query_string not in ['cmd=listkeys']

rhodecode/tests/lib/middleware/test_simplegit.py

@@ -1,142 +1,142 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
 import urlparse
 import mock
 import simplejson as json
 
 from rhodecode.lib.vcs.backends.base import Config
 from rhodecode.tests.lib.middleware import mock_scm_app
 import rhodecode.lib.middleware.simplegit as simplegit
 
 
 def get_environ(url, request_method):
     """Construct a minimum WSGI environ based on the URL."""
     parsed_url = urlparse.urlparse(url)
     environ = {
         'PATH_INFO': parsed_url.path,
         'QUERY_STRING': parsed_url.query,
         'REQUEST_METHOD': request_method,
     }
 
     return environ
 
 
 @pytest.mark.parametrize(
     'url, expected_action, request_method',
     [
         ('/foo/bar/info/refs?service=git-upload-pack', 'pull', 'GET'),
         ('/foo/bar/info/refs?service=git-receive-pack', 'push', 'GET'),
         ('/foo/bar/git-upload-pack', 'pull', 'GET'),
         ('/foo/bar/git-receive-pack', 'push', 'GET'),
         # Edge case: missing data for info/refs
         ('/foo/info/refs?service=', 'pull', 'GET'),
         ('/foo/info/refs', 'pull', 'GET'),
         # Edge case: git command comes with service argument
         ('/foo/git-upload-pack?service=git-receive-pack', 'pull', 'GET'),
         ('/foo/git-receive-pack?service=git-upload-pack', 'push', 'GET'),
         # Edge case: repo name conflicts with git commands
         ('/git-receive-pack/git-upload-pack', 'pull', 'GET'),
         ('/git-receive-pack/git-receive-pack', 'push', 'GET'),
         ('/git-upload-pack/git-upload-pack', 'pull', 'GET'),
         ('/git-upload-pack/git-receive-pack', 'push', 'GET'),
         ('/foo/git-receive-pack', 'push', 'GET'),
         # Edge case: not a smart protocol url
         ('/foo/bar', 'pull', 'GET'),
         # GIT LFS cases, batch
         ('/foo/bar/info/lfs/objects/batch', 'push', 'GET'),
         ('/foo/bar/info/lfs/objects/batch', 'pull', 'POST'),
         # GIT LFS oid, dl/upl
         ('/foo/bar/info/lfs/abcdeabcde', 'pull', 'GET'),
         ('/foo/bar/info/lfs/abcdeabcde', 'push', 'PUT'),
         ('/foo/bar/info/lfs/abcdeabcde', 'push', 'POST'),
         # Edge case: repo name conflicts with git commands
         ('/info/lfs/info/lfs/objects/batch', 'push', 'GET'),
         ('/info/lfs/info/lfs/objects/batch', 'pull', 'POST'),
 
     ])
-def test_get_action(url, expected_action, request_method, pylonsapp):
+def test_get_action(url, expected_action, request_method, pylonsapp, request_stub):
     app = simplegit.SimpleGit(application=None,
                               config={'auth_ret_code': '', 'base_path': ''},
-                              registry=None)
+                              registry=request_stub.registry)
     assert expected_action == app._get_action(get_environ(url, request_method))
 
 
 @pytest.mark.parametrize(
     'url, expected_repo_name, request_method',
     [
         ('/foo/info/refs?service=git-upload-pack', 'foo', 'GET'),
         ('/foo/bar/info/refs?service=git-receive-pack', 'foo/bar', 'GET'),
         ('/foo/git-upload-pack', 'foo', 'GET'),
         ('/foo/git-receive-pack', 'foo', 'GET'),
         ('/foo/bar/git-upload-pack', 'foo/bar', 'GET'),
         ('/foo/bar/git-receive-pack', 'foo/bar', 'GET'),
 
         # GIT LFS cases, batch
         ('/foo/bar/info/lfs/objects/batch', 'foo/bar', 'GET'),
         ('/example-git/info/lfs/objects/batch', 'example-git', 'POST'),
         # GIT LFS oid, dl/upl
         ('/foo/info/lfs/abcdeabcde', 'foo', 'GET'),
         ('/foo/bar/info/lfs/abcdeabcde', 'foo/bar', 'PUT'),
         ('/my-git-repo/info/lfs/abcdeabcde', 'my-git-repo', 'POST'),
         # Edge case: repo name conflicts with git commands
         ('/info/lfs/info/lfs/objects/batch', 'info/lfs', 'GET'),
         ('/info/lfs/info/lfs/objects/batch', 'info/lfs', 'POST'),
 
     ])
-def test_get_repository_name(url, expected_repo_name, request_method, pylonsapp):
+def test_get_repository_name(url, expected_repo_name, request_method, pylonsapp, request_stub):
     app = simplegit.SimpleGit(application=None,
                               config={'auth_ret_code': '', 'base_path': ''},
-                              registry=None)
+                              registry=request_stub.registry)
     assert expected_repo_name == app._get_repository_name(
         get_environ(url, request_method))
 
 
-def test_get_config(pylonsapp, user_util):
+def test_get_config(user_util, pylonsapp, request_stub):
     repo = user_util.create_repo(repo_type='git')
     app = simplegit.SimpleGit(application=None,
                               config={'auth_ret_code': '', 'base_path': ''},
-                              registry=None)
+                              registry=request_stub.registry)
     extras = {'foo': 'FOO', 'bar': 'BAR'}
 
     # We copy the extras as the method below will change the contents.
     git_config = app._create_config(dict(extras), repo_name=repo.repo_name)
 
     expected_config = dict(extras)
     expected_config.update({
         'git_update_server_info': False,
         'git_lfs_enabled': False,
         'git_lfs_store_path': git_config['git_lfs_store_path']
     })
 
     assert git_config == expected_config
 
 
-def test_create_wsgi_app_uses_scm_app_from_simplevcs(pylonsapp):
+def test_create_wsgi_app_uses_scm_app_from_simplevcs(pylonsapp, request_stub):
     config = {
         'auth_ret_code': '',
         'base_path': '',
         'vcs.scm_app_implementation':
             'rhodecode.tests.lib.middleware.mock_scm_app',
     }
-    app = simplegit.SimpleGit(application=None, config=config, registry=None)
+    app = simplegit.SimpleGit(application=None, config=config, registry=request_stub.registry)
     wsgi_app = app._create_wsgi_app('/tmp/test', 'test_repo', {})
     assert wsgi_app is mock_scm_app.mock_git_wsgi

rhodecode/tests/lib/middleware/test_simplehg.py

@@ -1,128 +1,129 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import urlparse
 
 import mock
 import pytest
 import simplejson as json
 
 from rhodecode.lib.vcs.backends.base import Config
 from rhodecode.tests.lib.middleware import mock_scm_app
 import rhodecode.lib.middleware.simplehg as simplehg
 
 
 def get_environ(url):
     """Construct a minimum WSGI environ based on the URL."""
     parsed_url = urlparse.urlparse(url)
     environ = {
         'PATH_INFO': parsed_url.path,
         'QUERY_STRING': parsed_url.query,
     }
 
     return environ
 
 
 @pytest.mark.parametrize(
     'url, expected_action',
     [
         ('/foo/bar?cmd=unbundle&key=tip', 'push'),
         ('/foo/bar?cmd=pushkey&key=tip', 'push'),
         ('/foo/bar?cmd=listkeys&key=tip', 'pull'),
         ('/foo/bar?cmd=changegroup&key=tip', 'pull'),
         # Edge case: unknown argument: assume pull
         ('/foo/bar?cmd=unknown&key=tip', 'pull'),
         ('/foo/bar?cmd=&key=tip', 'pull'),
         # Edge case: not cmd argument
         ('/foo/bar?key=tip', 'pull'),
     ])
-def test_get_action(url, expected_action):
+def test_get_action(url, expected_action, request_stub):
     app = simplehg.SimpleHg(application=None,
                             config={'auth_ret_code': '', 'base_path': ''},
-                            registry=None)
+                            registry=request_stub.registry)
     assert expected_action == app._get_action(get_environ(url))
 
 
 @pytest.mark.parametrize(
     'url, expected_repo_name',
     [
         ('/foo?cmd=unbundle&key=tip', 'foo'),
         ('/foo/bar?cmd=pushkey&key=tip', 'foo/bar'),
         ('/foo/bar/baz?cmd=listkeys&key=tip', 'foo/bar/baz'),
         # Repos with trailing slashes.
         ('/foo/?cmd=unbundle&key=tip', 'foo'),
         ('/foo/bar/?cmd=pushkey&key=tip', 'foo/bar'),
         ('/foo/bar/baz/?cmd=listkeys&key=tip', 'foo/bar/baz'),
     ])
-def test_get_repository_name(url, expected_repo_name):
+def test_get_repository_name(url, expected_repo_name, request_stub):
     app = simplehg.SimpleHg(application=None,
                             config={'auth_ret_code': '', 'base_path': ''},
-                            registry=None)
+                            registry=request_stub.registry)
     assert expected_repo_name == app._get_repository_name(get_environ(url))
 
 
-def test_get_config(pylonsapp, user_util):
+def test_get_config(user_util, pylonsapp, request_stub):
     repo = user_util.create_repo(repo_type='git')
     app = simplehg.SimpleHg(application=None,
                             config={'auth_ret_code': '', 'base_path': ''},
-                            registry=None)
+                            registry=request_stub.registry)
     extras = [('foo', 'FOO', 'bar', 'BAR')]
 
     hg_config = app._create_config(extras, repo_name=repo.repo_name)
 
     config = simplehg.utils.make_db_config(repo=repo.repo_name)
     config.set('rhodecode', 'RC_SCM_DATA', json.dumps(extras))
     hg_config_org = config
 
     expected_config = [
         ('vcs_svn_tag', 'ff89f8c714d135d865f44b90e5413b88de19a55f', '/tags/*'),
         ('web', 'push_ssl', 'False'),
         ('web', 'allow_push', '*'),
         ('web', 'allow_archive', 'gz zip bz2'),
         ('web', 'baseurl', '/'),
         ('vcs_git_lfs', 'store_location', hg_config_org.get('vcs_git_lfs', 'store_location')),
         ('vcs_svn_branch', '9aac1a38c3b8a0cdc4ae0f960a5f83332bc4fa5e', '/branches/*'),
         ('vcs_svn_branch', 'c7e6a611c87da06529fd0dd733308481d67c71a8', '/trunk'),
         ('largefiles', 'usercache', hg_config_org.get('largefiles', 'usercache')),
         ('hooks', 'preoutgoing.pre_pull', 'python:vcsserver.hooks.pre_pull'),
         ('hooks', 'prechangegroup.pre_push', 'python:vcsserver.hooks.pre_push'),
         ('hooks', 'outgoing.pull_logger', 'python:vcsserver.hooks.log_pull_action'),
         ('hooks', 'pretxnchangegroup.pre_push', 'python:vcsserver.hooks.pre_push'),
         ('hooks', 'changegroup.push_logger', 'python:vcsserver.hooks.log_push_action'),
         ('hooks', 'changegroup.repo_size', 'python:vcsserver.hooks.repo_size'),
         ('phases', 'publish', 'True'),
         ('extensions', 'largefiles', ''),
         ('paths', '/', hg_config_org.get('paths', '/')),
         ('rhodecode', 'RC_SCM_DATA', '[["foo", "FOO", "bar", "BAR"]]')
     ]
     for entry in expected_config:
         assert entry in hg_config
 
 
-def test_create_wsgi_app_uses_scm_app_from_simplevcs():
+def test_create_wsgi_app_uses_scm_app_from_simplevcs(request_stub):
     config = {
         'auth_ret_code': '',
         'base_path': '',
         'vcs.scm_app_implementation':
             'rhodecode.tests.lib.middleware.mock_scm_app',
     }
-    app = simplehg.SimpleHg(application=None, config=config, registry=None)
+    app = simplehg.SimpleHg(
+        application=None, config=config, registry=request_stub.registry)
     wsgi_app = app._create_wsgi_app('/tmp/test', 'test_repo', {})
     assert wsgi_app is mock_scm_app.mock_hg_wsgi

rhodecode/tests/lib/middleware/test_simplesvn.py

@@ -1,201 +1,201 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from StringIO import StringIO
 
 import pytest
 from mock import patch, Mock
 
 import rhodecode
 from rhodecode.lib.middleware.simplesvn import SimpleSvn, SimpleSvnApp
 
 
 class TestSimpleSvn(object):
     @pytest.fixture(autouse=True)
-    def simple_svn(self, pylonsapp):
+    def simple_svn(self, pylonsapp, request_stub):
         self.app = SimpleSvn(
             application='None',
             config={'auth_ret_code': '',
                     'base_path': rhodecode.CONFIG['base_path']},
-            registry=None)
+            registry=request_stub.registry)
 
     def test_get_config(self):
         extras = {'foo': 'FOO', 'bar': 'BAR'}
         config = self.app._create_config(extras, repo_name='test-repo')
         assert config == extras
 
     @pytest.mark.parametrize(
         'method', ['OPTIONS', 'PROPFIND', 'GET', 'REPORT'])
     def test_get_action_returns_pull(self, method):
         environment = {'REQUEST_METHOD': method}
         action = self.app._get_action(environment)
         assert action == 'pull'
 
     @pytest.mark.parametrize(
         'method', [
             'MKACTIVITY', 'PROPPATCH', 'PUT', 'CHECKOUT', 'MKCOL', 'MOVE',
             'COPY', 'DELETE', 'LOCK', 'UNLOCK', 'MERGE'
         ])
     def test_get_action_returns_push(self, method):
         environment = {'REQUEST_METHOD': method}
         action = self.app._get_action(environment)
         assert action == 'push'
 
     @pytest.mark.parametrize(
         'path, expected_name', [
             ('/hello-svn', 'hello-svn'),
             ('/hello-svn/', 'hello-svn'),
             ('/group/hello-svn/', 'group/hello-svn'),
             ('/group/hello-svn/!svn/vcc/default', 'group/hello-svn'),
         ])
     def test_get_repository_name(self, path, expected_name):
         environment = {'PATH_INFO': path}
         name = self.app._get_repository_name(environment)
         assert name == expected_name
 
     def test_get_repository_name_subfolder(self, backend_svn):
         repo = backend_svn.repo
         environment = {
             'PATH_INFO': '/{}/path/with/subfolders'.format(repo.repo_name)}
         name = self.app._get_repository_name(environment)
         assert name == repo.repo_name
 
     def test_create_wsgi_app(self):
         with patch.object(SimpleSvn, '_is_svn_enabled') as mock_method:
             mock_method.return_value = False
             with patch('rhodecode.lib.middleware.simplesvn.DisabledSimpleSvnApp') as (
                     wsgi_app_mock):
                 config = Mock()
                 wsgi_app = self.app._create_wsgi_app(
                     repo_path='', repo_name='', config=config)
 
             wsgi_app_mock.assert_called_once_with(config)
             assert wsgi_app == wsgi_app_mock()
 
     def test_create_wsgi_app_when_enabled(self):
         with patch.object(SimpleSvn, '_is_svn_enabled') as mock_method:
             mock_method.return_value = True
             with patch('rhodecode.lib.middleware.simplesvn.SimpleSvnApp') as (
                     wsgi_app_mock):
                 config = Mock()
                 wsgi_app = self.app._create_wsgi_app(
                     repo_path='', repo_name='', config=config)
 
             wsgi_app_mock.assert_called_once_with(config)
             assert wsgi_app == wsgi_app_mock()
 
 
 
 class TestSimpleSvnApp(object):
     data = '<xml></xml>'
     path = '/group/my-repo'
     wsgi_input = StringIO(data)
     environment = {
         'HTTP_DAV': (
             'http://subversion.tigris.org/xmlns/dav/svn/depth,'
             ' http://subversion.tigris.org/xmlns/dav/svn/mergeinfo'),
         'HTTP_USER_AGENT': 'SVN/1.8.11 (x86_64-linux) serf/1.3.8',
         'REQUEST_METHOD': 'OPTIONS',
         'PATH_INFO': path,
         'wsgi.input': wsgi_input,
         'CONTENT_TYPE': 'text/xml',
         'CONTENT_LENGTH': '130'
     }
 
     def setup_method(self, method):
         self.host = 'http://localhost/'
         self.app = SimpleSvnApp(
             config={'subversion_http_server_url': self.host})
 
     def test_get_request_headers_with_content_type(self):
         expected_headers = {
             'Dav': self.environment['HTTP_DAV'],
             'User-Agent': self.environment['HTTP_USER_AGENT'],
             'Content-Type': self.environment['CONTENT_TYPE'],
             'Content-Length': self.environment['CONTENT_LENGTH']
         }
         headers = self.app._get_request_headers(self.environment)
         assert headers == expected_headers
 
     def test_get_request_headers_without_content_type(self):
         environment = self.environment.copy()
         environment.pop('CONTENT_TYPE')
         expected_headers = {
             'Dav': environment['HTTP_DAV'],
             'Content-Length': self.environment['CONTENT_LENGTH'],
             'User-Agent': environment['HTTP_USER_AGENT'],
         }
         request_headers = self.app._get_request_headers(environment)
         assert request_headers == expected_headers
 
     def test_get_response_headers(self):
         headers = {
             'Connection': 'keep-alive',
             'Keep-Alive': 'timeout=5, max=100',
             'Transfer-Encoding': 'chunked',
             'Content-Encoding': 'gzip',
             'MS-Author-Via': 'DAV',
             'SVN-Supported-Posts': 'create-txn-with-props'
         }
         expected_headers = [
             ('MS-Author-Via', 'DAV'),
             ('SVN-Supported-Posts', 'create-txn-with-props'),
         ]
         response_headers = self.app._get_response_headers(headers)
         assert sorted(response_headers) == sorted(expected_headers)
 
     def test_get_url(self):
         url = self.app._get_url(self.path)
         expected_url = '{}{}'.format(self.host.strip('/'), self.path)
         assert url == expected_url
 
     def test_call(self):
         start_response = Mock()
         response_mock = Mock()
         response_mock.headers = {
             'Content-Encoding': 'gzip',
             'MS-Author-Via': 'DAV',
             'SVN-Supported-Posts': 'create-txn-with-props'
         }
         response_mock.status_code = 200
         response_mock.reason = 'OK'
         with patch('rhodecode.lib.middleware.simplesvn.requests.request') as (
                 request_mock):
             request_mock.return_value = response_mock
             self.app(self.environment, start_response)
 
         expected_url = '{}{}'.format(self.host.strip('/'), self.path)
         expected_request_headers = {
             'Dav': self.environment['HTTP_DAV'],
             'User-Agent': self.environment['HTTP_USER_AGENT'],
             'Content-Type': self.environment['CONTENT_TYPE'],
             'Content-Length': self.environment['CONTENT_LENGTH']
         }
         expected_response_headers = [
             ('SVN-Supported-Posts', 'create-txn-with-props'),
             ('MS-Author-Via', 'DAV'),
         ]
         request_mock.assert_called_once_with(
             self.environment['REQUEST_METHOD'], expected_url,
             data=self.data, headers=expected_request_headers)
         response_mock.iter_content.assert_called_once_with(chunk_size=1024)
         args, _ = start_response.call_args
         assert args[0] == '200 OK'
         assert sorted(args[1]) == sorted(expected_response_headers)

rhodecode/tests/lib/middleware/test_simplevcs.py

@@ -1,485 +1,498 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2017 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import base64
 
 import mock
 import pytest
 
+from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.tests.utils import CustomTestApp
 
 from rhodecode.lib.caching_query import FromCache
 from rhodecode.lib.hooks_daemon import DummyHooksCallbackDaemon
 from rhodecode.lib.middleware import simplevcs
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.model.db import User, _hash_key
 from rhodecode.model.meta import Session
 from rhodecode.tests import (
     HG_REPO, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS)
 from rhodecode.tests.lib.middleware import mock_scm_app
 
 
 class StubVCSController(simplevcs.SimpleVCS):
 
     SCM = 'hg'
     stub_response_body = tuple()
 
     def __init__(self, *args, **kwargs):
         super(StubVCSController, self).__init__(*args, **kwargs)
         self._action = 'pull'
         self._is_shadow_repo_dir = True
         self._name = HG_REPO
         self.set_repo_names(None)
 
     @property
     def is_shadow_repo_dir(self):
         return self._is_shadow_repo_dir
 
     def _get_repository_name(self, environ):
         return self._name
 
     def _get_action(self, environ):
         return self._action
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         def fake_app(environ, start_response):
             headers = [
                 ('Http-Accept', 'application/mercurial')
             ]
             start_response('200 OK', headers)
             return self.stub_response_body
         return fake_app
 
     def _create_config(self, extras, repo_name):
         return None
 
 
 @pytest.fixture
-def vcscontroller(pylonsapp, config_stub):
+def vcscontroller(pylonsapp, config_stub, request_stub):
     config_stub.testing_securitypolicy()
     config_stub.include('rhodecode.authentication')
 
-    #set_anonymous_access(True)
-    controller = StubVCSController(pylonsapp, pylonsapp.config, None)
+    controller = StubVCSController(
+        pylonsapp, pylonsapp.config, request_stub.registry)
     app = HttpsFixup(controller, pylonsapp.config)
     app = CustomTestApp(app)
 
     _remove_default_user_from_query_cache()
 
     # Sanity checks that things are set up correctly
     app.get('/' + HG_REPO, status=200)
 
     app.controller = controller
     return app
 
 
 def _remove_default_user_from_query_cache():
     user = User.get_default_user(cache=True)
     query = Session().query(User).filter(User.username == user.username)
     query = query.options(
         FromCache("sql_cache_short", "get_user_%s" % _hash_key(user.username)))
     query.invalidate()
     Session().expire(user)
 
 
 def test_handles_exceptions_during_permissions_checks(
         vcscontroller, disable_anonymous_user):
     user_and_pass = '%s:%s' % (TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS)
     auth_password = base64.encodestring(user_and_pass).strip()
     extra_environ = {
         'AUTH_TYPE': 'Basic',
         'HTTP_AUTHORIZATION': 'Basic %s' % auth_password,
         'REMOTE_USER': TEST_USER_ADMIN_LOGIN,
     }
 
     # Verify that things are hooked up correctly
     vcscontroller.get('/', status=200, extra_environ=extra_environ)
 
     # Simulate trouble during permission checks
     with mock.patch('rhodecode.model.db.User.get_by_username',
                     side_effect=Exception) as get_user:
         # Verify that a correct 500 is returned and check that the expected
         # code path was hit.
         vcscontroller.get('/', status=500, extra_environ=extra_environ)
         assert get_user.called
 
 
 def test_returns_forbidden_if_no_anonymous_access(
         vcscontroller, disable_anonymous_user):
     vcscontroller.get('/', status=401)
 
 
 class StubFailVCSController(simplevcs.SimpleVCS):
     def _handle_request(self, environ, start_response):
         raise Exception("BOOM")
 
 
 @pytest.fixture(scope='module')
 def fail_controller(pylonsapp):
-    controller = StubFailVCSController(pylonsapp, pylonsapp.config, None)
+    controller = StubFailVCSController(
+        pylonsapp, pylonsapp.config, pylonsapp.config)
     controller = HttpsFixup(controller, pylonsapp.config)
     controller = CustomTestApp(controller)
     return controller
 
 
 def test_handles_exceptions_as_internal_server_error(fail_controller):
     fail_controller.get('/', status=500)
 
 
 def test_provides_traceback_for_appenlight(fail_controller):
     response = fail_controller.get(
         '/', status=500, extra_environ={'appenlight.client': 'fake'})
     assert 'appenlight.__traceback' in response.request.environ
 
 
-def test_provides_utils_scm_app_as_scm_app_by_default(pylonsapp):
-    controller = StubVCSController(pylonsapp, pylonsapp.config, None)
+def test_provides_utils_scm_app_as_scm_app_by_default(pylonsapp, request_stub):
+    controller = StubVCSController(
+        pylonsapp, pylonsapp.config, request_stub.registry)
     assert controller.scm_app is scm_app_http
 
 
-def test_allows_to_override_scm_app_via_config(pylonsapp):
+def test_allows_to_override_scm_app_via_config(pylonsapp, request_stub):
     config = pylonsapp.config.copy()
     config['vcs.scm_app_implementation'] = (
         'rhodecode.tests.lib.middleware.mock_scm_app')
-    controller = StubVCSController(pylonsapp, config, None)
+    controller = StubVCSController(
+        pylonsapp, config, request_stub.registry)
     assert controller.scm_app is mock_scm_app
 
 
 @pytest.mark.parametrize('query_string, expected', [
     ('cmd=stub_command', True),
     ('cmd=listkeys', False),
 ])
 def test_should_check_locking(query_string, expected):
     result = simplevcs._should_check_locking(query_string)
     assert result == expected
 
 
 class TestShadowRepoRegularExpression(object):
     pr_segment = 'pull-request'
     shadow_segment = 'repository'
 
     @pytest.mark.parametrize('url, expected', [
         # repo with/without groups
         ('My-Repo/{pr_segment}/1/{shadow_segment}', True),
         ('Group/My-Repo/{pr_segment}/2/{shadow_segment}', True),
         ('Group/Sub-Group/My-Repo/{pr_segment}/3/{shadow_segment}', True),
         ('Group/Sub-Group1/Sub-Group2/My-Repo/{pr_segment}/3/{shadow_segment}', True),
 
         # pull request ID
         ('MyRepo/{pr_segment}/1/{shadow_segment}', True),
         ('MyRepo/{pr_segment}/1234567890/{shadow_segment}', True),
         ('MyRepo/{pr_segment}/-1/{shadow_segment}', False),
         ('MyRepo/{pr_segment}/invalid/{shadow_segment}', False),
 
         # unicode
         (u'Sp€çîál-Repö/{pr_segment}/1/{shadow_segment}', True),
         (u'Sp€çîál-Gröüp/Sp€çîál-Repö/{pr_segment}/1/{shadow_segment}', True),
 
         # trailing/leading slash
         ('/My-Repo/{pr_segment}/1/{shadow_segment}', False),
         ('My-Repo/{pr_segment}/1/{shadow_segment}/', False),
         ('/My-Repo/{pr_segment}/1/{shadow_segment}/', False),
 
         # misc
         ('My-Repo/{pr_segment}/1/{shadow_segment}/extra', False),
         ('My-Repo/{pr_segment}/1/{shadow_segment}extra', False),
     ])
     def test_shadow_repo_regular_expression(self, url, expected):
         from rhodecode.lib.middleware.simplevcs import SimpleVCS
         url = url.format(
             pr_segment=self.pr_segment,
             shadow_segment=self.shadow_segment)
         match_obj = SimpleVCS.shadow_repo_re.match(url)
         assert (match_obj is not None) == expected
 
 
 @pytest.mark.backends('git', 'hg')
 class TestShadowRepoExposure(object):
 
-    def test_pull_on_shadow_repo_propagates_to_wsgi_app(self, pylonsapp):
+    def test_pull_on_shadow_repo_propagates_to_wsgi_app(
+            self, pylonsapp, request_stub):
         """
         Check that a pull action to a shadow repo is propagated to the
         underlying wsgi app.
         """
-        controller = StubVCSController(pylonsapp, pylonsapp.config, None)
+        controller = StubVCSController(
+            pylonsapp, pylonsapp.config, request_stub.registry)
         controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'pull'
         controller._is_shadow_repo_dir = True
         controller.stub_response_body = 'dummy body value'
         environ_stub = {
             'HTTP_HOST': 'test.example.com',
             'HTTP_ACCEPT': 'application/mercurial',
             'REQUEST_METHOD': 'GET',
             'wsgi.url_scheme': 'http',
         }
 
         response = controller(environ_stub, mock.Mock())
         response_body = ''.join(response)
 
         # Assert that we got the response from the wsgi app.
         assert response_body == controller.stub_response_body
 
-    def test_pull_on_shadow_repo_that_is_missing(self, pylonsapp):
+    def test_pull_on_shadow_repo_that_is_missing(self, pylonsapp, request_stub):
         """
         Check that a pull action to a shadow repo is propagated to the
         underlying wsgi app.
         """
-        controller = StubVCSController(pylonsapp, pylonsapp.config, None)
+        controller = StubVCSController(
+            pylonsapp, pylonsapp.config, request_stub.registry)
         controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'pull'
         controller._is_shadow_repo_dir = False
         controller.stub_response_body = 'dummy body value'
         environ_stub = {
             'HTTP_HOST': 'test.example.com',
             'HTTP_ACCEPT': 'application/mercurial',
             'REQUEST_METHOD': 'GET',
             'wsgi.url_scheme': 'http',
         }
 
         response = controller(environ_stub, mock.Mock())
         response_body = ''.join(response)
 
         # Assert that we got the response from the wsgi app.
         assert '404 Not Found' in response_body
 
-    def test_push_on_shadow_repo_raises(self, pylonsapp):
+    def test_push_on_shadow_repo_raises(self, pylonsapp, request_stub):
         """
         Check that a push action to a shadow repo is aborted.
         """
-        controller = StubVCSController(pylonsapp, pylonsapp.config, None)
+        controller = StubVCSController(
+            pylonsapp, pylonsapp.config, request_stub.registry)
         controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'push'
         controller.stub_response_body = 'dummy body value'
         environ_stub = {
             'HTTP_HOST': 'test.example.com',
             'HTTP_ACCEPT': 'application/mercurial',
             'REQUEST_METHOD': 'GET',
             'wsgi.url_scheme': 'http',
         }
 
         response = controller(environ_stub, mock.Mock())
         response_body = ''.join(response)
 
         assert response_body != controller.stub_response_body
         # Assert that a 406 error is returned.
         assert '406 Not Acceptable' in response_body
 
-    def test_set_repo_names_no_shadow(self, pylonsapp):
+    def test_set_repo_names_no_shadow(self, pylonsapp, request_stub):
         """
         Check that the set_repo_names method sets all names to the one returned
         by the _get_repository_name method on a request to a non shadow repo.
         """
         environ_stub = {}
-        controller = StubVCSController(pylonsapp, pylonsapp.config, None)
+        controller = StubVCSController(
+            pylonsapp, pylonsapp.config, request_stub.registry)
         controller._name = 'RepoGroup/MyRepo'
         controller.set_repo_names(environ_stub)
         assert not controller.is_shadow_repo
         assert (controller.url_repo_name ==
                 controller.acl_repo_name ==
                 controller.vcs_repo_name ==
                 controller._get_repository_name(environ_stub))
 
-    def test_set_repo_names_with_shadow(self, pylonsapp, pr_util, config_stub):
+    def test_set_repo_names_with_shadow(
+            self, pylonsapp, pr_util, config_stub, request_stub):
         """
         Check that the set_repo_names method sets correct names on a request
         to a shadow repo.
         """
         from rhodecode.model.pull_request import PullRequestModel
 
         pull_request = pr_util.create_pull_request()
         shadow_url = '{target}/{pr_segment}/{pr_id}/{shadow_segment}'.format(
             target=pull_request.target_repo.repo_name,
             pr_id=pull_request.pull_request_id,
             pr_segment=TestShadowRepoRegularExpression.pr_segment,
             shadow_segment=TestShadowRepoRegularExpression.shadow_segment)
-        controller = StubVCSController(pylonsapp, pylonsapp.config, None)
+        controller = StubVCSController(
+            pylonsapp, pylonsapp.config, request_stub.registry)
         controller._name = shadow_url
         controller.set_repo_names({})
 
         # Get file system path to shadow repo for assertions.
         workspace_id = PullRequestModel()._workspace_id(pull_request)
         target_vcs = pull_request.target_repo.scm_instance()
         vcs_repo_name = target_vcs._get_shadow_repository_path(
             workspace_id)
 
         assert controller.vcs_repo_name == vcs_repo_name
         assert controller.url_repo_name == shadow_url
         assert controller.acl_repo_name == pull_request.target_repo.repo_name
         assert controller.is_shadow_repo
 
     def test_set_repo_names_with_shadow_but_missing_pr(
-            self, pylonsapp, pr_util, config_stub):
+            self, pylonsapp, pr_util, config_stub, request_stub):
         """
         Checks that the set_repo_names method enforces matching target repos
         and pull request IDs.
         """
         pull_request = pr_util.create_pull_request()
         shadow_url = '{target}/{pr_segment}/{pr_id}/{shadow_segment}'.format(
             target=pull_request.target_repo.repo_name,
             pr_id=999999999,
             pr_segment=TestShadowRepoRegularExpression.pr_segment,
             shadow_segment=TestShadowRepoRegularExpression.shadow_segment)
-        controller = StubVCSController(pylonsapp, pylonsapp.config, None)
+        controller = StubVCSController(
+            pylonsapp, pylonsapp.config, request_stub.registry)
         controller._name = shadow_url
         controller.set_repo_names({})
 
         assert not controller.is_shadow_repo
         assert (controller.url_repo_name ==
                 controller.acl_repo_name ==
                 controller.vcs_repo_name)
 
 
 @pytest.mark.usefixtures('db')
 class TestGenerateVcsResponse(object):
 
     def test_ensures_that_start_response_is_called_early_enough(self):
         self.call_controller_with_response_body(iter(['a', 'b']))
         assert self.start_response.called
 
     def test_invalidates_cache_after_body_is_consumed(self):
         result = self.call_controller_with_response_body(iter(['a', 'b']))
         assert not self.was_cache_invalidated()
         # Consume the result
         list(result)
         assert self.was_cache_invalidated()
 
     @mock.patch('rhodecode.lib.middleware.simplevcs.HTTPLockedRC')
     def test_handles_locking_exception(self, http_locked_rc):
         result = self.call_controller_with_response_body(
             self.raise_result_iter(vcs_kind='repo_locked'))
         assert not http_locked_rc.called
         # Consume the result
         list(result)
         assert http_locked_rc.called
 
     @mock.patch('rhodecode.lib.middleware.simplevcs.HTTPRequirementError')
     def test_handles_requirement_exception(self, http_requirement):
         result = self.call_controller_with_response_body(
             self.raise_result_iter(vcs_kind='requirement'))
         assert not http_requirement.called
         # Consume the result
         list(result)
         assert http_requirement.called
 
     @mock.patch('rhodecode.lib.middleware.simplevcs.HTTPLockedRC')
     def test_handles_locking_exception_in_app_call(self, http_locked_rc):
         app_factory_patcher = mock.patch.object(
             StubVCSController, '_create_wsgi_app')
         with app_factory_patcher as app_factory:
             app_factory().side_effect = self.vcs_exception()
             result = self.call_controller_with_response_body(['a'])
             list(result)
         assert http_locked_rc.called
 
     def test_raises_unknown_exceptions(self):
         result = self.call_controller_with_response_body(
             self.raise_result_iter(vcs_kind='unknown'))
         with pytest.raises(Exception):
             list(result)
 
     def test_prepare_callback_daemon_is_called(self):
         def side_effect(extras):
             return DummyHooksCallbackDaemon(), extras
 
         prepare_patcher = mock.patch.object(
             StubVCSController, '_prepare_callback_daemon')
         with prepare_patcher as prepare_mock:
             prepare_mock.side_effect = side_effect
             self.call_controller_with_response_body(iter(['a', 'b']))
         assert prepare_mock.called
         assert prepare_mock.call_count == 1
 
     def call_controller_with_response_body(self, response_body):
         settings = {
             'base_path': 'fake_base_path',
             'vcs.hooks.protocol': 'http',
             'vcs.hooks.direct_calls': False,
         }
-        controller = StubVCSController(None, settings, None)
+        registry = AttributeDict()
+        controller = StubVCSController(None, settings, registry)
         controller._invalidate_cache = mock.Mock()
         controller.stub_response_body = response_body
         self.start_response = mock.Mock()
         result = controller._generate_vcs_response(
             environ={}, start_response=self.start_response,
             repo_path='fake_repo_path',
             extras={}, action='push')
         self.controller = controller
         return result
 
     def raise_result_iter(self, vcs_kind='repo_locked'):
         """
         Simulates an exception due to a vcs raised exception if kind vcs_kind
         """
         raise self.vcs_exception(vcs_kind=vcs_kind)
         yield "never_reached"
 
     def vcs_exception(self, vcs_kind='repo_locked'):
         locked_exception = Exception('TEST_MESSAGE')
         locked_exception._vcs_kind = vcs_kind
         return locked_exception
 
     def was_cache_invalidated(self):
         return self.controller._invalidate_cache.called
 
 
 class TestInitializeGenerator(object):
 
     def test_drains_first_element(self):
         gen = self.factory(['__init__', 1, 2])
         result = list(gen)
         assert result == [1, 2]
 
     @pytest.mark.parametrize('values', [
         [],
         [1, 2],
     ])
     def test_raises_value_error(self, values):
         with pytest.raises(ValueError):
             self.factory(values)
 
     @simplevcs.initialize_generator
     def factory(self, iterable):
         for elem in iterable:
             yield elem
 
 
 class TestPrepareHooksDaemon(object):
-    def test_calls_imported_prepare_callback_daemon(self, app_settings):
+    def test_calls_imported_prepare_callback_daemon(self, app_settings, request_stub):
         expected_extras = {'extra1': 'value1'}
         daemon = DummyHooksCallbackDaemon()
 
-        controller = StubVCSController(None, app_settings, None)
+        controller = StubVCSController(None, app_settings, request_stub.registry)
         prepare_patcher = mock.patch.object(
             simplevcs, 'prepare_callback_daemon',
             return_value=(daemon, expected_extras))
         with prepare_patcher as prepare_mock:
             callback_daemon, extras = controller._prepare_callback_daemon(
                 expected_extras.copy())
         prepare_mock.assert_called_once_with(
             expected_extras,
             protocol=app_settings['vcs.hooks.protocol'],
             use_direct_calls=app_settings['vcs.hooks.direct_calls'])
 
         assert callback_daemon == daemon
         assert extras == extras