app: improve logging, and remove DB calls on app startup.

rhodecode/apps/admin/views/users.py

@@ -1,1414 +1,1418 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import datetime
 import formencode
 import formencode.htmlfill
 
 from pyramid.httpexceptions import HTTPFound
 from pyramid.view import view_config
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode import events
 from rhodecode.apps._base import BaseAppView, DataGridAppView, UserAppView
 from rhodecode.apps.ssh_support import SshKeyFileChangeEvent
 from rhodecode.authentication.base import get_authn_registry, RhodeCodeExternalAuthPlugin
 from rhodecode.authentication.plugins import auth_rhodecode
 from rhodecode.events import trigger
 from rhodecode.model.db import true, UserNotice
 
-from rhodecode.lib import audit_logger, rc_cache
+from rhodecode.lib import audit_logger, rc_cache, auth
 from rhodecode.lib.exceptions import (
     UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException,
     UserOwnsUserGroupsException, UserOwnsPullRequestsException,
     UserOwnsArtifactsException, DefaultUserException)
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.lib import helpers as h
 from rhodecode.lib.helpers import SqlPage
 from rhodecode.lib.utils2 import safe_int, safe_unicode, AttributeDict
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.forms import (
     UserForm, UserIndividualPermissionsForm, UserPermissionsForm,
     UserExtraEmailForm, UserExtraIpForm)
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.ssh_key import SshKeyModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.model.db import (
     or_, coalesce,IntegrityError, User, UserGroup, UserIpMap, UserEmailMap,
     UserApiKeys, UserSshKeys, RepoGroup)
 from rhodecode.model.meta import Session
 
 log = logging.getLogger(__name__)
 
 
 class AdminUsersView(BaseAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         return c
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='users', request_method='GET',
         renderer='rhodecode:templates/admin/users/users.mako')
     def users_list(self):
         c = self.load_default_context()
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         # renderer defined below
         route_name='users_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def users_list_data(self):
         self.load_default_context()
         column_map = {
             'first_name': 'name',
             'last_name': 'lastname',
         }
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(
             self.request, column_map=column_map)
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         def user_actions(user_id, username):
             return _render("user_actions", user_id, username)
 
         users_data_total_count = User.query()\
             .filter(User.username != User.DEFAULT_USER) \
             .count()
 
         users_data_total_inactive_count = User.query()\
             .filter(User.username != User.DEFAULT_USER) \
             .filter(User.active != true())\
             .count()
 
         # json generate
         base_q = User.query().filter(User.username != User.DEFAULT_USER)
         base_inactive_q = base_q.filter(User.active != true())
 
         if search_q:
             like_expression = u'%{}%'.format(safe_unicode(search_q))
             base_q = base_q.filter(or_(
                 User.username.ilike(like_expression),
                 User._email.ilike(like_expression),
                 User.name.ilike(like_expression),
                 User.lastname.ilike(like_expression),
             ))
             base_inactive_q = base_q.filter(User.active != true())
 
         users_data_total_filtered_count = base_q.count()
         users_data_total_filtered_inactive_count = base_inactive_q.count()
 
         sort_col = getattr(User, order_by, None)
         if sort_col:
             if order_dir == 'asc':
                 # handle null values properly to order by NULL last
                 if order_by in ['last_activity']:
                     sort_col = coalesce(sort_col, datetime.date.max)
                 sort_col = sort_col.asc()
             else:
                 # handle null values properly to order by NULL last
                 if order_by in ['last_activity']:
                     sort_col = coalesce(sort_col, datetime.date.min)
                 sort_col = sort_col.desc()
 
         base_q = base_q.order_by(sort_col)
         base_q = base_q.offset(start).limit(limit)
 
         users_list = base_q.all()
 
         users_data = []
         for user in users_list:
             users_data.append({
                 "username": h.gravatar_with_user(self.request, user.username),
                 "email": user.email,
                 "first_name": user.first_name,
                 "last_name": user.last_name,
                 "last_login": h.format_date(user.last_login),
                 "last_activity": h.format_date(user.last_activity),
                 "active": h.bool2icon(user.active),
                 "active_raw": user.active,
                 "admin": h.bool2icon(user.admin),
                 "extern_type": user.extern_type,
                 "extern_name": user.extern_name,
                 "action": user_actions(user.user_id, user.username),
             })
         data = ({
             'draw': draw,
             'data': users_data,
             'recordsTotal': users_data_total_count,
             'recordsFiltered': users_data_total_filtered_count,
             'recordsTotalInactive': users_data_total_inactive_count,
             'recordsFilteredInactive': users_data_total_filtered_inactive_count
         })
 
         return data
 
     def _set_personal_repo_group_template_vars(self, c_obj):
         DummyUser = AttributeDict({
             'username': '${username}',
             'user_id': '${user_id}',
         })
         c_obj.default_create_repo_group = RepoGroupModel() \
             .get_default_create_personal_repo_group()
         c_obj.personal_repo_group_name = RepoGroupModel() \
             .get_personal_group_name(DummyUser)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='users_new', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_add.mako')
     def users_new(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.default_extern_type = auth_rhodecode.RhodeCodeAuthPlugin.uid
         self._set_personal_repo_group_template_vars(c)
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='users_create', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_add.mako')
     def users_create(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.default_extern_type = auth_rhodecode.RhodeCodeAuthPlugin.uid
         user_model = UserModel()
         user_form = UserForm(self.request.translate)()
         try:
             form_result = user_form.to_python(dict(self.request.POST))
             user = user_model.create(form_result)
             Session().flush()
             creation_data = user.get_api_data()
             username = form_result['username']
 
             audit_logger.store_web(
                 'user.create', action_data={'data': creation_data},
                 user=c.rhodecode_user)
 
             user_link = h.link_to(
                 h.escape(username),
                 h.route_path('user_edit', user_id=user.user_id))
             h.flash(h.literal(_('Created user %(user_link)s')
                               % {'user_link': user_link}), category='success')
             Session().commit()
         except formencode.Invalid as errors:
             self._set_personal_repo_group_template_vars(c)
             data = render(
                 'rhodecode:templates/admin/users/user_add.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except UserCreationError as e:
             h.flash(e, 'error')
         except Exception:
             log.exception("Exception creation of user")
             h.flash(_('Error occurred during creation of user %s')
                     % self.request.POST.get('username'), category='error')
         raise HTTPFound(h.route_path('users'))
 
 
 class UsersView(UserAppView):
     ALLOW_SCOPED_TOKENS = False
     """
     This view has alternative version inside EE, if modified please take a look
     in there as well.
     """
 
     def get_auth_plugins(self):
         valid_plugins = []
         authn_registry = get_authn_registry(self.request.registry)
         for plugin in authn_registry.get_plugins_for_authentication():
             if isinstance(plugin, RhodeCodeExternalAuthPlugin):
                 valid_plugins.append(plugin)
             elif plugin.name == 'rhodecode':
                 valid_plugins.append(plugin)
 
         # extend our choices if user has set a bound plugin which isn't enabled at the
         # moment
         extern_type = self.db_user.extern_type
         if extern_type not in [x.uid for x in valid_plugins]:
             try:
                 plugin = authn_registry.get_plugin_by_uid(extern_type)
                 if plugin:
                     valid_plugins.append(plugin)
 
             except Exception:
                 log.exception(
                     'Could not extend user plugins with `{}`'.format(extern_type))
         return valid_plugins
 
     def load_default_context(self):
         req = self.request
 
         c = self._get_local_tmpl_context()
         c.allow_scoped_tokens = self.ALLOW_SCOPED_TOKENS
         c.allowed_languages = [
             ('en', 'English (en)'),
             ('de', 'German (de)'),
             ('fr', 'French (fr)'),
             ('it', 'Italian (it)'),
             ('ja', 'Japanese (ja)'),
             ('pl', 'Polish (pl)'),
             ('pt', 'Portuguese (pt)'),
             ('ru', 'Russian (ru)'),
             ('zh', 'Chinese (zh)'),
         ]
 
         c.allowed_extern_types = [
             (x.uid, x.get_display_name()) for x in self.get_auth_plugins()
         ]
+        perms = req.registry.settings.get('available_permissions')
+        if not perms:
+            # inject info about available permissions
+            auth.set_available_permissions(req.registry.settings)
 
         c.available_permissions = req.registry.settings['available_permissions']
         PermissionModel().set_global_permission_choices(
             c, gettext_translator=req.translate)
 
         return c
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='user_update', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_update(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         c.active = 'profile'
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
         available_languages = [x[0] for x in c.allowed_languages]
         _form = UserForm(self.request.translate, edit=True,
                          available_languages=available_languages,
                          old_data={'user_id': user_id,
                                    'email': c.user.email})()
         form_result = {}
         old_values = c.user.get_api_data()
         try:
             form_result = _form.to_python(dict(self.request.POST))
             skip_attrs = ['extern_name']
             # TODO: plugin should define if username can be updated
             if c.extern_type != "rhodecode":
                 # forbid updating username for external accounts
                 skip_attrs.append('username')
 
             UserModel().update_user(
                 user_id, skip_attrs=skip_attrs, **form_result)
 
             audit_logger.store_web(
                 'user.edit', action_data={'old_data': old_values},
                 user=c.rhodecode_user)
 
             Session().commit()
             h.flash(_('User updated successfully'), category='success')
         except formencode.Invalid as errors:
             data = render(
                 'rhodecode:templates/admin/users/user_edit.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except UserCreationError as e:
             h.flash(e, 'error')
         except Exception:
             log.exception("Exception updating user")
             h.flash(_('Error occurred during update of user %s')
                     % form_result.get('username'), category='error')
         raise HTTPFound(h.route_path('user_edit', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='user_delete', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         _repos = c.user.repositories
         _repo_groups = c.user.repository_groups
         _user_groups = c.user.user_groups
         _pull_requests = c.user.user_pull_requests
         _artifacts = c.user.artifacts
 
         handle_repos = None
         handle_repo_groups = None
         handle_user_groups = None
         handle_pull_requests = None
         handle_artifacts = None
 
         # calls for flash of handle based on handle case detach or delete
         def set_handle_flash_repos():
             handle = handle_repos
             if handle == 'detach':
                 h.flash(_('Detached %s repositories') % len(_repos),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s repositories') % len(_repos),
                         category='success')
 
         def set_handle_flash_repo_groups():
             handle = handle_repo_groups
             if handle == 'detach':
                 h.flash(_('Detached %s repository groups') % len(_repo_groups),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s repository groups') % len(_repo_groups),
                         category='success')
 
         def set_handle_flash_user_groups():
             handle = handle_user_groups
             if handle == 'detach':
                 h.flash(_('Detached %s user groups') % len(_user_groups),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s user groups') % len(_user_groups),
                         category='success')
 
         def set_handle_flash_pull_requests():
             handle = handle_pull_requests
             if handle == 'detach':
                 h.flash(_('Detached %s pull requests') % len(_pull_requests),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s pull requests') % len(_pull_requests),
                         category='success')
 
         def set_handle_flash_artifacts():
             handle = handle_artifacts
             if handle == 'detach':
                 h.flash(_('Detached %s artifacts') % len(_artifacts),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s artifacts') % len(_artifacts),
                         category='success')
 
         handle_user = User.get_first_super_admin()
         handle_user_id = safe_int(self.request.POST.get('detach_user_id'))
         if handle_user_id:
             # NOTE(marcink): we get new owner for objects...
             handle_user = User.get_or_404(handle_user_id)
 
         if _repos and self.request.POST.get('user_repos'):
             handle_repos = self.request.POST['user_repos']
 
         if _repo_groups and self.request.POST.get('user_repo_groups'):
             handle_repo_groups = self.request.POST['user_repo_groups']
 
         if _user_groups and self.request.POST.get('user_user_groups'):
             handle_user_groups = self.request.POST['user_user_groups']
 
         if _pull_requests and self.request.POST.get('user_pull_requests'):
             handle_pull_requests = self.request.POST['user_pull_requests']
 
         if _artifacts and self.request.POST.get('user_artifacts'):
             handle_artifacts = self.request.POST['user_artifacts']
 
         old_values = c.user.get_api_data()
 
         try:
 
             UserModel().delete(
                 c.user,
                 handle_repos=handle_repos,
                 handle_repo_groups=handle_repo_groups,
                 handle_user_groups=handle_user_groups,
                 handle_pull_requests=handle_pull_requests,
                 handle_artifacts=handle_artifacts,
                 handle_new_owner=handle_user
             )
 
             audit_logger.store_web(
                 'user.delete', action_data={'old_data': old_values},
                 user=c.rhodecode_user)
 
             Session().commit()
             set_handle_flash_repos()
             set_handle_flash_repo_groups()
             set_handle_flash_user_groups()
             set_handle_flash_pull_requests()
             set_handle_flash_artifacts()
             username = h.escape(old_values['username'])
             h.flash(_('Successfully deleted user `{}`').format(username), category='success')
         except (UserOwnsReposException, UserOwnsRepoGroupsException,
                 UserOwnsUserGroupsException, UserOwnsPullRequestsException,
                 UserOwnsArtifactsException, DefaultUserException) as e:
             h.flash(e, category='warning')
         except Exception:
             log.exception("Exception during deletion of user")
             h.flash(_('An error occurred during deletion of user'),
                     category='error')
         raise HTTPFound(h.route_path('users'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='user_edit', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_edit(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'profile'
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         defaults = c.user.get_dict()
         defaults.update({'language': c.user.user_data.get('language')})
 
         data = render(
             'rhodecode:templates/admin/users/user_edit.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='user_edit_advanced', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_edit_advanced(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         c.detach_user = User.get_first_super_admin()
         detach_user_id = safe_int(self.request.GET.get('detach_user_id'))
         if detach_user_id:
             c.detach_user = User.get_or_404(detach_user_id)
 
         c.active = 'advanced'
         c.personal_repo_group = RepoGroup.get_user_personal_repo_group(user_id)
         c.personal_repo_group_name = RepoGroupModel()\
             .get_personal_group_name(c.user)
 
         c.user_to_review_rules = sorted(
             (x.user for x in c.user.user_review_rules),
             key=lambda u: u.username.lower())
 
         defaults = c.user.get_dict()
 
         # Interim workaround if the user participated on any pull requests as a
         # reviewer.
         has_review = len(c.user.reviewer_pull_requests)
         c.can_delete_user = not has_review
         c.can_delete_user_message = ''
         inactive_link = h.link_to(
             'inactive', h.route_path('user_edit', user_id=user_id, _anchor='active'))
         if has_review == 1:
             c.can_delete_user_message = h.literal(_(
                 'The user participates as reviewer in {} pull request and '
                 'cannot be deleted. \nYou can set the user to '
                 '"{}" instead of deleting it.').format(
                 has_review, inactive_link))
         elif has_review:
             c.can_delete_user_message = h.literal(_(
                 'The user participates as reviewer in {} pull requests and '
                 'cannot be deleted. \nYou can set the user to '
                 '"{}" instead of deleting it.').format(
                 has_review, inactive_link))
 
         data = render(
             'rhodecode:templates/admin/users/user_edit.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='user_edit_global_perms', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_edit_global_perms(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'global_perms'
 
         c.default_user = User.get_default_user()
         defaults = c.user.get_dict()
         defaults.update(c.default_user.get_default_perms(suffix='_inherited'))
         defaults.update(c.default_user.get_default_perms())
         defaults.update(c.user.get_default_perms())
 
         data = render(
             'rhodecode:templates/admin/users/user_edit.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='user_edit_global_perms_update', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_edit_global_perms_update(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         c.active = 'global_perms'
         try:
             # first stage that verifies the checkbox
             _form = UserIndividualPermissionsForm(self.request.translate)
             form_result = _form.to_python(dict(self.request.POST))
             inherit_perms = form_result['inherit_default_permissions']
             c.user.inherit_default_permissions = inherit_perms
             Session().add(c.user)
 
             if not inherit_perms:
                 # only update the individual ones if we un check the flag
                 _form = UserPermissionsForm(
                     self.request.translate,
                     [x[0] for x in c.repo_create_choices],
                     [x[0] for x in c.repo_create_on_write_choices],
                     [x[0] for x in c.repo_group_create_choices],
                     [x[0] for x in c.user_group_create_choices],
                     [x[0] for x in c.fork_choices],
                     [x[0] for x in c.inherit_default_permission_choices])()
 
                 form_result = _form.to_python(dict(self.request.POST))
                 form_result.update({'perm_user_id': c.user.user_id})
 
                 PermissionModel().update_user_permissions(form_result)
 
             # TODO(marcink): implement global permissions
             # audit_log.store_web('user.edit.permissions')
 
             Session().commit()
 
             h.flash(_('User global permissions updated successfully'),
                     category='success')
 
         except formencode.Invalid as errors:
             data = render(
                 'rhodecode:templates/admin/users/user_edit.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except Exception:
             log.exception("Exception during permissions saving")
             h.flash(_('An error occurred during permissions saving'),
                     category='error')
 
         affected_user_ids = [user_id]
         PermissionModel().trigger_permission_flush(affected_user_ids)
         raise HTTPFound(h.route_path('user_edit_global_perms', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='user_enable_force_password_reset', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_enable_force_password_reset(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         try:
             c.user.update_userdata(force_password_change=True)
 
             msg = _('Force password change enabled for user')
             audit_logger.store_web('user.edit.password_reset.enabled',
                                    user=c.rhodecode_user)
 
             Session().commit()
             h.flash(msg, category='success')
         except Exception:
             log.exception("Exception during password reset for user")
             h.flash(_('An error occurred during password reset for user'),
                     category='error')
 
         raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='user_disable_force_password_reset', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_disable_force_password_reset(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         try:
             c.user.update_userdata(force_password_change=False)
 
             msg = _('Force password change disabled for user')
             audit_logger.store_web(
                 'user.edit.password_reset.disabled',
                 user=c.rhodecode_user)
 
             Session().commit()
             h.flash(msg, category='success')
         except Exception:
             log.exception("Exception during password reset for user")
             h.flash(_('An error occurred during password reset for user'),
                     category='error')
 
         raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='user_notice_dismiss', request_method='POST',
         renderer='json_ext', xhr=True)
     def user_notice_dismiss(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
         user_notice_id = safe_int(self.request.POST.get('notice_id'))
         notice = UserNotice().query()\
             .filter(UserNotice.user_id == user_id)\
             .filter(UserNotice.user_notice_id == user_notice_id)\
             .scalar()
         read = False
         if notice:
             notice.notice_read = True
             Session().add(notice)
             Session().commit()
             read = True
 
         return {'notice': user_notice_id, 'read': read}
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='user_create_personal_repo_group', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_create_personal_repo_group(self):
         """
         Create personal repository group for this user
         """
         from rhodecode.model.repo_group import RepoGroupModel
 
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         personal_repo_group = RepoGroup.get_user_personal_repo_group(
             c.user.user_id)
         if personal_repo_group:
             raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
         personal_repo_group_name = RepoGroupModel().get_personal_group_name(c.user)
         named_personal_group = RepoGroup.get_by_group_name(
             personal_repo_group_name)
         try:
 
             if named_personal_group and named_personal_group.user_id == c.user.user_id:
                 # migrate the same named group, and mark it as personal
                 named_personal_group.personal = True
                 Session().add(named_personal_group)
                 Session().commit()
                 msg = _('Linked repository group `%s` as personal' % (
                     personal_repo_group_name,))
                 h.flash(msg, category='success')
             elif not named_personal_group:
                 RepoGroupModel().create_personal_repo_group(c.user)
 
                 msg = _('Created repository group `%s`' % (
                     personal_repo_group_name,))
                 h.flash(msg, category='success')
             else:
                 msg = _('Repository group `%s` is already taken' % (
                     personal_repo_group_name,))
                 h.flash(msg, category='warning')
         except Exception:
             log.exception("Exception during repository group creation")
             msg = _(
                 'An error occurred during repository group creation for user')
             h.flash(msg, category='error')
             Session().rollback()
 
         raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_auth_tokens', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def auth_tokens(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'auth_tokens'
 
         c.lifetime_values = AuthTokenModel.get_lifetime_values(translator=_)
         c.role_values = [
             (x, AuthTokenModel.cls._get_role_name(x))
             for x in AuthTokenModel.cls.ROLES]
         c.role_options = [(c.role_values, _("Role"))]
         c.user_auth_tokens = AuthTokenModel().get_auth_tokens(
             c.user.user_id, show_expired=True)
         c.role_vcs = AuthTokenModel.cls.ROLE_VCS
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_auth_tokens_view', request_method='POST',
         renderer='json_ext', xhr=True)
     def auth_tokens_view(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         auth_token_id = self.request.POST.get('auth_token_id')
 
         if auth_token_id:
             token = UserApiKeys.get_or_404(auth_token_id)
 
             return {
                 'auth_token': token.api_key
             }
 
     def maybe_attach_token_scope(self, token):
         # implemented in EE edition
         pass
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_auth_tokens_add', request_method='POST')
     def auth_tokens_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
         lifetime = safe_int(self.request.POST.get('lifetime'), -1)
         description = self.request.POST.get('description')
         role = self.request.POST.get('role')
 
         token = UserModel().add_auth_token(
             user=c.user.user_id,
             lifetime_minutes=lifetime, role=role, description=description,
             scope_callback=self.maybe_attach_token_scope)
         token_data = token.get_api_data()
 
         audit_logger.store_web(
             'user.edit.token.add', action_data={
                 'data': {'token': token_data, 'user': user_data}},
             user=self._rhodecode_user, )
         Session().commit()
 
         h.flash(_("Auth token successfully created"), category='success')
         return HTTPFound(h.route_path('edit_user_auth_tokens', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_auth_tokens_delete', request_method='POST')
     def auth_tokens_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
 
         del_auth_token = self.request.POST.get('del_auth_token')
 
         if del_auth_token:
             token = UserApiKeys.get_or_404(del_auth_token)
             token_data = token.get_api_data()
 
             AuthTokenModel().delete(del_auth_token, c.user.user_id)
             audit_logger.store_web(
                 'user.edit.token.delete', action_data={
                     'data': {'token': token_data, 'user': user_data}},
                 user=self._rhodecode_user,)
             Session().commit()
             h.flash(_("Auth token successfully deleted"), category='success')
 
         return HTTPFound(h.route_path('edit_user_auth_tokens', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_ssh_keys', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def ssh_keys(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'ssh_keys'
         c.default_key = self.request.GET.get('default_key')
         c.user_ssh_keys = SshKeyModel().get_ssh_keys(c.user.user_id)
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_ssh_keys_generate_keypair', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def ssh_keys_generate_keypair(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         c.user = self.db_user
 
         c.active = 'ssh_keys_generate'
         comment = 'RhodeCode-SSH {}'.format(c.user.email or '')
         private_format = self.request.GET.get('private_format') \
                          or SshKeyModel.DEFAULT_PRIVATE_KEY_FORMAT
         c.private, c.public = SshKeyModel().generate_keypair(
             comment=comment, private_format=private_format)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_ssh_keys_add', request_method='POST')
     def ssh_keys_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
         key_data = self.request.POST.get('key_data')
         description = self.request.POST.get('description')
 
         fingerprint = 'unknown'
         try:
             if not key_data:
                 raise ValueError('Please add a valid public key')
 
             key = SshKeyModel().parse_key(key_data.strip())
             fingerprint = key.hash_md5()
 
             ssh_key = SshKeyModel().create(
                 c.user.user_id, fingerprint, key.keydata, description)
             ssh_key_data = ssh_key.get_api_data()
 
             audit_logger.store_web(
                 'user.edit.ssh_key.add', action_data={
                     'data': {'ssh_key': ssh_key_data, 'user': user_data}},
                 user=self._rhodecode_user, )
             Session().commit()
 
             # Trigger an event on change of keys.
             trigger(SshKeyFileChangeEvent(), self.request.registry)
 
             h.flash(_("Ssh Key successfully created"), category='success')
 
         except IntegrityError:
             log.exception("Exception during ssh key saving")
             err = 'Such key with fingerprint `{}` already exists, ' \
                   'please use a different one'.format(fingerprint)
             h.flash(_('An error occurred during ssh key saving: {}').format(err),
                     category='error')
         except Exception as e:
             log.exception("Exception during ssh key saving")
             h.flash(_('An error occurred during ssh key saving: {}').format(e),
                     category='error')
 
         return HTTPFound(
             h.route_path('edit_user_ssh_keys', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_ssh_keys_delete', request_method='POST')
     def ssh_keys_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
 
         del_ssh_key = self.request.POST.get('del_ssh_key')
 
         if del_ssh_key:
             ssh_key = UserSshKeys.get_or_404(del_ssh_key)
             ssh_key_data = ssh_key.get_api_data()
 
             SshKeyModel().delete(del_ssh_key, c.user.user_id)
             audit_logger.store_web(
                 'user.edit.ssh_key.delete', action_data={
                     'data': {'ssh_key': ssh_key_data, 'user': user_data}},
                 user=self._rhodecode_user,)
             Session().commit()
             # Trigger an event on change of keys.
             trigger(SshKeyFileChangeEvent(), self.request.registry)
             h.flash(_("Ssh key successfully deleted"), category='success')
 
         return HTTPFound(h.route_path('edit_user_ssh_keys', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_emails', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def emails(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'emails'
         c.user_email_map = UserEmailMap.query() \
             .filter(UserEmailMap.user == c.user).all()
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_emails_add', request_method='POST')
     def emails_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         email = self.request.POST.get('new_email')
         user_data = c.user.get_api_data()
         try:
 
             form = UserExtraEmailForm(self.request.translate)()
             data = form.to_python({'email': email})
             email = data['email']
 
             UserModel().add_extra_email(c.user.user_id, email)
             audit_logger.store_web(
                 'user.edit.email.add',
                 action_data={'email': email, 'user': user_data},
                 user=self._rhodecode_user)
             Session().commit()
             h.flash(_("Added new email address `%s` for user account") % email,
                     category='success')
         except formencode.Invalid as error:
             h.flash(h.escape(error.error_dict['email']), category='error')
         except IntegrityError:
             log.warning("Email %s already exists", email)
             h.flash(_('Email `{}` is already registered for another user.').format(email),
                     category='error')
         except Exception:
             log.exception("Exception during email saving")
             h.flash(_('An error occurred during email saving'),
                     category='error')
         raise HTTPFound(h.route_path('edit_user_emails', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_emails_delete', request_method='POST')
     def emails_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         email_id = self.request.POST.get('del_email_id')
         user_model = UserModel()
 
         email = UserEmailMap.query().get(email_id).email
         user_data = c.user.get_api_data()
         user_model.delete_extra_email(c.user.user_id, email_id)
         audit_logger.store_web(
             'user.edit.email.delete',
             action_data={'email': email, 'user': user_data},
             user=self._rhodecode_user)
         Session().commit()
         h.flash(_("Removed email address from user account"),
                 category='success')
         raise HTTPFound(h.route_path('edit_user_emails', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_ips', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def ips(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'ips'
         c.user_ip_map = UserIpMap.query() \
             .filter(UserIpMap.user == c.user).all()
 
         c.inherit_default_ips = c.user.inherit_default_permissions
         c.default_user_ip_map = UserIpMap.query() \
             .filter(UserIpMap.user == User.get_default_user()).all()
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_ips_add', request_method='POST')
     # NOTE(marcink): this view is allowed for default users, as we can
     # edit their IP white list
     def ips_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_model = UserModel()
         desc = self.request.POST.get('description')
         try:
             ip_list = user_model.parse_ip_range(
                 self.request.POST.get('new_ip'))
         except Exception as e:
             ip_list = []
             log.exception("Exception during ip saving")
             h.flash(_('An error occurred during ip saving:%s' % (e,)),
                     category='error')
         added = []
         user_data = c.user.get_api_data()
         for ip in ip_list:
             try:
                 form = UserExtraIpForm(self.request.translate)()
                 data = form.to_python({'ip': ip})
                 ip = data['ip']
 
                 user_model.add_extra_ip(c.user.user_id, ip, desc)
                 audit_logger.store_web(
                     'user.edit.ip.add',
                     action_data={'ip': ip, 'user': user_data},
                     user=self._rhodecode_user)
                 Session().commit()
                 added.append(ip)
             except formencode.Invalid as error:
                 msg = error.error_dict['ip']
                 h.flash(msg, category='error')
             except Exception:
                 log.exception("Exception during ip saving")
                 h.flash(_('An error occurred during ip saving'),
                         category='error')
         if added:
             h.flash(
                 _("Added ips %s to user whitelist") % (', '.join(ip_list), ),
                 category='success')
         if 'default_user' in self.request.POST:
             # case for editing global IP list we do it for 'DEFAULT' user
             raise HTTPFound(h.route_path('admin_permissions_ips'))
         raise HTTPFound(h.route_path('edit_user_ips', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_ips_delete', request_method='POST')
     # NOTE(marcink): this view is allowed for default users, as we can
     # edit their IP white list
     def ips_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         ip_id = self.request.POST.get('del_ip_id')
         user_model = UserModel()
         user_data = c.user.get_api_data()
         ip = UserIpMap.query().get(ip_id).ip_addr
         user_model.delete_extra_ip(c.user.user_id, ip_id)
         audit_logger.store_web(
             'user.edit.ip.delete', action_data={'ip': ip, 'user': user_data},
             user=self._rhodecode_user)
         Session().commit()
         h.flash(_("Removed ip address from user whitelist"), category='success')
 
         if 'default_user' in self.request.POST:
             # case for editing global IP list we do it for 'DEFAULT' user
             raise HTTPFound(h.route_path('admin_permissions_ips'))
         raise HTTPFound(h.route_path('edit_user_ips', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_groups_management', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def groups_management(self):
         c = self.load_default_context()
         c.user = self.db_user
         c.data = c.user.group_member
 
         groups = [UserGroupModel.get_user_groups_as_dict(group.users_group)
                   for group in c.user.group_member]
         c.groups = json.dumps(groups)
         c.active = 'groups'
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_groups_management_updates', request_method='POST')
     def groups_management_updates(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_groups = set(self.request.POST.getall('users_group_id'))
         user_groups_objects = []
 
         for ugid in user_groups:
             user_groups_objects.append(
                 UserGroupModel().get_group(safe_int(ugid)))
         user_group_model = UserGroupModel()
         added_to_groups, removed_from_groups = \
             user_group_model.change_groups(c.user, user_groups_objects)
 
         user_data = c.user.get_api_data()
         for user_group_id in added_to_groups:
             user_group = UserGroup.get(user_group_id)
             old_values = user_group.get_api_data()
             audit_logger.store_web(
                 'user_group.edit.member.add',
                 action_data={'user': user_data, 'old_data': old_values},
                 user=self._rhodecode_user)
 
         for user_group_id in removed_from_groups:
             user_group = UserGroup.get(user_group_id)
             old_values = user_group.get_api_data()
             audit_logger.store_web(
                 'user_group.edit.member.delete',
                 action_data={'user': user_data, 'old_data': old_values},
                 user=self._rhodecode_user)
 
         Session().commit()
         c.active = 'user_groups_management'
         h.flash(_("Groups successfully changed"), category='success')
 
         return HTTPFound(h.route_path(
             'edit_user_groups_management', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_audit_logs', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_audit_logs(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'audit'
 
         p = safe_int(self.request.GET.get('page', 1), 1)
 
         filter_term = self.request.GET.get('filter')
         user_log = UserModel().get_user_log(c.user, filter_term)
 
         def url_generator(page_num):
             query_params = {
                 'page': page_num
             }
             if filter_term:
                 query_params['filter'] = filter_term
             return self.request.current_route_path(_query=query_params)
 
         c.audit_logs = SqlPage(
             user_log, page=p, items_per_page=10, url_maker=url_generator)
         c.filter_term = filter_term
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_audit_logs_download', request_method='GET',
         renderer='string')
     def user_audit_logs_download(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         user_log = UserModel().get_user_log(c.user, filter_term=None)
 
         audit_log_data = {}
         for entry in user_log:
             audit_log_data[entry.user_log_id] = entry.get_dict()
 
         response = Response(json.dumps(audit_log_data, indent=4))
         response.content_disposition = str(
             'attachment; filename=%s' % 'user_{}_audit_logs.json'.format(c.user.user_id))
         response.content_type = 'application/json'
 
         return response
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_perms_summary', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_perms_summary(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'perms_summary'
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_perms_summary_json', request_method='GET',
         renderer='json_ext')
     def user_perms_summary_json(self):
         self.load_default_context()
         perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr)
 
         return perm_user.permissions
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='edit_user_caches', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
     def user_caches(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'caches'
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
         c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
         c.backend = c.region.backend
         c.user_keys = sorted(c.region.backend.list_keys(prefix=cache_namespace_uid))
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='edit_user_caches_update', request_method='POST')
     def user_caches_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'caches'
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
         del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid)
 
         h.flash(_("Deleted {} cache keys").format(del_keys), category='success')
 
         return HTTPFound(h.route_path(
             'edit_user_caches', user_id=c.user.user_id))

rhodecode/config/middleware.py

@@ -1,768 +1,766 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import sys
 import logging
 import collections
 import tempfile
 import time
 
 from paste.gzipper import make_gzip_middleware
 import pyramid.events
 from pyramid.wsgi import wsgiapp
 from pyramid.authorization import ACLAuthorizationPolicy
 from pyramid.config import Configurator
 from pyramid.settings import asbool, aslist
 from pyramid.httpexceptions import (
     HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
 from pyramid.renderers import render_to_response
 
 from rhodecode.model import meta
 from rhodecode.config import patches
 from rhodecode.config import utils as config_utils
 from rhodecode.config.environment import load_pyramid_environment
 
 import rhodecode.events
 from rhodecode.lib.middleware.vcs import VCSMiddleware
 from rhodecode.lib.request import Request
 from rhodecode.lib.vcs import VCSCommunicationError
 from rhodecode.lib.exceptions import VCSServerUnavailable
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.celerylib.loader import configure_celery
 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
 from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict
 from rhodecode.lib.exc_tracking import store_exception
 from rhodecode.subscribers import (
     scan_repositories_if_enabled, write_js_routes_if_enabled,
-    write_metadata_if_needed, write_usage_data, inject_app_settings)
+    write_metadata_if_needed, write_usage_data)
 
 
 log = logging.getLogger(__name__)
 
 
 def is_http_error(response):
     # error which should have traceback
     return response.status_code > 499
 
 
 def should_load_all():
     """
     Returns if all application components should be loaded. In some cases it's
     desired to skip apps loading for faster shell script execution
     """
     ssh_cmd = os.environ.get('RC_CMD_SSH_WRAPPER')
     if ssh_cmd:
         return False
 
     return True
 
 
 def make_pyramid_app(global_config, **settings):
     """
     Constructs the WSGI application based on Pyramid.
 
     Specials:
 
     * The application can also be integrated like a plugin via the call to
       `includeme`. This is accompanied with the other utility functions which
       are called. Changing this should be done with great care to not break
       cases when these fragments are assembled from another place.
 
     """
 
     # Allows to use format style "{ENV_NAME}" placeholders in the configuration. It
     # will be replaced by the value of the environment variable "NAME" in this case.
     start_time = time.time()
 
     debug = asbool(global_config.get('debug'))
     if debug:
         enable_debug()
 
     environ = {'ENV_{}'.format(key): value for key, value in os.environ.items()}
 
     global_config = _substitute_values(global_config, environ)
     settings = _substitute_values(settings, environ)
 
     sanitize_settings_and_apply_defaults(global_config, settings)
 
     config = Configurator(settings=settings)
 
     # Apply compatibility patches
     patches.inspect_getargspec()
 
     load_pyramid_environment(global_config, settings)
 
     # Static file view comes first
     includeme_first(config)
 
     includeme(config)
 
     pyramid_app = config.make_wsgi_app()
     pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
     pyramid_app.config = config
 
     config.configure_celery(global_config['__file__'])
     # creating the app uses a connection - return it after we are done
     meta.Session.remove()
     total_time = time.time() - start_time
     log.info('Pyramid app `%s` created and configured in %.2fs',
              pyramid_app.func_name, total_time)
 
     return pyramid_app
 
 
 def not_found_view(request):
     """
     This creates the view which should be registered as not-found-view to
     pyramid.
     """
 
     if not getattr(request, 'vcs_call', None):
         # handle like regular case with our error_handler
         return error_handler(HTTPNotFound(), request)
 
     # handle not found view as a vcs call
     settings = request.registry.settings
     ae_client = getattr(request, 'ae_client', None)
     vcs_app = VCSMiddleware(
         HTTPNotFound(), request.registry, settings,
         appenlight_client=ae_client)
 
     return wsgiapp(vcs_app)(None, request)
 
 
 def error_handler(exception, request):
     import rhodecode
     from rhodecode.lib import helpers
 
     rhodecode_title = rhodecode.CONFIG.get('rhodecode_title') or 'RhodeCode'
 
     base_response = HTTPInternalServerError()
     # prefer original exception for the response since it may have headers set
     if isinstance(exception, HTTPException):
         base_response = exception
     elif isinstance(exception, VCSCommunicationError):
         base_response = VCSServerUnavailable()
 
     if is_http_error(base_response):
         log.exception(
             'error occurred handling this request for path: %s', request.path)
 
     error_explanation = base_response.explanation or str(base_response)
     if base_response.status_code == 404:
         error_explanation += " Optionally you don't have permission to access this page."
     c = AttributeDict()
     c.error_message = base_response.status
     c.error_explanation = error_explanation
     c.visual = AttributeDict()
 
     c.visual.rhodecode_support_url = (
         request.registry.settings.get('rhodecode_support_url') or
         request.route_url('rhodecode_support')
     )
     c.redirect_time = 0
     c.rhodecode_name = rhodecode_title
     if not c.rhodecode_name:
         c.rhodecode_name = 'Rhodecode'
 
     c.causes = []
     if is_http_error(base_response):
         c.causes.append('Server is overloaded.')
         c.causes.append('Server database connection is lost.')
         c.causes.append('Server expected unhandled error.')
 
     if hasattr(base_response, 'causes'):
         c.causes = base_response.causes
 
     c.messages = helpers.flash.pop_messages(request=request)
 
     exc_info = sys.exc_info()
     c.exception_id = id(exc_info)
     c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \
                           or base_response.status_code > 499
     c.exception_id_url = request.route_url(
         'admin_settings_exception_tracker_show', exception_id=c.exception_id)
 
     if c.show_exception_id:
         store_exception(c.exception_id, exc_info)
 
     response = render_to_response(
         '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
         response=base_response)
 
     return response
 
 
 def includeme_first(config):
     # redirect automatic browser favicon.ico requests to correct place
     def favicon_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/images/favicon.ico'))
 
     config.add_view(favicon_redirect, route_name='favicon')
     config.add_route('favicon', '/favicon.ico')
 
     def robots_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/robots.txt'))
 
     config.add_view(robots_redirect, route_name='robots')
     config.add_route('robots', '/robots.txt')
 
     config.add_static_view(
         '_static/deform', 'deform:static')
     config.add_static_view(
         '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
 
 
 def includeme(config):
     log.debug('Initializing main includeme from %s', os.path.basename(__file__))
     settings = config.registry.settings
     config.set_request_factory(Request)
 
     # plugin information
     config.registry.rhodecode_plugins = collections.OrderedDict()
 
     config.add_directive(
         'register_rhodecode_plugin', register_rhodecode_plugin)
 
     config.add_directive('configure_celery', configure_celery)
 
     if asbool(settings.get('appenlight', 'false')):
         config.include('appenlight_client.ext.pyramid_tween')
 
     load_all = should_load_all()
 
     # Includes which are required. The application would fail without them.
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
 
     config.include('rhodecode.apps._base.navigation')
     config.include('rhodecode.apps._base.subscribers')
     config.include('rhodecode.tweens')
     config.include('rhodecode.authentication')
 
     if load_all:
         config.include('rhodecode.integrations')
 
     if load_all:
         # load CE authentication plugins
         config.include('rhodecode.authentication.plugins.auth_crowd')
         config.include('rhodecode.authentication.plugins.auth_headers')
         config.include('rhodecode.authentication.plugins.auth_jasig_cas')
         config.include('rhodecode.authentication.plugins.auth_ldap')
         config.include('rhodecode.authentication.plugins.auth_pam')
         config.include('rhodecode.authentication.plugins.auth_rhodecode')
         config.include('rhodecode.authentication.plugins.auth_token')
 
         # Auto discover authentication plugins and include their configuration.
         if asbool(settings.get('auth_plugin.import_legacy_plugins', 'true')):
             from rhodecode.authentication import discover_legacy_plugins
             discover_legacy_plugins(config)
 
     # apps
     if load_all:
         config.include('rhodecode.apps._base')
         config.include('rhodecode.apps.hovercards')
         config.include('rhodecode.apps.ops')
         config.include('rhodecode.apps.admin')
         config.include('rhodecode.apps.channelstream')
         config.include('rhodecode.apps.file_store')
         config.include('rhodecode.apps.login')
         config.include('rhodecode.apps.home')
         config.include('rhodecode.apps.journal')
         config.include('rhodecode.apps.repository')
         config.include('rhodecode.apps.repo_group')
         config.include('rhodecode.apps.user_group')
         config.include('rhodecode.apps.search')
         config.include('rhodecode.apps.user_profile')
         config.include('rhodecode.apps.user_group_profile')
         config.include('rhodecode.apps.my_account')
         config.include('rhodecode.apps.svn_support')
         config.include('rhodecode.apps.ssh_support')
         config.include('rhodecode.apps.gist')
         config.include('rhodecode.apps.debug_style')
         config.include('rhodecode.api')
 
     config.add_route('rhodecode_support', 'https://rhodecode.com/help/', static=True)
     config.add_translation_dirs('rhodecode:i18n/')
     settings['default_locale_name'] = settings.get('lang', 'en')
 
     # Add subscribers.
     if load_all:
-        config.add_subscriber(inject_app_settings,
-                              pyramid.events.ApplicationCreated)
         config.add_subscriber(scan_repositories_if_enabled,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_metadata_if_needed,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_usage_data,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_js_routes_if_enabled,
                               pyramid.events.ApplicationCreated)
 
     # request custom methods
     config.add_request_method(
         'rhodecode.lib.partial_renderer.get_partial_renderer',
         'get_partial_renderer')
 
     config.add_request_method(
         'rhodecode.lib.request_counter.get_request_counter',
         'request_count')
 
     # Set the authorization policy.
     authz_policy = ACLAuthorizationPolicy()
     config.set_authorization_policy(authz_policy)
 
     # Set the default renderer for HTML templates to mako.
     config.add_mako_renderer('.html')
 
     config.add_renderer(
         name='json_ext',
         factory='rhodecode.lib.ext_json_renderer.pyramid_ext_json')
 
     config.add_renderer(
         name='string_html',
         factory='rhodecode.lib.string_renderer.html')
 
     # include RhodeCode plugins
     includes = aslist(settings.get('rhodecode.includes', []))
     for inc in includes:
         config.include(inc)
 
     # custom not found view, if our pyramid app doesn't know how to handle
     # the request pass it to potential VCS handling ap
     config.add_notfound_view(not_found_view)
     if not settings.get('debugtoolbar.enabled', False):
         # disabled debugtoolbar handle all exceptions via the error_handlers
         config.add_view(error_handler, context=Exception)
 
     # all errors including 403/404/50X
     config.add_view(error_handler, context=HTTPError)
 
 
 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
     """
     Apply outer WSGI middlewares around the application.
     """
     registry = config.registry
     settings = registry.settings
 
     # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     pyramid_app = HttpsFixup(pyramid_app, settings)
 
     pyramid_app, _ae_client = wrap_in_appenlight_if_enabled(
         pyramid_app, settings)
     registry.ae_client = _ae_client
 
     if settings['gzip_responses']:
         pyramid_app = make_gzip_middleware(
             pyramid_app, settings, compress_level=1)
 
     # this should be the outer most middleware in the wsgi stack since
     # middleware like Routes make database calls
     def pyramid_app_with_cleanup(environ, start_response):
         try:
             return pyramid_app(environ, start_response)
         finally:
             # Dispose current database session and rollback uncommitted
             # transactions.
             meta.Session.remove()
 
             # In a single threaded mode server, on non sqlite db we should have
             # '0 Current Checked out connections' at the end of a request,
             # if not, then something, somewhere is leaving a connection open
             pool = meta.Base.metadata.bind.engine.pool
             log.debug('sa pool status: %s', pool.status())
             log.debug('Request processing finalized')
 
     return pyramid_app_with_cleanup
 
 
 def sanitize_settings_and_apply_defaults(global_config, settings):
     """
     Applies settings defaults and does all type conversion.
 
     We would move all settings parsing and preparation into this place, so that
     we have only one place left which deals with this part. The remaining parts
     of the application would start to rely fully on well prepared settings.
 
     This piece would later be split up per topic to avoid a big fat monster
     function.
     """
 
     settings.setdefault('rhodecode.edition', 'Community Edition')
     settings.setdefault('rhodecode.edition_id', 'CE')
 
     if 'mako.default_filters' not in settings:
         # set custom default filters if we don't have it defined
         settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
         settings['mako.default_filters'] = 'h_filter'
 
     if 'mako.directories' not in settings:
         mako_directories = settings.setdefault('mako.directories', [
             # Base templates of the original application
             'rhodecode:templates',
         ])
         log.debug(
             "Using the following Mako template directories: %s",
             mako_directories)
 
     # NOTE(marcink): fix redis requirement for schema of connection since 3.X
     if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
         raw_url = settings['beaker.session.url']
         if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
             settings['beaker.session.url'] = 'redis://' + raw_url
 
     # Default includes, possible to change as a user
     pyramid_includes = settings.setdefault('pyramid.includes', [])
     log.debug(
         "Using the following pyramid.includes: %s",
         pyramid_includes)
 
     # TODO: johbo: Re-think this, usually the call to config.include
     # should allow to pass in a prefix.
     settings.setdefault('rhodecode.api.url', '/_admin/api')
     settings.setdefault('__file__', global_config.get('__file__'))
 
     # Sanitize generic settings.
     _list_setting(settings, 'default_encoding', 'UTF-8')
     _bool_setting(settings, 'is_test', 'false')
     _bool_setting(settings, 'gzip_responses', 'false')
 
     # Call split out functions that sanitize settings for each topic.
     _sanitize_appenlight_settings(settings)
     _sanitize_vcs_settings(settings)
     _sanitize_cache_settings(settings)
 
     # configure instance id
     config_utils.set_instance_id(settings)
 
     return settings
 
 
 def enable_debug():
     """
     Helper to enable debug on running instance
     :return:
     """
     import tempfile
     import textwrap
     import logging.config
 
     ini_template = textwrap.dedent("""
     #####################################
     ### DEBUG LOGGING CONFIGURATION  ####
     #####################################
     [loggers]
     keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
     [handlers]
     keys = console, console_sql
 
     [formatters]
     keys = generic, color_formatter, color_formatter_sql
 
     #############
     ## LOGGERS ##
     #############
     [logger_root]
     level = NOTSET
     handlers = console
 
     [logger_sqlalchemy]
     level = INFO
     handlers = console_sql
     qualname = sqlalchemy.engine
     propagate = 0
 
     [logger_beaker]
     level = DEBUG
     handlers =
     qualname = beaker.container
     propagate = 1
 
     [logger_rhodecode]
     level = DEBUG
     handlers =
     qualname = rhodecode
     propagate = 1
 
     [logger_ssh_wrapper]
     level = DEBUG
     handlers =
     qualname = ssh_wrapper
     propagate = 1
 
     [logger_celery]
     level = DEBUG
     handlers =
     qualname = celery
 
 
     ##############
     ## HANDLERS ##
     ##############
 
     [handler_console]
     class = StreamHandler
     args = (sys.stderr, )
     level = DEBUG
     formatter = color_formatter
 
     [handler_console_sql]
     # "level = DEBUG" logs SQL queries and results.
     # "level = INFO" logs SQL queries.
     # "level = WARN" logs neither.  (Recommended for production systems.)
     class = StreamHandler
     args = (sys.stderr, )
     level = WARN
     formatter = color_formatter_sql
 
     ################
     ## FORMATTERS ##
     ################
 
     [formatter_generic]
     class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
     format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s
     datefmt = %Y-%m-%d %H:%M:%S
 
     [formatter_color_formatter]
     class = rhodecode.lib.logging_formatter.ColorRequestTrackingFormatter
     format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s
     datefmt = %Y-%m-%d %H:%M:%S
 
     [formatter_color_formatter_sql]
     class = rhodecode.lib.logging_formatter.ColorFormatterSql
     format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
     datefmt = %Y-%m-%d %H:%M:%S    
     """)
 
     with tempfile.NamedTemporaryFile(prefix='rc_debug_logging_', suffix='.ini',
                                      delete=False) as f:
         log.info('Saved Temporary DEBUG config at %s', f.name)
         f.write(ini_template)
 
     logging.config.fileConfig(f.name)
     log.debug('DEBUG MODE ON')
     os.remove(f.name)
 
 
 def _sanitize_appenlight_settings(settings):
     _bool_setting(settings, 'appenlight', 'false')
 
 
 def _sanitize_vcs_settings(settings):
     """
     Applies settings defaults and does type conversion for all VCS related
     settings.
     """
     _string_setting(settings, 'vcs.svn.compatible_version', '')
     _string_setting(settings, 'vcs.hooks.protocol', 'http')
     _string_setting(settings, 'vcs.hooks.host', '127.0.0.1')
     _string_setting(settings, 'vcs.scm_app_implementation', 'http')
     _string_setting(settings, 'vcs.server', '')
     _string_setting(settings, 'vcs.server.protocol', 'http')
     _bool_setting(settings, 'startup.import_repos', 'false')
     _bool_setting(settings, 'vcs.hooks.direct_calls', 'false')
     _bool_setting(settings, 'vcs.server.enable', 'true')
     _bool_setting(settings, 'vcs.start_server', 'false')
     _list_setting(settings, 'vcs.backends', 'hg, git, svn')
     _int_setting(settings, 'vcs.connection_timeout', 3600)
 
     # Support legacy values of vcs.scm_app_implementation. Legacy
     # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
     # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
     scm_app_impl = settings['vcs.scm_app_implementation']
     if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
         settings['vcs.scm_app_implementation'] = 'http'
 
 
 def _sanitize_cache_settings(settings):
     temp_store = tempfile.gettempdir()
     default_cache_dir = os.path.join(temp_store, 'rc_cache')
 
     # save default, cache dir, and use it for all backends later.
     default_cache_dir = _string_setting(
         settings,
         'cache_dir',
         default_cache_dir, lower=False, default_when_empty=True)
 
     # ensure we have our dir created
     if not os.path.isdir(default_cache_dir):
         os.makedirs(default_cache_dir, mode=0o755)
 
     # exception store cache
     _string_setting(
         settings,
         'exception_tracker.store_path',
         temp_store, lower=False, default_when_empty=True)
     _bool_setting(
         settings,
         'exception_tracker.send_email',
         'false')
     _string_setting(
         settings,
         'exception_tracker.email_prefix',
         '[RHODECODE ERROR]', lower=False, default_when_empty=True)
 
     # cache_perms
     _string_setting(
         settings,
         'rc_cache.cache_perms.backend',
         'dogpile.cache.rc.file_namespace', lower=False)
     _int_setting(
         settings,
         'rc_cache.cache_perms.expiration_time',
         60)
     _string_setting(
         settings,
         'rc_cache.cache_perms.arguments.filename',
         os.path.join(default_cache_dir, 'rc_cache_1'), lower=False)
 
     # cache_repo
     _string_setting(
         settings,
         'rc_cache.cache_repo.backend',
         'dogpile.cache.rc.file_namespace', lower=False)
     _int_setting(
         settings,
         'rc_cache.cache_repo.expiration_time',
         60)
     _string_setting(
         settings,
         'rc_cache.cache_repo.arguments.filename',
         os.path.join(default_cache_dir, 'rc_cache_2'), lower=False)
 
     # cache_license
     _string_setting(
         settings,
         'rc_cache.cache_license.backend',
         'dogpile.cache.rc.file_namespace', lower=False)
     _int_setting(
         settings,
         'rc_cache.cache_license.expiration_time',
         5*60)
     _string_setting(
         settings,
         'rc_cache.cache_license.arguments.filename',
         os.path.join(default_cache_dir, 'rc_cache_3'), lower=False)
 
     # cache_repo_longterm memory, 96H
     _string_setting(
         settings,
         'rc_cache.cache_repo_longterm.backend',
         'dogpile.cache.rc.memory_lru', lower=False)
     _int_setting(
         settings,
         'rc_cache.cache_repo_longterm.expiration_time',
         345600)
     _int_setting(
         settings,
         'rc_cache.cache_repo_longterm.max_size',
         10000)
 
     # sql_cache_short
     _string_setting(
         settings,
         'rc_cache.sql_cache_short.backend',
         'dogpile.cache.rc.memory_lru', lower=False)
     _int_setting(
         settings,
         'rc_cache.sql_cache_short.expiration_time',
         30)
     _int_setting(
         settings,
         'rc_cache.sql_cache_short.max_size',
         10000)
 
 
 def _int_setting(settings, name, default):
     settings[name] = int(settings.get(name, default))
     return settings[name]
 
 
 def _bool_setting(settings, name, default):
     input_val = settings.get(name, default)
     if isinstance(input_val, unicode):
         input_val = input_val.encode('utf8')
     settings[name] = asbool(input_val)
     return settings[name]
 
 
 def _list_setting(settings, name, default):
     raw_value = settings.get(name, default)
 
     old_separator = ','
     if old_separator in raw_value:
         # If we get a comma separated list, pass it to our own function.
         settings[name] = rhodecode_aslist(raw_value, sep=old_separator)
     else:
         # Otherwise we assume it uses pyramids space/newline separation.
         settings[name] = aslist(raw_value)
     return settings[name]
 
 
 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
     value = settings.get(name, default)
 
     if default_when_empty and not value:
         # use default value when value is empty
         value = default
 
     if lower:
         value = value.lower()
     settings[name] = value
     return settings[name]
 
 
 def _substitute_values(mapping, substitutions):
     result = {}
 
     try:
         for key, value in mapping.items():
             # initialize without substitution first
             result[key] = value
 
             # Note: Cannot use regular replacements, since they would clash
             # with the implementation of ConfigParser. Using "format" instead.
             try:
                 result[key] = value.format(**substitutions)
             except KeyError as e:
                 env_var = '{}'.format(e.args[0])
 
                 msg = 'Failed to substitute: `{key}={{{var}}}` with environment entry. ' \
                       'Make sure your environment has {var} set, or remove this ' \
                       'variable from config file'.format(key=key, var=env_var)
 
                 if env_var.startswith('ENV_'):
                     raise ValueError(msg)
                 else:
                     log.warning(msg)
 
     except ValueError as e:
         log.warning('Failed to substitute ENV variable: %s', e)
         result = mapping
 
     return result

rhodecode/lib/rc_cache/__init__.py

@@ -1,82 +1,86 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2015-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 from dogpile.cache import register_backend
 
 register_backend(
     "dogpile.cache.rc.memory_lru", "rhodecode.lib.rc_cache.backends",
     "LRUMemoryBackend")
 
 register_backend(
     "dogpile.cache.rc.file_namespace", "rhodecode.lib.rc_cache.backends",
     "FileNamespaceBackend")
 
 register_backend(
     "dogpile.cache.rc.redis", "rhodecode.lib.rc_cache.backends",
     "RedisPickleBackend")
 
 register_backend(
     "dogpile.cache.rc.redis_msgpack", "rhodecode.lib.rc_cache.backends",
     "RedisMsgPackBackend")
 
 
 log = logging.getLogger(__name__)
 
 from . import region_meta
 from .utils import (
     get_default_cache_settings, backend_key_generator, get_or_create_region,
     clear_cache_namespace, make_region, InvalidationContext,
     FreshRegionCache, ActiveRegionCache)
 
 
 FILE_TREE_CACHE_VER = 'v4'
 LICENSE_CACHE_VER = 'v2'
 
 
 def configure_dogpile_cache(settings):
     cache_dir = settings.get('cache_dir')
     if cache_dir:
         region_meta.dogpile_config_defaults['cache_dir'] = cache_dir
 
     rc_cache_data = get_default_cache_settings(settings, prefixes=['rc_cache.'])
 
     # inspect available namespaces
     avail_regions = set()
     for key in rc_cache_data.keys():
         namespace_name = key.split('.', 1)[0]
         avail_regions.add(namespace_name)
     log.debug('dogpile: found following cache regions: %s', avail_regions)
 
     # register them into namespace
     for region_name in avail_regions:
         new_region = make_region(
             name=region_name,
             function_key_generator=None
         )
 
         new_region.configure_from_config(settings, 'rc_cache.{}.'.format(region_name))
         new_region.function_key_generator = backend_key_generator(new_region.actual_backend)
-        log.debug('dogpile: registering a new region %s[%s]', region_name, new_region.__dict__)
+        if log.isEnabledFor(logging.DEBUG):
+            region_args = dict(backend=new_region.actual_backend.__class__,
+                               region_invalidator=new_region.region_invalidator.__class__)
+            log.debug('dogpile: registering a new region `%s` %s', region_name, region_args)
+
         region_meta.dogpile_cache_regions[region_name] = new_region
 
 
 def includeme(config):
     configure_dogpile_cache(config.registry.settings)

rhodecode/subscribers.py

@@ -1,395 +1,389 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import io
 import math
 import re
 import os
 import datetime
 import logging
 import Queue
 import subprocess32
 
 
 from dateutil.parser import parse
 from pyramid.threadlocal import get_current_request
 from pyramid.interfaces import IRoutesMapper
 from pyramid.settings import asbool
 from pyramid.path import AssetResolver
 from threading import Thread
 
 from rhodecode.translation import _ as tsf
 from rhodecode.config.jsroutes import generate_jsroutes_content
 from rhodecode.lib import auth
 from rhodecode.lib.base import get_auth_user
 
 import rhodecode
 
 
 log = logging.getLogger(__name__)
 
 
 def add_renderer_globals(event):
     from rhodecode.lib import helpers
 
     # TODO: When executed in pyramid view context the request is not available
     # in the event. Find a better solution to get the request.
     request = event['request'] or get_current_request()
 
     # Add Pyramid translation as '_' to context
     event['_'] = request.translate
     event['_ungettext'] = request.plularize
     event['h'] = helpers
 
 
 def add_localizer(event):
     request = event.request
     localizer = request.localizer
 
     def auto_translate(*args, **kwargs):
         return localizer.translate(tsf(*args, **kwargs))
 
     request.translate = auto_translate
     request.plularize = localizer.pluralize
 
 
 def set_user_lang(event):
     request = event.request
     cur_user = getattr(request, 'user', None)
 
     if cur_user:
         user_lang = cur_user.get_instance().user_data.get('language')
         if user_lang:
             log.debug('lang: setting current user:%s language to: %s', cur_user, user_lang)
             event.request._LOCALE_ = user_lang
 
 
 def add_request_user_context(event):
     """
     Adds auth user into request context
     """
     request = event.request
     # access req_id as soon as possible
     req_id = request.req_id
 
     if hasattr(request, 'vcs_call'):
         # skip vcs calls
         return
 
     if hasattr(request, 'rpc_method'):
         # skip api calls
         return
 
     auth_user, auth_token = get_auth_user(request)
     request.user = auth_user
     request.user_auth_token = auth_token
     request.environ['rc_auth_user'] = auth_user
     request.environ['rc_auth_user_id'] = auth_user.user_id
     request.environ['rc_req_id'] = req_id
 
 
-def inject_app_settings(event):
-    settings = event.app.registry.settings
-    # inject info about available permissions
-    auth.set_available_permissions(settings)
-
-
 def scan_repositories_if_enabled(event):
     """
     This is subscribed to the `pyramid.events.ApplicationCreated` event. It
     does a repository scan if enabled in the settings.
     """
     settings = event.app.registry.settings
     vcs_server_enabled = settings['vcs.server.enable']
     import_on_startup = settings['startup.import_repos']
     if vcs_server_enabled and import_on_startup:
         from rhodecode.model.scm import ScmModel
         from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_base_path
         repositories = ScmModel().repo_scan(get_rhodecode_base_path())
         repo2db_mapper(repositories, remove_obsolete=False)
 
 
 def write_metadata_if_needed(event):
     """
     Writes upgrade metadata
     """
     import rhodecode
     from rhodecode.lib import system_info
     from rhodecode.lib import ext_json
 
     fname = '.rcmetadata.json'
     ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
     metadata_destination = os.path.join(ini_loc, fname)
 
     def get_update_age():
         now = datetime.datetime.utcnow()
 
         with open(metadata_destination, 'rb') as f:
             data = ext_json.json.loads(f.read())
             if 'created_on' in data:
                 update_date = parse(data['created_on'])
                 diff = now - update_date
                 return diff.total_seconds() / 60.0
 
         return 0
 
     def write():
         configuration = system_info.SysInfo(
             system_info.rhodecode_config)()['value']
         license_token = configuration['config']['license_token']
 
         setup = dict(
             workers=configuration['config']['server:main'].get(
                 'workers', '?'),
             worker_type=configuration['config']['server:main'].get(
                 'worker_class', 'sync'),
         )
         dbinfo = system_info.SysInfo(system_info.database_info)()['value']
         del dbinfo['url']
 
         metadata = dict(
             desc='upgrade metadata info',
             license_token=license_token,
             created_on=datetime.datetime.utcnow().isoformat(),
             usage=system_info.SysInfo(system_info.usage_info)()['value'],
             platform=system_info.SysInfo(system_info.platform_type)()['value'],
             database=dbinfo,
             cpu=system_info.SysInfo(system_info.cpu)()['value'],
             memory=system_info.SysInfo(system_info.memory)()['value'],
             setup=setup
         )
 
         with open(metadata_destination, 'wb') as f:
             f.write(ext_json.json.dumps(metadata))
 
     settings = event.app.registry.settings
     if settings.get('metadata.skip'):
         return
 
     # only write this every 24h, workers restart caused unwanted delays
     try:
         age_in_min = get_update_age()
     except Exception:
         age_in_min = 0
 
     if age_in_min > 60 * 60 * 24:
         return
 
     try:
         write()
     except Exception:
         pass
 
 
 def write_usage_data(event):
     import rhodecode
     from rhodecode.lib import system_info
     from rhodecode.lib import ext_json
 
     settings = event.app.registry.settings
     instance_tag = settings.get('metadata.write_usage_tag')
     if not settings.get('metadata.write_usage'):
         return
 
     def get_update_age(dest_file):
         now = datetime.datetime.utcnow()
 
         with open(dest_file, 'rb') as f:
             data = ext_json.json.loads(f.read())
             if 'created_on' in data:
                 update_date = parse(data['created_on'])
                 diff = now - update_date
                 return math.ceil(diff.total_seconds() / 60.0)
 
         return 0
 
     utc_date = datetime.datetime.utcnow()
     hour_quarter = int(math.ceil((utc_date.hour + utc_date.minute/60.0) / 6.))
     fname = '.rc_usage_{date.year}{date.month:02d}{date.day:02d}_{hour}.json'.format(
         date=utc_date, hour=hour_quarter)
     ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
 
     usage_dir = os.path.join(ini_loc, '.rcusage')
     if not os.path.isdir(usage_dir):
         os.makedirs(usage_dir)
     usage_metadata_destination = os.path.join(usage_dir, fname)
 
     try:
         age_in_min = get_update_age(usage_metadata_destination)
     except Exception:
         age_in_min = 0
 
     # write every 6th hour
     if age_in_min and age_in_min < 60 * 6:
         log.debug('Usage file created %s minutes ago, skipping (threashold: %s)...',
                   age_in_min, 60 * 6)
         return
 
     def write(dest_file):
         configuration = system_info.SysInfo(system_info.rhodecode_config)()['value']
         license_token = configuration['config']['license_token']
 
         metadata = dict(
             desc='Usage data',
             instance_tag=instance_tag,
             license_token=license_token,
             created_on=datetime.datetime.utcnow().isoformat(),
             usage=system_info.SysInfo(system_info.usage_info)()['value'],
         )
 
         with open(dest_file, 'wb') as f:
             f.write(ext_json.json.dumps(metadata, indent=2, sort_keys=True))
 
     try:
         log.debug('Writing usage file at: %s', usage_metadata_destination)
         write(usage_metadata_destination)
     except Exception:
         pass
 
 
 def write_js_routes_if_enabled(event):
     registry = event.app.registry
 
     mapper = registry.queryUtility(IRoutesMapper)
     _argument_prog = re.compile('\{(.*?)\}|:\((.*)\)')
 
     def _extract_route_information(route):
         """
         Convert a route into tuple(name, path, args), eg:
             ('show_user', '/profile/%(username)s', ['username'])
         """
 
         routepath = route.pattern
         pattern = route.pattern
 
         def replace(matchobj):
             if matchobj.group(1):
                 return "%%(%s)s" % matchobj.group(1).split(':')[0]
             else:
                 return "%%(%s)s" % matchobj.group(2)
 
         routepath = _argument_prog.sub(replace, routepath)
 
         if not routepath.startswith('/'):
             routepath = '/'+routepath
 
         return (
             route.name,
             routepath,
             [(arg[0].split(':')[0] if arg[0] != '' else arg[1])
               for arg in _argument_prog.findall(pattern)]
         )
 
     def get_routes():
         # pyramid routes
         for route in mapper.get_routes():
             if not route.name.startswith('__'):
                 yield _extract_route_information(route)
 
     if asbool(registry.settings.get('generate_js_files', 'false')):
         static_path = AssetResolver().resolve('rhodecode:public').abspath()
         jsroutes = get_routes()
         jsroutes_file_content = generate_jsroutes_content(jsroutes)
         jsroutes_file_path = os.path.join(
             static_path, 'js', 'rhodecode', 'routes.js')
 
         try:
             with io.open(jsroutes_file_path, 'w', encoding='utf-8') as f:
                 f.write(jsroutes_file_content)
         except Exception:
             log.exception('Failed to write routes.js into %s', jsroutes_file_path)
 
 
 class Subscriber(object):
     """
     Base class for subscribers to the pyramid event system.
     """
     def __call__(self, event):
         self.run(event)
 
     def run(self, event):
         raise NotImplementedError('Subclass has to implement this.')
 
 
 class AsyncSubscriber(Subscriber):
     """
     Subscriber that handles the execution of events in a separate task to not
     block the execution of the code which triggers the event. It puts the
     received events into a queue from which the worker process takes them in
     order.
     """
     def __init__(self):
         self._stop = False
         self._eventq = Queue.Queue()
         self._worker = self.create_worker()
         self._worker.start()
 
     def __call__(self, event):
         self._eventq.put(event)
 
     def create_worker(self):
         worker = Thread(target=self.do_work)
         worker.daemon = True
         return worker
 
     def stop_worker(self):
         self._stop = False
         self._eventq.put(None)
         self._worker.join()
 
     def do_work(self):
         while not self._stop:
             event = self._eventq.get()
             if event is not None:
                 self.run(event)
 
 
 class AsyncSubprocessSubscriber(AsyncSubscriber):
     """
     Subscriber that uses the subprocess32 module to execute a command if an
     event is received. Events are handled asynchronously.
     """
 
     def __init__(self, cmd, timeout=None):
         super(AsyncSubprocessSubscriber, self).__init__()
         self._cmd = cmd
         self._timeout = timeout
 
     def run(self, event):
         cmd = self._cmd
         timeout = self._timeout
         log.debug('Executing command %s.', cmd)
 
         try:
             output = subprocess32.check_output(
                 cmd, timeout=timeout, stderr=subprocess32.STDOUT)
             log.debug('Command finished %s', cmd)
             if output:
                 log.debug('Command output: %s', output)
         except subprocess32.TimeoutExpired as e:
             log.exception('Timeout while executing command.')
             if e.output:
                 log.error('Command output: %s', e.output)
         except subprocess32.CalledProcessError as e:
             log.exception('Error while executing command.')
             if e.output:
                 log.error('Command output: %s', e.output)
         except:
             log.exception(
                 'Exception while executing command %s.', cmd)