json: fixed calls to json after orjson implementation

rhodecode/apps/admin/views/users.py

@@ -1,1322 +1,1321 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import datetime
 import formencode
 import formencode.htmlfill
 
 from pyramid.httpexceptions import HTTPFound
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode import events
 from rhodecode.apps._base import BaseAppView, DataGridAppView, UserAppView
 from rhodecode.apps.ssh_support import SshKeyFileChangeEvent
 from rhodecode.authentication.base import get_authn_registry, RhodeCodeExternalAuthPlugin
 from rhodecode.authentication.plugins import auth_rhodecode
 from rhodecode.events import trigger
 from rhodecode.model.db import true, UserNotice
 
 from rhodecode.lib import audit_logger, rc_cache, auth
 from rhodecode.lib.exceptions import (
     UserCreationError, UserOwnsReposException, UserOwnsRepoGroupsException,
     UserOwnsUserGroupsException, UserOwnsPullRequestsException,
     UserOwnsArtifactsException, DefaultUserException)
-from rhodecode.lib.ext_json import json
+from rhodecode.lib import ext_json
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.lib import helpers as h
 from rhodecode.lib.helpers import SqlPage
 from rhodecode.lib.utils2 import safe_int, safe_unicode, AttributeDict
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.forms import (
     UserForm, UserIndividualPermissionsForm, UserPermissionsForm,
     UserExtraEmailForm, UserExtraIpForm)
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.ssh_key import SshKeyModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.model.db import (
     or_, coalesce,IntegrityError, User, UserGroup, UserIpMap, UserEmailMap,
     UserApiKeys, UserSshKeys, RepoGroup)
 from rhodecode.model.meta import Session
 
 log = logging.getLogger(__name__)
 
 
 class AdminUsersView(BaseAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         return c
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def users_list(self):
         c = self.load_default_context()
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def users_list_data(self):
         self.load_default_context()
         column_map = {
             'first_name': 'name',
             'last_name': 'lastname',
         }
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(
             self.request, column_map=column_map)
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         def user_actions(user_id, username):
             return _render("user_actions", user_id, username)
 
         users_data_total_count = User.query()\
             .filter(User.username != User.DEFAULT_USER) \
             .count()
 
         users_data_total_inactive_count = User.query()\
             .filter(User.username != User.DEFAULT_USER) \
             .filter(User.active != true())\
             .count()
 
         # json generate
         base_q = User.query().filter(User.username != User.DEFAULT_USER)
         base_inactive_q = base_q.filter(User.active != true())
 
         if search_q:
             like_expression = u'%{}%'.format(safe_unicode(search_q))
             base_q = base_q.filter(or_(
                 User.username.ilike(like_expression),
                 User._email.ilike(like_expression),
                 User.name.ilike(like_expression),
                 User.lastname.ilike(like_expression),
             ))
             base_inactive_q = base_q.filter(User.active != true())
 
         users_data_total_filtered_count = base_q.count()
         users_data_total_filtered_inactive_count = base_inactive_q.count()
 
         sort_col = getattr(User, order_by, None)
         if sort_col:
             if order_dir == 'asc':
                 # handle null values properly to order by NULL last
                 if order_by in ['last_activity']:
                     sort_col = coalesce(sort_col, datetime.date.max)
                 sort_col = sort_col.asc()
             else:
                 # handle null values properly to order by NULL last
                 if order_by in ['last_activity']:
                     sort_col = coalesce(sort_col, datetime.date.min)
                 sort_col = sort_col.desc()
 
         base_q = base_q.order_by(sort_col)
         base_q = base_q.offset(start).limit(limit)
 
         users_list = base_q.all()
 
         users_data = []
         for user in users_list:
             users_data.append({
                 "username": h.gravatar_with_user(self.request, user.username),
                 "email": user.email,
                 "first_name": user.first_name,
                 "last_name": user.last_name,
                 "last_login": h.format_date(user.last_login),
                 "last_activity": h.format_date(user.last_activity),
                 "active": h.bool2icon(user.active),
                 "active_raw": user.active,
                 "admin": h.bool2icon(user.admin),
                 "extern_type": user.extern_type,
                 "extern_name": user.extern_name,
                 "action": user_actions(user.user_id, user.username),
             })
         data = ({
             'draw': draw,
             'data': users_data,
             'recordsTotal': users_data_total_count,
             'recordsFiltered': users_data_total_filtered_count,
             'recordsTotalInactive': users_data_total_inactive_count,
             'recordsFilteredInactive': users_data_total_filtered_inactive_count
         })
 
         return data
 
     def _set_personal_repo_group_template_vars(self, c_obj):
         DummyUser = AttributeDict({
             'username': '${username}',
             'user_id': '${user_id}',
         })
         c_obj.default_create_repo_group = RepoGroupModel() \
             .get_default_create_personal_repo_group()
         c_obj.personal_repo_group_name = RepoGroupModel() \
             .get_personal_group_name(DummyUser)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def users_new(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.default_extern_type = auth_rhodecode.RhodeCodeAuthPlugin.uid
         self._set_personal_repo_group_template_vars(c)
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def users_create(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.default_extern_type = auth_rhodecode.RhodeCodeAuthPlugin.uid
         user_model = UserModel()
         user_form = UserForm(self.request.translate)()
         try:
             form_result = user_form.to_python(dict(self.request.POST))
             user = user_model.create(form_result)
             Session().flush()
             creation_data = user.get_api_data()
             username = form_result['username']
 
             audit_logger.store_web(
                 'user.create', action_data={'data': creation_data},
                 user=c.rhodecode_user)
 
             user_link = h.link_to(
                 h.escape(username),
                 h.route_path('user_edit', user_id=user.user_id))
             h.flash(h.literal(_('Created user %(user_link)s')
                               % {'user_link': user_link}), category='success')
             Session().commit()
         except formencode.Invalid as errors:
             self._set_personal_repo_group_template_vars(c)
             data = render(
                 'rhodecode:templates/admin/users/user_add.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except UserCreationError as e:
             h.flash(e, 'error')
         except Exception:
             log.exception("Exception creation of user")
             h.flash(_('Error occurred during creation of user %s')
                     % self.request.POST.get('username'), category='error')
         raise HTTPFound(h.route_path('users'))
 
 
 class UsersView(UserAppView):
     ALLOW_SCOPED_TOKENS = False
     """
     This view has alternative version inside EE, if modified please take a look
     in there as well.
     """
 
     def get_auth_plugins(self):
         valid_plugins = []
         authn_registry = get_authn_registry(self.request.registry)
         for plugin in authn_registry.get_plugins_for_authentication():
             if isinstance(plugin, RhodeCodeExternalAuthPlugin):
                 valid_plugins.append(plugin)
             elif plugin.name == 'rhodecode':
                 valid_plugins.append(plugin)
 
         # extend our choices if user has set a bound plugin which isn't enabled at the
         # moment
         extern_type = self.db_user.extern_type
         if extern_type not in [x.uid for x in valid_plugins]:
             try:
                 plugin = authn_registry.get_plugin_by_uid(extern_type)
                 if plugin:
                     valid_plugins.append(plugin)
 
             except Exception:
                 log.exception(
                     'Could not extend user plugins with `{}`'.format(extern_type))
         return valid_plugins
 
     def load_default_context(self):
         req = self.request
 
         c = self._get_local_tmpl_context()
         c.allow_scoped_tokens = self.ALLOW_SCOPED_TOKENS
         c.allowed_languages = [
             ('en', 'English (en)'),
             ('de', 'German (de)'),
             ('fr', 'French (fr)'),
             ('it', 'Italian (it)'),
             ('ja', 'Japanese (ja)'),
             ('pl', 'Polish (pl)'),
             ('pt', 'Portuguese (pt)'),
             ('ru', 'Russian (ru)'),
             ('zh', 'Chinese (zh)'),
         ]
 
         c.allowed_extern_types = [
             (x.uid, x.get_display_name()) for x in self.get_auth_plugins()
         ]
         perms = req.registry.settings.get('available_permissions')
         if not perms:
             # inject info about available permissions
             auth.set_available_permissions(req.registry.settings)
 
         c.available_permissions = req.registry.settings['available_permissions']
         PermissionModel().set_global_permission_choices(
             c, gettext_translator=req.translate)
 
         return c
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_update(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         c.active = 'profile'
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
         available_languages = [x[0] for x in c.allowed_languages]
         _form = UserForm(self.request.translate, edit=True,
                          available_languages=available_languages,
                          old_data={'user_id': user_id,
                                    'email': c.user.email})()
 
         c.edit_mode = self.request.POST.get('edit') == '1'
         form_result = {}
         old_values = c.user.get_api_data()
         try:
             form_result = _form.to_python(dict(self.request.POST))
             skip_attrs = ['extern_name']
             # TODO: plugin should define if username can be updated
 
             if c.extern_type != "rhodecode" and not c.edit_mode:
                 # forbid updating username for external accounts
                 skip_attrs.append('username')
 
             UserModel().update_user(
                 user_id, skip_attrs=skip_attrs, **form_result)
 
             audit_logger.store_web(
                 'user.edit', action_data={'old_data': old_values},
                 user=c.rhodecode_user)
 
             Session().commit()
             h.flash(_('User updated successfully'), category='success')
         except formencode.Invalid as errors:
             data = render(
                 'rhodecode:templates/admin/users/user_edit.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except UserCreationError as e:
             h.flash(e, 'error')
         except Exception:
             log.exception("Exception updating user")
             h.flash(_('Error occurred during update of user %s')
                     % form_result.get('username'), category='error')
         raise HTTPFound(h.route_path('user_edit', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         _repos = c.user.repositories
         _repo_groups = c.user.repository_groups
         _user_groups = c.user.user_groups
         _pull_requests = c.user.user_pull_requests
         _artifacts = c.user.artifacts
 
         handle_repos = None
         handle_repo_groups = None
         handle_user_groups = None
         handle_pull_requests = None
         handle_artifacts = None
 
         # calls for flash of handle based on handle case detach or delete
         def set_handle_flash_repos():
             handle = handle_repos
             if handle == 'detach':
                 h.flash(_('Detached %s repositories') % len(_repos),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s repositories') % len(_repos),
                         category='success')
 
         def set_handle_flash_repo_groups():
             handle = handle_repo_groups
             if handle == 'detach':
                 h.flash(_('Detached %s repository groups') % len(_repo_groups),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s repository groups') % len(_repo_groups),
                         category='success')
 
         def set_handle_flash_user_groups():
             handle = handle_user_groups
             if handle == 'detach':
                 h.flash(_('Detached %s user groups') % len(_user_groups),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s user groups') % len(_user_groups),
                         category='success')
 
         def set_handle_flash_pull_requests():
             handle = handle_pull_requests
             if handle == 'detach':
                 h.flash(_('Detached %s pull requests') % len(_pull_requests),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s pull requests') % len(_pull_requests),
                         category='success')
 
         def set_handle_flash_artifacts():
             handle = handle_artifacts
             if handle == 'detach':
                 h.flash(_('Detached %s artifacts') % len(_artifacts),
                         category='success')
             elif handle == 'delete':
                 h.flash(_('Deleted %s artifacts') % len(_artifacts),
                         category='success')
 
         handle_user = User.get_first_super_admin()
         handle_user_id = safe_int(self.request.POST.get('detach_user_id'))
         if handle_user_id:
             # NOTE(marcink): we get new owner for objects...
             handle_user = User.get_or_404(handle_user_id)
 
         if _repos and self.request.POST.get('user_repos'):
             handle_repos = self.request.POST['user_repos']
 
         if _repo_groups and self.request.POST.get('user_repo_groups'):
             handle_repo_groups = self.request.POST['user_repo_groups']
 
         if _user_groups and self.request.POST.get('user_user_groups'):
             handle_user_groups = self.request.POST['user_user_groups']
 
         if _pull_requests and self.request.POST.get('user_pull_requests'):
             handle_pull_requests = self.request.POST['user_pull_requests']
 
         if _artifacts and self.request.POST.get('user_artifacts'):
             handle_artifacts = self.request.POST['user_artifacts']
 
         old_values = c.user.get_api_data()
 
         try:
 
             UserModel().delete(
                 c.user,
                 handle_repos=handle_repos,
                 handle_repo_groups=handle_repo_groups,
                 handle_user_groups=handle_user_groups,
                 handle_pull_requests=handle_pull_requests,
                 handle_artifacts=handle_artifacts,
                 handle_new_owner=handle_user
             )
 
             audit_logger.store_web(
                 'user.delete', action_data={'old_data': old_values},
                 user=c.rhodecode_user)
 
             Session().commit()
             set_handle_flash_repos()
             set_handle_flash_repo_groups()
             set_handle_flash_user_groups()
             set_handle_flash_pull_requests()
             set_handle_flash_artifacts()
             username = h.escape(old_values['username'])
             h.flash(_('Successfully deleted user `{}`').format(username), category='success')
         except (UserOwnsReposException, UserOwnsRepoGroupsException,
                 UserOwnsUserGroupsException, UserOwnsPullRequestsException,
                 UserOwnsArtifactsException, DefaultUserException) as e:
             h.flash(e, category='warning')
         except Exception:
             log.exception("Exception during deletion of user")
             h.flash(_('An error occurred during deletion of user'),
                     category='error')
         raise HTTPFound(h.route_path('users'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_edit(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'profile'
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
         c.edit_mode = self.request.GET.get('edit') == '1'
 
         defaults = c.user.get_dict()
         defaults.update({'language': c.user.user_data.get('language')})
 
         data = render(
             'rhodecode:templates/admin/users/user_edit.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_edit_advanced(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         c.detach_user = User.get_first_super_admin()
         detach_user_id = safe_int(self.request.GET.get('detach_user_id'))
         if detach_user_id:
             c.detach_user = User.get_or_404(detach_user_id)
 
         c.active = 'advanced'
         c.personal_repo_group = RepoGroup.get_user_personal_repo_group(user_id)
         c.personal_repo_group_name = RepoGroupModel()\
             .get_personal_group_name(c.user)
 
         c.user_to_review_rules = sorted(
             (x.user for x in c.user.user_review_rules),
             key=lambda u: u.username.lower())
 
         defaults = c.user.get_dict()
 
         # Interim workaround if the user participated on any pull requests as a
         # reviewer.
         has_review = len(c.user.reviewer_pull_requests)
         c.can_delete_user = not has_review
         c.can_delete_user_message = ''
         inactive_link = h.link_to(
             'inactive', h.route_path('user_edit', user_id=user_id, _anchor='active'))
         if has_review == 1:
             c.can_delete_user_message = h.literal(_(
                 'The user participates as reviewer in {} pull request and '
                 'cannot be deleted. \nYou can set the user to '
                 '"{}" instead of deleting it.').format(
                 has_review, inactive_link))
         elif has_review:
             c.can_delete_user_message = h.literal(_(
                 'The user participates as reviewer in {} pull requests and '
                 'cannot be deleted. \nYou can set the user to '
                 '"{}" instead of deleting it.').format(
                 has_review, inactive_link))
 
         data = render(
             'rhodecode:templates/admin/users/user_edit.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_edit_global_perms(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'global_perms'
 
         c.default_user = User.get_default_user()
         defaults = c.user.get_dict()
         defaults.update(c.default_user.get_default_perms(suffix='_inherited'))
         defaults.update(c.default_user.get_default_perms())
         defaults.update(c.user.get_default_perms())
 
         data = render(
             'rhodecode:templates/admin/users/user_edit.mako',
             self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_edit_global_perms_update(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         c.active = 'global_perms'
         try:
             # first stage that verifies the checkbox
             _form = UserIndividualPermissionsForm(self.request.translate)
             form_result = _form.to_python(dict(self.request.POST))
             inherit_perms = form_result['inherit_default_permissions']
             c.user.inherit_default_permissions = inherit_perms
             Session().add(c.user)
 
             if not inherit_perms:
                 # only update the individual ones if we un check the flag
                 _form = UserPermissionsForm(
                     self.request.translate,
                     [x[0] for x in c.repo_create_choices],
                     [x[0] for x in c.repo_create_on_write_choices],
                     [x[0] for x in c.repo_group_create_choices],
                     [x[0] for x in c.user_group_create_choices],
                     [x[0] for x in c.fork_choices],
                     [x[0] for x in c.inherit_default_permission_choices])()
 
                 form_result = _form.to_python(dict(self.request.POST))
                 form_result.update({'perm_user_id': c.user.user_id})
 
                 PermissionModel().update_user_permissions(form_result)
 
             # TODO(marcink): implement global permissions
             # audit_log.store_web('user.edit.permissions')
 
             Session().commit()
 
             h.flash(_('User global permissions updated successfully'),
                     category='success')
 
         except formencode.Invalid as errors:
             data = render(
                 'rhodecode:templates/admin/users/user_edit.mako',
                 self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
         except Exception:
             log.exception("Exception during permissions saving")
             h.flash(_('An error occurred during permissions saving'),
                     category='error')
 
         affected_user_ids = [user_id]
         PermissionModel().trigger_permission_flush(affected_user_ids)
         raise HTTPFound(h.route_path('user_edit_global_perms', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_enable_force_password_reset(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         try:
             c.user.update_userdata(force_password_change=True)
 
             msg = _('Force password change enabled for user')
             audit_logger.store_web('user.edit.password_reset.enabled',
                                    user=c.rhodecode_user)
 
             Session().commit()
             h.flash(msg, category='success')
         except Exception:
             log.exception("Exception during password reset for user")
             h.flash(_('An error occurred during password reset for user'),
                     category='error')
 
         raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_disable_force_password_reset(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         try:
             c.user.update_userdata(force_password_change=False)
 
             msg = _('Force password change disabled for user')
             audit_logger.store_web(
                 'user.edit.password_reset.disabled',
                 user=c.rhodecode_user)
 
             Session().commit()
             h.flash(msg, category='success')
         except Exception:
             log.exception("Exception during password reset for user")
             h.flash(_('An error occurred during password reset for user'),
                     category='error')
 
         raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_notice_dismiss(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
         user_notice_id = safe_int(self.request.POST.get('notice_id'))
         notice = UserNotice().query()\
             .filter(UserNotice.user_id == user_id)\
             .filter(UserNotice.user_notice_id == user_notice_id)\
             .scalar()
         read = False
         if notice:
             notice.notice_read = True
             Session().add(notice)
             Session().commit()
             read = True
 
         return {'notice': user_notice_id, 'read': read}
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_create_personal_repo_group(self):
         """
         Create personal repository group for this user
         """
         from rhodecode.model.repo_group import RepoGroupModel
 
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         personal_repo_group = RepoGroup.get_user_personal_repo_group(
             c.user.user_id)
         if personal_repo_group:
             raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
         personal_repo_group_name = RepoGroupModel().get_personal_group_name(c.user)
         named_personal_group = RepoGroup.get_by_group_name(
             personal_repo_group_name)
         try:
 
             if named_personal_group and named_personal_group.user_id == c.user.user_id:
                 # migrate the same named group, and mark it as personal
                 named_personal_group.personal = True
                 Session().add(named_personal_group)
                 Session().commit()
                 msg = _('Linked repository group `%s` as personal' % (
                     personal_repo_group_name,))
                 h.flash(msg, category='success')
             elif not named_personal_group:
                 RepoGroupModel().create_personal_repo_group(c.user)
 
                 msg = _('Created repository group `%s`' % (
                     personal_repo_group_name,))
                 h.flash(msg, category='success')
             else:
                 msg = _('Repository group `%s` is already taken' % (
                     personal_repo_group_name,))
                 h.flash(msg, category='warning')
         except Exception:
             log.exception("Exception during repository group creation")
             msg = _(
                 'An error occurred during repository group creation for user')
             h.flash(msg, category='error')
             Session().rollback()
 
         raise HTTPFound(h.route_path('user_edit_advanced', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def auth_tokens(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'auth_tokens'
 
         c.lifetime_values = AuthTokenModel.get_lifetime_values(translator=_)
         c.role_values = [
             (x, AuthTokenModel.cls._get_role_name(x))
             for x in AuthTokenModel.cls.ROLES]
         c.role_options = [(c.role_values, _("Role"))]
         c.user_auth_tokens = AuthTokenModel().get_auth_tokens(
             c.user.user_id, show_expired=True)
         c.role_vcs = AuthTokenModel.cls.ROLE_VCS
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def auth_tokens_view(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         auth_token_id = self.request.POST.get('auth_token_id')
 
         if auth_token_id:
             token = UserApiKeys.get_or_404(auth_token_id)
 
             return {
                 'auth_token': token.api_key
             }
 
     def maybe_attach_token_scope(self, token):
         # implemented in EE edition
         pass
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def auth_tokens_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
         lifetime = safe_int(self.request.POST.get('lifetime'), -1)
         description = self.request.POST.get('description')
         role = self.request.POST.get('role')
 
         token = UserModel().add_auth_token(
             user=c.user.user_id,
             lifetime_minutes=lifetime, role=role, description=description,
             scope_callback=self.maybe_attach_token_scope)
         token_data = token.get_api_data()
 
         audit_logger.store_web(
             'user.edit.token.add', action_data={
                 'data': {'token': token_data, 'user': user_data}},
             user=self._rhodecode_user, )
         Session().commit()
 
         h.flash(_("Auth token successfully created"), category='success')
         return HTTPFound(h.route_path('edit_user_auth_tokens', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def auth_tokens_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
 
         del_auth_token = self.request.POST.get('del_auth_token')
 
         if del_auth_token:
             token = UserApiKeys.get_or_404(del_auth_token)
             token_data = token.get_api_data()
 
             AuthTokenModel().delete(del_auth_token, c.user.user_id)
             audit_logger.store_web(
                 'user.edit.token.delete', action_data={
                     'data': {'token': token_data, 'user': user_data}},
                 user=self._rhodecode_user,)
             Session().commit()
             h.flash(_("Auth token successfully deleted"), category='success')
 
         return HTTPFound(h.route_path('edit_user_auth_tokens', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def ssh_keys(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'ssh_keys'
         c.default_key = self.request.GET.get('default_key')
         c.user_ssh_keys = SshKeyModel().get_ssh_keys(c.user.user_id)
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def ssh_keys_generate_keypair(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         c.user = self.db_user
 
         c.active = 'ssh_keys_generate'
         comment = 'RhodeCode-SSH {}'.format(c.user.email or '')
         private_format = self.request.GET.get('private_format') \
                          or SshKeyModel.DEFAULT_PRIVATE_KEY_FORMAT
         c.private, c.public = SshKeyModel().generate_keypair(
             comment=comment, private_format=private_format)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def ssh_keys_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
         key_data = self.request.POST.get('key_data')
         description = self.request.POST.get('description')
 
         fingerprint = 'unknown'
         try:
             if not key_data:
                 raise ValueError('Please add a valid public key')
 
             key = SshKeyModel().parse_key(key_data.strip())
             fingerprint = key.hash_md5()
 
             ssh_key = SshKeyModel().create(
                 c.user.user_id, fingerprint, key.keydata, description)
             ssh_key_data = ssh_key.get_api_data()
 
             audit_logger.store_web(
                 'user.edit.ssh_key.add', action_data={
                     'data': {'ssh_key': ssh_key_data, 'user': user_data}},
                 user=self._rhodecode_user, )
             Session().commit()
 
             # Trigger an event on change of keys.
             trigger(SshKeyFileChangeEvent(), self.request.registry)
 
             h.flash(_("Ssh Key successfully created"), category='success')
 
         except IntegrityError:
             log.exception("Exception during ssh key saving")
             err = 'Such key with fingerprint `{}` already exists, ' \
                   'please use a different one'.format(fingerprint)
             h.flash(_('An error occurred during ssh key saving: {}').format(err),
                     category='error')
         except Exception as e:
             log.exception("Exception during ssh key saving")
             h.flash(_('An error occurred during ssh key saving: {}').format(e),
                     category='error')
 
         return HTTPFound(
             h.route_path('edit_user_ssh_keys', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def ssh_keys_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_data = c.user.get_api_data()
 
         del_ssh_key = self.request.POST.get('del_ssh_key')
 
         if del_ssh_key:
             ssh_key = UserSshKeys.get_or_404(del_ssh_key)
             ssh_key_data = ssh_key.get_api_data()
 
             SshKeyModel().delete(del_ssh_key, c.user.user_id)
             audit_logger.store_web(
                 'user.edit.ssh_key.delete', action_data={
                     'data': {'ssh_key': ssh_key_data, 'user': user_data}},
                 user=self._rhodecode_user,)
             Session().commit()
             # Trigger an event on change of keys.
             trigger(SshKeyFileChangeEvent(), self.request.registry)
             h.flash(_("Ssh key successfully deleted"), category='success')
 
         return HTTPFound(h.route_path('edit_user_ssh_keys', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def emails(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'emails'
         c.user_email_map = UserEmailMap.query() \
             .filter(UserEmailMap.user == c.user).all()
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def emails_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         email = self.request.POST.get('new_email')
         user_data = c.user.get_api_data()
         try:
 
             form = UserExtraEmailForm(self.request.translate)()
             data = form.to_python({'email': email})
             email = data['email']
 
             UserModel().add_extra_email(c.user.user_id, email)
             audit_logger.store_web(
                 'user.edit.email.add',
                 action_data={'email': email, 'user': user_data},
                 user=self._rhodecode_user)
             Session().commit()
             h.flash(_("Added new email address `%s` for user account") % email,
                     category='success')
         except formencode.Invalid as error:
             h.flash(h.escape(error.error_dict['email']), category='error')
         except IntegrityError:
             log.warning("Email %s already exists", email)
             h.flash(_('Email `{}` is already registered for another user.').format(email),
                     category='error')
         except Exception:
             log.exception("Exception during email saving")
             h.flash(_('An error occurred during email saving'),
                     category='error')
         raise HTTPFound(h.route_path('edit_user_emails', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def emails_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         email_id = self.request.POST.get('del_email_id')
         user_model = UserModel()
 
         email = UserEmailMap.query().get(email_id).email
         user_data = c.user.get_api_data()
         user_model.delete_extra_email(c.user.user_id, email_id)
         audit_logger.store_web(
             'user.edit.email.delete',
             action_data={'email': email, 'user': user_data},
             user=self._rhodecode_user)
         Session().commit()
         h.flash(_("Removed email address from user account"),
                 category='success')
         raise HTTPFound(h.route_path('edit_user_emails', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def ips(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'ips'
         c.user_ip_map = UserIpMap.query() \
             .filter(UserIpMap.user == c.user).all()
 
         c.inherit_default_ips = c.user.inherit_default_permissions
         c.default_user_ip_map = UserIpMap.query() \
             .filter(UserIpMap.user == User.get_default_user()).all()
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     # NOTE(marcink): this view is allowed for default users, as we can
     # edit their IP white list
     def ips_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_model = UserModel()
         desc = self.request.POST.get('description')
         try:
             ip_list = user_model.parse_ip_range(
                 self.request.POST.get('new_ip'))
         except Exception as e:
             ip_list = []
             log.exception("Exception during ip saving")
             h.flash(_('An error occurred during ip saving:%s' % (e,)),
                     category='error')
         added = []
         user_data = c.user.get_api_data()
         for ip in ip_list:
             try:
                 form = UserExtraIpForm(self.request.translate)()
                 data = form.to_python({'ip': ip})
                 ip = data['ip']
 
                 user_model.add_extra_ip(c.user.user_id, ip, desc)
                 audit_logger.store_web(
                     'user.edit.ip.add',
                     action_data={'ip': ip, 'user': user_data},
                     user=self._rhodecode_user)
                 Session().commit()
                 added.append(ip)
             except formencode.Invalid as error:
                 msg = error.error_dict['ip']
                 h.flash(msg, category='error')
             except Exception:
                 log.exception("Exception during ip saving")
                 h.flash(_('An error occurred during ip saving'),
                         category='error')
         if added:
             h.flash(
                 _("Added ips %s to user whitelist") % (', '.join(ip_list), ),
                 category='success')
         if 'default_user' in self.request.POST:
             # case for editing global IP list we do it for 'DEFAULT' user
             raise HTTPFound(h.route_path('admin_permissions_ips'))
         raise HTTPFound(h.route_path('edit_user_ips', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     # NOTE(marcink): this view is allowed for default users, as we can
     # edit their IP white list
     def ips_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         ip_id = self.request.POST.get('del_ip_id')
         user_model = UserModel()
         user_data = c.user.get_api_data()
         ip = UserIpMap.query().get(ip_id).ip_addr
         user_model.delete_extra_ip(c.user.user_id, ip_id)
         audit_logger.store_web(
             'user.edit.ip.delete', action_data={'ip': ip, 'user': user_data},
             user=self._rhodecode_user)
         Session().commit()
         h.flash(_("Removed ip address from user whitelist"), category='success')
 
         if 'default_user' in self.request.POST:
             # case for editing global IP list we do it for 'DEFAULT' user
             raise HTTPFound(h.route_path('admin_permissions_ips'))
         raise HTTPFound(h.route_path('edit_user_ips', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def groups_management(self):
         c = self.load_default_context()
         c.user = self.db_user
         c.data = c.user.group_member
 
         groups = [UserGroupModel.get_user_groups_as_dict(group.users_group)
                   for group in c.user.group_member]
-        c.groups = json.dumps(groups)
+        c.groups = ext_json.str_json(groups)
         c.active = 'groups'
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def groups_management_updates(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         user_id = self.db_user_id
         c.user = self.db_user
 
         user_groups = set(self.request.POST.getall('users_group_id'))
         user_groups_objects = []
 
         for ugid in user_groups:
             user_groups_objects.append(
                 UserGroupModel().get_group(safe_int(ugid)))
         user_group_model = UserGroupModel()
         added_to_groups, removed_from_groups = \
             user_group_model.change_groups(c.user, user_groups_objects)
 
         user_data = c.user.get_api_data()
         for user_group_id in added_to_groups:
             user_group = UserGroup.get(user_group_id)
             old_values = user_group.get_api_data()
             audit_logger.store_web(
                 'user_group.edit.member.add',
                 action_data={'user': user_data, 'old_data': old_values},
                 user=self._rhodecode_user)
 
         for user_group_id in removed_from_groups:
             user_group = UserGroup.get(user_group_id)
             old_values = user_group.get_api_data()
             audit_logger.store_web(
                 'user_group.edit.member.delete',
                 action_data={'user': user_data, 'old_data': old_values},
                 user=self._rhodecode_user)
 
         Session().commit()
         c.active = 'user_groups_management'
         h.flash(_("Groups successfully changed"), category='success')
 
         return HTTPFound(h.route_path(
             'edit_user_groups_management', user_id=user_id))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_audit_logs(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'audit'
 
         p = safe_int(self.request.GET.get('page', 1), 1)
 
         filter_term = self.request.GET.get('filter')
         user_log = UserModel().get_user_log(c.user, filter_term)
 
         def url_generator(page_num):
             query_params = {
                 'page': page_num
             }
             if filter_term:
                 query_params['filter'] = filter_term
             return self.request.current_route_path(_query=query_params)
 
         c.audit_logs = SqlPage(
             user_log, page=p, items_per_page=10, url_maker=url_generator)
         c.filter_term = filter_term
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_audit_logs_download(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         user_log = UserModel().get_user_log(c.user, filter_term=None)
 
         audit_log_data = {}
         for entry in user_log:
             audit_log_data[entry.user_log_id] = entry.get_dict()
 
-        response = Response(json.dumps(audit_log_data, indent=4))
-        response.content_disposition = str(
-            'attachment; filename=%s' % 'user_{}_audit_logs.json'.format(c.user.user_id))
+        response = Response(ext_json.formatted_str_json(audit_log_data))
+        response.content_disposition = f'attachment; filename=user_{c.user.user_id}_audit_logs.json'
         response.content_type = 'application/json'
 
         return response
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_perms_summary(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'perms_summary'
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_perms_summary_json(self):
         self.load_default_context()
         perm_user = self.db_user.AuthUser(ip_addr=self.request.remote_addr)
 
         return perm_user.permissions
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def user_caches(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'caches'
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
         c.region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
         c.backend = c.region.backend
         c.user_keys = sorted(c.region.backend.list_keys(prefix=cache_namespace_uid))
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def user_caches_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.user = self.db_user
 
         c.active = 'caches'
         c.perm_user = c.user.AuthUser(ip_addr=self.request.remote_addr)
 
         cache_namespace_uid = 'cache_user_auth.{}'.format(self.db_user.user_id)
         del_keys = rc_cache.clear_cache_namespace('cache_perms', cache_namespace_uid)
 
         h.flash(_("Deleted {} cache keys").format(del_keys), category='success')
 
         return HTTPFound(h.route_path(
             'edit_user_caches', user_id=c.user.user_id))

rhodecode/apps/channelstream/__init__.py

@@ -1,106 +1,107 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 
 from pyramid.events import ApplicationCreated
 from pyramid.settings import asbool
 
 from rhodecode.apps._base import ADMIN_PREFIX
 from rhodecode.lib.ext_json import json
+from rhodecode.lib.str_utils import safe_str
 
 
 def url_gen(request):
     registry = request.registry
     longpoll_url = registry.settings.get('channelstream.longpoll_url', '')
     ws_url = registry.settings.get('channelstream.ws_url', '')
     proxy_url = request.route_url('channelstream_proxy')
     urls = {
         'connect': request.route_path('channelstream_connect'),
         'subscribe': request.route_path('channelstream_subscribe'),
         'longpoll': longpoll_url or proxy_url,
         'ws': ws_url or proxy_url.replace('http', 'ws')
     }
-    return json.dumps(urls)
+    return safe_str(json.dumps(urls))
 
 
 PLUGIN_DEFINITION = {
     'name': 'channelstream',
     'config': {
         'javascript': [],
         'css': [],
         'template_hooks': {
             'plugin_init_template': 'rhodecode:templates/channelstream/plugin_init.mako'
         },
         'url_gen': url_gen,
         'static': None,
         'enabled': False,
         'server': '',
         'secret': ''
     }
 }
 
 
 def maybe_create_history_store(event):
     # create plugin history location
     settings = event.app.registry.settings
     history_dir = settings.get('channelstream.history.location', '')
     if history_dir and not os.path.exists(history_dir):
         os.makedirs(history_dir, 0o750)
 
 
 def includeme(config):
     from rhodecode.apps.channelstream.views import ChannelstreamView
 
     settings = config.registry.settings
     PLUGIN_DEFINITION['config']['enabled'] = asbool(
         settings.get('channelstream.enabled'))
     PLUGIN_DEFINITION['config']['server'] = settings.get(
         'channelstream.server', '')
     PLUGIN_DEFINITION['config']['secret'] = settings.get(
         'channelstream.secret', '')
     PLUGIN_DEFINITION['config']['history.location'] = settings.get(
         'channelstream.history.location', '')
     config.register_rhodecode_plugin(
         PLUGIN_DEFINITION['name'],
         PLUGIN_DEFINITION['config']
     )
     config.add_subscriber(maybe_create_history_store, ApplicationCreated)
 
     config.add_route(
         name='channelstream_connect',
         pattern=ADMIN_PREFIX + '/channelstream/connect')
     config.add_view(
         ChannelstreamView,
         attr='channelstream_connect',
         route_name='channelstream_connect', renderer='json_ext')
 
     config.add_route(
         name='channelstream_subscribe',
         pattern=ADMIN_PREFIX + '/channelstream/subscribe')
     config.add_view(
         ChannelstreamView,
         attr='channelstream_subscribe',
         route_name='channelstream_subscribe', renderer='json_ext')
 
     config.add_route(
         name='channelstream_proxy',
         pattern=settings.get('channelstream.proxy_path') or '/_channelstream')
 

rhodecode/apps/gist/views.py

@@ -1,386 +1,386 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2013-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import time
 import logging
 
 import formencode
 import formencode.htmlfill
 import peppercorn
 
 from pyramid.httpexceptions import HTTPNotFound, HTTPFound, HTTPBadRequest
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.apps._base import BaseAppView
-from rhodecode.lib import helpers as h
+from rhodecode.lib import helpers as h, ext_json
 from rhodecode.lib.auth import LoginRequired, NotAnonymous, CSRFRequired
 from rhodecode.lib.utils2 import time_to_datetime
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.vcs.exceptions import VCSError, NodeNotChangedError
 from rhodecode.model.gist import GistModel
 from rhodecode.model.meta import Session
 from rhodecode.model.db import Gist, User, or_
 from rhodecode.model import validation_schema
 from rhodecode.model.validation_schema.schemas import gist_schema
 
 
 log = logging.getLogger(__name__)
 
 
 class GistView(BaseAppView):
 
     def load_default_context(self):
         _ = self.request.translate
         c = self._get_local_tmpl_context()
         c.user = c.auth_user.get_instance()
 
         c.lifetime_values = [
             (-1, _('forever')),
             (5, _('5 minutes')),
             (60, _('1 hour')),
             (60 * 24, _('1 day')),
             (60 * 24 * 30, _('1 month')),
         ]
 
         c.lifetime_options = [(c.lifetime_values, _("Lifetime"))]
         c.acl_options = [
             (Gist.ACL_LEVEL_PRIVATE, _("Requires registered account")),
             (Gist.ACL_LEVEL_PUBLIC, _("Can be accessed by anonymous users"))
         ]
 
         return c
 
     @LoginRequired()
     def gist_show_all(self):
         c = self.load_default_context()
 
         not_default_user = self._rhodecode_user.username != User.DEFAULT_USER
         c.show_private = self.request.GET.get('private') and not_default_user
         c.show_public = self.request.GET.get('public') and not_default_user
         c.show_all = self.request.GET.get('all') and self._rhodecode_user.admin
 
         gists = _gists = Gist().query()\
             .filter(or_(Gist.gist_expires == -1, Gist.gist_expires >= time.time()))\
             .order_by(Gist.created_on.desc())
 
         c.active = 'public'
         # MY private
         if c.show_private and not c.show_public:
             gists = _gists.filter(Gist.gist_type == Gist.GIST_PRIVATE)\
                         .filter(Gist.gist_owner == self._rhodecode_user.user_id)
             c.active = 'my_private'
         # MY public
         elif c.show_public and not c.show_private:
             gists = _gists.filter(Gist.gist_type == Gist.GIST_PUBLIC)\
                         .filter(Gist.gist_owner == self._rhodecode_user.user_id)
             c.active = 'my_public'
         # MY public+private
         elif c.show_private and c.show_public:
             gists = _gists.filter(or_(Gist.gist_type == Gist.GIST_PUBLIC,
                                       Gist.gist_type == Gist.GIST_PRIVATE))\
                         .filter(Gist.gist_owner == self._rhodecode_user.user_id)
             c.active = 'my_all'
         # Show all by super-admin
         elif c.show_all:
             c.active = 'all'
             gists = _gists
 
         # default show ALL public gists
         if not c.show_public and not c.show_private and not c.show_all:
             gists = _gists.filter(Gist.gist_type == Gist.GIST_PUBLIC)
             c.active = 'public'
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         data = []
 
         for gist in gists:
             data.append({
                 'created_on': _render('gist_created', gist.created_on),
                 'created_on_raw': gist.created_on,
                 'type': _render('gist_type', gist.gist_type),
                 'access_id': _render('gist_access_id', gist.gist_access_id, gist.owner.full_contact),
                 'author': _render('gist_author', gist.owner.full_contact, gist.created_on, gist.gist_expires),
                 'author_raw': h.escape(gist.owner.full_contact),
                 'expires': _render('gist_expires', gist.gist_expires),
                 'description': _render('gist_description', gist.gist_description)
             })
-        c.data = json.dumps(data)
+        c.data = ext_json.str_json(data)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     def gist_new(self):
         c = self.load_default_context()
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def gist_create(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         data = dict(self.request.POST)
         data['filename'] = data.get('filename') or Gist.DEFAULT_FILENAME
 
         data['nodes'] = [{
             'filename': data['filename'],
             'content': data.get('content'),
             'mimetype': data.get('mimetype')  # None is autodetect
         }]
         gist_type = {
             'public': Gist.GIST_PUBLIC,
             'private': Gist.GIST_PRIVATE
         }.get(data.get('gist_type')) or Gist.GIST_PRIVATE
 
         data['gist_type'] = gist_type
 
         data['gist_acl_level'] = (
             data.get('gist_acl_level') or Gist.ACL_LEVEL_PRIVATE)
 
         schema = gist_schema.GistSchema().bind(
             lifetime_options=[x[0] for x in c.lifetime_values])
 
         try:
 
             schema_data = schema.deserialize(data)
             # convert to safer format with just KEYs so we sure no duplicates
             schema_data['nodes'] = gist_schema.sequence_to_nodes(
                 schema_data['nodes'])
 
             gist = GistModel().create(
                 gist_id=schema_data['gistid'],  # custom access id not real ID
                 description=schema_data['description'],
                 owner=self._rhodecode_user.user_id,
                 gist_mapping=schema_data['nodes'],
                 gist_type=schema_data['gist_type'],
                 lifetime=schema_data['lifetime'],
                 gist_acl_level=schema_data['gist_acl_level']
             )
             Session().commit()
             new_gist_id = gist.gist_access_id
         except validation_schema.Invalid as errors:
             defaults = data
             errors = errors.asdict()
 
             if 'nodes.0.content' in errors:
                 errors['content'] = errors['nodes.0.content']
                 del errors['nodes.0.content']
             if 'nodes.0.filename' in errors:
                 errors['filename'] = errors['nodes.0.filename']
                 del errors['nodes.0.filename']
 
             data = render('rhodecode:templates/admin/gists/gist_new.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=defaults,
                 errors=errors,
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         except Exception:
             log.exception("Exception while trying to create a gist")
             h.flash(_('Error occurred during gist creation'), category='error')
             raise HTTPFound(h.route_url('gists_new'))
         raise HTTPFound(h.route_url('gist_show', gist_id=new_gist_id))
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def gist_delete(self):
         _ = self.request.translate
         gist_id = self.request.matchdict['gist_id']
 
         c = self.load_default_context()
         c.gist = Gist.get_or_404(gist_id)
 
         owner = c.gist.gist_owner == self._rhodecode_user.user_id
         if not (h.HasPermissionAny('hg.admin')() or owner):
             log.warning('Deletion of Gist was forbidden '
                         'by unauthorized user: `%s`', self._rhodecode_user)
             raise HTTPNotFound()
 
         GistModel().delete(c.gist)
         Session().commit()
         h.flash(_('Deleted gist %s') % c.gist.gist_access_id, category='success')
 
         raise HTTPFound(h.route_url('gists_show'))
 
     def _get_gist(self, gist_id):
 
         gist = Gist.get_or_404(gist_id)
 
         # Check if this gist is expired
         if gist.gist_expires != -1:
             if time.time() > gist.gist_expires:
                 log.error(
                     'Gist expired at %s', time_to_datetime(gist.gist_expires))
                 raise HTTPNotFound()
 
         # check if this gist requires a login
         is_default_user = self._rhodecode_user.username == User.DEFAULT_USER
         if gist.acl_level == Gist.ACL_LEVEL_PRIVATE and is_default_user:
             log.error("Anonymous user %s tried to access protected gist `%s`",
                       self._rhodecode_user, gist_id)
             raise HTTPNotFound()
         return gist
 
     @LoginRequired()
     def gist_show(self):
         gist_id = self.request.matchdict['gist_id']
 
         # TODO(marcink): expose those via matching dict
         revision = self.request.matchdict.get('revision', 'tip')
         f_path = self.request.matchdict.get('f_path', None)
         return_format = self.request.matchdict.get('format')
 
         c = self.load_default_context()
         c.gist = self._get_gist(gist_id)
         c.render = not self.request.GET.get('no-render', False)
 
         try:
             c.file_last_commit, c.files = GistModel().get_gist_files(
                 gist_id, revision=revision)
         except VCSError:
             log.exception("Exception in gist show")
             raise HTTPNotFound()
 
         if return_format == 'raw':
             content = '\n\n'.join([f.content for f in c.files
                                    if (f_path is None or f.path == f_path)])
             response = Response(content)
             response.content_type = 'text/plain'
             return response
         elif return_format:
             raise HTTPBadRequest()
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     def gist_edit(self):
         _ = self.request.translate
         gist_id = self.request.matchdict['gist_id']
         c = self.load_default_context()
         c.gist = self._get_gist(gist_id)
 
         owner = c.gist.gist_owner == self._rhodecode_user.user_id
         if not (h.HasPermissionAny('hg.admin')() or owner):
             raise HTTPNotFound()
 
         try:
             c.file_last_commit, c.files = GistModel().get_gist_files(gist_id)
         except VCSError:
             log.exception("Exception in gist edit")
             raise HTTPNotFound()
 
         if c.gist.gist_expires == -1:
             expiry = _('never')
         else:
             # this cannot use timeago, since it's used in select2 as a value
             expiry = h.age(h.time_to_datetime(c.gist.gist_expires))
 
         c.lifetime_values.append(
             (0, _('%(expiry)s - current value') % {'expiry': _(expiry)})
         )
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def gist_update(self):
         _ = self.request.translate
         gist_id = self.request.matchdict['gist_id']
         c = self.load_default_context()
         c.gist = self._get_gist(gist_id)
 
         owner = c.gist.gist_owner == self._rhodecode_user.user_id
         if not (h.HasPermissionAny('hg.admin')() or owner):
             raise HTTPNotFound()
 
         data = peppercorn.parse(self.request.POST.items())
 
         schema = gist_schema.GistSchema()
         schema = schema.bind(
             # '0' is special value to leave lifetime untouched
             lifetime_options=[x[0] for x in c.lifetime_values] + [0],
         )
 
         try:
             schema_data = schema.deserialize(data)
             # convert to safer format with just KEYs so we sure no duplicates
             schema_data['nodes'] = gist_schema.sequence_to_nodes(
                 schema_data['nodes'])
 
             GistModel().update(
                 gist=c.gist,
                 description=schema_data['description'],
                 owner=c.gist.owner,
                 gist_mapping=schema_data['nodes'],
                 lifetime=schema_data['lifetime'],
                 gist_acl_level=schema_data['gist_acl_level']
             )
 
             Session().commit()
             h.flash(_('Successfully updated gist content'), category='success')
         except NodeNotChangedError:
             # raised if nothing was changed in repo itself. We anyway then
             # store only DB stuff for gist
             Session().commit()
             h.flash(_('Successfully updated gist data'), category='success')
         except validation_schema.Invalid as errors:
             errors = h.escape(errors.asdict())
             h.flash(_('Error occurred during update of gist {}: {}').format(
                 gist_id, errors), category='error')
         except Exception:
             log.exception("Exception in gist edit")
             h.flash(_('Error occurred during update of gist %s') % gist_id,
                     category='error')
 
         raise HTTPFound(h.route_url('gist_show', gist_id=gist_id))
 
     @LoginRequired()
     @NotAnonymous()
     def gist_edit_check_revision(self):
         _ = self.request.translate
         gist_id = self.request.matchdict['gist_id']
         c = self.load_default_context()
         c.gist = self._get_gist(gist_id)
 
         last_rev = c.gist.scm_instance().get_commit()
         success = True
         revision = self.request.GET.get('revision')
 
         if revision != last_rev.raw_id:
             log.error('Last revision %s is different then submitted %s',
                       revision, last_rev)
             # our gist has newer version than we
             success = False
 
         return {'success': success}

rhodecode/apps/my_account/views/my_account.py

@@ -1,783 +1,783 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import datetime
 import string
 
 import formencode
 import formencode.htmlfill
 import peppercorn
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 
 from rhodecode.apps._base import BaseAppView, DataGridAppView
 from rhodecode import forms
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
-from rhodecode.lib.ext_json import json
+from rhodecode.lib import ext_json
 from rhodecode.lib.auth import (
     LoginRequired, NotAnonymous, CSRFRequired,
     HasRepoPermissionAny, HasRepoGroupPermissionAny, AuthUser)
 from rhodecode.lib.channelstream import (
     channelstream_request, ChannelstreamException)
 from rhodecode.lib.utils2 import safe_int, md5, str2bool
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
     IntegrityError, or_, in_filter_generator,
     Repository, UserEmailMap, UserApiKeys, UserFollowing,
     PullRequest, UserBookmark, RepoGroup, ChangesetStatus)
 from rhodecode.model.meta import Session
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.model.validation_schema.schemas import user_schema
 
 log = logging.getLogger(__name__)
 
 
 class MyAccountView(BaseAppView, DataGridAppView):
     ALLOW_SCOPED_TOKENS = False
     """
     This view has alternative version inside EE, if modified please take a look
     in there as well.
     """
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         c.user = c.auth_user.get_instance()
         c.allow_scoped_tokens = self.ALLOW_SCOPED_TOKENS
         return c
 
     @LoginRequired()
     @NotAnonymous()
     def my_account_profile(self):
         c = self.load_default_context()
         c.active = 'profile'
         c.extern_type = c.user.extern_type
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     def my_account_edit(self):
         c = self.load_default_context()
         c.active = 'profile_edit'
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
 
         schema = user_schema.UserProfileSchema().bind(
             username=c.user.username, user_emails=c.user.emails)
         appstruct = {
             'username': c.user.username,
             'email': c.user.email,
             'firstname': c.user.firstname,
             'lastname': c.user.lastname,
             'description': c.user.description,
         }
         c.form = forms.RcForm(
             schema, appstruct=appstruct,
             action=h.route_path('my_account_update'),
             buttons=(forms.buttons.save, forms.buttons.reset))
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def my_account_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'profile_edit'
         c.perm_user = c.auth_user
         c.extern_type = c.user.extern_type
         c.extern_name = c.user.extern_name
 
         schema = user_schema.UserProfileSchema().bind(
             username=c.user.username, user_emails=c.user.emails)
         form = forms.RcForm(
             schema, buttons=(forms.buttons.save, forms.buttons.reset))
 
         controls = self.request.POST.items()
         try:
             valid_data = form.validate(controls)
             skip_attrs = ['admin', 'active', 'extern_type', 'extern_name',
                           'new_password', 'password_confirmation']
             if c.extern_type != "rhodecode":
                 # forbid updating username for external accounts
                 skip_attrs.append('username')
             old_email = c.user.email
             UserModel().update_user(
                      self._rhodecode_user.user_id, skip_attrs=skip_attrs,
                      **valid_data)
             if old_email != valid_data['email']:
                 old = UserEmailMap.query() \
                     .filter(UserEmailMap.user == c.user)\
                     .filter(UserEmailMap.email == valid_data['email'])\
                     .first()
                 old.email = old_email
             h.flash(_('Your account was updated successfully'), category='success')
             Session().commit()
         except forms.ValidationFailure as e:
             c.form = e
             return self._get_template_context(c)
         except Exception:
             log.exception("Exception updating user")
             h.flash(_('Error occurred during update of user'),
                     category='error')
         raise HTTPFound(h.route_path('my_account_profile'))
 
     @LoginRequired()
     @NotAnonymous()
     def my_account_password(self):
         c = self.load_default_context()
         c.active = 'password'
         c.extern_type = c.user.extern_type
 
         schema = user_schema.ChangePasswordSchema().bind(
             username=c.user.username)
 
         form = forms.Form(
             schema,
             action=h.route_path('my_account_password_update'),
             buttons=(forms.buttons.save, forms.buttons.reset))
 
         c.form = form
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def my_account_password_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'password'
         c.extern_type = c.user.extern_type
 
         schema = user_schema.ChangePasswordSchema().bind(
             username=c.user.username)
 
         form = forms.Form(
             schema, buttons=(forms.buttons.save, forms.buttons.reset))
 
         if c.extern_type != 'rhodecode':
             raise HTTPFound(self.request.route_path('my_account_password'))
 
         controls = self.request.POST.items()
         try:
             valid_data = form.validate(controls)
             UserModel().update_user(c.user.user_id, **valid_data)
             c.user.update_userdata(force_password_change=False)
             Session().commit()
         except forms.ValidationFailure as e:
             c.form = e
             return self._get_template_context(c)
 
         except Exception:
             log.exception("Exception updating password")
             h.flash(_('Error occurred during update of user password'),
                     category='error')
         else:
             instance = c.auth_user.get_instance()
             self.session.setdefault('rhodecode_user', {}).update(
                 {'password': md5(instance.password)})
             self.session.save()
             h.flash(_("Successfully updated password"), category='success')
 
         raise HTTPFound(self.request.route_path('my_account_password'))
 
     @LoginRequired()
     @NotAnonymous()
     def my_account_auth_tokens(self):
         _ = self.request.translate
 
         c = self.load_default_context()
         c.active = 'auth_tokens'
         c.lifetime_values = AuthTokenModel.get_lifetime_values(translator=_)
         c.role_values = [
             (x, AuthTokenModel.cls._get_role_name(x))
             for x in AuthTokenModel.cls.ROLES]
         c.role_options = [(c.role_values, _("Role"))]
         c.user_auth_tokens = AuthTokenModel().get_auth_tokens(
             c.user.user_id, show_expired=True)
         c.role_vcs = AuthTokenModel.cls.ROLE_VCS
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def my_account_auth_tokens_view(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         auth_token_id = self.request.POST.get('auth_token_id')
 
         if auth_token_id:
             token = UserApiKeys.get_or_404(auth_token_id)
             if token.user.user_id != c.user.user_id:
                 raise HTTPNotFound()
 
             return {
                 'auth_token': token.api_key
             }
 
     def maybe_attach_token_scope(self, token):
         # implemented in EE edition
         pass
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def my_account_auth_tokens_add(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         lifetime = safe_int(self.request.POST.get('lifetime'), -1)
         description = self.request.POST.get('description')
         role = self.request.POST.get('role')
 
         token = UserModel().add_auth_token(
             user=c.user.user_id,
             lifetime_minutes=lifetime, role=role, description=description,
             scope_callback=self.maybe_attach_token_scope)
         token_data = token.get_api_data()
 
         audit_logger.store_web(
             'user.edit.token.add', action_data={
                 'data': {'token': token_data, 'user': 'self'}},
             user=self._rhodecode_user, )
         Session().commit()
 
         h.flash(_("Auth token successfully created"), category='success')
         return HTTPFound(h.route_path('my_account_auth_tokens'))
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def my_account_auth_tokens_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         del_auth_token = self.request.POST.get('del_auth_token')
 
         if del_auth_token:
             token = UserApiKeys.get_or_404(del_auth_token)
             token_data = token.get_api_data()
 
             AuthTokenModel().delete(del_auth_token, c.user.user_id)
             audit_logger.store_web(
                 'user.edit.token.delete', action_data={
                     'data': {'token': token_data, 'user': 'self'}},
                 user=self._rhodecode_user,)
             Session().commit()
             h.flash(_("Auth token successfully deleted"), category='success')
 
         return HTTPFound(h.route_path('my_account_auth_tokens'))
 
     @LoginRequired()
     @NotAnonymous()
     def my_account_emails(self):
         _ = self.request.translate
 
         c = self.load_default_context()
         c.active = 'emails'
 
         c.user_email_map = UserEmailMap.query()\
             .filter(UserEmailMap.user == c.user).all()
 
         schema = user_schema.AddEmailSchema().bind(
             username=c.user.username, user_emails=c.user.emails)
 
         form = forms.RcForm(schema,
                             action=h.route_path('my_account_emails_add'),
                             buttons=(forms.buttons.save, forms.buttons.reset))
 
         c.form = form
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def my_account_emails_add(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'emails'
 
         schema = user_schema.AddEmailSchema().bind(
             username=c.user.username, user_emails=c.user.emails)
 
         form = forms.RcForm(
             schema, action=h.route_path('my_account_emails_add'),
             buttons=(forms.buttons.save, forms.buttons.reset))
 
         controls = self.request.POST.items()
         try:
             valid_data = form.validate(controls)
             UserModel().add_extra_email(c.user.user_id, valid_data['email'])
             audit_logger.store_web(
                 'user.edit.email.add', action_data={
                     'data': {'email': valid_data['email'], 'user': 'self'}},
                 user=self._rhodecode_user,)
             Session().commit()
         except formencode.Invalid as error:
             h.flash(h.escape(error.error_dict['email']), category='error')
         except forms.ValidationFailure as e:
             c.user_email_map = UserEmailMap.query() \
                 .filter(UserEmailMap.user == c.user).all()
             c.form = e
             return self._get_template_context(c)
         except Exception:
             log.exception("Exception adding email")
             h.flash(_('Error occurred during adding email'),
                     category='error')
         else:
             h.flash(_("Successfully added email"), category='success')
 
         raise HTTPFound(self.request.route_path('my_account_emails'))
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def my_account_emails_delete(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         del_email_id = self.request.POST.get('del_email_id')
         if del_email_id:
             email = UserEmailMap.get_or_404(del_email_id).email
             UserModel().delete_extra_email(c.user.user_id, del_email_id)
             audit_logger.store_web(
                 'user.edit.email.delete', action_data={
                     'data': {'email': email, 'user': 'self'}},
                 user=self._rhodecode_user,)
             Session().commit()
             h.flash(_("Email successfully deleted"),
                     category='success')
         return HTTPFound(h.route_path('my_account_emails'))
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def my_account_notifications_test_channelstream(self):
         message = 'Test message sent via Channelstream by user: {}, on {}'.format(
             self._rhodecode_user.username, datetime.datetime.now())
         payload = {
             # 'channel': 'broadcast',
             'type': 'message',
             'timestamp': datetime.datetime.utcnow(),
             'user': 'system',
             'pm_users': [self._rhodecode_user.username],
             'message': {
                 'message': message,
                 'level': 'info',
                 'topic': '/notifications'
             }
         }
 
         registry = self.request.registry
         rhodecode_plugins = getattr(registry, 'rhodecode_plugins', {})
         channelstream_config = rhodecode_plugins.get('channelstream', {})
 
         try:
             channelstream_request(channelstream_config, [payload], '/message')
         except ChannelstreamException as e:
             log.exception('Failed to send channelstream data')
             return {"response": 'ERROR: {}'.format(e.__class__.__name__)}
         return {"response": 'Channelstream data sent. '
                             'You should see a new live message now.'}
 
     def _load_my_repos_data(self, watched=False):
 
         allowed_ids = [-1] + self._rhodecode_user.repo_acl_ids_from_stack(AuthUser.repo_read_perms)
 
         if watched:
             # repos user watch
             repo_list = Session().query(
                     Repository
                 ) \
                 .join(
                     (UserFollowing, UserFollowing.follows_repo_id == Repository.repo_id)
                 ) \
                 .filter(
                     UserFollowing.user_id == self._rhodecode_user.user_id
                 ) \
                 .filter(or_(
                     # generate multiple IN to fix limitation problems
                     *in_filter_generator(Repository.repo_id, allowed_ids))
                 ) \
                 .order_by(Repository.repo_name) \
                 .all()
 
         else:
             # repos user is owner of
             repo_list = Session().query(
                     Repository
                 ) \
                 .filter(
                     Repository.user_id == self._rhodecode_user.user_id
                 ) \
                 .filter(or_(
                     # generate multiple IN to fix limitation problems
                     *in_filter_generator(Repository.repo_id, allowed_ids))
                 ) \
                 .order_by(Repository.repo_name) \
                 .all()
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         def repo_lnk(name, rtype, rstate, private, archived, fork_of):
             return _render('repo_name', name, rtype, rstate, private, archived, fork_of,
                            short_name=False, admin=False)
 
         repos_data = []
         for repo in repo_list:
             row = {
                 "name": repo_lnk(repo.repo_name, repo.repo_type, repo.repo_state,
                                  repo.private, repo.archived, repo.fork),
                 "name_raw": repo.repo_name.lower(),
             }
 
             repos_data.append(row)
 
         # json used to render the grid
-        return json.dumps(repos_data)
+        return ext_json.str_json(repos_data)
 
     @LoginRequired()
     @NotAnonymous()
     def my_account_repos(self):
         c = self.load_default_context()
         c.active = 'repos'
 
         # json used to render the grid
         c.data = self._load_my_repos_data()
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     def my_account_watched(self):
         c = self.load_default_context()
         c.active = 'watched'
 
         # json used to render the grid
         c.data = self._load_my_repos_data(watched=True)
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     def my_account_bookmarks(self):
         c = self.load_default_context()
         c.active = 'bookmarks'
         c.bookmark_items = UserBookmark.get_bookmarks_for_user(
             self._rhodecode_db_user.user_id, cache=False)
         return self._get_template_context(c)
 
     def _process_bookmark_entry(self, entry, user_id):
         position = safe_int(entry.get('position'))
         cur_position = safe_int(entry.get('cur_position'))
         if position is None:
             return
 
         # check if this is an existing entry
         is_new = False
         db_entry = UserBookmark().get_by_position_for_user(cur_position, user_id)
 
         if db_entry and str2bool(entry.get('remove')):
             log.debug('Marked bookmark %s for deletion', db_entry)
             Session().delete(db_entry)
             return
 
         if not db_entry:
             # new
             db_entry = UserBookmark()
             is_new = True
 
         should_save = False
         default_redirect_url = ''
 
         # save repo
         if entry.get('bookmark_repo') and safe_int(entry.get('bookmark_repo')):
             repo = Repository.get(entry['bookmark_repo'])
             perm_check = HasRepoPermissionAny(
                 'repository.read', 'repository.write', 'repository.admin')
             if repo and perm_check(repo_name=repo.repo_name):
                 db_entry.repository = repo
                 should_save = True
                 default_redirect_url = '${repo_url}'
         # save repo group
         elif entry.get('bookmark_repo_group') and safe_int(entry.get('bookmark_repo_group')):
             repo_group = RepoGroup.get(entry['bookmark_repo_group'])
             perm_check = HasRepoGroupPermissionAny(
                 'group.read', 'group.write', 'group.admin')
 
             if repo_group and perm_check(group_name=repo_group.group_name):
                 db_entry.repository_group = repo_group
                 should_save = True
                 default_redirect_url = '${repo_group_url}'
         # save generic info
         elif entry.get('title') and entry.get('redirect_url'):
             should_save = True
 
         if should_save:
             # mark user and position
             db_entry.user_id = user_id
             db_entry.position = position
             db_entry.title = entry.get('title')
             db_entry.redirect_url = entry.get('redirect_url') or default_redirect_url
             log.debug('Saving bookmark %s, new:%s', db_entry, is_new)
 
             Session().add(db_entry)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def my_account_bookmarks_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'bookmarks'
 
         controls = peppercorn.parse(self.request.POST.items())
         user_id = c.user.user_id
 
         # validate positions
         positions = {}
         for entry in controls.get('bookmarks', []):
             position = safe_int(entry['position'])
             if position is None:
                 continue
 
             if position in positions:
                 h.flash(_("Position {} is defined twice. "
                           "Please correct this error.").format(position), category='error')
                 return HTTPFound(h.route_path('my_account_bookmarks'))
 
             entry['position'] = position
             entry['cur_position'] = safe_int(entry.get('cur_position'))
             positions[position] = entry
 
         try:
             for entry in positions.values():
                 self._process_bookmark_entry(entry, user_id)
 
             Session().commit()
             h.flash(_("Update Bookmarks"), category='success')
         except IntegrityError:
             h.flash(_("Failed to update bookmarks. "
                       "Make sure an unique position is used."), category='error')
 
         return HTTPFound(h.route_path('my_account_bookmarks'))
 
     @LoginRequired()
     @NotAnonymous()
     def my_account_goto_bookmark(self):
 
         bookmark_id = self.request.matchdict['bookmark_id']
         user_bookmark = UserBookmark().query()\
             .filter(UserBookmark.user_id == self.request.user.user_id) \
             .filter(UserBookmark.position == bookmark_id).scalar()
 
         redirect_url = h.route_path('my_account_bookmarks')
         if not user_bookmark:
             raise HTTPFound(redirect_url)
 
         # repository set
         if user_bookmark.repository:
             repo_name = user_bookmark.repository.repo_name
             base_redirect_url = h.route_path(
                 'repo_summary', repo_name=repo_name)
             if user_bookmark.redirect_url and \
                     '${repo_url}' in user_bookmark.redirect_url:
                 redirect_url = string.Template(user_bookmark.redirect_url)\
                     .safe_substitute({'repo_url': base_redirect_url})
             else:
                 redirect_url = base_redirect_url
         # repository group set
         elif user_bookmark.repository_group:
             repo_group_name = user_bookmark.repository_group.group_name
             base_redirect_url = h.route_path(
                 'repo_group_home', repo_group_name=repo_group_name)
             if user_bookmark.redirect_url and \
                     '${repo_group_url}' in user_bookmark.redirect_url:
                 redirect_url = string.Template(user_bookmark.redirect_url)\
                     .safe_substitute({'repo_group_url': base_redirect_url})
             else:
                 redirect_url = base_redirect_url
         # custom URL set
         elif user_bookmark.redirect_url:
             server_url = h.route_url('home').rstrip('/')
             redirect_url = string.Template(user_bookmark.redirect_url) \
                 .safe_substitute({'server_url': server_url})
 
         log.debug('Redirecting bookmark %s to %s', user_bookmark, redirect_url)
         raise HTTPFound(redirect_url)
 
     @LoginRequired()
     @NotAnonymous()
     def my_account_perms(self):
         c = self.load_default_context()
         c.active = 'perms'
 
         c.perm_user = c.auth_user
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     def my_notifications(self):
         c = self.load_default_context()
         c.active = 'notifications'
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def my_notifications_toggle_visibility(self):
         user = self._rhodecode_db_user
         new_status = not user.user_data.get('notification_status', True)
         user.update_userdata(notification_status=new_status)
         Session().commit()
         return user.user_data['notification_status']
 
     def _get_pull_requests_list(self, statuses, filter_type=None):
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(self.request)
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         if filter_type == 'awaiting_my_review':
             pull_requests = PullRequestModel().get_im_participating_in_for_review(
                 user_id=self._rhodecode_user.user_id,
                 statuses=statuses, query=search_q,
                 offset=start, length=limit, order_by=order_by,
                 order_dir=order_dir)
 
             pull_requests_total_count = PullRequestModel().count_im_participating_in_for_review(
                 user_id=self._rhodecode_user.user_id, statuses=statuses, query=search_q)
         else:
             pull_requests = PullRequestModel().get_im_participating_in(
                 user_id=self._rhodecode_user.user_id,
                 statuses=statuses, query=search_q,
                 offset=start, length=limit, order_by=order_by,
                 order_dir=order_dir)
 
             pull_requests_total_count = PullRequestModel().count_im_participating_in(
                 user_id=self._rhodecode_user.user_id, statuses=statuses, query=search_q)
 
         data = []
         comments_model = CommentsModel()
         for pr in pull_requests:
             repo_id = pr.target_repo_id
             comments_count = comments_model.get_all_comments(
                 repo_id, pull_request=pr, include_drafts=False, count_only=True)
             owned = pr.user_id == self._rhodecode_user.user_id
 
             review_statuses = pr.reviewers_statuses(user=self._rhodecode_db_user)
             my_review_status = ChangesetStatus.STATUS_NOT_REVIEWED
             if review_statuses and review_statuses[4]:
                 _review_obj, _user, _reasons, _mandatory, statuses = review_statuses
                 my_review_status = statuses[0][1].status
 
             data.append({
                 'target_repo': _render('pullrequest_target_repo',
                                        pr.target_repo.repo_name),
                 'name': _render('pullrequest_name',
                                 pr.pull_request_id, pr.pull_request_state,
                                 pr.work_in_progress, pr.target_repo.repo_name,
                                 short=True),
                 'name_raw': pr.pull_request_id,
                 'status': _render('pullrequest_status',
                                   pr.calculated_review_status()),
                 'my_status': _render('pullrequest_status',
                                      my_review_status),
                 'title': _render('pullrequest_title', pr.title, pr.description),
                 'description': h.escape(pr.description),
                 'updated_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.updated_on),
                                       pr.versions_count),
                 'updated_on_raw': h.datetime_to_time(pr.updated_on),
                 'created_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.created_on)),
                 'created_on_raw': h.datetime_to_time(pr.created_on),
                 'state': pr.pull_request_state,
                 'author': _render('pullrequest_author',
                                   pr.author.full_contact, ),
                 'author_raw': pr.author.full_name,
                 'comments': _render('pullrequest_comments', comments_count),
                 'comments_raw': comments_count,
                 'closed': pr.is_closed(),
                 'owned': owned
             })
 
         # json used to render the grid
         data = ({
             'draw': draw,
             'data': data,
             'recordsTotal': pull_requests_total_count,
             'recordsFiltered': pull_requests_total_count,
         })
         return data
 
     @LoginRequired()
     @NotAnonymous()
     def my_account_pullrequests(self):
         c = self.load_default_context()
         c.active = 'pullrequests'
         req_get = self.request.GET
 
         c.closed = str2bool(req_get.get('closed'))
         c.awaiting_my_review = str2bool(req_get.get('awaiting_my_review'))
 
         c.selected_filter = 'all'
         if c.closed:
             c.selected_filter = 'all_closed'
         if c.awaiting_my_review:
             c.selected_filter = 'awaiting_my_review'
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     def my_account_pullrequests_data(self):
         self.load_default_context()
         req_get = self.request.GET
 
         awaiting_my_review = str2bool(req_get.get('awaiting_my_review'))
         closed = str2bool(req_get.get('closed'))
 
         statuses = [PullRequest.STATUS_NEW, PullRequest.STATUS_OPEN]
         if closed:
             statuses += [PullRequest.STATUS_CLOSED]
 
         filter_type = \
             'awaiting_my_review' if awaiting_my_review \
             else None
 
         data = self._get_pull_requests_list(statuses=statuses, filter_type=filter_type)
         return data
 
     @LoginRequired()
     @NotAnonymous()
     def my_account_user_group_membership(self):
         c = self.load_default_context()
         c.active = 'user_group_membership'
         groups = [UserGroupModel.get_user_groups_as_dict(group.users_group)
                   for group in self._rhodecode_db_user.group_member]
-        c.user_groups = json.dumps(groups)
+        c.user_groups = ext_json.str_json(groups)
         return self._get_template_context(c)

rhodecode/apps/repository/views/repo_bookmarks.py

@@ -1,51 +1,50 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import logging
 
 from pyramid.httpexceptions import HTTPNotFound
 
-
 from rhodecode.apps._base import BaseReferencesView
-from rhodecode.lib.ext_json import json
+from rhodecode.lib import ext_json
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (LoginRequired, HasRepoPermissionAnyDecorator)
 
 
 log = logging.getLogger(__name__)
 
 
 class RepoBookmarksView(BaseReferencesView):
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def bookmarks(self):
         c = self.load_default_context()
 
         if not h.is_hg(self.db_repo):
             raise HTTPNotFound()
 
         ref_items = self.rhodecode_vcs_repo.bookmarks.items()
         data = self.load_refs_context(
             ref_items=ref_items, partials_template='bookmarks/bookmarks_data.mako')
 
         c.has_references = bool(data)
-        c.data = json.dumps(data)
+        c.data = ext_json.str_json(data)
         return self._get_template_context(c)

rhodecode/apps/repository/views/repo_branches.py

@@ -1,46 +1,46 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 
 from rhodecode.apps._base import BaseReferencesView
-from rhodecode.lib.ext_json import json
+from rhodecode.lib import ext_json
 from rhodecode.lib.auth import (LoginRequired, HasRepoPermissionAnyDecorator)
 
 
 log = logging.getLogger(__name__)
 
 
 class RepoBranchesView(BaseReferencesView):
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def branches(self):
         c = self.load_default_context()
 
         ref_items = self.rhodecode_vcs_repo.branches_all.items()
         data = self.load_refs_context(
             ref_items=ref_items, partials_template='branches/branches_data.mako')
 
         c.has_references = bool(data)
-        c.data = json.dumps(data)
+        c.data = ext_json.str_json(data)
         return self._get_template_context(c)

rhodecode/apps/repository/views/repo_changelog.py

@@ -1,355 +1,356 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import logging
 
 from pyramid.httpexceptions import HTTPNotFound, HTTPFound
 
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.apps._base import RepoAppView
 import rhodecode.lib.helpers as h
+from rhodecode.lib import ext_json
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator)
 
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.graphmod import _colored, _dagwalker
 from rhodecode.lib.helpers import RepoPage
 from rhodecode.lib.utils2 import safe_int, safe_str, str2bool, safe_unicode
 from rhodecode.lib.vcs.exceptions import (
     RepositoryError, CommitDoesNotExistError,
     CommitError, NodeDoesNotExistError, EmptyRepositoryError)
 
 log = logging.getLogger(__name__)
 
 DEFAULT_CHANGELOG_SIZE = 20
 
 
 class RepoChangelogView(RepoAppView):
 
     def _get_commit_or_redirect(self, commit_id, redirect_after=True):
         """
         This is a safe way to get commit. If an error occurs it redirects to
         tip with proper message
 
         :param commit_id: id of commit to fetch
         :param redirect_after: toggle redirection
         """
         _ = self.request.translate
 
         try:
             return self.rhodecode_vcs_repo.get_commit(commit_id)
         except EmptyRepositoryError:
             if not redirect_after:
                 return None
 
             h.flash(h.literal(
                 _('There are no commits yet')), category='warning')
             raise HTTPFound(
                 h.route_path('repo_summary', repo_name=self.db_repo_name))
 
         except (CommitDoesNotExistError, LookupError):
             msg = _('No such commit exists for this repository')
             h.flash(msg, category='error')
             raise HTTPNotFound()
         except RepositoryError as e:
             h.flash(h.escape(safe_str(e)), category='error')
             raise HTTPNotFound()
 
     def _graph(self, repo, commits, prev_data=None, next_data=None):
         """
         Generates a DAG graph for repo
 
         :param repo: repo instance
         :param commits: list of commits
         """
         if not commits:
             return json.dumps([]), json.dumps([])
 
         def serialize(commit, parents=True):
             data = dict(
                 raw_id=commit.raw_id,
                 idx=commit.idx,
                 branch=None,
             )
             if parents:
                 data['parents'] = [
                     serialize(x, parents=False) for x in commit.parents]
             return data
 
         prev_data = prev_data or []
         next_data = next_data or []
 
         current = [serialize(x) for x in commits]
         commits = prev_data + current + next_data
 
         dag = _dagwalker(repo, commits)
 
         data = [[commit_id, vtx, edges, branch]
                 for commit_id, vtx, edges, branch in _colored(dag)]
-        return json.dumps(data), json.dumps(current)
+        return ext_json.str_json(data), ext_json.str_json(current)
 
     def _check_if_valid_branch(self, branch_name, repo_name, f_path):
         if branch_name not in self.rhodecode_vcs_repo.branches_all:
             h.flash(u'Branch {} is not found.'.format(h.escape(safe_unicode(branch_name))),
                     category='warning')
             redirect_url = h.route_path(
                 'repo_commits_file', repo_name=repo_name,
                 commit_id=branch_name, f_path=f_path or '')
             raise HTTPFound(redirect_url)
 
     def _load_changelog_data(
             self, c, collection, page, chunk_size, branch_name=None,
             dynamic=False, f_path=None, commit_id=None):
 
         def url_generator(page_num):
             query_params = {
                 'page': page_num
             }
 
             if branch_name:
                 query_params.update({
                     'branch': branch_name
                 })
 
             if f_path:
                 # changelog for file
                 return h.route_path(
                     'repo_commits_file',
                     repo_name=c.rhodecode_db_repo.repo_name,
                     commit_id=commit_id, f_path=f_path,
                     _query=query_params)
             else:
                 return h.route_path(
                     'repo_commits',
                     repo_name=c.rhodecode_db_repo.repo_name, _query=query_params)
 
         c.total_cs = len(collection)
         c.showing_commits = min(chunk_size, c.total_cs)
         c.pagination = RepoPage(collection, page=page, item_count=c.total_cs,
                                 items_per_page=chunk_size, url_maker=url_generator)
 
         c.next_page = c.pagination.next_page
         c.prev_page = c.pagination.previous_page
 
         if dynamic:
             if self.request.GET.get('chunk') != 'next':
                 c.next_page = None
             if self.request.GET.get('chunk') != 'prev':
                 c.prev_page = None
 
         page_commit_ids = [x.raw_id for x in c.pagination]
         c.comments = c.rhodecode_db_repo.get_comments(page_commit_ids)
         c.statuses = c.rhodecode_db_repo.statuses(page_commit_ids)
 
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
 
         c.rhodecode_repo = self.rhodecode_vcs_repo
 
         return c
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_changelog(self):
         c = self.load_default_context()
 
         commit_id = self.request.matchdict.get('commit_id')
         f_path = self._get_f_path(self.request.matchdict)
         show_hidden = str2bool(self.request.GET.get('evolve'))
 
         chunk_size = 20
 
         c.branch_name = branch_name = self.request.GET.get('branch') or ''
         c.book_name = book_name = self.request.GET.get('bookmark') or ''
         c.f_path = f_path
         c.commit_id = commit_id
         c.show_hidden = show_hidden
 
         hist_limit = safe_int(self.request.GET.get('limit')) or None
 
         p = safe_int(self.request.GET.get('page', 1), 1)
 
         c.selected_name = branch_name or book_name
         if not commit_id and branch_name:
             self._check_if_valid_branch(branch_name, self.db_repo_name, f_path)
 
         c.changelog_for_path = f_path
         pre_load = self.get_commit_preload_attrs()
 
         partial_xhr = self.request.environ.get('HTTP_X_PARTIAL_XHR')
 
         try:
             if f_path:
                 log.debug('generating changelog for path %s', f_path)
                 # get the history for the file !
                 base_commit = self.rhodecode_vcs_repo.get_commit(commit_id)
 
                 try:
                     collection = base_commit.get_path_history(
                         f_path, limit=hist_limit, pre_load=pre_load)
                     if collection and partial_xhr:
                         # for ajax call we remove first one since we're looking
                         # at it right now in the context of a file commit
                         collection.pop(0)
                 except (NodeDoesNotExistError, CommitError):
                     # this node is not present at tip!
                     try:
                         commit = self._get_commit_or_redirect(commit_id)
                         collection = commit.get_path_history(f_path)
                     except RepositoryError as e:
                         h.flash(safe_str(e), category='warning')
                         redirect_url = h.route_path(
                             'repo_commits', repo_name=self.db_repo_name)
                         raise HTTPFound(redirect_url)
                 collection = list(reversed(collection))
             else:
                 collection = self.rhodecode_vcs_repo.get_commits(
                     branch_name=branch_name, show_hidden=show_hidden,
                     pre_load=pre_load, translate_tags=False)
 
             self._load_changelog_data(
                 c, collection, p, chunk_size, c.branch_name,
                 f_path=f_path, commit_id=commit_id)
 
         except EmptyRepositoryError as e:
             h.flash(h.escape(safe_str(e)), category='warning')
             raise HTTPFound(
                 h.route_path('repo_summary', repo_name=self.db_repo_name))
         except HTTPFound:
             raise
         except (RepositoryError, CommitDoesNotExistError, Exception) as e:
             log.exception(safe_str(e))
             h.flash(h.escape(safe_str(e)), category='error')
 
             if commit_id:
                 # from single commit page, we redirect to main commits
                 raise HTTPFound(
                     h.route_path('repo_commits', repo_name=self.db_repo_name))
             else:
                 # otherwise we redirect to summary
                 raise HTTPFound(
                     h.route_path('repo_summary', repo_name=self.db_repo_name))
 
 
 
         if partial_xhr or self.request.environ.get('HTTP_X_PJAX'):
             # case when loading dynamic file history in file view
             # loading from ajax, we don't want the first result, it's popped
             # in the code above
             html = render(
                 'rhodecode:templates/commits/changelog_file_history.mako',
                 self._get_template_context(c), self.request)
             return Response(html)
 
         commit_ids = []
         if not f_path:
             # only load graph data when not in file history mode
             commit_ids = c.pagination
 
         c.graph_data, c.graph_commits = self._graph(
             self.rhodecode_vcs_repo, commit_ids)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commits_elements(self):
         c = self.load_default_context()
         commit_id = self.request.matchdict.get('commit_id')
         f_path = self._get_f_path(self.request.matchdict)
         show_hidden = str2bool(self.request.GET.get('evolve'))
 
         chunk_size = 20
         hist_limit = safe_int(self.request.GET.get('limit')) or None
 
         def wrap_for_error(err):
             html = '<tr>' \
                    '<td colspan="9" class="alert alert-error">ERROR: {}</td>' \
                    '</tr>'.format(err)
             return Response(html)
 
         c.branch_name = branch_name = self.request.GET.get('branch') or ''
         c.book_name = book_name = self.request.GET.get('bookmark') or ''
         c.f_path = f_path
         c.commit_id = commit_id
         c.show_hidden = show_hidden
 
         c.selected_name = branch_name or book_name
         if branch_name and branch_name not in self.rhodecode_vcs_repo.branches_all:
             return wrap_for_error(
                 safe_str('Branch: {} is not valid'.format(branch_name)))
 
         pre_load = self.get_commit_preload_attrs()
 
         if f_path:
             try:
                 base_commit = self.rhodecode_vcs_repo.get_commit(commit_id)
             except (RepositoryError, CommitDoesNotExistError, Exception) as e:
                 log.exception(safe_str(e))
                 raise HTTPFound(
                     h.route_path('repo_commits', repo_name=self.db_repo_name))
 
             collection = base_commit.get_path_history(
                 f_path, limit=hist_limit, pre_load=pre_load)
             collection = list(reversed(collection))
         else:
             collection = self.rhodecode_vcs_repo.get_commits(
                 branch_name=branch_name, show_hidden=show_hidden, pre_load=pre_load,
                 translate_tags=False)
 
         p = safe_int(self.request.GET.get('page', 1), 1)
         try:
             self._load_changelog_data(
                 c, collection, p, chunk_size, dynamic=True,
                 f_path=f_path, commit_id=commit_id)
         except EmptyRepositoryError as e:
             return wrap_for_error(safe_str(e))
         except (RepositoryError, CommitDoesNotExistError, Exception) as e:
             log.exception('Failed to fetch commits')
             return wrap_for_error(safe_str(e))
 
         prev_data = None
         next_data = None
 
         try:
             prev_graph = json.loads(self.request.POST.get('graph') or '{}')
         except json.JSONDecodeError:
             prev_graph = {}
 
         if self.request.GET.get('chunk') == 'prev':
             next_data = prev_graph
         elif self.request.GET.get('chunk') == 'next':
             prev_data = prev_graph
 
         commit_ids = []
         if not f_path:
             # only load graph data when not in file history mode
             commit_ids = c.pagination
 
         c.graph_data, c.graph_commits = self._graph(
             self.rhodecode_vcs_repo, commit_ids,
             prev_data=prev_data, next_data=next_data)
 
         return self._get_template_context(c)

rhodecode/apps/repository/views/repo_commits.py

@@ -1,819 +1,819 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import collections
 
 from pyramid.httpexceptions import (
     HTTPNotFound, HTTPBadRequest, HTTPFound, HTTPForbidden, HTTPConflict)
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.apps._base import RepoAppView
 from rhodecode.apps.file_store import utils as store_utils
 from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException
 
 from rhodecode.lib import diffs, codeblocks, channelstream
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, NotAnonymous, CSRFRequired)
-from rhodecode.lib.ext_json import json
+from rhodecode.lib import ext_json
 from collections import OrderedDict
 from rhodecode.lib.diffs import (
     cache_diff, load_cached_diff, diff_cache_exist, get_diff_context,
     get_diff_whitespace_flag)
 from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError, CommentVersionMismatch
 import rhodecode.lib.helpers as h
 from rhodecode.lib.utils2 import safe_unicode, str2bool, StrictAttributeDict, safe_str
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.exceptions import (
     RepositoryError, CommitDoesNotExistError)
 from rhodecode.model.db import ChangesetComment, ChangesetStatus, FileStore, \
     ChangesetCommentHistory
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def _update_with_GET(params, request):
     for k in ['diff1', 'diff2', 'diff']:
         params[k] += request.GET.getall(k)
 
 
 class RepoCommitsView(RepoAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.rhodecode_repo = self.rhodecode_vcs_repo
 
         return c
 
     def _is_diff_cache_enabled(self, target_repo):
         caching_enabled = self._get_general_setting(
             target_repo, 'rhodecode_diff_cache')
         log.debug('Diff caching enabled: %s', caching_enabled)
         return caching_enabled
 
     def _commit(self, commit_id_range, method):
         _ = self.request.translate
         c = self.load_default_context()
         c.fulldiff = self.request.GET.get('fulldiff')
         redirect_to_combined = str2bool(self.request.GET.get('redirect_combined'))
 
         # fetch global flags of ignore ws or context lines
         diff_context = get_diff_context(self.request)
         hide_whitespace_changes = get_diff_whitespace_flag(self.request)
 
         # diff_limit will cut off the whole diff if the limit is applied
         # otherwise it will just hide the big files from the front-end
         diff_limit = c.visual.cut_off_limit_diff
         file_limit = c.visual.cut_off_limit_file
 
         # get ranges of commit ids if preset
         commit_range = commit_id_range.split('...')[:2]
 
         try:
             pre_load = ['affected_files', 'author', 'branch', 'date',
                         'message', 'parents']
             if self.rhodecode_vcs_repo.alias == 'hg':
                 pre_load += ['hidden', 'obsolete', 'phase']
 
             if len(commit_range) == 2:
                 commits = self.rhodecode_vcs_repo.get_commits(
                     start_id=commit_range[0], end_id=commit_range[1],
                     pre_load=pre_load, translate_tags=False)
                 commits = list(commits)
             else:
                 commits = [self.rhodecode_vcs_repo.get_commit(
                     commit_id=commit_id_range, pre_load=pre_load)]
 
             c.commit_ranges = commits
             if not c.commit_ranges:
                 raise RepositoryError('The commit range returned an empty result')
         except CommitDoesNotExistError as e:
             msg = _('No such commit exists. Org exception: `{}`').format(safe_str(e))
             h.flash(msg, category='error')
             raise HTTPNotFound()
         except Exception:
             log.exception("General failure")
             raise HTTPNotFound()
         single_commit = len(c.commit_ranges) == 1
 
         if redirect_to_combined and not single_commit:
             source_ref = getattr(c.commit_ranges[0].parents[0]
                                  if c.commit_ranges[0].parents else h.EmptyCommit(), 'raw_id')
             target_ref = c.commit_ranges[-1].raw_id
             next_url = h.route_path(
                 'repo_compare',
                 repo_name=c.repo_name,
                 source_ref_type='rev',
                 source_ref=source_ref,
                 target_ref_type='rev',
                 target_ref=target_ref)
             raise HTTPFound(next_url)
 
         c.changes = OrderedDict()
         c.lines_added = 0
         c.lines_deleted = 0
 
         # auto collapse if we have more than limit
         collapse_limit = diffs.DiffProcessor._collapse_commits_over
         c.collapse_all_commits = len(c.commit_ranges) > collapse_limit
 
         c.commit_statuses = ChangesetStatus.STATUSES
         c.inline_comments = []
         c.files = []
 
         c.comments = []
         c.unresolved_comments = []
         c.resolved_comments = []
 
         # Single commit
         if single_commit:
             commit = c.commit_ranges[0]
             c.comments = CommentsModel().get_comments(
                 self.db_repo.repo_id,
                 revision=commit.raw_id)
 
             # comments from PR
             statuses = ChangesetStatusModel().get_statuses(
                 self.db_repo.repo_id, commit.raw_id,
                 with_revisions=True)
 
             prs = set()
             reviewers = list()
             reviewers_duplicates = set()  # to not have duplicates from multiple votes
             for c_status in statuses:
 
                 # extract associated pull-requests from votes
                 if c_status.pull_request:
                     prs.add(c_status.pull_request)
 
                 # extract reviewers
                 _user_id = c_status.author.user_id
                 if _user_id not in reviewers_duplicates:
                     reviewers.append(
                         StrictAttributeDict({
                             'user': c_status.author,
 
                             # fake attributed for commit, page that we don't have
                             # but we share the display with PR page
                             'mandatory': False,
                             'reasons': [],
                             'rule_user_group_data': lambda: None
                         })
                     )
                     reviewers_duplicates.add(_user_id)
 
             c.reviewers_count = len(reviewers)
             c.observers_count = 0
 
             # from associated statuses, check the pull requests, and
             # show comments from them
             for pr in prs:
                 c.comments.extend(pr.comments)
 
             c.unresolved_comments = CommentsModel()\
                 .get_commit_unresolved_todos(commit.raw_id)
             c.resolved_comments = CommentsModel()\
                 .get_commit_resolved_todos(commit.raw_id)
 
             c.inline_comments_flat = CommentsModel()\
                 .get_commit_inline_comments(commit.raw_id)
 
             review_statuses = ChangesetStatusModel().aggregate_votes_by_user(
                 statuses, reviewers)
 
             c.commit_review_status = ChangesetStatus.STATUS_NOT_REVIEWED
 
             c.commit_set_reviewers_data_json = collections.OrderedDict({'reviewers': []})
 
             for review_obj, member, reasons, mandatory, status in review_statuses:
                 member_reviewer = h.reviewer_as_json(
                     member, reasons=reasons, mandatory=mandatory, role=None,
                     user_group=None
                 )
 
                 current_review_status = status[0][1].status if status else ChangesetStatus.STATUS_NOT_REVIEWED
                 member_reviewer['review_status'] = current_review_status
                 member_reviewer['review_status_label'] = h.commit_status_lbl(current_review_status)
                 member_reviewer['allowed_to_update'] = False
                 c.commit_set_reviewers_data_json['reviewers'].append(member_reviewer)
 
-            c.commit_set_reviewers_data_json = json.dumps(c.commit_set_reviewers_data_json)
+            c.commit_set_reviewers_data_json = ext_json.str_json(c.commit_set_reviewers_data_json)
 
             # NOTE(marcink): this uses the same voting logic as in pull-requests
             c.commit_review_status = ChangesetStatusModel().calculate_status(review_statuses)
             c.commit_broadcast_channel = channelstream.comment_channel(c.repo_name, commit_obj=commit)
 
         diff = None
         # Iterate over ranges (default commit view is always one commit)
         for commit in c.commit_ranges:
             c.changes[commit.raw_id] = []
 
             commit2 = commit
             commit1 = commit.first_parent
 
             if method == 'show':
                 inline_comments = CommentsModel().get_inline_comments(
                     self.db_repo.repo_id, revision=commit.raw_id)
                 c.inline_cnt = len(CommentsModel().get_inline_comments_as_list(
                     inline_comments))
                 c.inline_comments = inline_comments
 
                 cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(
                     self.db_repo)
                 cache_file_path = diff_cache_exist(
                     cache_path, 'diff', commit.raw_id,
                     hide_whitespace_changes, diff_context, c.fulldiff)
 
                 caching_enabled = self._is_diff_cache_enabled(self.db_repo)
                 force_recache = str2bool(self.request.GET.get('force_recache'))
 
                 cached_diff = None
                 if caching_enabled:
                     cached_diff = load_cached_diff(cache_file_path)
 
                 has_proper_diff_cache = cached_diff and cached_diff.get('diff')
                 if not force_recache and has_proper_diff_cache:
                     diffset = cached_diff['diff']
                 else:
                     vcs_diff = self.rhodecode_vcs_repo.get_diff(
                         commit1, commit2,
                         ignore_whitespace=hide_whitespace_changes,
                         context=diff_context)
 
                     diff_processor = diffs.DiffProcessor(
                         vcs_diff, format='newdiff', diff_limit=diff_limit,
                         file_limit=file_limit, show_full_diff=c.fulldiff)
 
                     _parsed = diff_processor.prepare()
 
                     diffset = codeblocks.DiffSet(
                         repo_name=self.db_repo_name,
                         source_node_getter=codeblocks.diffset_node_getter(commit1),
                         target_node_getter=codeblocks.diffset_node_getter(commit2))
 
                     diffset = self.path_filter.render_patchset_filtered(
                         diffset, _parsed, commit1.raw_id, commit2.raw_id)
 
                     # save cached diff
                     if caching_enabled:
                         cache_diff(cache_file_path, diffset, None)
 
                 c.limited_diff = diffset.limited_diff
                 c.changes[commit.raw_id] = diffset
             else:
                 # TODO(marcink): no cache usage here...
                 _diff = self.rhodecode_vcs_repo.get_diff(
                     commit1, commit2,
                     ignore_whitespace=hide_whitespace_changes, context=diff_context)
                 diff_processor = diffs.DiffProcessor(
                     _diff, format='newdiff', diff_limit=diff_limit,
                     file_limit=file_limit, show_full_diff=c.fulldiff)
                 # downloads/raw we only need RAW diff nothing else
                 diff = self.path_filter.get_raw_patch(diff_processor)
                 c.changes[commit.raw_id] = [None, None, None, None, diff, None, None]
 
         # sort comments by how they were generated
         c.comments = sorted(c.comments, key=lambda x: x.comment_id)
         c.at_version_num = None
 
         if len(c.commit_ranges) == 1:
             c.commit = c.commit_ranges[0]
             c.parent_tmpl = ''.join(
                 '# Parent  %s\n' % x.raw_id for x in c.commit.parents)
 
         if method == 'download':
             response = Response(diff)
             response.content_type = 'text/plain'
             response.content_disposition = (
                 'attachment; filename=%s.diff' % commit_id_range[:12])
             return response
         elif method == 'patch':
             c.diff = safe_unicode(diff)
             patch = render(
                 'rhodecode:templates/changeset/patch_changeset.mako',
                 self._get_template_context(c), self.request)
             response = Response(patch)
             response.content_type = 'text/plain'
             return response
         elif method == 'raw':
             response = Response(diff)
             response.content_type = 'text/plain'
             return response
         elif method == 'show':
             if len(c.commit_ranges) == 1:
                 html = render(
                     'rhodecode:templates/changeset/changeset.mako',
                     self._get_template_context(c), self.request)
                 return Response(html)
             else:
                 c.ancestor = None
                 c.target_repo = self.db_repo
                 html = render(
                     'rhodecode:templates/changeset/changeset_range.mako',
                     self._get_template_context(c), self.request)
                 return Response(html)
 
         raise HTTPBadRequest()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commit_show(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='show')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commit_raw(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='raw')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commit_patch(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='patch')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commit_download(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='download')
 
     def _commit_comments_create(self, commit_id, comments):
         _ = self.request.translate
         data = {}
         if not comments:
             return
 
         commit = self.db_repo.get_commit(commit_id)
 
         all_drafts = len([x for x in comments if str2bool(x['is_draft'])]) == len(comments)
         for entry in comments:
             c = self.load_default_context()
             comment_type = entry['comment_type']
             text = entry['text']
             status = entry['status']
             is_draft = str2bool(entry['is_draft'])
             resolves_comment_id = entry['resolves_comment_id']
             f_path = entry['f_path']
             line_no = entry['line']
             target_elem_id = 'file-{}'.format(h.safeid(h.safe_unicode(f_path)))
 
             if status:
                 text = text or (_('Status change %(transition_icon)s %(status)s')
                                 % {'transition_icon': '>',
                                    'status': ChangesetStatus.get_status_lbl(status)})
 
             comment = CommentsModel().create(
                 text=text,
                 repo=self.db_repo.repo_id,
                 user=self._rhodecode_db_user.user_id,
                 commit_id=commit_id,
                 f_path=f_path,
                 line_no=line_no,
                 status_change=(ChangesetStatus.get_status_lbl(status)
                                if status else None),
                 status_change_type=status,
                 comment_type=comment_type,
                 is_draft=is_draft,
                 resolves_comment_id=resolves_comment_id,
                 auth_user=self._rhodecode_user,
                 send_email=not is_draft,  # skip notification for draft comments
             )
             is_inline = comment.is_inline
 
             # get status if set !
             if status:
                 # `dont_allow_on_closed_pull_request = True` means
                 # if latest status was from pull request and it's closed
                 # disallow changing status !
 
                 try:
                     ChangesetStatusModel().set_status(
                         self.db_repo.repo_id,
                         status,
                         self._rhodecode_db_user.user_id,
                         comment,
                         revision=commit_id,
                         dont_allow_on_closed_pull_request=True
                     )
                 except StatusChangeOnClosedPullRequestError:
                     msg = _('Changing the status of a commit associated with '
                             'a closed pull request is not allowed')
                     log.exception(msg)
                     h.flash(msg, category='warning')
                     raise HTTPFound(h.route_path(
                         'repo_commit', repo_name=self.db_repo_name,
                         commit_id=commit_id))
 
             Session().flush()
             # this is somehow required to get access to some relationship
             # loaded on comment
             Session().refresh(comment)
 
             # skip notifications for drafts
             if not is_draft:
                 CommentsModel().trigger_commit_comment_hook(
                     self.db_repo, self._rhodecode_user, 'create',
                     data={'comment': comment, 'commit': commit})
 
             comment_id = comment.comment_id
             data[comment_id] = {
                 'target_id': target_elem_id
             }
             Session().flush()
 
             c.co = comment
             c.at_version_num = 0
             c.is_new = True
             rendered_comment = render(
                 'rhodecode:templates/changeset/changeset_comment_block.mako',
                 self._get_template_context(c), self.request)
 
             data[comment_id].update(comment.get_dict())
             data[comment_id].update({'rendered_text': rendered_comment})
 
         # finalize, commit and redirect
         Session().commit()
 
         # skip channelstream for draft comments
         if not all_drafts:
             comment_broadcast_channel = channelstream.comment_channel(
                 self.db_repo_name, commit_obj=commit)
 
             comment_data = data
             posted_comment_type = 'inline' if is_inline else 'general'
             if len(data) == 1:
                 msg = _('posted {} new {} comment').format(len(data), posted_comment_type)
             else:
                 msg = _('posted {} new {} comments').format(len(data), posted_comment_type)
 
             channelstream.comment_channelstream_push(
                 self.request, comment_broadcast_channel, self._rhodecode_user, msg,
                 comment_data=comment_data)
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_commit_comment_create(self):
         _ = self.request.translate
         commit_id = self.request.matchdict['commit_id']
 
         multi_commit_ids = []
         for _commit_id in self.request.POST.get('commit_ids', '').split(','):
             if _commit_id not in ['', None, EmptyCommit.raw_id]:
                 if _commit_id not in multi_commit_ids:
                     multi_commit_ids.append(_commit_id)
 
         commit_ids = multi_commit_ids or [commit_id]
 
         data = []
         # Multiple comments for each passed commit id
         for current_id in filter(None, commit_ids):
             comment_data = {
                 'comment_type': self.request.POST.get('comment_type'),
                 'text': self.request.POST.get('text'),
                 'status': self.request.POST.get('changeset_status', None),
                 'is_draft': self.request.POST.get('draft'),
                 'resolves_comment_id': self.request.POST.get('resolves_comment_id', None),
                 'close_pull_request': self.request.POST.get('close_pull_request'),
                 'f_path': self.request.POST.get('f_path'),
                 'line': self.request.POST.get('line'),
             }
             comment = self._commit_comments_create(commit_id=current_id, comments=[comment_data])
             data.append(comment)
 
         return data if len(data) > 1 else data[0]
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_commit_comment_preview(self):
         # Technically a CSRF token is not needed as no state changes with this
         # call. However, as this is a POST is better to have it, so automated
         # tools don't flag it as potential CSRF.
         # Post is required because the payload could be bigger than the maximum
         # allowed by GET.
 
         text = self.request.POST.get('text')
         renderer = self.request.POST.get('renderer') or 'rst'
         if text:
             return h.render(text, renderer=renderer, mentions=True,
                             repo_name=self.db_repo_name)
         return ''
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_commit_comment_history_view(self):
         c = self.load_default_context()
         comment_id = self.request.matchdict['comment_id']
         comment_history_id = self.request.matchdict['comment_history_id']
 
         comment = ChangesetComment.get_or_404(comment_id)
         comment_owner = (comment.author.user_id == self._rhodecode_db_user.user_id)
         if comment.draft and not comment_owner:
             # if we see draft comments history, we only allow this for owner
             raise HTTPNotFound()
 
         comment_history = ChangesetCommentHistory.get_or_404(comment_history_id)
         is_repo_comment = comment_history.comment.repo.repo_id == self.db_repo.repo_id
 
         if is_repo_comment:
             c.comment_history = comment_history
 
             rendered_comment = render(
                 'rhodecode:templates/changeset/comment_history.mako',
                 self._get_template_context(c), self.request)
             return rendered_comment
         else:
             log.warning('No permissions for user %s to show comment_history_id: %s',
                         self._rhodecode_db_user, comment_history_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_commit_comment_attachment_upload(self):
         c = self.load_default_context()
         upload_key = 'attachment'
 
         file_obj = self.request.POST.get(upload_key)
 
         if file_obj is None:
             self.request.response.status = 400
             return {'store_fid': None,
                     'access_path': None,
                     'error': '{} data field is missing'.format(upload_key)}
 
         if not hasattr(file_obj, 'filename'):
             self.request.response.status = 400
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'filename cannot be read from the data field'}
 
         filename = file_obj.filename
         file_display_name = filename
 
         metadata = {
             'user_uploaded': {'username': self._rhodecode_user.username,
                               'user_id': self._rhodecode_user.user_id,
                               'ip': self._rhodecode_user.ip_addr}}
 
         # TODO(marcink): allow .ini configuration for allowed_extensions, and file-size
         allowed_extensions = [
             'gif', '.jpeg', '.jpg', '.png', '.docx', '.gz', '.log', '.pdf',
             '.pptx', '.txt', '.xlsx', '.zip']
         max_file_size = 10 * 1024 * 1024  # 10MB, also validated via dropzone.js
 
         try:
             storage = store_utils.get_file_storage(self.request.registry.settings)
             store_uid, metadata = storage.save_file(
                 file_obj.file, filename, extra_metadata=metadata,
                 extensions=allowed_extensions, max_filesize=max_file_size)
         except FileNotAllowedException:
             self.request.response.status = 400
             permitted_extensions = ', '.join(allowed_extensions)
             error_msg = 'File `{}` is not allowed. ' \
                         'Only following extensions are permitted: {}'.format(
                             filename, permitted_extensions)
             return {'store_fid': None,
                     'access_path': None,
                     'error': error_msg}
         except FileOverSizeException:
             self.request.response.status = 400
             limit_mb = h.format_byte_size_binary(max_file_size)
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'File {} is exceeding allowed limit of {}.'.format(
                         filename, limit_mb)}
 
         try:
             entry = FileStore.create(
                 file_uid=store_uid, filename=metadata["filename"],
                 file_hash=metadata["sha256"], file_size=metadata["size"],
                 file_display_name=file_display_name,
                 file_description=u'comment attachment `{}`'.format(safe_unicode(filename)),
                 hidden=True, check_acl=True, user_id=self._rhodecode_user.user_id,
                 scope_repo_id=self.db_repo.repo_id
             )
             Session().add(entry)
             Session().commit()
             log.debug('Stored upload in DB as %s', entry)
         except Exception:
             log.exception('Failed to store file %s', filename)
             self.request.response.status = 400
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'File {} failed to store in DB.'.format(filename)}
 
         Session().commit()
 
         return {
             'store_fid': store_uid,
             'access_path': h.route_path(
                 'download_file', fid=store_uid),
             'fqn_access_path': h.route_url(
                 'download_file', fid=store_uid),
             'repo_access_path': h.route_path(
                 'repo_artifacts_get', repo_name=self.db_repo_name, uid=store_uid),
             'repo_fqn_access_path': h.route_url(
                 'repo_artifacts_get', repo_name=self.db_repo_name, uid=store_uid),
         }
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_commit_comment_delete(self):
         commit_id = self.request.matchdict['commit_id']
         comment_id = self.request.matchdict['comment_id']
 
         comment = ChangesetComment.get_or_404(comment_id)
         if not comment:
             log.debug('Comment with id:%s not found, skipping', comment_id)
             # comment already deleted in another call probably
             return True
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = (comment.author.user_id == self._rhodecode_db_user.user_id)
         is_repo_comment = comment.repo.repo_id == self.db_repo.repo_id
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if comment.draft and not comment_owner:
             # We never allow to delete draft comments for other than owners
             raise HTTPNotFound()
 
         if super_admin or comment_owner or comment_repo_admin:
             CommentsModel().delete(comment=comment, auth_user=self._rhodecode_user)
             Session().commit()
             return True
         else:
             log.warning('No permissions for user %s to delete comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_commit_comment_edit(self):
         self.load_default_context()
 
         commit_id = self.request.matchdict['commit_id']
         comment_id = self.request.matchdict['comment_id']
         comment = ChangesetComment.get_or_404(comment_id)
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = (comment.author.user_id == self._rhodecode_db_user.user_id)
         is_repo_comment = comment.repo.repo_id == self.db_repo.repo_id
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if super_admin or comment_owner or comment_repo_admin:
             text = self.request.POST.get('text')
             version = self.request.POST.get('version')
             if text == comment.text:
                 log.warning(
                     'Comment(repo): '
                     'Trying to create new version '
                     'with the same comment body {}'.format(
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             if version.isdigit():
                 version = int(version)
             else:
                 log.warning(
                     'Comment(repo): Wrong version type {} {} '
                     'for comment {}'.format(
                         version,
                         type(version),
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             try:
                 comment_history = CommentsModel().edit(
                     comment_id=comment_id,
                     text=text,
                     auth_user=self._rhodecode_user,
                     version=version,
                 )
             except CommentVersionMismatch:
                 raise HTTPConflict()
 
             if not comment_history:
                 raise HTTPNotFound()
 
             if not comment.draft:
                 commit = self.db_repo.get_commit(commit_id)
                 CommentsModel().trigger_commit_comment_hook(
                     self.db_repo, self._rhodecode_user, 'edit',
                     data={'comment': comment, 'commit': commit})
 
             Session().commit()
             return {
                 'comment_history_id': comment_history.comment_history_id,
                 'comment_id': comment.comment_id,
                 'comment_version': comment_history.version,
                 'comment_author_username': comment_history.author.username,
                 'comment_author_gravatar': h.gravatar_url(comment_history.author.email, 16),
                 'comment_created_on': h.age_component(comment_history.created_on,
                                                       time_is_local=True),
             }
         else:
             log.warning('No permissions for user %s to edit comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commit_data(self):
         commit_id = self.request.matchdict['commit_id']
         self.load_default_context()
 
         try:
             return self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
         except CommitDoesNotExistError as e:
             return EmptyCommit(message=str(e))
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commit_children(self):
         commit_id = self.request.matchdict['commit_id']
         self.load_default_context()
 
         try:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
             children = commit.children
         except CommitDoesNotExistError:
             children = []
 
         result = {"results": children}
         return result
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commit_parents(self):
         commit_id = self.request.matchdict['commit_id']
         self.load_default_context()
 
         try:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
             parents = commit.parents
         except CommitDoesNotExistError:
             parents = []
         result = {"results": parents}
         return result

rhodecode/apps/repository/views/repo_pull_requests.py

@@ -1,1877 +1,1877 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import collections
 
 import formencode
 import formencode.htmlfill
 import peppercorn
 from pyramid.httpexceptions import (
     HTTPFound, HTTPNotFound, HTTPForbidden, HTTPBadRequest, HTTPConflict)
 
 from pyramid.renderers import render
 
 from rhodecode.apps._base import RepoAppView, DataGridAppView
 
 from rhodecode.lib import helpers as h, diffs, codeblocks, channelstream
 from rhodecode.lib.base import vcs_operation_context
 from rhodecode.lib.diffs import load_cached_diff, cache_diff, diff_cache_exist
 from rhodecode.lib.exceptions import CommentVersionMismatch
-from rhodecode.lib.ext_json import json
+from rhodecode.lib import ext_json
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAny, HasRepoPermissionAnyDecorator,
     NotAnonymous, CSRFRequired)
 from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode, safe_int, aslist, retry
 from rhodecode.lib.vcs.backends.base import (
     EmptyCommit, UpdateFailureReason, unicode_to_reference)
 from rhodecode.lib.vcs.exceptions import (
     CommitDoesNotExistError, RepositoryRequirementError, EmptyRepositoryError)
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
     func, false, or_, PullRequest, ChangesetComment, ChangesetStatus, Repository,
     PullRequestReviewers)
 from rhodecode.model.forms import PullRequestForm
 from rhodecode.model.meta import Session
 from rhodecode.model.pull_request import PullRequestModel, MergeCheck
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoPullRequestsView(RepoAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.REVIEW_STATUS_APPROVED = ChangesetStatus.STATUS_APPROVED
         c.REVIEW_STATUS_REJECTED = ChangesetStatus.STATUS_REJECTED
         # backward compat., we use for OLD PRs a plain renderer
         c.renderer = 'plain'
         return c
 
     def _get_pull_requests_list(
             self, repo_name, source, filter_type, opened_by, statuses):
 
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(self.request)
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         # pagination
 
         if filter_type == 'awaiting_review':
             pull_requests = PullRequestModel().get_awaiting_review(
                 repo_name,
                 search_q=search_q, statuses=statuses,
                 offset=start, length=limit, order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_awaiting_review(
                 repo_name,
                 search_q=search_q, statuses=statuses)
         elif filter_type == 'awaiting_my_review':
             pull_requests = PullRequestModel().get_awaiting_my_review(
                 repo_name, self._rhodecode_user.user_id,
                 search_q=search_q, statuses=statuses,
                 offset=start, length=limit, order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_awaiting_my_review(
                 repo_name, self._rhodecode_user.user_id,
                 search_q=search_q, statuses=statuses)
         else:
             pull_requests = PullRequestModel().get_all(
                 repo_name, search_q=search_q, source=source, opened_by=opened_by,
                 statuses=statuses, offset=start, length=limit,
                 order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_all(
                 repo_name, search_q=search_q, source=source, statuses=statuses,
                 opened_by=opened_by)
 
         data = []
         comments_model = CommentsModel()
         for pr in pull_requests:
             comments_count = comments_model.get_all_comments(
                 self.db_repo.repo_id, pull_request=pr,
                 include_drafts=False, count_only=True)
 
             review_statuses = pr.reviewers_statuses(user=self._rhodecode_db_user)
             my_review_status = ChangesetStatus.STATUS_NOT_REVIEWED
             if review_statuses and review_statuses[4]:
                 _review_obj, _user, _reasons, _mandatory, statuses = review_statuses
                 my_review_status = statuses[0][1].status
 
             data.append({
                 'name': _render('pullrequest_name',
                                 pr.pull_request_id, pr.pull_request_state,
                                 pr.work_in_progress, pr.target_repo.repo_name,
                                 short=True),
                 'name_raw': pr.pull_request_id,
                 'status': _render('pullrequest_status',
                                   pr.calculated_review_status()),
                 'my_status': _render('pullrequest_status',
                                      my_review_status),
                 'title': _render('pullrequest_title', pr.title, pr.description),
                 'description': h.escape(pr.description),
                 'updated_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.updated_on),
                                       pr.versions_count),
                 'updated_on_raw': h.datetime_to_time(pr.updated_on),
                 'created_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.created_on)),
                 'created_on_raw': h.datetime_to_time(pr.created_on),
                 'state': pr.pull_request_state,
                 'author': _render('pullrequest_author',
                                   pr.author.full_contact, ),
                 'author_raw': pr.author.full_name,
                 'comments': _render('pullrequest_comments', comments_count),
                 'comments_raw': comments_count,
                 'closed': pr.is_closed(),
             })
 
         data = ({
             'draw': draw,
             'data': data,
             'recordsTotal': pull_requests_total_count,
             'recordsFiltered': pull_requests_total_count,
         })
         return data
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_list(self):
         c = self.load_default_context()
 
         req_get = self.request.GET
         c.source = str2bool(req_get.get('source'))
         c.closed = str2bool(req_get.get('closed'))
         c.my = str2bool(req_get.get('my'))
         c.awaiting_review = str2bool(req_get.get('awaiting_review'))
         c.awaiting_my_review = str2bool(req_get.get('awaiting_my_review'))
 
         c.active = 'open'
         if c.my:
             c.active = 'my'
         if c.closed:
             c.active = 'closed'
         if c.awaiting_review and not c.source:
             c.active = 'awaiting'
         if c.source and not c.awaiting_review:
             c.active = 'source'
         if c.awaiting_my_review:
             c.active = 'awaiting_my'
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_list_data(self):
         self.load_default_context()
 
         # additional filters
         req_get = self.request.GET
         source = str2bool(req_get.get('source'))
         closed = str2bool(req_get.get('closed'))
         my = str2bool(req_get.get('my'))
         awaiting_review = str2bool(req_get.get('awaiting_review'))
         awaiting_my_review = str2bool(req_get.get('awaiting_my_review'))
 
         filter_type = 'awaiting_review' if awaiting_review \
             else 'awaiting_my_review' if awaiting_my_review \
             else None
 
         opened_by = None
         if my:
             opened_by = [self._rhodecode_user.user_id]
 
         statuses = [PullRequest.STATUS_NEW, PullRequest.STATUS_OPEN]
         if closed:
             statuses = [PullRequest.STATUS_CLOSED]
 
         data = self._get_pull_requests_list(
             repo_name=self.db_repo_name, source=source,
             filter_type=filter_type, opened_by=opened_by, statuses=statuses)
 
         return data
 
     def _is_diff_cache_enabled(self, target_repo):
         caching_enabled = self._get_general_setting(
             target_repo, 'rhodecode_diff_cache')
         log.debug('Diff caching enabled: %s', caching_enabled)
         return caching_enabled
 
     def _get_diffset(self, source_repo_name, source_repo,
                      ancestor_commit,
                      source_ref_id, target_ref_id,
                      target_commit, source_commit, diff_limit, file_limit,
                      fulldiff, hide_whitespace_changes, diff_context, use_ancestor=True):
 
         target_commit_final = target_commit
         source_commit_final = source_commit
 
         if use_ancestor:
             # we might want to not use it for versions
             target_ref_id = ancestor_commit.raw_id
             target_commit_final = ancestor_commit
 
         vcs_diff = PullRequestModel().get_diff(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context)
 
         diff_processor = diffs.DiffProcessor(
             vcs_diff, format='newdiff', diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=fulldiff)
 
         _parsed = diff_processor.prepare()
 
         diffset = codeblocks.DiffSet(
             repo_name=self.db_repo_name,
             source_repo_name=source_repo_name,
             source_node_getter=codeblocks.diffset_node_getter(target_commit_final),
             target_node_getter=codeblocks.diffset_node_getter(source_commit_final),
         )
         diffset = self.path_filter.render_patchset_filtered(
             diffset, _parsed, target_ref_id, source_ref_id)
 
         return diffset
 
     def _get_range_diffset(self, source_scm, source_repo,
                            commit1, commit2, diff_limit, file_limit,
                            fulldiff, hide_whitespace_changes, diff_context):
         vcs_diff = source_scm.get_diff(
             commit1, commit2,
             ignore_whitespace=hide_whitespace_changes,
             context=diff_context)
 
         diff_processor = diffs.DiffProcessor(
             vcs_diff, format='newdiff', diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=fulldiff)
 
         _parsed = diff_processor.prepare()
 
         diffset = codeblocks.DiffSet(
             repo_name=source_repo.repo_name,
             source_node_getter=codeblocks.diffset_node_getter(commit1),
             target_node_getter=codeblocks.diffset_node_getter(commit2))
 
         diffset = self.path_filter.render_patchset_filtered(
             diffset, _parsed, commit1.raw_id, commit2.raw_id)
 
         return diffset
 
     def register_comments_vars(self, c, pull_request, versions, include_drafts=True):
         comments_model = CommentsModel()
 
         # GENERAL COMMENTS with versions #
         q = comments_model._all_general_comments_of_pull_request(pull_request)
         q = q.order_by(ChangesetComment.comment_id.asc())
         if not include_drafts:
             q = q.filter(ChangesetComment.draft == false())
         general_comments = q
 
         # pick comments we want to render at current version
         c.comment_versions = comments_model.aggregate_comments(
             general_comments, versions, c.at_version_num)
 
         # INLINE COMMENTS with versions  #
         q = comments_model._all_inline_comments_of_pull_request(pull_request)
         q = q.order_by(ChangesetComment.comment_id.asc())
         if not include_drafts:
             q = q.filter(ChangesetComment.draft == false())
         inline_comments = q
 
         c.inline_versions = comments_model.aggregate_comments(
             inline_comments, versions, c.at_version_num, inline=True)
 
         # Comments inline+general
         if c.at_version:
             c.inline_comments_flat = c.inline_versions[c.at_version_num]['display']
             c.comments = c.comment_versions[c.at_version_num]['display']
         else:
             c.inline_comments_flat = c.inline_versions[c.at_version_num]['until']
             c.comments = c.comment_versions[c.at_version_num]['until']
 
         return general_comments, inline_comments
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_show(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
 
         c.state_progressing = pull_request.is_state_changing()
         c.pr_broadcast_channel = channelstream.pr_channel(pull_request)
 
         _new_state = {
             'created': PullRequest.STATE_CREATED,
         }.get(self.request.GET.get('force_state'))
         can_force_state = c.is_super_admin or HasRepoPermissionAny('repository.admin')(c.repo_name)
 
         if can_force_state and _new_state:
             with pull_request.set_state(PullRequest.STATE_UPDATING, final_state=_new_state):
                 h.flash(
                     _('Pull Request state was force changed to `{}`').format(_new_state),
                     category='success')
                 Session().commit()
 
             raise HTTPFound(h.route_path(
                 'pullrequest_show', repo_name=self.db_repo_name,
                 pull_request_id=pull_request_id))
 
         version = self.request.GET.get('version')
         from_version = self.request.GET.get('from_version') or version
         merge_checks = self.request.GET.get('merge_checks')
         c.fulldiff = str2bool(self.request.GET.get('fulldiff'))
         force_refresh = str2bool(self.request.GET.get('force_refresh'))
         c.range_diff_on = self.request.GET.get('range-diff') == "1"
 
         # fetch global flags of ignore ws or context lines
         diff_context = diffs.get_diff_context(self.request)
         hide_whitespace_changes = diffs.get_diff_whitespace_flag(self.request)
 
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
 
         pr_closed = pull_request_latest.is_closed()
 
         if pr_closed and (version or from_version):
             # not allow to browse versions for closed PR
             raise HTTPFound(h.route_path(
                 'pullrequest_show', repo_name=self.db_repo_name,
                 pull_request_id=pull_request_id))
 
         versions = pull_request_display_obj.versions()
 
         c.commit_versions = PullRequestModel().pr_commits_versions(versions)
 
         # used to store per-commit range diffs
         c.changes = collections.OrderedDict()
 
         c.at_version = at_version
         c.at_version_num = (at_version
                             if at_version and at_version != PullRequest.LATEST_VER
                             else None)
 
         c.at_version_index = ChangesetComment.get_index_from_version(
             c.at_version_num, versions)
 
         (prev_pull_request_latest,
          prev_pull_request_at_ver,
          prev_pull_request_display_obj,
          prev_at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=from_version)
 
         c.from_version = prev_at_version
         c.from_version_num = (prev_at_version
                               if prev_at_version and prev_at_version != PullRequest.LATEST_VER
                               else None)
         c.from_version_index = ChangesetComment.get_index_from_version(
             c.from_version_num, versions)
 
         # define if we're in COMPARE mode or VIEW at version mode
         compare = at_version != prev_at_version
 
         # pull_requests repo_name we opened it against
         # ie. target_repo must match
         if self.db_repo_name != pull_request_at_ver.target_repo.repo_name:
             log.warning('Mismatch between the current repo: %s, and target %s',
                         self.db_repo_name, pull_request_at_ver.target_repo.repo_name)
             raise HTTPNotFound()
 
         c.shadow_clone_url = PullRequestModel().get_shadow_clone_url(pull_request_at_ver)
 
         c.pull_request = pull_request_display_obj
         c.renderer = pull_request_at_ver.description_renderer or c.renderer
         c.pull_request_latest = pull_request_latest
 
         # inject latest version
         latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
         c.versions = versions + [latest_ver]
 
         if compare or (at_version and not at_version == PullRequest.LATEST_VER):
             c.allowed_to_change_status = False
             c.allowed_to_update = False
             c.allowed_to_merge = False
             c.allowed_to_delete = False
             c.allowed_to_comment = False
             c.allowed_to_close = False
         else:
             can_change_status = PullRequestModel().check_user_change_status(
                 pull_request_at_ver, self._rhodecode_user)
             c.allowed_to_change_status = can_change_status and not pr_closed
 
             c.allowed_to_update = PullRequestModel().check_user_update(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_merge = PullRequestModel().check_user_merge(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_delete = PullRequestModel().check_user_delete(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_comment = not pr_closed
             c.allowed_to_close = c.allowed_to_merge and not pr_closed
 
         c.forbid_adding_reviewers = False
 
         if pull_request_latest.reviewer_data and \
                         'rules' in pull_request_latest.reviewer_data:
             rules = pull_request_latest.reviewer_data['rules'] or {}
             try:
                 c.forbid_adding_reviewers = rules.get('forbid_adding_reviewers')
             except Exception:
                 pass
 
         # check merge capabilities
         _merge_check = MergeCheck.validate(
             pull_request_latest, auth_user=self._rhodecode_user,
             translator=self.request.translate,
             force_shadow_repo_refresh=force_refresh)
 
         c.pr_merge_errors = _merge_check.error_details
         c.pr_merge_possible = not _merge_check.failed
         c.pr_merge_message = _merge_check.merge_msg
         c.pr_merge_source_commit = _merge_check.source_commit
         c.pr_merge_target_commit = _merge_check.target_commit
 
         c.pr_merge_info = MergeCheck.get_merge_conditions(
             pull_request_latest, translator=self.request.translate)
 
         c.pull_request_review_status = _merge_check.review_status
         if merge_checks:
             self.request.override_renderer = \
                 'rhodecode:templates/pullrequests/pullrequest_merge_checks.mako'
             return self._get_template_context(c)
 
         c.reviewers_count = pull_request.reviewers_count
         c.observers_count = pull_request.observers_count
 
         # reviewers and statuses
-        c.pull_request_default_reviewers_data_json = json.dumps(pull_request.reviewer_data)
+        c.pull_request_default_reviewers_data_json = ext_json.str_json(pull_request.reviewer_data)
         c.pull_request_set_reviewers_data_json = collections.OrderedDict({'reviewers': []})
         c.pull_request_set_observers_data_json = collections.OrderedDict({'observers': []})
 
         for review_obj, member, reasons, mandatory, status in pull_request_at_ver.reviewers_statuses():
             member_reviewer = h.reviewer_as_json(
                 member, reasons=reasons, mandatory=mandatory,
                 role=review_obj.role,
                 user_group=review_obj.rule_user_group_data()
             )
 
             current_review_status = status[0][1].status if status else ChangesetStatus.STATUS_NOT_REVIEWED
             member_reviewer['review_status'] = current_review_status
             member_reviewer['review_status_label'] = h.commit_status_lbl(current_review_status)
             member_reviewer['allowed_to_update'] = c.allowed_to_update
             c.pull_request_set_reviewers_data_json['reviewers'].append(member_reviewer)
 
-        c.pull_request_set_reviewers_data_json = json.dumps(c.pull_request_set_reviewers_data_json)
+        c.pull_request_set_reviewers_data_json = ext_json.str_json(c.pull_request_set_reviewers_data_json)
 
         for observer_obj, member in pull_request_at_ver.observers():
             member_observer = h.reviewer_as_json(
                 member, reasons=[], mandatory=False,
                 role=observer_obj.role,
                 user_group=observer_obj.rule_user_group_data()
             )
             member_observer['allowed_to_update'] = c.allowed_to_update
             c.pull_request_set_observers_data_json['observers'].append(member_observer)
 
-        c.pull_request_set_observers_data_json = json.dumps(c.pull_request_set_observers_data_json)
+        c.pull_request_set_observers_data_json = ext_json.str_json(c.pull_request_set_observers_data_json)
 
         general_comments, inline_comments = \
             self.register_comments_vars(c, pull_request_latest, versions)
 
         # TODOs
         c.unresolved_comments = CommentsModel() \
             .get_pull_request_unresolved_todos(pull_request_latest)
         c.resolved_comments = CommentsModel() \
             .get_pull_request_resolved_todos(pull_request_latest)
 
         # Drafts
         c.draft_comments = CommentsModel().get_pull_request_drafts(
             self._rhodecode_db_user.user_id,
             pull_request_latest)
 
         # if we use version, then do not show later comments
         # than current version
         display_inline_comments = collections.defaultdict(
             lambda: collections.defaultdict(list))
         for co in inline_comments:
             if c.at_version_num:
                 # pick comments that are at least UPTO given version, so we
                 # don't render comments for higher version
                 should_render = co.pull_request_version_id and \
                                 co.pull_request_version_id <= c.at_version_num
             else:
                 # showing all, for 'latest'
                 should_render = True
 
             if should_render:
                 display_inline_comments[co.f_path][co.line_no].append(co)
 
         # load diff data into template context, if we use compare mode then
         # diff is calculated based on changes between versions of PR
 
         source_repo = pull_request_at_ver.source_repo
         source_ref_id = pull_request_at_ver.source_ref_parts.commit_id
 
         target_repo = pull_request_at_ver.target_repo
         target_ref_id = pull_request_at_ver.target_ref_parts.commit_id
 
         if compare:
             # in compare switch the diff base to latest commit from prev version
             target_ref_id = prev_pull_request_display_obj.revisions[0]
 
         # despite opening commits for bookmarks/branches/tags, we always
         # convert this to rev to prevent changes after bookmark or branch change
         c.source_ref_type = 'rev'
         c.source_ref = source_ref_id
 
         c.target_ref_type = 'rev'
         c.target_ref = target_ref_id
 
         c.source_repo = source_repo
         c.target_repo = target_repo
 
         c.commit_ranges = []
         source_commit = EmptyCommit()
         target_commit = EmptyCommit()
         c.missing_requirements = False
 
         source_scm = source_repo.scm_instance()
         target_scm = target_repo.scm_instance()
 
         shadow_scm = None
         try:
             shadow_scm = pull_request_latest.get_shadow_repo()
         except Exception:
             log.debug('Failed to get shadow repo', exc_info=True)
         # try first the existing source_repo, and then shadow
         # repo if we can obtain one
         commits_source_repo = source_scm
         if shadow_scm:
             commits_source_repo = shadow_scm
 
         c.commits_source_repo = commits_source_repo
         c.ancestor = None  # set it to None, to hide it from PR view
 
         # empty version means latest, so we keep this to prevent
         # double caching
         version_normalized = version or PullRequest.LATEST_VER
         from_version_normalized = from_version or PullRequest.LATEST_VER
 
         cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(target_repo)
         cache_file_path = diff_cache_exist(
             cache_path, 'pull_request', pull_request_id, version_normalized,
             from_version_normalized, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context, c.fulldiff)
 
         caching_enabled = self._is_diff_cache_enabled(c.target_repo)
         force_recache = self.get_recache_flag()
 
         cached_diff = None
         if caching_enabled:
             cached_diff = load_cached_diff(cache_file_path)
 
         has_proper_commit_cache = (
                 cached_diff and cached_diff.get('commits')
                 and len(cached_diff.get('commits', [])) == 5
                 and cached_diff.get('commits')[0]
                 and cached_diff.get('commits')[3])
 
         if not force_recache and not c.range_diff_on and has_proper_commit_cache:
             diff_commit_cache = \
                 (ancestor_commit, commit_cache, missing_requirements,
                  source_commit, target_commit) = cached_diff['commits']
         else:
             # NOTE(marcink): we reach potentially unreachable errors when a PR has
             # merge errors resulting in potentially hidden commits in the shadow repo.
             maybe_unreachable = _merge_check.MERGE_CHECK in _merge_check.error_details \
                                 and _merge_check.merge_response
             maybe_unreachable = maybe_unreachable \
                                 and _merge_check.merge_response.metadata.get('unresolved_files')
             log.debug("Using unreachable commits due to MERGE_CHECK in merge simulation")
             diff_commit_cache = \
                 (ancestor_commit, commit_cache, missing_requirements,
                  source_commit, target_commit) = self.get_commits(
                     commits_source_repo,
                     pull_request_at_ver,
                     source_commit,
                     source_ref_id,
                     source_scm,
                     target_commit,
                     target_ref_id,
                     target_scm,
                 maybe_unreachable=maybe_unreachable)
 
         # register our commit range
         for comm in commit_cache.values():
             c.commit_ranges.append(comm)
 
         c.missing_requirements = missing_requirements
         c.ancestor_commit = ancestor_commit
         c.statuses = source_repo.statuses(
             [x.raw_id for x in c.commit_ranges])
 
         # auto collapse if we have more than limit
         collapse_limit = diffs.DiffProcessor._collapse_commits_over
         c.collapse_all_commits = len(c.commit_ranges) > collapse_limit
         c.compare_mode = compare
 
         # diff_limit is the old behavior, will cut off the whole diff
         # if the limit is applied  otherwise will just hide the
         # big files from the front-end
         diff_limit = c.visual.cut_off_limit_diff
         file_limit = c.visual.cut_off_limit_file
 
         c.missing_commits = False
         if (c.missing_requirements
             or isinstance(source_commit, EmptyCommit)
             or source_commit == target_commit):
 
             c.missing_commits = True
         else:
             c.inline_comments = display_inline_comments
 
             use_ancestor = True
             if from_version_normalized != version_normalized:
                 use_ancestor = False
 
             has_proper_diff_cache = cached_diff and cached_diff.get('commits')
             if not force_recache and has_proper_diff_cache:
                 c.diffset = cached_diff['diff']
             else:
                 try:
                     c.diffset = self._get_diffset(
                         c.source_repo.repo_name, commits_source_repo,
                         c.ancestor_commit,
                         source_ref_id, target_ref_id,
                         target_commit, source_commit,
                         diff_limit, file_limit, c.fulldiff,
                         hide_whitespace_changes, diff_context,
                         use_ancestor=use_ancestor
                     )
 
                     # save cached diff
                     if caching_enabled:
                         cache_diff(cache_file_path, c.diffset, diff_commit_cache)
                 except CommitDoesNotExistError:
                     log.exception('Failed to generate diffset')
                     c.missing_commits = True
 
             if not c.missing_commits:
 
                 c.limited_diff = c.diffset.limited_diff
 
                 # calculate removed files that are bound to comments
                 comment_deleted_files = [
                     fname for fname in display_inline_comments
                     if fname not in c.diffset.file_stats]
 
                 c.deleted_files_comments = collections.defaultdict(dict)
                 for fname, per_line_comments in display_inline_comments.items():
                     if fname in comment_deleted_files:
                         c.deleted_files_comments[fname]['stats'] = 0
                         c.deleted_files_comments[fname]['comments'] = list()
                         for lno, comments in per_line_comments.items():
                             c.deleted_files_comments[fname]['comments'].extend(comments)
 
                 # maybe calculate the range diff
                 if c.range_diff_on:
                     # TODO(marcink): set whitespace/context
                     context_lcl = 3
                     ign_whitespace_lcl = False
 
                     for commit in c.commit_ranges:
                         commit2 = commit
                         commit1 = commit.first_parent
 
                         range_diff_cache_file_path = diff_cache_exist(
                             cache_path, 'diff', commit.raw_id,
                             ign_whitespace_lcl, context_lcl, c.fulldiff)
 
                         cached_diff = None
                         if caching_enabled:
                             cached_diff = load_cached_diff(range_diff_cache_file_path)
 
                         has_proper_diff_cache = cached_diff and cached_diff.get('diff')
                         if not force_recache and has_proper_diff_cache:
                             diffset = cached_diff['diff']
                         else:
                             diffset = self._get_range_diffset(
                                 commits_source_repo, source_repo,
                                 commit1, commit2, diff_limit, file_limit,
                                 c.fulldiff, ign_whitespace_lcl, context_lcl
                             )
 
                         # save cached diff
                         if caching_enabled:
                             cache_diff(range_diff_cache_file_path, diffset, None)
 
                         c.changes[commit.raw_id] = diffset
 
         # this is a hack to properly display links, when creating PR, the
         # compare view and others uses different notation, and
         # compare_commits.mako renders links based on the target_repo.
         # We need to swap that here to generate it properly on the html side
         c.target_repo = c.source_repo
 
         c.commit_statuses = ChangesetStatus.STATUSES
 
         c.show_version_changes = not pr_closed
         if c.show_version_changes:
             cur_obj = pull_request_at_ver
             prev_obj = prev_pull_request_at_ver
 
             old_commit_ids = prev_obj.revisions
             new_commit_ids = cur_obj.revisions
             commit_changes = PullRequestModel()._calculate_commit_id_changes(
                 old_commit_ids, new_commit_ids)
             c.commit_changes_summary = commit_changes
 
             # calculate the diff for commits between versions
             c.commit_changes = []
 
             def mark(cs, fw):
                 return list(h.itertools.zip_longest([], cs, fillvalue=fw))
 
             for c_type, raw_id in mark(commit_changes.added, 'a') \
                                 + mark(commit_changes.removed, 'r') \
                                 + mark(commit_changes.common, 'c'):
 
                 if raw_id in commit_cache:
                     commit = commit_cache[raw_id]
                 else:
                     try:
                         commit = commits_source_repo.get_commit(raw_id)
                     except CommitDoesNotExistError:
                         # in case we fail extracting still use "dummy" commit
                         # for display in commit diff
                         commit = h.AttributeDict(
                             {'raw_id': raw_id,
                              'message': 'EMPTY or MISSING COMMIT'})
                 c.commit_changes.append([c_type, commit])
 
             # current user review statuses for each version
             c.review_versions = {}
             is_reviewer = PullRequestModel().is_user_reviewer(
                 pull_request, self._rhodecode_user)
             if is_reviewer:
                 for co in general_comments:
                     if co.author.user_id == self._rhodecode_user.user_id:
                         status = co.status_change
                         if status:
                             _ver_pr = status[0].comment.pull_request_version_id
                             c.review_versions[_ver_pr] = status[0]
 
         return self._get_template_context(c)
 
     def get_commits(
             self, commits_source_repo, pull_request_at_ver, source_commit,
             source_ref_id, source_scm, target_commit, target_ref_id, target_scm,
             maybe_unreachable=False):
 
         commit_cache = collections.OrderedDict()
         missing_requirements = False
 
         try:
             pre_load = ["author", "date", "message", "branch", "parents"]
 
             pull_request_commits = pull_request_at_ver.revisions
             log.debug('Loading %s commits from %s',
                       len(pull_request_commits), commits_source_repo)
 
             for rev in pull_request_commits:
                 comm = commits_source_repo.get_commit(commit_id=rev, pre_load=pre_load,
                                                       maybe_unreachable=maybe_unreachable)
                 commit_cache[comm.raw_id] = comm
 
             # Order here matters, we first need to get target, and then
             # the source
             target_commit = commits_source_repo.get_commit(
                 commit_id=safe_str(target_ref_id))
 
             source_commit = commits_source_repo.get_commit(
                 commit_id=safe_str(source_ref_id), maybe_unreachable=True)
         except CommitDoesNotExistError:
             log.warning('Failed to get commit from `{}` repo'.format(
                 commits_source_repo), exc_info=True)
         except RepositoryRequirementError:
             log.warning('Failed to get all required data from repo', exc_info=True)
             missing_requirements = True
 
         pr_ancestor_id = pull_request_at_ver.common_ancestor_id
 
         try:
             ancestor_commit = source_scm.get_commit(pr_ancestor_id)
         except Exception:
             ancestor_commit = None
 
         return ancestor_commit, commit_cache, missing_requirements, source_commit, target_commit
 
     def assure_not_empty_repo(self):
         _ = self.request.translate
 
         try:
             self.db_repo.scm_instance().get_commit()
         except EmptyRepositoryError:
             h.flash(h.literal(_('There are no commits yet')),
                     category='warning')
             raise HTTPFound(
                 h.route_path('repo_summary', repo_name=self.db_repo.repo_name))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_new(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         self.assure_not_empty_repo()
         source_repo = self.db_repo
 
         commit_id = self.request.GET.get('commit')
         branch_ref = self.request.GET.get('branch')
         bookmark_ref = self.request.GET.get('bookmark')
 
         try:
             source_repo_data = PullRequestModel().generate_repo_data(
                 source_repo, commit_id=commit_id,
                 branch=branch_ref, bookmark=bookmark_ref,
                 translator=self.request.translate)
         except CommitDoesNotExistError as e:
             log.exception(e)
             h.flash(_('Commit does not exist'), 'error')
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=source_repo.repo_name))
 
         default_target_repo = source_repo
 
         if source_repo.parent and c.has_origin_repo_read_perm:
             parent_vcs_obj = source_repo.parent.scm_instance()
             if parent_vcs_obj and not parent_vcs_obj.is_empty():
                 # change default if we have a parent repo
                 default_target_repo = source_repo.parent
 
         target_repo_data = PullRequestModel().generate_repo_data(
             default_target_repo, translator=self.request.translate)
 
         selected_source_ref = source_repo_data['refs']['selected_ref']
         title_source_ref = ''
         if selected_source_ref:
             title_source_ref = selected_source_ref.split(':', 2)[1]
         c.default_title = PullRequestModel().generate_pullrequest_title(
             source=source_repo.repo_name,
             source_ref=title_source_ref,
             target=default_target_repo.repo_name
         )
 
         c.default_repo_data = {
             'source_repo_name': source_repo.repo_name,
-            'source_refs_json': json.dumps(source_repo_data),
+            'source_refs_json': ext_json.str_json(source_repo_data),
             'target_repo_name': default_target_repo.repo_name,
-            'target_refs_json': json.dumps(target_repo_data),
+            'target_refs_json': ext_json.str_json(target_repo_data),
         }
         c.default_source_ref = selected_source_ref
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pull_request_repo_refs(self):
         self.load_default_context()
         target_repo_name = self.request.matchdict['target_repo_name']
         repo = Repository.get_by_repo_name(target_repo_name)
         if not repo:
             raise HTTPNotFound()
 
         target_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             target_repo_name)
         if not target_perm:
             raise HTTPNotFound()
 
         return PullRequestModel().generate_repo_data(
             repo, translator=self.request.translate)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pullrequest_repo_targets(self):
         _ = self.request.translate
         filter_query = self.request.GET.get('query')
 
         # get the parents
         parent_target_repos = []
         if self.db_repo.parent:
             parents_query = Repository.query() \
                 .order_by(func.length(Repository.repo_name)) \
                 .filter(Repository.fork_id == self.db_repo.parent.repo_id)
 
             if filter_query:
                 ilike_expression = u'%{}%'.format(safe_unicode(filter_query))
                 parents_query = parents_query.filter(
                     Repository.repo_name.ilike(ilike_expression))
             parents = parents_query.limit(20).all()
 
             for parent in parents:
                 parent_vcs_obj = parent.scm_instance()
                 if parent_vcs_obj and not parent_vcs_obj.is_empty():
                     parent_target_repos.append(parent)
 
         # get other forks, and repo itself
         query = Repository.query() \
             .order_by(func.length(Repository.repo_name)) \
             .filter(
                 or_(Repository.repo_id == self.db_repo.repo_id,  # repo itself
                     Repository.fork_id == self.db_repo.repo_id)  # forks of this repo
             )  \
             .filter(~Repository.repo_id.in_([x.repo_id for x in parent_target_repos]))
 
         if filter_query:
             ilike_expression = u'%{}%'.format(safe_unicode(filter_query))
             query = query.filter(Repository.repo_name.ilike(ilike_expression))
 
         limit = max(20 - len(parent_target_repos), 5)  # not less then 5
         target_repos = query.limit(limit).all()
 
         all_target_repos = target_repos + parent_target_repos
 
         repos = []
         # This checks permissions to the repositories
         for obj in ScmModel().get_repos(all_target_repos):
             repos.append({
                 'id': obj['name'],
                 'text': obj['name'],
                 'type': 'repo',
                 'repo_id': obj['dbrepo']['repo_id'],
                 'repo_type': obj['dbrepo']['repo_type'],
                 'private': obj['dbrepo']['private'],
 
             })
 
         data = {
             'more': False,
             'results': [{
                 'text': _('Repositories'),
                 'children': repos
             }] if repos else []
         }
         return data
 
     @classmethod
     def get_comment_ids(cls, post_data):
         return filter(lambda e: e > 0, map(safe_int, aslist(post_data.get('comments'), ',')))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pullrequest_comments(self):
         self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
         version = self.request.GET.get('version')
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/base/sidebar.mako')
         c = _render.get_call_context()
 
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
         versions = pull_request_display_obj.versions()
         latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
         c.versions = versions + [latest_ver]
 
         c.at_version = at_version
         c.at_version_num = (at_version
                             if at_version and at_version != PullRequest.LATEST_VER
                             else None)
 
         self.register_comments_vars(c, pull_request_latest, versions, include_drafts=False)
         all_comments = c.inline_comments_flat + c.comments
 
         existing_ids = self.get_comment_ids(self.request.POST)
         return _render('comments_table', all_comments, len(all_comments),
                        existing_ids=existing_ids)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pullrequest_todos(self):
         self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
         version = self.request.GET.get('version')
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/base/sidebar.mako')
         c = _render.get_call_context()
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
         versions = pull_request_display_obj.versions()
         latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
         c.versions = versions + [latest_ver]
 
         c.at_version = at_version
         c.at_version_num = (at_version
                             if at_version and at_version != PullRequest.LATEST_VER
                             else None)
 
         c.unresolved_comments = CommentsModel() \
             .get_pull_request_unresolved_todos(pull_request, include_drafts=False)
         c.resolved_comments = CommentsModel() \
             .get_pull_request_resolved_todos(pull_request, include_drafts=False)
 
         all_comments = c.unresolved_comments + c.resolved_comments
         existing_ids = self.get_comment_ids(self.request.POST)
         return _render('comments_table', all_comments, len(c.unresolved_comments),
                        todo_comments=True, existing_ids=existing_ids)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def pullrequest_drafts(self):
         self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
         version = self.request.GET.get('version')
 
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/base/sidebar.mako')
         c = _render.get_call_context()
 
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
         versions = pull_request_display_obj.versions()
         latest_ver = PullRequest.get_pr_display_object(pull_request_latest, pull_request_latest)
         c.versions = versions + [latest_ver]
 
         c.at_version = at_version
         c.at_version_num = (at_version
                             if at_version and at_version != PullRequest.LATEST_VER
                             else None)
 
         c.draft_comments = CommentsModel() \
             .get_pull_request_drafts(self._rhodecode_db_user.user_id, pull_request)
 
         all_comments = c.draft_comments
 
         existing_ids = self.get_comment_ids(self.request.POST)
         return _render('comments_table', all_comments, len(all_comments),
                        existing_ids=existing_ids, draft_comments=True)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_create(self):
         _ = self.request.translate
         self.assure_not_empty_repo()
         self.load_default_context()
 
         controls = peppercorn.parse(self.request.POST.items())
 
         try:
             form = PullRequestForm(
                 self.request.translate, self.db_repo.repo_id)()
             _form = form.to_python(controls)
         except formencode.Invalid as errors:
             if errors.error_dict.get('revisions'):
                 msg = 'Revisions: %s' % errors.error_dict['revisions']
             elif errors.error_dict.get('pullrequest_title'):
                 msg = errors.error_dict.get('pullrequest_title')
             else:
                 msg = _('Error creating pull request: {}').format(errors)
             log.exception(msg)
             h.flash(msg, 'error')
 
             # would rather just go back to form ...
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name))
 
         source_repo = _form['source_repo']
         source_ref = _form['source_ref']
         target_repo = _form['target_repo']
         target_ref = _form['target_ref']
         commit_ids = _form['revisions'][::-1]
         common_ancestor_id = _form['common_ancestor']
 
         # find the ancestor for this pr
         source_db_repo = Repository.get_by_repo_name(_form['source_repo'])
         target_db_repo = Repository.get_by_repo_name(_form['target_repo'])
 
         if not (source_db_repo or target_db_repo):
             h.flash(_('source_repo or target repo not found'), category='error')
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name))
 
         # re-check permissions again here
         # source_repo we must have read permissions
 
         source_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             source_db_repo.repo_name)
         if not source_perm:
             msg = _('Not Enough permissions to source repo `{}`.'.format(
                 source_db_repo.repo_name))
             h.flash(msg, category='error')
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         # target repo we must have read permissions, and also later on
         # we want to check branch permissions here
         target_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             target_db_repo.repo_name)
         if not target_perm:
             msg = _('Not Enough permissions to target repo `{}`.'.format(
                 target_db_repo.repo_name))
             h.flash(msg, category='error')
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         source_scm = source_db_repo.scm_instance()
         target_scm = target_db_repo.scm_instance()
 
         source_ref_obj = unicode_to_reference(source_ref)
         target_ref_obj = unicode_to_reference(target_ref)
 
         source_commit = source_scm.get_commit(source_ref_obj.commit_id)
         target_commit = target_scm.get_commit(target_ref_obj.commit_id)
 
         ancestor = source_scm.get_common_ancestor(
             source_commit.raw_id, target_commit.raw_id, target_scm)
 
         # recalculate target ref based on ancestor
         target_ref = ':'.join((target_ref_obj.type, target_ref_obj.name, ancestor))
 
         get_default_reviewers_data, validate_default_reviewers, validate_observers = \
             PullRequestModel().get_reviewer_functions()
 
         # recalculate reviewers logic, to make sure we can validate this
         reviewer_rules = get_default_reviewers_data(
             self._rhodecode_db_user,
             source_db_repo,
             source_ref_obj,
             target_db_repo,
             target_ref_obj,
             include_diff_info=False)
 
         reviewers = validate_default_reviewers(_form['review_members'], reviewer_rules)
         observers = validate_observers(_form['observer_members'], reviewer_rules)
 
         pullrequest_title = _form['pullrequest_title']
         title_source_ref = source_ref_obj.name
         if not pullrequest_title:
             pullrequest_title = PullRequestModel().generate_pullrequest_title(
                 source=source_repo,
                 source_ref=title_source_ref,
                 target=target_repo
             )
 
         description = _form['pullrequest_desc']
         description_renderer = _form['description_renderer']
 
         try:
             pull_request = PullRequestModel().create(
                 created_by=self._rhodecode_user.user_id,
                 source_repo=source_repo,
                 source_ref=source_ref,
                 target_repo=target_repo,
                 target_ref=target_ref,
                 revisions=commit_ids,
                 common_ancestor_id=common_ancestor_id,
                 reviewers=reviewers,
                 observers=observers,
                 title=pullrequest_title,
                 description=description,
                 description_renderer=description_renderer,
                 reviewer_data=reviewer_rules,
                 auth_user=self._rhodecode_user
             )
             Session().commit()
 
             h.flash(_('Successfully opened new pull request'),
                     category='success')
         except Exception:
             msg = _('Error occurred during creation of this pull request.')
             log.exception(msg)
             h.flash(msg, category='error')
 
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         raise HTTPFound(
             h.route_path('pullrequest_show', repo_name=target_repo,
                          pull_request_id=pull_request.pull_request_id))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_update(self):
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         _ = self.request.translate
 
         c = self.load_default_context()
         redirect_url = None
         # we do this check as first, because we want to know ASAP in the flow that
         # pr is updating currently
         is_state_changing = pull_request.is_state_changing()
 
         if pull_request.is_closed():
             log.debug('update: forbidden because pull request is closed')
             msg = _(u'Cannot update closed pull requests.')
             h.flash(msg, category='error')
             return {'response': True,
                     'redirect_url': redirect_url}
 
         c.pr_broadcast_channel = channelstream.pr_channel(pull_request)
 
         # only owner or admin can update it
         allowed_to_update = PullRequestModel().check_user_update(
             pull_request, self._rhodecode_user)
 
         if allowed_to_update:
             controls = peppercorn.parse(self.request.POST.items())
             force_refresh = str2bool(self.request.POST.get('force_refresh', 'false'))
             do_update_commits = str2bool(self.request.POST.get('update_commits', 'false'))
 
             if 'review_members' in controls:
                 self._update_reviewers(
                     c,
                     pull_request, controls['review_members'],
                     pull_request.reviewer_data,
                     PullRequestReviewers.ROLE_REVIEWER)
             elif 'observer_members' in controls:
                 self._update_reviewers(
                     c,
                     pull_request, controls['observer_members'],
                     pull_request.reviewer_data,
                     PullRequestReviewers.ROLE_OBSERVER)
             elif do_update_commits:
                 if is_state_changing:
                     log.debug('commits update: forbidden because pull request is in state %s',
                               pull_request.pull_request_state)
                     msg = _(u'Cannot update pull requests commits in state other than `{}`. '
                             u'Current state is: `{}`').format(
                         PullRequest.STATE_CREATED, pull_request.pull_request_state)
                     h.flash(msg, category='error')
                     return {'response': True,
                             'redirect_url': redirect_url}
 
                 self._update_commits(c, pull_request)
                 if force_refresh:
                     redirect_url = h.route_path(
                         'pullrequest_show', repo_name=self.db_repo_name,
                         pull_request_id=pull_request.pull_request_id,
                         _query={"force_refresh": 1})
             elif str2bool(self.request.POST.get('edit_pull_request', 'false')):
                 self._edit_pull_request(pull_request)
             else:
                 log.error('Unhandled update data.')
                 raise HTTPBadRequest()
 
             return {'response': True,
                     'redirect_url': redirect_url}
         raise HTTPForbidden()
 
     def _edit_pull_request(self, pull_request):
         """
         Edit title and description
         """
         _ = self.request.translate
 
         try:
             PullRequestModel().edit(
                 pull_request,
                 self.request.POST.get('title'),
                 self.request.POST.get('description'),
                 self.request.POST.get('description_renderer'),
                 self._rhodecode_user)
         except ValueError:
             msg = _(u'Cannot update closed pull requests.')
             h.flash(msg, category='error')
             return
         else:
             Session().commit()
 
         msg = _(u'Pull request title & description updated.')
         h.flash(msg, category='success')
         return
 
     def _update_commits(self, c, pull_request):
         _ = self.request.translate
         log.debug('pull-request: running update commits actions')
 
         @retry(exception=Exception, n_tries=3, delay=2)
         def commits_update():
             return PullRequestModel().update_commits(
                 pull_request, self._rhodecode_db_user)
 
         with pull_request.set_state(PullRequest.STATE_UPDATING):
             resp = commits_update()  # retry x3
 
         if resp.executed:
 
             if resp.target_changed and resp.source_changed:
                 changed = 'target and source repositories'
             elif resp.target_changed and not resp.source_changed:
                 changed = 'target repository'
             elif not resp.target_changed and resp.source_changed:
                 changed = 'source repository'
             else:
                 changed = 'nothing'
 
             msg = _(u'Pull request updated to "{source_commit_id}" with '
                     u'{count_added} added, {count_removed} removed commits. '
                     u'Source of changes: {change_source}.')
             msg = msg.format(
                 source_commit_id=pull_request.source_ref_parts.commit_id,
                 count_added=len(resp.changes.added),
                 count_removed=len(resp.changes.removed),
                 change_source=changed)
             h.flash(msg, category='success')
             channelstream.pr_update_channelstream_push(
                 self.request, c.pr_broadcast_channel, self._rhodecode_user, msg)
         else:
             msg = PullRequestModel.UPDATE_STATUS_MESSAGES[resp.reason]
             warning_reasons = [
                 UpdateFailureReason.NO_CHANGE,
                 UpdateFailureReason.WRONG_REF_TYPE,
             ]
             category = 'warning' if resp.reason in warning_reasons else 'error'
             h.flash(msg, category=category)
 
     def _update_reviewers(self, c, pull_request, review_members, reviewer_rules, role):
         _ = self.request.translate
 
         get_default_reviewers_data, validate_default_reviewers, validate_observers = \
             PullRequestModel().get_reviewer_functions()
 
         if role == PullRequestReviewers.ROLE_REVIEWER:
             try:
                 reviewers = validate_default_reviewers(review_members, reviewer_rules)
             except ValueError as e:
                 log.error('Reviewers Validation: {}'.format(e))
                 h.flash(e, category='error')
                 return
 
             old_calculated_status = pull_request.calculated_review_status()
             PullRequestModel().update_reviewers(
                 pull_request, reviewers, self._rhodecode_db_user)
 
             Session().commit()
 
             msg = _('Pull request reviewers updated.')
             h.flash(msg, category='success')
             channelstream.pr_update_channelstream_push(
                 self.request, c.pr_broadcast_channel, self._rhodecode_user, msg)
 
             # trigger status changed if change in reviewers changes the status
             calculated_status = pull_request.calculated_review_status()
             if old_calculated_status != calculated_status:
                 PullRequestModel().trigger_pull_request_hook(
                     pull_request, self._rhodecode_user, 'review_status_change',
                     data={'status': calculated_status})
 
         elif role == PullRequestReviewers.ROLE_OBSERVER:
             try:
                 observers = validate_observers(review_members, reviewer_rules)
             except ValueError as e:
                 log.error('Observers Validation: {}'.format(e))
                 h.flash(e, category='error')
                 return
 
             PullRequestModel().update_observers(
                 pull_request, observers, self._rhodecode_db_user)
 
             Session().commit()
             msg = _('Pull request observers updated.')
             h.flash(msg, category='success')
             channelstream.pr_update_channelstream_push(
                 self.request, c.pr_broadcast_channel, self._rhodecode_user, msg)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_merge(self):
         """
         Merge will perform a server-side merge of the specified
         pull request, if the pull request is approved and mergeable.
         After successful merging, the pull request is automatically
         closed, with a relevant comment.
         """
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         _ = self.request.translate
 
         if pull_request.is_state_changing():
             log.debug('show: forbidden because pull request is in state %s',
                       pull_request.pull_request_state)
             msg = _(u'Cannot merge pull requests in state other than `{}`. '
                     u'Current state is: `{}`').format(PullRequest.STATE_CREATED,
                                                       pull_request.pull_request_state)
             h.flash(msg, category='error')
             raise HTTPFound(
                 h.route_path('pullrequest_show',
                              repo_name=pull_request.target_repo.repo_name,
                              pull_request_id=pull_request.pull_request_id))
 
         self.load_default_context()
 
         with pull_request.set_state(PullRequest.STATE_UPDATING):
             check = MergeCheck.validate(
                 pull_request, auth_user=self._rhodecode_user,
                 translator=self.request.translate)
         merge_possible = not check.failed
 
         for err_type, error_msg in check.errors:
             h.flash(error_msg, category=err_type)
 
         if merge_possible:
             log.debug("Pre-conditions checked, trying to merge.")
             extras = vcs_operation_context(
                 self.request.environ, repo_name=pull_request.target_repo.repo_name,
                 username=self._rhodecode_db_user.username, action='push',
                 scm=pull_request.target_repo.repo_type)
             with pull_request.set_state(PullRequest.STATE_UPDATING):
                 self._merge_pull_request(
                     pull_request, self._rhodecode_db_user, extras)
         else:
             log.debug("Pre-conditions failed, NOT merging.")
 
         raise HTTPFound(
             h.route_path('pullrequest_show',
                          repo_name=pull_request.target_repo.repo_name,
                          pull_request_id=pull_request.pull_request_id))
 
     def _merge_pull_request(self, pull_request, user, extras):
         _ = self.request.translate
         merge_resp = PullRequestModel().merge_repo(pull_request, user, extras=extras)
 
         if merge_resp.executed:
             log.debug("The merge was successful, closing the pull request.")
             PullRequestModel().close_pull_request(
                 pull_request.pull_request_id, user)
             Session().commit()
             msg = _('Pull request was successfully merged and closed.')
             h.flash(msg, category='success')
         else:
             log.debug(
                 "The merge was not successful. Merge response: %s", merge_resp)
             msg = merge_resp.merge_status_message
             h.flash(msg, category='error')
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_delete(self):
         _ = self.request.translate
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         self.load_default_context()
 
         pr_closed = pull_request.is_closed()
         allowed_to_delete = PullRequestModel().check_user_delete(
             pull_request, self._rhodecode_user) and not pr_closed
 
         # only owner can delete it !
         if allowed_to_delete:
             PullRequestModel().delete(pull_request, self._rhodecode_user)
             Session().commit()
             h.flash(_('Successfully deleted pull request'),
                     category='success')
             raise HTTPFound(h.route_path('pullrequest_show_all',
                                          repo_name=self.db_repo_name))
 
         log.warning('user %s tried to delete pull request without access',
                     self._rhodecode_user)
         raise HTTPNotFound()
 
     def _pull_request_comments_create(self, pull_request, comments):
         _ = self.request.translate
         data = {}
         if not comments:
             return
         pull_request_id = pull_request.pull_request_id
 
         all_drafts = len([x for x in comments if str2bool(x['is_draft'])]) == len(comments)
 
         for entry in comments:
             c = self.load_default_context()
             comment_type = entry['comment_type']
             text = entry['text']
             status = entry['status']
             is_draft = str2bool(entry['is_draft'])
             resolves_comment_id = entry['resolves_comment_id']
             close_pull_request = entry['close_pull_request']
             f_path = entry['f_path']
             line_no = entry['line']
             target_elem_id = 'file-{}'.format(h.safeid(h.safe_unicode(f_path)))
 
             # the logic here should work like following, if we submit close
             # pr comment, use `close_pull_request_with_comment` function
             # else handle regular comment logic
 
             if close_pull_request:
                 # only owner or admin or person with write permissions
                 allowed_to_close = PullRequestModel().check_user_update(
                     pull_request, self._rhodecode_user)
                 if not allowed_to_close:
                     log.debug('comment: forbidden because not allowed to close '
                               'pull request %s', pull_request_id)
                     raise HTTPForbidden()
 
                 # This also triggers `review_status_change`
                 comment, status = PullRequestModel().close_pull_request_with_comment(
                     pull_request, self._rhodecode_user, self.db_repo, message=text,
                     auth_user=self._rhodecode_user)
                 Session().flush()
                 is_inline = comment.is_inline
 
                 PullRequestModel().trigger_pull_request_hook(
                     pull_request, self._rhodecode_user, 'comment',
                     data={'comment': comment})
 
             else:
                 # regular comment case, could be inline, or one with status.
                 # for that one we check also permissions
                 # Additionally ENSURE if somehow draft is sent we're then unable to change status
                 allowed_to_change_status = PullRequestModel().check_user_change_status(
                     pull_request, self._rhodecode_user) and not is_draft
 
                 if status and allowed_to_change_status:
                     message = (_('Status change %(transition_icon)s %(status)s')
                                % {'transition_icon': '>',
                                   'status': ChangesetStatus.get_status_lbl(status)})
                     text = text or message
 
                 comment = CommentsModel().create(
                     text=text,
                     repo=self.db_repo.repo_id,
                     user=self._rhodecode_user.user_id,
                     pull_request=pull_request,
                     f_path=f_path,
                     line_no=line_no,
                     status_change=(ChangesetStatus.get_status_lbl(status)
                                    if status and allowed_to_change_status else None),
                     status_change_type=(status
                                         if status and allowed_to_change_status else None),
                     comment_type=comment_type,
                     is_draft=is_draft,
                     resolves_comment_id=resolves_comment_id,
                     auth_user=self._rhodecode_user,
                     send_email=not is_draft,  # skip notification for draft comments
                 )
                 is_inline = comment.is_inline
 
                 if allowed_to_change_status:
                     # calculate old status before we change it
                     old_calculated_status = pull_request.calculated_review_status()
 
                     # get status if set !
                     if status:
                         ChangesetStatusModel().set_status(
                             self.db_repo.repo_id,
                             status,
                             self._rhodecode_user.user_id,
                             comment,
                             pull_request=pull_request
                         )
 
                     Session().flush()
                     # this is somehow required to get access to some relationship
                     # loaded on comment
                     Session().refresh(comment)
 
                     # skip notifications for drafts
                     if not is_draft:
                         PullRequestModel().trigger_pull_request_hook(
                             pull_request, self._rhodecode_user, 'comment',
                             data={'comment': comment})
 
                     # we now calculate the status of pull request, and based on that
                     # calculation we set the commits status
                     calculated_status = pull_request.calculated_review_status()
                     if old_calculated_status != calculated_status:
                         PullRequestModel().trigger_pull_request_hook(
                             pull_request, self._rhodecode_user, 'review_status_change',
                             data={'status': calculated_status})
 
             comment_id = comment.comment_id
             data[comment_id] = {
                 'target_id': target_elem_id
             }
             Session().flush()
 
             c.co = comment
             c.at_version_num = None
             c.is_new = True
             rendered_comment = render(
                 'rhodecode:templates/changeset/changeset_comment_block.mako',
                 self._get_template_context(c), self.request)
 
             data[comment_id].update(comment.get_dict())
             data[comment_id].update({'rendered_text': rendered_comment})
 
         Session().commit()
 
         # skip channelstream for draft comments
         if not all_drafts:
             comment_broadcast_channel = channelstream.comment_channel(
                 self.db_repo_name, pull_request_obj=pull_request)
 
             comment_data = data
             posted_comment_type = 'inline' if is_inline else 'general'
             if len(data) == 1:
                 msg = _('posted {} new {} comment').format(len(data), posted_comment_type)
             else:
                 msg = _('posted {} new {} comments').format(len(data), posted_comment_type)
 
             channelstream.comment_channelstream_push(
                 self.request, comment_broadcast_channel, self._rhodecode_user, msg,
                 comment_data=comment_data)
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_comment_create(self):
         _ = self.request.translate
 
         pull_request = PullRequest.get_or_404(self.request.matchdict['pull_request_id'])
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         allowed_to_comment = PullRequestModel().check_user_comment(
             pull_request, self._rhodecode_user)
         if not allowed_to_comment:
             log.debug('comment: forbidden because pull request is from forbidden repo')
             raise HTTPForbidden()
 
         comment_data = {
             'comment_type': self.request.POST.get('comment_type'),
             'text': self.request.POST.get('text'),
             'status': self.request.POST.get('changeset_status', None),
             'is_draft': self.request.POST.get('draft'),
             'resolves_comment_id': self.request.POST.get('resolves_comment_id', None),
             'close_pull_request': self.request.POST.get('close_pull_request'),
             'f_path': self.request.POST.get('f_path'),
             'line': self.request.POST.get('line'),
         }
         data = self._pull_request_comments_create(pull_request, [comment_data])
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_comment_delete(self):
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
 
         comment = ChangesetComment.get_or_404(
             self.request.matchdict['comment_id'])
         comment_id = comment.comment_id
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         if not comment:
             log.debug('Comment with id:%s not found, skipping', comment_id)
             # comment already deleted in another call probably
             return True
 
         if comment.pull_request.is_closed():
             # don't allow deleting comments on closed pull request
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = comment.author.user_id == self._rhodecode_user.user_id
         is_repo_comment = comment.repo.repo_name == self.db_repo_name
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if comment.draft and not comment_owner:
             # We never allow to delete draft comments for other than owners
             raise HTTPNotFound()
 
         if super_admin or comment_owner or comment_repo_admin:
             old_calculated_status = comment.pull_request.calculated_review_status()
             CommentsModel().delete(comment=comment, auth_user=self._rhodecode_user)
             Session().commit()
             calculated_status = comment.pull_request.calculated_review_status()
             if old_calculated_status != calculated_status:
                 PullRequestModel().trigger_pull_request_hook(
                     comment.pull_request, self._rhodecode_user, 'review_status_change',
                     data={'status': calculated_status})
             return True
         else:
             log.warning('No permissions for user %s to delete comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def pull_request_comment_edit(self):
         self.load_default_context()
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id']
         )
         comment = ChangesetComment.get_or_404(
             self.request.matchdict['comment_id']
         )
         comment_id = comment.comment_id
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         if comment.pull_request.is_closed():
             # don't allow deleting comments on closed pull request
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = comment.author.user_id == self._rhodecode_user.user_id
         is_repo_comment = comment.repo.repo_name == self.db_repo_name
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if super_admin or comment_owner or comment_repo_admin:
             text = self.request.POST.get('text')
             version = self.request.POST.get('version')
             if text == comment.text:
                 log.warning(
                     'Comment(PR): '
                     'Trying to create new version '
                     'with the same comment body {}'.format(
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             if version.isdigit():
                 version = int(version)
             else:
                 log.warning(
                     'Comment(PR): Wrong version type {} {} '
                     'for comment {}'.format(
                         version,
                         type(version),
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             try:
                 comment_history = CommentsModel().edit(
                     comment_id=comment_id,
                     text=text,
                     auth_user=self._rhodecode_user,
                     version=version,
                 )
             except CommentVersionMismatch:
                 raise HTTPConflict()
 
             if not comment_history:
                 raise HTTPNotFound()
 
             Session().commit()
             if not comment.draft:
                 PullRequestModel().trigger_pull_request_hook(
                     pull_request, self._rhodecode_user, 'comment_edit',
                     data={'comment': comment})
 
             return {
                 'comment_history_id': comment_history.comment_history_id,
                 'comment_id': comment.comment_id,
                 'comment_version': comment_history.version,
                 'comment_author_username': comment_history.author.username,
                 'comment_author_gravatar': h.gravatar_url(comment_history.author.email, 16),
                 'comment_created_on': h.age_component(comment_history.created_on,
                                                       time_is_local=True),
             }
         else:
             log.warning('No permissions for user %s to edit comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()

rhodecode/apps/repository/views/repo_tags.py

@@ -1,44 +1,44 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 from rhodecode.apps._base import BaseReferencesView
-from rhodecode.lib.ext_json import json
+from rhodecode.lib import ext_json
 from rhodecode.lib.auth import (LoginRequired, HasRepoPermissionAnyDecorator)
 
 log = logging.getLogger(__name__)
 
 
 class RepoTagsView(BaseReferencesView):
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def tags(self):
         c = self.load_default_context()
 
         ref_items = self.rhodecode_vcs_repo.tags.items()
         data = self.load_refs_context(
             ref_items=ref_items, partials_template='tags/tags_data.mako')
 
         c.has_references = bool(data)
-        c.data = json.dumps(data)
+        c.data = ext_json.str_json(data)
         return self._get_template_context(c)

rhodecode/lib/ext_json_renderer.py

@@ -1,40 +1,41 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-from rhodecode.lib.ext_json import json
+from rhodecode.lib import ext_json
 
 
 def pyramid_ext_json(info):
     """
     Custom json renderer for pyramid to use our ext_json lib
     """
     def _render(value, system):
         request = system.get('request')
         indent = None
         if request is not None:
             response = request.response
             ct = response.content_type
             if ct == response.default_content_type:
                 response.content_type = 'application/json'
             indent = getattr(request, 'ext_json_indent', None)
-
-        return json.dumps(value, indent=indent)
+        if indent:
+            return ext_json.formatted_json(value)
+        return ext_json.json.dumps(value)
 
     return _render

rhodecode/lib/helpers.py

@@ -1,2155 +1,2165 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Helper functions
 
 Consists of functions to typically be used within templates, but also
 available to Controllers. This module is available to both as 'h'.
 """
 import base64
 import collections
 
 import os
 import random
 import hashlib
 from io import StringIO
 import textwrap
 import urllib.request, urllib.parse, urllib.error
 import math
 import logging
 import re
 import time
 import string
 import hashlib
 import regex
 from collections import OrderedDict
 
 import pygments
 import itertools
 import fnmatch
 import bleach
 
 from datetime import datetime
 from functools import partial
 from pygments.formatters.html import HtmlFormatter
 from pygments.lexers import (
     get_lexer_by_name, get_lexer_for_filename, get_lexer_for_mimetype)
 
 from pyramid.threadlocal import get_current_request
 from tempita import looper
 from webhelpers2.html import literal, HTML, escape
 from webhelpers2.html._autolink import _auto_link_urls
 from webhelpers2.html.tools import (
     button_to, highlight, js_obfuscate, strip_links, strip_tags)
 
 from webhelpers2.text import (
     chop_at, collapse, convert_accented_entities,
     convert_misc_entities, lchop, plural, rchop, remove_formatting,
     replace_whitespace, urlify, truncate, wrap_paragraphs)
 from webhelpers2.date import time_ago_in_words
 
 from webhelpers2.html.tags import (
     _input, NotGiven, _make_safe_id_component as safeid,
     form as insecure_form,
     auto_discovery_link, checkbox, end_form, file,
     hidden, image, javascript_link, link_to, link_to_if, link_to_unless, ol,
     select as raw_select, stylesheet_link, submit, text, password, textarea,
     ul, radio, Options)
 
 from webhelpers2.number import format_byte_size
 
 from rhodecode.lib.action_parser import action_parser
 from rhodecode.lib.pagination import Page, RepoPage, SqlPage
+from rhodecode.lib import ext_json
 from rhodecode.lib.ext_json import json
+from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.lib.utils import repo_name_slug, get_custom_lexer
 from rhodecode.lib.utils2 import (
     str2bool, safe_unicode, safe_str,
     get_commit_safe, datetime_to_time, time_to_datetime, time_to_utcdatetime,
     AttributeDict, safe_int, md5, md5_safe, get_host_info)
 from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
 from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError
 from rhodecode.lib.vcs.backends.base import BaseChangeset, EmptyCommit
 from rhodecode.lib.vcs.conf.settings import ARCHIVE_SPECS
 from rhodecode.lib.index.search_utils import get_matching_line_offsets
 from rhodecode.config.conf import DATE_FORMAT, DATETIME_FORMAT
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.db import Permission, User, Repository, UserApiKeys, FileStore
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import IssueTrackerSettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 DEFAULT_USER = User.DEFAULT_USER
 DEFAULT_USER_EMAIL = User.DEFAULT_USER_EMAIL
 
 
 def asset(path, ver=None, **kwargs):
     """
     Helper to generate a static asset file path for rhodecode assets
 
     eg. h.asset('images/image.png', ver='3923')
 
     :param path: path of asset
     :param ver: optional version query param to append as ?ver=
     """
     request = get_current_request()
     query = {}
     query.update(kwargs)
     if ver:
         query = {'ver': ver}
     return request.static_path(
         'rhodecode:public/{}'.format(path), _query=query)
 
 
 default_html_escape_table = {
     ord('&'): u'&amp;',
     ord('<'): u'&lt;',
     ord('>'): u'&gt;',
     ord('"'): u'&quot;',
     ord("'"): u'&#39;',
 }
 
 
 def html_escape(text, html_escape_table=default_html_escape_table):
     """Produce entities within text."""
     return text.translate(html_escape_table)
 
 
+def str_json(*args, **kwargs):
+    return ext_json.str_json(*args, **kwargs)
+
+
+def formatted_str_json(*args, **kwargs):
+    return ext_json.formatted_str_json(*args, **kwargs)
+
+
 def chop_at_smart(s, sub, inclusive=False, suffix_if_chopped=None):
     """
     Truncate string ``s`` at the first occurrence of ``sub``.
 
     If ``inclusive`` is true, truncate just after ``sub`` rather than at it.
     """
     suffix_if_chopped = suffix_if_chopped or ''
     pos = s.find(sub)
     if pos == -1:
         return s
 
     if inclusive:
         pos += len(sub)
 
     chopped = s[:pos]
     left = s[pos:].strip()
 
     if left and suffix_if_chopped:
         chopped += suffix_if_chopped
 
     return chopped
 
 
 def shorter(text, size=20, prefix=False):
     postfix = '...'
     if len(text) > size:
         if prefix:
             # shorten in front
             return postfix + text[-(size - len(postfix)):]
         else:
             return text[:size - len(postfix)] + postfix
     return text
 
 
 def reset(name, value=None, id=NotGiven, type="reset", **attrs):
     """
     Reset button
     """
     return _input(type, name, value, id, attrs)
 
 
 def select(name, selected_values, options, id=NotGiven, **attrs):
 
     if isinstance(options, (list, tuple)):
         options_iter = options
         # Handle old value,label lists ... where value also can be value,label lists
         options = Options()
         for opt in options_iter:
             if isinstance(opt, tuple) and len(opt) == 2:
                 value, label = opt
             elif isinstance(opt, str):
                 value = label = opt
             else:
                 raise ValueError('invalid select option type %r' % type(opt))
 
             if isinstance(value, (list, tuple)):
                 option_group = options.add_optgroup(label)
                 for opt2 in value:
                     if isinstance(opt2, tuple) and len(opt2) == 2:
                         group_value, group_label = opt2
                     elif isinstance(opt2, str):
                         group_value = group_label = opt2
                     else:
                         raise ValueError('invalid select option type %r' % type(opt2))
 
                     option_group.add_option(group_label, group_value)
             else:
                 options.add_option(label, value)
 
     return raw_select(name, selected_values, options, id=id, **attrs)
 
 
 def branding(name, length=40):
     return truncate(name, length, indicator="")
 
 
 def FID(raw_id, path):
     """
     Creates a unique ID for filenode based on it's hash of path and commit
     it's safe to use in urls
 
     :param raw_id:
     :param path:
     """
 
     return 'c-%s-%s' % (short_id(raw_id), md5_safe(path)[:12])
 
 
 class _GetError(object):
     """Get error from form_errors, and represent it as span wrapped error
     message
 
     :param field_name: field to fetch errors for
     :param form_errors: form errors dict
     """
 
     def __call__(self, field_name, form_errors):
         tmpl = """<span class="error_msg">%s</span>"""
         if form_errors and field_name in form_errors:
             return literal(tmpl % form_errors.get(field_name))
 
 
 get_error = _GetError()
 
 
 class _ToolTip(object):
 
     def __call__(self, tooltip_title, trim_at=50):
         """
         Special function just to wrap our text into nice formatted
         autowrapped text
 
         :param tooltip_title:
         """
         tooltip_title = escape(tooltip_title)
         tooltip_title = tooltip_title.replace('<', '&lt;').replace('>', '&gt;')
         return tooltip_title
 
 
 tooltip = _ToolTip()
 
 files_icon = u'<i class="file-breadcrumb-copy tooltip icon-clipboard clipboard-action" data-clipboard-text="{}" title="Copy file path"></i>'
 
 
 def files_breadcrumbs(repo_name, repo_type, commit_id, file_path, landing_ref_name=None, at_ref=None,
                       limit_items=False, linkify_last_item=False, hide_last_item=False,
                       copy_path_icon=True):
     if isinstance(file_path, str):
         file_path = safe_unicode(file_path)
 
     if at_ref:
         route_qry = {'at': at_ref}
         default_landing_ref = at_ref or landing_ref_name or commit_id
     else:
         route_qry = None
         default_landing_ref = commit_id
 
     # first segment is a `HOME` link to repo files root location
     root_name = literal(u'<i class="icon-home"></i>')
 
     url_segments = [
         link_to(
             root_name,
             repo_files_by_ref_url(
                 repo_name,
                 repo_type,
                 f_path=None,  # None here is a special case for SVN repos,
                               # that won't prefix with a ref
                 ref_name=default_landing_ref,
                 commit_id=commit_id,
                 query=route_qry
             )
         )]
 
     path_segments = file_path.split('/')
     last_cnt = len(path_segments) - 1
     for cnt, segment in enumerate(path_segments):
         if not segment:
             continue
         segment_html = escape(segment)
 
         last_item = cnt == last_cnt
 
         if last_item and hide_last_item:
             # iterate over and hide last element
             continue
 
         if last_item and linkify_last_item is False:
             # plain version
             url_segments.append(segment_html)
         else:
             url_segments.append(
                 link_to(
                     segment_html,
                     repo_files_by_ref_url(
                         repo_name,
                         repo_type,
                         f_path='/'.join(path_segments[:cnt + 1]),
                         ref_name=default_landing_ref,
                         commit_id=commit_id,
                         query=route_qry
                     ),
                 ))
 
     limited_url_segments = url_segments[:1] + ['...'] + url_segments[-5:]
     if limit_items and len(limited_url_segments) < len(url_segments):
         url_segments = limited_url_segments
 
     full_path = file_path
     if copy_path_icon:
         icon = files_icon.format(escape(full_path))
     else:
         icon = ''
 
     if file_path == '':
         return root_name
     else:
         return literal(' / '.join(url_segments) + icon)
 
 
 def files_url_data(request):
     import urllib.request, urllib.parse, urllib.error
     matchdict = request.matchdict
 
     if 'f_path' not in matchdict:
         matchdict['f_path'] = ''
     else:
         matchdict['f_path'] = urllib.parse.quote(safe_str(matchdict['f_path']))
     if 'commit_id' not in matchdict:
         matchdict['commit_id'] = 'tip'
 
-    return json.dumps(matchdict)
+    return ext_json.str_json(matchdict)
 
 
 def repo_files_by_ref_url(db_repo_name, db_repo_type, f_path, ref_name, commit_id, query=None, ):
     _is_svn = is_svn(db_repo_type)
     final_f_path = f_path
 
     if _is_svn:
         """
         For SVN the ref_name cannot be used as a commit_id, it needs to be prefixed with
         actually commit_id followed by the ref_name. This should be done only in case
         This is a initial landing url, without additional paths.
 
         like: /1000/tags/1.0.0/?at=tags/1.0.0
         """
 
         if ref_name and ref_name != 'tip':
             # NOTE(marcink): for svn the ref_name is actually the stored path, so we prefix it
             # for SVN we only do this magic prefix if it's root, .eg landing revision
             # of files link. If we are in the tree we don't need this since we traverse the url
             # that has everything stored
             if f_path in ['', '/']:
                 final_f_path = '/'.join([ref_name, f_path])
 
         # SVN always needs a commit_id explicitly, without a named REF
         default_commit_id = commit_id
     else:
         """
         For git and mercurial we construct a new URL using the names instead of commit_id
         like: /master/some_path?at=master
         """
         # We currently do not support branches with slashes
         if '/' in ref_name:
             default_commit_id = commit_id
         else:
             default_commit_id = ref_name
 
     # sometimes we pass f_path as None, to indicate explicit no prefix,
     # we translate it to string to not have None
     final_f_path = final_f_path or ''
 
     files_url = route_path(
         'repo_files',
         repo_name=db_repo_name,
         commit_id=default_commit_id,
         f_path=final_f_path,
         _query=query
     )
     return files_url
 
 
 def code_highlight(code, lexer, formatter, use_hl_filter=False):
     """
     Lex ``code`` with ``lexer`` and format it with the formatter ``formatter``.
 
     If ``outfile`` is given and a valid file object (an object
     with a ``write`` method), the result will be written to it, otherwise
     it is returned as a string.
     """
     if use_hl_filter:
         # add HL filter
         from rhodecode.lib.index import search_utils
         lexer.add_filter(search_utils.ElasticSearchHLFilter())
     return pygments.format(pygments.lex(code, lexer), formatter)
 
 
 class CodeHtmlFormatter(HtmlFormatter):
     """
     My code Html Formatter for source codes
     """
 
     def wrap(self, source, outfile):
         return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<div id="L%s">%s</div>' % (cnt + 1, t)
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         dummyoutfile = StringIO.StringIO()
         lncount = 0
         for t, line in inner:
             if t:
                 lncount += 1
             dummyoutfile.write(line)
 
         fl = self.linenostart
         mw = len(str(lncount + fl - 1))
         sp = self.linenospecial
         st = self.linenostep
         la = self.lineanchors
         aln = self.anchorlinenos
         nocls = self.noclasses
         if sp:
             lines = []
 
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if i % sp == 0:
                         if aln:
                             lines.append('<a href="#%s%d" class="special">%*d</a>' %
                                          (la, i, mw, i))
                         else:
                             lines.append('<span class="special">%*d</span>' % (mw, i))
                     else:
                         if aln:
                             lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                         else:
                             lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
         else:
             lines = []
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if aln:
                         lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                     else:
                         lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
 
         # in case you wonder about the seemingly redundant <div> here: since the
         # content in the other cell also is wrapped in a div, some browsers in
         # some configurations seem to mess up the formatting...
         if nocls:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td><div class="linenodiv" '
                       'style="background-color: #f0f0f0; padding-right: 10px">'
                       '<pre style="line-height: 125%">' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         else:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td class="linenos"><div class="linenodiv"><pre>' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         yield 0, dummyoutfile.getvalue()
         yield 0, '</td></tr></table>'
 
 
 class SearchContentCodeHtmlFormatter(CodeHtmlFormatter):
     def __init__(self, **kw):
         # only show these line numbers if set
         self.only_lines = kw.pop('only_line_numbers', [])
         self.query_terms = kw.pop('query_terms', [])
         self.max_lines = kw.pop('max_lines', 5)
         self.line_context = kw.pop('line_context', 3)
         self.url = kw.pop('url', None)
 
         super(CodeHtmlFormatter, self).__init__(**kw)
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<pre>%s</pre>' % t
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         yield 0, '<table class="code-highlight %stable">' % self.cssclass
 
         last_shown_line_number = 0
         current_line_number = 1
 
         for t, line in inner:
             if not t:
                 yield t, line
                 continue
 
             if current_line_number in self.only_lines:
                 if last_shown_line_number + 1 != current_line_number:
                     yield 0, '<tr>'
                     yield 0, '<td class="line">...</td>'
                     yield 0, '<td id="hlcode" class="code"></td>'
                     yield 0, '</tr>'
 
                 yield 0, '<tr>'
                 if self.url:
                     yield 0, '<td class="line"><a href="%s#L%i">%i</a></td>' % (
                         self.url, current_line_number, current_line_number)
                 else:
                     yield 0, '<td class="line"><a href="">%i</a></td>' % (
                         current_line_number)
                 yield 0, '<td id="hlcode" class="code">' + line + '</td>'
                 yield 0, '</tr>'
 
                 last_shown_line_number = current_line_number
 
             current_line_number += 1
 
         yield 0, '</table>'
 
 
 def hsv_to_rgb(h, s, v):
     """ Convert hsv color values to rgb """
 
     if s == 0.0:
         return v, v, v
     i = int(h * 6.0)  # XXX assume int() truncates!
     f = (h * 6.0) - i
     p = v * (1.0 - s)
     q = v * (1.0 - s * f)
     t = v * (1.0 - s * (1.0 - f))
     i = i % 6
     if i == 0:
         return v, t, p
     if i == 1:
         return q, v, p
     if i == 2:
         return p, v, t
     if i == 3:
         return p, q, v
     if i == 4:
         return t, p, v
     if i == 5:
         return v, p, q
 
 
 def unique_color_generator(n=10000, saturation=0.10, lightness=0.95):
     """
     Generator for getting n of evenly distributed colors using
     hsv color and golden ratio. It always return same order of colors
 
     :param n: number of colors to generate
     :param saturation: saturation of returned colors
     :param lightness: lightness of returned colors
     :returns: RGB tuple
     """
 
     golden_ratio = 0.618033988749895
     h = 0.22717784590367374
 
     for _ in range(n):
         h += golden_ratio
         h %= 1
         HSV_tuple = [h, saturation, lightness]
         RGB_tuple = hsv_to_rgb(*HSV_tuple)
         yield map(lambda x: str(int(x * 256)), RGB_tuple)
 
 
 def color_hasher(n=10000, saturation=0.10, lightness=0.95):
     """
     Returns a function which when called with an argument returns a unique
     color for that argument, eg.
 
     :param n: number of colors to generate
     :param saturation: saturation of returned colors
     :param lightness: lightness of returned colors
     :returns: css RGB string
 
     >>> color_hash = color_hasher()
     >>> color_hash('hello')
     'rgb(34, 12, 59)'
     >>> color_hash('hello')
     'rgb(34, 12, 59)'
     >>> color_hash('other')
     'rgb(90, 224, 159)'
     """
 
     color_dict = {}
     cgenerator = unique_color_generator(
         saturation=saturation, lightness=lightness)
 
     def get_color_string(thing):
         if thing in color_dict:
             col = color_dict[thing]
         else:
             col = color_dict[thing] = next(cgenerator)
         return "rgb(%s)" % (', '.join(col))
 
     return get_color_string
 
 
 def get_lexer_safe(mimetype=None, filepath=None):
     """
     Tries to return a relevant pygments lexer using mimetype/filepath name,
     defaulting to plain text if none could be found
     """
     lexer = None
     try:
         if mimetype:
             lexer = get_lexer_for_mimetype(mimetype)
         if not lexer:
             lexer = get_lexer_for_filename(filepath)
     except pygments.util.ClassNotFound:
         pass
 
     if not lexer:
         lexer = get_lexer_by_name('text')
 
     return lexer
 
 
 def get_lexer_for_filenode(filenode):
     lexer = get_custom_lexer(filenode.extension) or filenode.lexer
     return lexer
 
 
 def pygmentize(filenode, **kwargs):
     """
     pygmentize function using pygments
 
     :param filenode:
     """
     lexer = get_lexer_for_filenode(filenode)
     return literal(code_highlight(filenode.content, lexer,
                                   CodeHtmlFormatter(**kwargs)))
 
 
 def is_following_repo(repo_name, user_id):
     from rhodecode.model.scm import ScmModel
     return ScmModel().is_following_repo(repo_name, user_id)
 
 
 class _Message(object):
     """A message returned by ``Flash.pop_messages()``.
 
     Converting the message to a string returns the message text. Instances
     also have the following attributes:
 
     * ``message``: the message text.
     * ``category``: the category specified when the message was created.
     """
 
     def __init__(self, category, message, sub_data=None):
         self.category = category
         self.message = message
         self.sub_data = sub_data or {}
 
     def __str__(self):
         return self.message
 
     __unicode__ = __str__
 
     def __html__(self):
         return escape(safe_unicode(self.message))
 
 
 class Flash(object):
     # List of allowed categories.  If None, allow any category.
     categories = ["warning", "notice", "error", "success"]
 
     # Default category if none is specified.
     default_category = "notice"
 
     def __init__(self, session_key="flash", categories=None,
                  default_category=None):
         """
         Instantiate a ``Flash`` object.
 
         ``session_key`` is the key to save the messages under in the user's
         session.
 
         ``categories`` is an optional list which overrides the default list
         of categories.
 
         ``default_category`` overrides the default category used for messages
         when none is specified.
         """
         self.session_key = session_key
         if categories is not None:
             self.categories = categories
         if default_category is not None:
             self.default_category = default_category
         if self.categories and self.default_category not in self.categories:
             raise ValueError(
                 "unrecognized default category %r" % (self.default_category,))
 
     def pop_messages(self, session=None, request=None):
         """
         Return all accumulated messages and delete them from the session.
 
         The return value is a list of ``Message`` objects.
         """
         messages = []
 
         if not session:
             if not request:
                 request = get_current_request()
             session = request.session
 
         # Pop the 'old' pylons flash messages. They are tuples of the form
         # (category, message)
         for cat, msg in session.pop(self.session_key, []):
             messages.append(_Message(cat, msg))
 
         # Pop the 'new' pyramid flash messages for each category as list
         # of strings.
         for cat in self.categories:
             for msg in session.pop_flash(queue=cat):
                 sub_data = {}
                 if hasattr(msg, 'rsplit'):
                     flash_data = msg.rsplit('|DELIM|', 1)
                     org_message = flash_data[0]
                     if len(flash_data) > 1:
                         sub_data = json.loads(flash_data[1])
                 else:
                     org_message = msg
 
                 messages.append(_Message(cat, org_message, sub_data=sub_data))
 
         # Map messages from the default queue to the 'notice' category.
         for msg in session.pop_flash():
             messages.append(_Message('notice', msg))
 
         session.save()
         return messages
 
     def json_alerts(self, session=None, request=None):
         payloads = []
         messages = flash.pop_messages(session=session, request=request) or []
         for message in messages:
             payloads.append({
                 'message': {
                     'message': u'{}'.format(message.message),
                     'level': message.category,
                     'force': True,
                     'subdata': message.sub_data
                 }
             })
-        return json.dumps(payloads)
+        return safe_str(json.dumps(payloads))
 
     def __call__(self, message, category=None, ignore_duplicate=True,
                  session=None, request=None):
 
         if not session:
             if not request:
                 request = get_current_request()
             session = request.session
 
         session.flash(
             message, queue=category, allow_duplicate=not ignore_duplicate)
 
 
 flash = Flash()
 
 #==============================================================================
 # SCM FILTERS available via h.
 #==============================================================================
 from rhodecode.lib.vcs.utils import author_name, author_email
 from rhodecode.lib.utils2 import age, age_from_seconds
 from rhodecode.model.db import User, ChangesetStatus
 
 
 email = author_email
 
 
 def capitalize(raw_text):
     return raw_text.capitalize()
 
 
 def short_id(long_id):
     return long_id[:12]
 
 
 def hide_credentials(url):
     from rhodecode.lib.utils2 import credentials_filter
     return credentials_filter(url)
 
 
 import pytz
 import tzlocal
 local_timezone = tzlocal.get_localzone()
 
 
 def get_timezone(datetime_iso, time_is_local=False):
     tzinfo = '+00:00'
 
     # detect if we have a timezone info, otherwise, add it
     if time_is_local and isinstance(datetime_iso, datetime) and not datetime_iso.tzinfo:
         force_timezone = os.environ.get('RC_TIMEZONE', '')
         if force_timezone:
             force_timezone = pytz.timezone(force_timezone)
         timezone = force_timezone or local_timezone
         offset = timezone.localize(datetime_iso).strftime('%z')
         tzinfo = '{}:{}'.format(offset[:-2], offset[-2:])
     return tzinfo
 
 
 def age_component(datetime_iso, value=None, time_is_local=False, tooltip=True):
     title = value or format_date(datetime_iso)
     tzinfo = get_timezone(datetime_iso, time_is_local=time_is_local)
 
     return literal(
         '<time class="timeago {cls}" title="{tt_title}" datetime="{dt}{tzinfo}">{title}</time>'.format(
             cls='tooltip' if tooltip else '',
             tt_title=('{title}{tzinfo}'.format(title=title, tzinfo=tzinfo)) if tooltip else '',
             title=title, dt=datetime_iso, tzinfo=tzinfo
         ))
 
 
 def _shorten_commit_id(commit_id, commit_len=None):
     if commit_len is None:
         request = get_current_request()
         commit_len = request.call_context.visual.show_sha_length
     return commit_id[:commit_len]
 
 
 def show_id(commit, show_idx=None, commit_len=None):
     """
     Configurable function that shows ID
     by default it's r123:fffeeefffeee
 
     :param commit: commit instance
     """
     if show_idx is None:
         request = get_current_request()
         show_idx = request.call_context.visual.show_revision_number
 
     raw_id = _shorten_commit_id(commit.raw_id, commit_len=commit_len)
     if show_idx:
         return 'r%s:%s' % (commit.idx, raw_id)
     else:
         return '%s' % (raw_id, )
 
 
 def format_date(date):
     """
     use a standardized formatting for dates used in RhodeCode
 
     :param date: date/datetime object
     :return: formatted date
     """
 
     if date:
         _fmt = "%a, %d %b %Y %H:%M:%S"
         return safe_unicode(date.strftime(_fmt))
 
     return u""
 
 
 class _RepoChecker(object):
 
     def __init__(self, backend_alias):
         self._backend_alias = backend_alias
 
     def __call__(self, repository):
         if hasattr(repository, 'alias'):
             _type = repository.alias
         elif hasattr(repository, 'repo_type'):
             _type = repository.repo_type
         else:
             _type = repository
         return _type == self._backend_alias
 
 
 is_git = _RepoChecker('git')
 is_hg = _RepoChecker('hg')
 is_svn = _RepoChecker('svn')
 
 
 def get_repo_type_by_name(repo_name):
     repo = Repository.get_by_repo_name(repo_name)
     if repo:
         return repo.repo_type
 
 
 def is_svn_without_proxy(repository):
     if is_svn(repository):
         from rhodecode.model.settings import VcsSettingsModel
         conf = VcsSettingsModel().get_ui_settings_as_config_obj()
         return not str2bool(conf.get('vcs_svn_proxy', 'http_requests_enabled'))
     return False
 
 
 def discover_user(author):
     """
     Tries to discover RhodeCode User based on the author string. Author string
     is typically `FirstName LastName <[e-mail]>`
     """
 
     # if author is already an instance use it for extraction
     if isinstance(author, User):
         return author
 
     # Valid email in the attribute passed, see if they're in the system
     _email = author_email(author)
     if _email != '':
         user = User.get_by_email(_email, case_insensitive=True, cache=True)
         if user is not None:
             return user
 
     # Maybe it's a username, we try to extract it and fetch by username ?
     _author = author_name(author)
     user = User.get_by_username(_author, case_insensitive=True, cache=True)
     if user is not None:
         return user
 
     return None
 
 
 def email_or_none(author):
     # extract email from the commit string
     _email = author_email(author)
 
     # If we have an email, use it, otherwise
     # see if it contains a username we can get an email from
     if _email != '':
         return _email
     else:
         user = User.get_by_username(
             author_name(author), case_insensitive=True, cache=True)
 
     if user is not None:
             return user.email
 
     # No valid email, not a valid user in the system, none!
     return None
 
 
 def link_to_user(author, length=0, **kwargs):
     user = discover_user(author)
     # user can be None, but if we have it already it means we can re-use it
     # in the person() function, so we save 1 intensive-query
     if user:
         author = user
 
     display_person = person(author, 'username_or_name_or_email')
     if length:
         display_person = shorter(display_person, length)
 
     if user and user.username != user.DEFAULT_USER:
         return link_to(
             escape(display_person),
             route_path('user_profile', username=user.username),
             **kwargs)
     else:
         return escape(display_person)
 
 
 def link_to_group(users_group_name, **kwargs):
     return link_to(
         escape(users_group_name),
         route_path('user_group_profile', user_group_name=users_group_name),
         **kwargs)
 
 
 def person(author, show_attr="username_and_name"):
     user = discover_user(author)
     if user:
         return getattr(user, show_attr)
     else:
         _author = author_name(author)
         _email = email(author)
         return _author or _email
 
 
 def author_string(email):
     if email:
         user = User.get_by_email(email, case_insensitive=True, cache=True)
         if user:
             if user.first_name or user.last_name:
                 return '%s %s &lt;%s&gt;' % (
                     user.first_name, user.last_name, email)
             else:
                 return email
         else:
             return email
     else:
         return None
 
 
 def person_by_id(id_, show_attr="username_and_name"):
     # attr to return from fetched user
     person_getter = lambda usr: getattr(usr, show_attr)
 
     #maybe it's an ID ?
     if str(id_).isdigit() or isinstance(id_, int):
         id_ = int(id_)
         user = User.get(id_)
         if user is not None:
             return person_getter(user)
     return id_
 
 
 def gravatar_with_user(request, author, show_disabled=False, tooltip=False):
     _render = request.get_partial_renderer('rhodecode:templates/base/base.mako')
     return _render('gravatar_with_user', author, show_disabled=show_disabled, tooltip=tooltip)
 
 
 tags_paterns = OrderedDict((
     ('lang', (re.compile(r'\[(lang|language)\ \=\&gt;\ *([a-zA-Z\-\/\#\+\.]*)\]'),
              '<div class="metatag" tag="lang">\\2</div>')),
 
     ('see', (re.compile(r'\[see\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]'),
             '<div class="metatag" tag="see">see: \\1 </div>')),
 
     ('url', (re.compile(r'\[url\ \=\&gt;\ \[([a-zA-Z0-9\ \.\-\_]+)\]\((http://|https://|/)(.*?)\)\]'),
             '<div class="metatag" tag="url"> <a href="\\2\\3">\\1</a> </div>')),
 
     ('license', (re.compile(r'\[license\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]'),
                 '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/\\1">\\1</a></div>')),
 
     ('ref', (re.compile(r'\[(requires|recommends|conflicts|base)\ \=\&gt;\ *([a-zA-Z0-9\-\/]*)\]'),
             '<div class="metatag" tag="ref \\1">\\1: <a href="/\\2">\\2</a></div>')),
 
     ('state', (re.compile(r'\[(stable|featured|stale|dead|dev|deprecated)\]'),
               '<div class="metatag" tag="state \\1">\\1</div>')),
 
     # label in grey
     ('label', (re.compile(r'\[([a-z]+)\]'),
               '<div class="metatag" tag="label">\\1</div>')),
 
     # generic catch all in grey
     ('generic', (re.compile(r'\[([a-zA-Z0-9\.\-\_]+)\]'),
                 '<div class="metatag" tag="generic">\\1</div>')),
 ))
 
 
 def extract_metatags(value):
     """
     Extract supported meta-tags from given text value
     """
     tags = []
     if not value:
         return tags, ''
 
     for key, val in tags_paterns.items():
         pat, replace_html = val
         tags.extend([(key, x.group()) for x in pat.finditer(value)])
         value = pat.sub('', value)
 
     return tags, value
 
 
 def style_metatag(tag_type, value):
     """
     converts tags from value into html equivalent
     """
     if not value:
         return ''
 
     html_value = value
     tag_data = tags_paterns.get(tag_type)
     if tag_data:
         pat, replace_html = tag_data
         # convert to plain `unicode` instead of a markup tag to be used in
         # regex expressions. safe_unicode doesn't work here
         html_value = pat.sub(replace_html, value)
 
     return html_value
 
 
 def bool2icon(value, show_at_false=True):
     """
     Returns boolean value of a given value, represented as html element with
     classes that will represent icons
 
     :param value: given value to convert to html node
     """
 
     if value:  # does bool conversion
         return HTML.tag('i', class_="icon-true", title='True')
     else:  # not true as bool
         if show_at_false:
             return HTML.tag('i', class_="icon-false", title='False')
         return HTML.tag('i')
 
 
 def b64(inp):
     return base64.b64encode(inp)
 
 #==============================================================================
 # PERMS
 #==============================================================================
 from rhodecode.lib.auth import (
     HasPermissionAny, HasPermissionAll,
     HasRepoPermissionAny, HasRepoPermissionAll, HasRepoGroupPermissionAll,
     HasRepoGroupPermissionAny, HasRepoPermissionAnyApi, get_csrf_token,
     csrf_token_key, AuthUser)
 
 
 #==============================================================================
 # GRAVATAR URL
 #==============================================================================
 class InitialsGravatar(object):
     def __init__(self, email_address, first_name, last_name, size=30,
                  background=None, text_color='#fff'):
         self.size = size
         self.first_name = first_name
         self.last_name = last_name
         self.email_address = email_address
         self.background = background or self.str2color(email_address)
         self.text_color = text_color
 
     def get_color_bank(self):
         """
         returns a predefined list of colors that gravatars can use.
         Those are randomized distinct colors that guarantee readability and
         uniqueness.
 
         generated with: http://phrogz.net/css/distinct-colors.html
         """
         return [
             '#bf3030', '#a67f53', '#00ff00', '#5989b3', '#392040', '#d90000',
             '#402910', '#204020', '#79baf2', '#a700b3', '#bf6060', '#7f5320',
             '#008000', '#003059', '#ee00ff', '#ff0000', '#8c4b00', '#007300',
             '#005fb3', '#de73e6', '#ff4040', '#ffaa00', '#3df255', '#203140',
             '#47004d', '#591616', '#664400', '#59b365', '#0d2133', '#83008c',
             '#592d2d', '#bf9f60', '#73e682', '#1d3f73', '#73006b', '#402020',
             '#b2862d', '#397341', '#597db3', '#e600d6', '#a60000', '#736039',
             '#00b318', '#79aaf2', '#330d30', '#ff8080', '#403010', '#16591f',
             '#002459', '#8c4688', '#e50000', '#ffbf40', '#00732e', '#102340',
             '#bf60ac', '#8c4646', '#cc8800', '#00a642', '#1d3473', '#b32d98',
             '#660e00', '#ffd580', '#80ffb2', '#7391e6', '#733967', '#d97b6c',
             '#8c5e00', '#59b389', '#3967e6', '#590047', '#73281d', '#665200',
             '#00e67a', '#2d50b3', '#8c2377', '#734139', '#b2982d', '#16593a',
             '#001859', '#ff00aa', '#a65e53', '#ffcc00', '#0d3321', '#2d3959',
             '#731d56', '#401610', '#4c3d00', '#468c6c', '#002ca6', '#d936a3',
             '#d94c36', '#403920', '#36d9a3', '#0d1733', '#592d4a', '#993626',
             '#cca300', '#00734d', '#46598c', '#8c005e', '#7f1100', '#8c7000',
             '#00a66f', '#7382e6', '#b32d74', '#d9896c', '#ffe680', '#1d7362',
             '#364cd9', '#73003d', '#d93a00', '#998a4d', '#59b3a1', '#5965b3',
             '#e5007a', '#73341d', '#665f00', '#00b38f', '#0018b3', '#59163a',
             '#b2502d', '#bfb960', '#00ffcc', '#23318c', '#a6537f', '#734939',
             '#b2a700', '#104036', '#3d3df2', '#402031', '#e56739', '#736f39',
             '#79f2ea', '#000059', '#401029', '#4c1400', '#ffee00', '#005953',
             '#101040', '#990052', '#402820', '#403d10', '#00ffee', '#0000d9',
             '#ff80c4', '#a66953', '#eeff00', '#00ccbe', '#8080ff', '#e673a1',
             '#a62c00', '#474d00', '#1a3331', '#46468c', '#733950', '#662900',
             '#858c23', '#238c85', '#0f0073', '#b20047', '#d9986c', '#becc00',
             '#396f73', '#281d73', '#ff0066', '#ff6600', '#dee673', '#59adb3',
             '#6559b3', '#590024', '#b2622d', '#98b32d', '#36ced9', '#332d59',
             '#40001a', '#733f1d', '#526600', '#005359', '#242040', '#bf6079',
             '#735039', '#cef23d', '#007780', '#5630bf', '#66001b', '#b24700',
             '#acbf60', '#1d6273', '#25008c', '#731d34', '#a67453', '#50592d',
             '#00ccff', '#6600ff', '#ff0044', '#4c1f00', '#8a994d', '#79daf2',
             '#a173e6', '#d93662', '#402310', '#aaff00', '#2d98b3', '#8c40ff',
             '#592d39', '#ff8c40', '#354020', '#103640', '#1a0040', '#331a20',
             '#331400', '#334d00', '#1d5673', '#583973', '#7f0022', '#4c3626',
             '#88cc00', '#36a3d9', '#3d0073', '#d9364c', '#33241a', '#698c23',
             '#5995b3', '#300059', '#e57382', '#7f3300', '#366600', '#00aaff',
             '#3a1659', '#733941', '#663600', '#74b32d', '#003c59', '#7f53a6',
             '#73000f', '#ff8800', '#baf279', '#79caf2', '#291040', '#a6293a',
             '#b2742d', '#587339', '#0077b3', '#632699', '#400009', '#d9a66c',
             '#294010', '#2d4a59', '#aa00ff', '#4c131b', '#b25f00', '#5ce600',
             '#267399', '#a336d9', '#990014', '#664e33', '#86bf60', '#0088ff',
             '#7700b3', '#593a16', '#073300', '#1d4b73', '#ac60bf', '#e59539',
             '#4f8c46', '#368dd9', '#5c0073'
         ]
 
     def rgb_to_hex_color(self, rgb_tuple):
         """
         Converts an rgb_tuple passed to an hex color.
 
         :param rgb_tuple: tuple with 3 ints represents rgb color space
         """
         return '#' + ("".join(map(chr, rgb_tuple)).encode('hex'))
 
     def email_to_int_list(self, email_str):
         """
         Get every byte of the hex digest value of email and turn it to integer.
         It's going to be always between 0-255
         """
         digest = md5_safe(email_str.lower())
         return [int(digest[i * 2:i * 2 + 2], 16) for i in range(16)]
 
     def pick_color_bank_index(self, email_str, color_bank):
         return self.email_to_int_list(email_str)[0] % len(color_bank)
 
     def str2color(self, email_str):
         """
         Tries to map in a stable algorithm an email to color
 
         :param email_str:
         """
         color_bank = self.get_color_bank()
         # pick position (module it's length so we always find it in the
         # bank even if it's smaller than 256 values
         pos = self.pick_color_bank_index(email_str, color_bank)
         return color_bank[pos]
 
     def normalize_email(self, email_address):
         import unicodedata
         # default host used to fill in the fake/missing email
         default_host = 'localhost'
 
         if not email_address:
             email_address = '%s@%s' % (User.DEFAULT_USER, default_host)
 
         email_address = safe_unicode(email_address)
 
         if u'@' not in email_address:
             email_address = u'%s@%s' % (email_address, default_host)
 
         if email_address.endswith(u'@'):
             email_address = u'%s%s' % (email_address, default_host)
 
         email_address = unicodedata.normalize('NFKD', email_address)\
             .encode('ascii', 'ignore')
         return email_address
 
     def get_initials(self):
         """
         Returns 2 letter initials calculated based on the input.
         The algorithm picks first given email address, and takes first letter
         of part before @, and then the first letter of server name. In case
         the part before @ is in a format of `somestring.somestring2` it replaces
         the server letter with first letter of somestring2
 
         In case function was initialized with both first and lastname, this
         overrides the extraction from email by first letter of the first and
         last name. We add special logic to that functionality, In case Full name
         is compound, like Guido Von Rossum, we use last part of the last name
         (Von Rossum) picking `R`.
 
         Function also normalizes the non-ascii characters to they ascii
         representation, eg Ą => A
         """
         import unicodedata
         # replace non-ascii to ascii
         first_name = unicodedata.normalize(
             'NFKD', safe_unicode(self.first_name)).encode('ascii', 'ignore')
         last_name = unicodedata.normalize(
             'NFKD', safe_unicode(self.last_name)).encode('ascii', 'ignore')
 
         # do NFKD encoding, and also make sure email has proper format
         email_address = self.normalize_email(self.email_address)
 
         # first push the email initials
         prefix, server = email_address.split('@', 1)
 
         # check if prefix is maybe a 'first_name.last_name' syntax
         _dot_split = prefix.rsplit('.', 1)
         if len(_dot_split) == 2 and _dot_split[1]:
             initials = [_dot_split[0][0], _dot_split[1][0]]
         else:
             initials = [prefix[0], server[0]]
 
         # then try to replace either first_name or last_name
         fn_letter = (first_name or " ")[0].strip()
         ln_letter = (last_name.split(' ', 1)[-1] or " ")[0].strip()
 
         if fn_letter:
             initials[0] = fn_letter
 
         if ln_letter:
             initials[1] = ln_letter
 
         return ''.join(initials).upper()
 
     def get_img_data_by_type(self, font_family, img_type):
         default_user = """
         <svg xmlns="http://www.w3.org/2000/svg"
         version="1.1" x="0px" y="0px" width="{size}" height="{size}"
         viewBox="-15 -10 439.165 429.164"
 
         xml:space="preserve"
         style="background:{background};" >
 
         <path d="M204.583,216.671c50.664,0,91.74-48.075,
                  91.74-107.378c0-82.237-41.074-107.377-91.74-107.377
                  c-50.668,0-91.74,25.14-91.74,107.377C112.844,
                  168.596,153.916,216.671,
                  204.583,216.671z" fill="{text_color}"/>
         <path d="M407.164,374.717L360.88,
                  270.454c-2.117-4.771-5.836-8.728-10.465-11.138l-71.83-37.392
                  c-1.584-0.823-3.502-0.663-4.926,0.415c-20.316,
                  15.366-44.203,23.488-69.076,23.488c-24.877,
                  0-48.762-8.122-69.078-23.488
                  c-1.428-1.078-3.346-1.238-4.93-0.415L58.75,
                  259.316c-4.631,2.41-8.346,6.365-10.465,11.138L2.001,374.717
                  c-3.191,7.188-2.537,15.412,1.75,22.005c4.285,
                  6.592,11.537,10.526,19.4,10.526h362.861c7.863,0,15.117-3.936,
                  19.402-10.527 C409.699,390.129,
                  410.355,381.902,407.164,374.717z" fill="{text_color}"/>
         </svg>""".format(
             size=self.size,
             background='#979797',  # @grey4
             text_color=self.text_color,
             font_family=font_family)
 
         return {
             "default_user": default_user
         }[img_type]
 
     def get_img_data(self, svg_type=None):
         """
         generates the svg metadata for image
         """
         fonts = [
             '-apple-system',
              'BlinkMacSystemFont',
              'Segoe UI',
              'Roboto',
              'Oxygen-Sans',
              'Ubuntu',
              'Cantarell',
              'Helvetica Neue',
              'sans-serif'
         ]
         font_family = ','.join(fonts)
         if svg_type:
             return self.get_img_data_by_type(font_family, svg_type)
 
         initials = self.get_initials()
         img_data = """
         <svg xmlns="http://www.w3.org/2000/svg" pointer-events="none"
              width="{size}" height="{size}"
              style="width: 100%; height: 100%; background-color: {background}"
              viewBox="0 0 {size} {size}">
             <text text-anchor="middle" y="50%" x="50%" dy="0.35em"
                   pointer-events="auto" fill="{text_color}"
                   font-family="{font_family}"
                   style="font-weight: 400; font-size: {f_size}px;">{text}
             </text>
         </svg>""".format(
             size=self.size,
             f_size=self.size/2.05,  # scale the text inside the box nicely
             background=self.background,
             text_color=self.text_color,
             text=initials.upper(),
             font_family=font_family)
 
         return img_data
 
     def generate_svg(self, svg_type=None):
         img_data = self.get_img_data(svg_type)
         return "data:image/svg+xml;base64,%s" % base64.b64encode(img_data)
 
 
 def initials_gravatar(request, email_address, first_name, last_name, size=30, store_on_disk=False):
 
     svg_type = None
     if email_address == User.DEFAULT_USER_EMAIL:
         svg_type = 'default_user'
 
     klass = InitialsGravatar(email_address, first_name, last_name, size)
 
     if store_on_disk:
         from rhodecode.apps.file_store import utils as store_utils
         from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
             FileOverSizeException
         from rhodecode.model.db import Session
 
         image_key = md5_safe(email_address.lower()
                              + first_name.lower() + last_name.lower())
 
         storage = store_utils.get_file_storage(request.registry.settings)
         filename = '{}.svg'.format(image_key)
         subdir = 'gravatars'
         # since final name has a counter, we apply the 0
         uid = storage.apply_counter(0, store_utils.uid_filename(filename, randomized=False))
         store_uid = os.path.join(subdir, uid)
 
         db_entry = FileStore.get_by_store_uid(store_uid)
         if db_entry:
             return request.route_path('download_file', fid=store_uid)
 
         img_data = klass.get_img_data(svg_type=svg_type)
         img_file = store_utils.bytes_to_file_obj(img_data)
 
         try:
             store_uid, metadata = storage.save_file(
                 img_file, filename, directory=subdir,
                 extensions=['.svg'], randomized_name=False)
         except (FileNotAllowedException, FileOverSizeException):
             raise
 
         try:
             entry = FileStore.create(
                 file_uid=store_uid, filename=metadata["filename"],
                 file_hash=metadata["sha256"], file_size=metadata["size"],
                 file_display_name=filename,
                 file_description=u'user gravatar `{}`'.format(safe_unicode(filename)),
                 hidden=True, check_acl=False, user_id=1
             )
             Session().add(entry)
             Session().commit()
             log.debug('Stored upload in DB as %s', entry)
         except Exception:
             raise
 
         return request.route_path('download_file', fid=store_uid)
 
     else:
         return klass.generate_svg(svg_type=svg_type)
 
 
 def gravatar_external(request, gravatar_url_tmpl, email_address, size=30):
     return safe_str(gravatar_url_tmpl)\
         .replace('{email}', email_address) \
         .replace('{md5email}', md5_safe(email_address.lower())) \
         .replace('{netloc}', request.host) \
         .replace('{scheme}', request.scheme) \
         .replace('{size}', safe_str(size))
 
 
 def gravatar_url(email_address, size=30, request=None):
     request = request or get_current_request()
     _use_gravatar = request.call_context.visual.use_gravatar
 
     email_address = email_address or User.DEFAULT_USER_EMAIL
     if isinstance(email_address, str):
         # hashlib crashes on unicode items
         email_address = safe_str(email_address)
 
     # empty email or default user
     if not email_address or email_address == User.DEFAULT_USER_EMAIL:
         return initials_gravatar(request, User.DEFAULT_USER_EMAIL, '', '', size=size)
 
     if _use_gravatar:
         gravatar_url_tmpl = request.call_context.visual.gravatar_url \
                             or User.DEFAULT_GRAVATAR_URL
         return gravatar_external(request, gravatar_url_tmpl, email_address, size=size)
 
     else:
         return initials_gravatar(request, email_address, '', '', size=size)
 
 
 def breadcrumb_repo_link(repo):
     """
     Makes a breadcrumbs path link to repo
 
     ex::
         group >> subgroup >> repo
 
     :param repo: a Repository instance
     """
 
     path = [
         link_to(group.name, route_path('repo_group_home', repo_group_name=group.group_name),
                 title='last change:{}'.format(format_date(group.last_commit_change)))
         for group in repo.groups_with_parents
     ] + [
         link_to(repo.just_name, route_path('repo_summary', repo_name=repo.repo_name),
                 title='last change:{}'.format(format_date(repo.last_commit_change)))
     ]
 
     return literal(' &raquo; '.join(path))
 
 
 def breadcrumb_repo_group_link(repo_group):
     """
     Makes a breadcrumbs path link to repo
 
     ex::
         group >> subgroup
 
     :param repo_group: a Repository Group instance
     """
 
     path = [
         link_to(group.name,
                 route_path('repo_group_home', repo_group_name=group.group_name),
                 title='last change:{}'.format(format_date(group.last_commit_change)))
         for group in repo_group.parents
     ] + [
         link_to(repo_group.name,
                 route_path('repo_group_home', repo_group_name=repo_group.group_name),
                 title='last change:{}'.format(format_date(repo_group.last_commit_change)))
     ]
 
     return literal(' &raquo; '.join(path))
 
 
 def format_byte_size_binary(file_size):
     """
     Formats file/folder sizes to standard.
     """
     if file_size is None:
         file_size = 0
 
     formatted_size = format_byte_size(file_size, binary=True)
     return formatted_size
 
 
 def urlify_text(text_, safe=True, **href_attrs):
     """
     Extract urls from text and make html links out of them
     """
 
     url_pat = re.compile(r'''(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@#.&+]'''
                          '''|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)''')
 
     def url_func(match_obj):
         url_full = match_obj.groups()[0]
         a_options = dict(href_attrs)
         a_options['href'] = url_full
         a_text = url_full
         return HTML.tag("a", a_text, **a_options)
 
     _new_text = url_pat.sub(url_func, text_)
 
     if safe:
         return literal(_new_text)
     return _new_text
 
 
 def urlify_commits(text_, repo_name):
     """
     Extract commit ids from text and make link from them
 
     :param text_:
     :param repo_name: repo name to build the URL with
     """
 
     url_pat = re.compile(r'(^|\s)([0-9a-fA-F]{12,40})($|\s)')
 
     def url_func(match_obj):
         commit_id = match_obj.groups()[1]
         pref = match_obj.groups()[0]
         suf = match_obj.groups()[2]
 
         tmpl = (
             '%(pref)s<a class="tooltip-hovercard %(cls)s" href="%(url)s" data-hovercard-alt="%(hovercard_alt)s" data-hovercard-url="%(hovercard_url)s">'
             '%(commit_id)s</a>%(suf)s'
         )
         return tmpl % {
             'pref': pref,
             'cls': 'revision-link',
             'url': route_url(
                 'repo_commit', repo_name=repo_name, commit_id=commit_id),
             'commit_id': commit_id,
             'suf': suf,
             'hovercard_alt': 'Commit: {}'.format(commit_id),
             'hovercard_url': route_url(
                 'hovercard_repo_commit', repo_name=repo_name, commit_id=commit_id)
         }
 
     new_text = url_pat.sub(url_func, text_)
 
     return new_text
 
 
 def _process_url_func(match_obj, repo_name, uid, entry,
                       return_raw_data=False, link_format='html'):
     pref = ''
     if match_obj.group().startswith(' '):
         pref = ' '
 
     issue_id = ''.join(match_obj.groups())
 
     if link_format == 'html':
         tmpl = (
             '%(pref)s<a class="tooltip %(cls)s" href="%(url)s" title="%(title)s">'
             '%(issue-prefix)s%(id-repr)s'
             '</a>')
     elif link_format == 'html+hovercard':
         tmpl = (
             '%(pref)s<a class="tooltip-hovercard %(cls)s" href="%(url)s" data-hovercard-url="%(hovercard_url)s">'
             '%(issue-prefix)s%(id-repr)s'
             '</a>')
     elif link_format in ['rst', 'rst+hovercard']:
         tmpl = '`%(issue-prefix)s%(id-repr)s <%(url)s>`_'
     elif link_format in ['markdown', 'markdown+hovercard']:
         tmpl = '[%(pref)s%(issue-prefix)s%(id-repr)s](%(url)s)'
     else:
         raise ValueError('Bad link_format:{}'.format(link_format))
 
     (repo_name_cleaned,
      parent_group_name) = RepoGroupModel()._get_group_name_and_parent(repo_name)
 
     # variables replacement
     named_vars = {
         'id': issue_id,
         'repo': repo_name,
         'repo_name': repo_name_cleaned,
         'group_name': parent_group_name,
         # set dummy keys so we always have them
         'hostname': '',
         'netloc': '',
         'scheme': ''
     }
 
     request = get_current_request()
     if request:
         # exposes, hostname, netloc, scheme
         host_data = get_host_info(request)
         named_vars.update(host_data)
 
     # named regex variables
     named_vars.update(match_obj.groupdict())
     _url = string.Template(entry['url']).safe_substitute(**named_vars)
     desc = string.Template(escape(entry['desc'])).safe_substitute(**named_vars)
     hovercard_url = string.Template(entry.get('hovercard_url', '')).safe_substitute(**named_vars)
 
     def quote_cleaner(input_str):
         """Remove quotes as it's HTML"""
         return input_str.replace('"', '')
 
     data = {
         'pref': pref,
         'cls': quote_cleaner('issue-tracker-link'),
         'url': quote_cleaner(_url),
         'id-repr': issue_id,
         'issue-prefix': entry['pref'],
         'serv': entry['url'],
         'title': bleach.clean(desc, strip=True),
         'hovercard_url': hovercard_url
     }
 
     if return_raw_data:
         return {
             'id': issue_id,
             'url': _url
         }
     return tmpl % data
 
 
 def get_active_pattern_entries(repo_name):
     repo = None
     if repo_name:
         # Retrieving repo_name to avoid invalid repo_name to explode on
         # IssueTrackerSettingsModel but still passing invalid name further down
         repo = Repository.get_by_repo_name(repo_name, cache=True)
 
     settings_model = IssueTrackerSettingsModel(repo=repo)
     active_entries = settings_model.get_settings(cache=True)
     return active_entries
 
 
 pr_pattern_re = regex.compile(r'(?:(?:^!)|(?: !))(\d+)')
 
 allowed_link_formats = [
     'html', 'rst', 'markdown', 'html+hovercard', 'rst+hovercard', 'markdown+hovercard']
 
 compile_cache = {
 
 }
 
 
 def process_patterns(text_string, repo_name, link_format='html', active_entries=None):
 
     if link_format not in allowed_link_formats:
         raise ValueError('Link format can be only one of:{} got {}'.format(
                          allowed_link_formats, link_format))
     issues_data = []
     errors = []
     new_text = text_string
 
     if active_entries is None:
         log.debug('Fetch active issue tracker patterns for repo: %s', repo_name)
         active_entries = get_active_pattern_entries(repo_name)
 
     log.debug('Got %s pattern entries to process', len(active_entries))
 
     for uid, entry in active_entries.items():
 
         if not (entry['pat'] and entry['url']):
             log.debug('skipping due to missing data')
             continue
 
         log.debug('issue tracker entry: uid: `%s` PAT:%s URL:%s PREFIX:%s',
                   uid, entry['pat'], entry['url'], entry['pref'])
 
         if entry.get('pat_compiled'):
             pattern = entry['pat_compiled']
         elif entry['pat'] in compile_cache:
             pattern = compile_cache[entry['pat']]
         else:
             try:
                 pattern = regex.compile(r'%s' % entry['pat'])
             except regex.error as e:
                 regex_err = ValueError('{}:{}'.format(entry['pat'], e))
                 log.exception('issue tracker pattern: `%s` failed to compile', regex_err)
                 errors.append(regex_err)
                 continue
             compile_cache[entry['pat']] = pattern
 
         data_func = partial(
             _process_url_func, repo_name=repo_name, entry=entry, uid=uid,
             return_raw_data=True)
 
         for match_obj in pattern.finditer(text_string):
             issues_data.append(data_func(match_obj))
 
         url_func = partial(
             _process_url_func, repo_name=repo_name, entry=entry, uid=uid,
             link_format=link_format)
 
         new_text = pattern.sub(url_func, new_text)
         log.debug('processed prefix:uid `%s`', uid)
 
     # finally use global replace, eg !123 -> pr-link, those will not catch
     # if already similar pattern exists
     server_url = '${scheme}://${netloc}'
     pr_entry = {
         'pref': '!',
         'url': server_url + '/_admin/pull-requests/${id}',
         'desc': 'Pull Request !${id}',
         'hovercard_url': server_url + '/_hovercard/pull_request/${id}'
     }
     pr_url_func = partial(
         _process_url_func, repo_name=repo_name, entry=pr_entry, uid=None,
         link_format=link_format+'+hovercard')
     new_text = pr_pattern_re.sub(pr_url_func, new_text)
     log.debug('processed !pr pattern')
 
     return new_text, issues_data, errors
 
 
 def urlify_commit_message(commit_text, repository=None, active_pattern_entries=None,
                           issues_container_callback=None, error_container=None):
     """
     Parses given text message and makes proper links.
     issues are linked to given issue-server, and rest is a commit link
     """
 
     def escaper(_text):
         return _text.replace('<', '&lt;').replace('>', '&gt;')
 
     new_text = escaper(commit_text)
 
     # extract http/https links and make them real urls
     new_text = urlify_text(new_text, safe=False)
 
     # urlify commits - extract commit ids and make link out of them, if we have
     # the scope of repository present.
     if repository:
         new_text = urlify_commits(new_text, repository)
 
     # process issue tracker patterns
     new_text, issues, errors = process_patterns(
         new_text, repository or '', active_entries=active_pattern_entries)
 
     if issues_container_callback is not None:
         for issue in issues:
             issues_container_callback(issue)
 
     if error_container is not None:
         error_container.extend(errors)
 
     return literal(new_text)
 
 
 def render_binary(repo_name, file_obj):
     """
     Choose how to render a binary file
     """
 
     # unicode
     filename = file_obj.name
 
     # images
     for ext in ['*.png', '*.jpeg', '*.jpg', '*.ico', '*.gif']:
         if fnmatch.fnmatch(filename, pat=ext):
             src = route_path(
                 'repo_file_raw', repo_name=repo_name,
                 commit_id=file_obj.commit.raw_id,
                 f_path=file_obj.path)
 
             return literal(
                 '<img class="rendered-binary" alt="rendered-image" src="{}">'.format(src))
 
 
 def renderer_from_filename(filename, exclude=None):
     """
     choose a renderer based on filename, this works only for text based files
     """
 
     # ipython
     for ext in ['*.ipynb']:
         if fnmatch.fnmatch(filename, pat=ext):
             return 'jupyter'
 
     is_markup = MarkupRenderer.renderer_from_filename(filename, exclude=exclude)
     if is_markup:
         return is_markup
     return None
 
 
 def render(source, renderer='rst', mentions=False, relative_urls=None,
            repo_name=None, active_pattern_entries=None, issues_container_callback=None):
 
     def maybe_convert_relative_links(html_source):
         if relative_urls:
             return relative_links(html_source, relative_urls)
         return html_source
 
     if renderer == 'plain':
         return literal(
             MarkupRenderer.plain(source, leading_newline=False))
 
     elif renderer == 'rst':
         if repo_name:
             # process patterns on comments if we pass in repo name
             source, issues, errors = process_patterns(
                 source, repo_name, link_format='rst',
                 active_entries=active_pattern_entries)
             if issues_container_callback is not None:
                 for issue in issues:
                     issues_container_callback(issue)
 
         return literal(
             '<div class="rst-block">%s</div>' %
             maybe_convert_relative_links(
                 MarkupRenderer.rst(source, mentions=mentions)))
 
     elif renderer == 'markdown':
         if repo_name:
             # process patterns on comments if we pass in repo name
             source, issues, errors = process_patterns(
                 source, repo_name, link_format='markdown',
                 active_entries=active_pattern_entries)
             if issues_container_callback is not None:
                 for issue in issues:
                     issues_container_callback(issue)
 
 
         return literal(
             '<div class="markdown-block">%s</div>' %
             maybe_convert_relative_links(
                 MarkupRenderer.markdown(source, flavored=True,
                                         mentions=mentions)))
 
     elif renderer == 'jupyter':
         return literal(
             '<div class="ipynb">%s</div>' %
             maybe_convert_relative_links(
                 MarkupRenderer.jupyter(source)))
 
     # None means just show the file-source
     return None
 
 
 def commit_status(repo, commit_id):
     return ChangesetStatusModel().get_status(repo, commit_id)
 
 
 def commit_status_lbl(commit_status):
     return dict(ChangesetStatus.STATUSES).get(commit_status)
 
 
 def commit_time(repo_name, commit_id):
     repo = Repository.get_by_repo_name(repo_name)
     commit = repo.get_commit(commit_id=commit_id)
     return commit.date
 
 
 def get_permission_name(key):
     return dict(Permission.PERMS).get(key)
 
 
 def journal_filter_help(request):
     _ = request.translate
     from rhodecode.lib.audit_logger import ACTIONS
     actions = '\n'.join(textwrap.wrap(', '.join(sorted(ACTIONS.keys())), 80))
 
     return _(
         'Example filter terms:\n' +
         '     repository:vcs\n' +
         '     username:marcin\n' +
         '     username:(NOT marcin)\n' +
         '     action:*push*\n' +
         '     ip:127.0.0.1\n' +
         '     date:20120101\n' +
         '     date:[20120101100000 TO 20120102]\n' +
         '\n' +
         'Actions: {actions}\n' +
         '\n' +
         'Generate wildcards using \'*\' character:\n' +
         '     "repository:vcs*" - search everything starting with \'vcs\'\n' +
         '     "repository:*vcs*" - search for repository containing \'vcs\'\n' +
         '\n' +
         'Optional AND / OR operators in queries\n' +
         '     "repository:vcs OR repository:test"\n' +
         '     "username:test AND repository:test*"\n'
     ).format(actions=actions)
 
 
 def not_mapped_error(repo_name):
     from rhodecode.translation import _
     flash(_('%s repository is not mapped to db perhaps'
             ' it was created or renamed from the filesystem'
             ' please run the application again'
             ' in order to rescan repositories') % repo_name, category='error')
 
 
 def ip_range(ip_addr):
     from rhodecode.model.db import UserIpMap
     s, e = UserIpMap._get_ip_range(ip_addr)
     return '%s - %s' % (s, e)
 
 
 def form(url, method='post', needs_csrf_token=True, **attrs):
     """Wrapper around webhelpers.tags.form to prevent CSRF attacks."""
     if method.lower() != 'get' and needs_csrf_token:
         raise Exception(
             'Forms to POST/PUT/DELETE endpoints should have (in general) a ' +
             'CSRF token. If the endpoint does not require such token you can ' +
             'explicitly set the parameter needs_csrf_token to false.')
 
     return insecure_form(url, method=method, **attrs)
 
 
 def secure_form(form_url, method="POST", multipart=False, **attrs):
     """Start a form tag that points the action to an url. This
     form tag will also include the hidden field containing
     the auth token.
 
     The url options should be given either as a string, or as a
     ``url()`` function. The method for the form defaults to POST.
 
     Options:
 
     ``multipart``
         If set to True, the enctype is set to "multipart/form-data".
     ``method``
         The method to use when submitting the form, usually either
         "GET" or "POST". If "PUT", "DELETE", or another verb is used, a
         hidden input with name _method is added to simulate the verb
         over POST.
 
     """
 
     if 'request' in attrs:
         session = attrs['request'].session
         del attrs['request']
     else:
         raise ValueError(
             'Calling this form requires request= to be passed as argument')
 
     _form = insecure_form(form_url, method, multipart, **attrs)
     token = literal(
         '<input type="hidden" name="{}" value="{}">'.format(
             csrf_token_key, get_csrf_token(session)))
 
     return literal("%s\n%s" % (_form, token))
 
 
 def dropdownmenu(name, selected, options, enable_filter=False, **attrs):
     select_html = select(name, selected, options, **attrs)
 
     select2 = """
         <script>
             $(document).ready(function() {
                   $('#%s').select2({
                       containerCssClass: 'drop-menu %s',
                       dropdownCssClass: 'drop-menu-dropdown',
                       dropdownAutoWidth: true%s
                   });
             });
         </script>
     """
 
     filter_option = """,
                         minimumResultsForSearch: -1
     """
     input_id = attrs.get('id') or name
     extra_classes = ' '.join(attrs.pop('extra_classes', []))
     filter_enabled = "" if enable_filter else filter_option
     select_script = literal(select2 % (input_id, extra_classes, filter_enabled))
 
     return literal(select_html+select_script)
 
 
 def get_visual_attr(tmpl_context_var, attr_name):
     """
     A safe way to get a variable from visual variable of template context
 
     :param tmpl_context_var: instance of tmpl_context, usually present as `c`
     :param attr_name: name of the attribute we fetch from the c.visual
     """
     visual = getattr(tmpl_context_var, 'visual', None)
     if not visual:
         return
     else:
         return getattr(visual, attr_name, None)
 
 
 def get_last_path_part(file_node):
     if not file_node.path:
         return u'/'
 
     path = safe_unicode(file_node.path.split('/')[-1])
     return u'../' + path
 
 
 def route_url(*args, **kwargs):
     """
     Wrapper around pyramids `route_url` (fully qualified url) function.
     """
     req = get_current_request()
     return req.route_url(*args, **kwargs)
 
 
 def route_path(*args, **kwargs):
     """
     Wrapper around pyramids `route_path` function.
     """
     req = get_current_request()
     return req.route_path(*args, **kwargs)
 
 
 def route_path_or_none(*args, **kwargs):
     try:
         return route_path(*args, **kwargs)
     except KeyError:
         return None
 
 
 def current_route_path(request, **kw):
     new_args = request.GET.mixed()
     new_args.update(kw)
     return request.current_route_path(_query=new_args)
 
 
 def curl_api_example(method, args):
     args_json = json.dumps(OrderedDict([
         ('id', 1),
         ('auth_token', 'SECRET'),
         ('method', method),
         ('args', args)
     ]))
 
     return "curl {api_url} -X POST -H 'content-type:text/plain' --data-binary '{args_json}'".format(
         api_url=route_url('apiv2'),
         args_json=args_json
     )
 
 
 def api_call_example(method, args):
     """
     Generates an API call example via CURL
     """
     curl_call = curl_api_example(method, args)
 
     return literal(
         curl_call +
         "<br/><br/>SECRET can be found in <a href=\"{token_url}\">auth-tokens</a> page, "
         "and needs to be of `api calls` role."
         .format(token_url=route_url('my_account_auth_tokens')))
 
 
 def notification_description(notification, request):
     """
     Generate notification human readable description based on notification type
     """
     from rhodecode.model.notification import NotificationModel
     return NotificationModel().make_description(
         notification, translate=request.translate)
 
 
 def go_import_header(request, db_repo=None):
     """
     Creates a header for go-import functionality in Go Lang
     """
 
     if not db_repo:
         return
     if 'go-get' not in request.GET:
         return
 
     clone_url = db_repo.clone_url()
     prefix = re.split(r'^https?:\/\/', clone_url)[-1]
     # we have a repo and go-get flag,
     return literal('<meta name="go-import" content="{} {} {}">'.format(
         prefix, db_repo.repo_type, clone_url))
 
 
 def reviewer_as_json(*args, **kwargs):
     from rhodecode.apps.repository.utils import reviewer_as_json as _reviewer_as_json
     return _reviewer_as_json(*args, **kwargs)
 
 
 def get_repo_view_type(request):
     route_name = request.matched_route.name
     route_to_view_type = {
         'repo_changelog': 'commits',
         'repo_commits': 'commits',
         'repo_files': 'files',
         'repo_summary': 'summary',
         'repo_commit': 'commit'
     }
 
     return route_to_view_type.get(route_name)
 
 
 def is_active(menu_entry, selected):
     """
     Returns active class for selecting menus in templates
     <li class=${h.is_active('settings', current_active)}></li>
     """
     if not isinstance(menu_entry, list):
         menu_entry = [menu_entry]
 
     if selected in menu_entry:
         return "active"
 
 
 class IssuesRegistry(object):
     """
     issue_registry = IssuesRegistry()
     some_func(issues_callback=issues_registry(...))
     """
 
     def __init__(self):
         self.issues = []
         self.unique_issues = collections.defaultdict(lambda: [])
 
     def __call__(self, commit_dict=None):
         def callback(issue):
             if commit_dict and issue:
                 issue['commit'] = commit_dict
             self.issues.append(issue)
             self.unique_issues[issue['id']].append(issue)
         return callback
 
     def get_issues(self):
         return self.issues
 
     @property
     def issues_unique_count(self):
         return len(set(i['id'] for i in self.issues))

rhodecode/subscribers.py

@@ -1,398 +1,398 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import io
 import shlex
 
 import math
 import re
 import os
 import datetime
 import logging
 import queue
 import subprocess
 
 
 from dateutil.parser import parse
 from pyramid.threadlocal import get_current_request
 from pyramid.interfaces import IRoutesMapper
 from pyramid.settings import asbool
 from pyramid.path import AssetResolver
 from threading import Thread
 
 from rhodecode.config.jsroutes import generate_jsroutes_content
 from rhodecode.lib.base import get_auth_user
 
 import rhodecode
 
 
 log = logging.getLogger(__name__)
 
 
 def add_renderer_globals(event):
     from rhodecode.lib import helpers
 
     # TODO: When executed in pyramid view context the request is not available
     # in the event. Find a better solution to get the request.
     request = event['request'] or get_current_request()
 
     # Add Pyramid translation as '_' to context
     event['_'] = request.translate
     event['_ungettext'] = request.plularize
     event['h'] = helpers
 
 
 def set_user_lang(event):
     request = event.request
     cur_user = getattr(request, 'user', None)
 
     if cur_user:
         user_lang = cur_user.get_instance().user_data.get('language')
         if user_lang:
             log.debug('lang: setting current user:%s language to: %s', cur_user, user_lang)
             event.request._LOCALE_ = user_lang
 
 
 def update_celery_conf(event):
     from rhodecode.lib.celerylib.loader import set_celery_conf
     log.debug('Setting celery config from new request')
     set_celery_conf(request=event.request, registry=event.request.registry)
 
 
 def add_request_user_context(event):
     """
     Adds auth user into request context
     """
     request = event.request
     # access req_id as soon as possible
     req_id = request.req_id
 
     if hasattr(request, 'vcs_call'):
         # skip vcs calls
         return
 
     if hasattr(request, 'rpc_method'):
         # skip api calls
         return
 
     auth_user, auth_token = get_auth_user(request)
     request.user = auth_user
     request.user_auth_token = auth_token
     request.environ['rc_auth_user'] = auth_user
     request.environ['rc_auth_user_id'] = auth_user.user_id
     request.environ['rc_req_id'] = req_id
 
 
 def reset_log_bucket(event):
     """
     reset the log bucket on new request
     """
     request = event.request
     request.req_id_records_init()
 
 
 def scan_repositories_if_enabled(event):
     """
     This is subscribed to the `pyramid.events.ApplicationCreated` event. It
     does a repository scan if enabled in the settings.
     """
     settings = event.app.registry.settings
     vcs_server_enabled = settings['vcs.server.enable']
     import_on_startup = settings['startup.import_repos']
     if vcs_server_enabled and import_on_startup:
         from rhodecode.model.scm import ScmModel
         from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_base_path
         repositories = ScmModel().repo_scan(get_rhodecode_base_path())
         repo2db_mapper(repositories, remove_obsolete=False)
 
 
 def write_metadata_if_needed(event):
     """
     Writes upgrade metadata
     """
     import rhodecode
     from rhodecode.lib import system_info
     from rhodecode.lib import ext_json
 
     fname = '.rcmetadata.json'
     ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
     metadata_destination = os.path.join(ini_loc, fname)
 
     def get_update_age():
         now = datetime.datetime.utcnow()
 
         with open(metadata_destination, 'rb') as f:
             data = ext_json.json.loads(f.read())
             if 'created_on' in data:
                 update_date = parse(data['created_on'])
                 diff = now - update_date
                 return diff.total_seconds() / 60.0
 
         return 0
 
     def write():
         configuration = system_info.SysInfo(
             system_info.rhodecode_config)()['value']
         license_token = configuration['config']['license_token']
 
         setup = dict(
             workers=configuration['config']['server:main'].get(
                 'workers', '?'),
             worker_type=configuration['config']['server:main'].get(
                 'worker_class', 'sync'),
         )
         dbinfo = system_info.SysInfo(system_info.database_info)()['value']
         del dbinfo['url']
 
         metadata = dict(
             desc='upgrade metadata info',
             license_token=license_token,
             created_on=datetime.datetime.utcnow().isoformat(),
             usage=system_info.SysInfo(system_info.usage_info)()['value'],
             platform=system_info.SysInfo(system_info.platform_type)()['value'],
             database=dbinfo,
             cpu=system_info.SysInfo(system_info.cpu)()['value'],
             memory=system_info.SysInfo(system_info.memory)()['value'],
             setup=setup
         )
 
         with open(metadata_destination, 'wb') as f:
             f.write(ext_json.json.dumps(metadata))
 
     settings = event.app.registry.settings
     if settings.get('metadata.skip'):
         return
 
     # only write this every 24h, workers restart caused unwanted delays
     try:
         age_in_min = get_update_age()
     except Exception:
         age_in_min = 0
 
     if age_in_min > 60 * 60 * 24:
         return
 
     try:
         write()
     except Exception:
         pass
 
 
 def write_usage_data(event):
     import rhodecode
     from rhodecode.lib import system_info
     from rhodecode.lib import ext_json
 
     settings = event.app.registry.settings
     instance_tag = settings.get('metadata.write_usage_tag')
     if not settings.get('metadata.write_usage'):
         return
 
     def get_update_age(dest_file):
         now = datetime.datetime.utcnow()
 
         with open(dest_file, 'rb') as f:
             data = ext_json.json.loads(f.read())
             if 'created_on' in data:
                 update_date = parse(data['created_on'])
                 diff = now - update_date
                 return math.ceil(diff.total_seconds() / 60.0)
 
         return 0
 
     utc_date = datetime.datetime.utcnow()
     hour_quarter = int(math.ceil((utc_date.hour + utc_date.minute/60.0) / 6.))
     fname = '.rc_usage_{date.year}{date.month:02d}{date.day:02d}_{hour}.json'.format(
         date=utc_date, hour=hour_quarter)
     ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
 
     usage_dir = os.path.join(ini_loc, '.rcusage')
     if not os.path.isdir(usage_dir):
         os.makedirs(usage_dir)
     usage_metadata_destination = os.path.join(usage_dir, fname)
 
     try:
         age_in_min = get_update_age(usage_metadata_destination)
     except Exception:
         age_in_min = 0
 
     # write every 6th hour
     if age_in_min and age_in_min < 60 * 6:
         log.debug('Usage file created %s minutes ago, skipping (threshold: %s minutes)...',
                   age_in_min, 60 * 6)
         return
 
     def write(dest_file):
         configuration = system_info.SysInfo(system_info.rhodecode_config)()['value']
         license_token = configuration['config']['license_token']
 
         metadata = dict(
             desc='Usage data',
             instance_tag=instance_tag,
             license_token=license_token,
             created_on=datetime.datetime.utcnow().isoformat(),
             usage=system_info.SysInfo(system_info.usage_info)()['value'],
         )
 
         with open(dest_file, 'wb') as f:
-            f.write(ext_json.json.dumps(metadata, indent=2, sort_keys=True))
+            f.write(ext_json.formatted_json(metadata))
 
     try:
         log.debug('Writing usage file at: %s', usage_metadata_destination)
         write(usage_metadata_destination)
     except Exception:
         pass
 
 
 def write_js_routes_if_enabled(event):
     registry = event.app.registry
 
     mapper = registry.queryUtility(IRoutesMapper)
     _argument_prog = re.compile(r'\{(.*?)\}|:\((.*)\)')
 
     def _extract_route_information(route):
         """
         Convert a route into tuple(name, path, args), eg:
             ('show_user', '/profile/%(username)s', ['username'])
         """
 
         routepath = route.pattern
         pattern = route.pattern
 
         def replace(matchobj):
             if matchobj.group(1):
                 return "%%(%s)s" % matchobj.group(1).split(':')[0]
             else:
                 return "%%(%s)s" % matchobj.group(2)
 
         routepath = _argument_prog.sub(replace, routepath)
 
         if not routepath.startswith('/'):
             routepath = '/'+routepath
 
         return (
             route.name,
             routepath,
             [(arg[0].split(':')[0] if arg[0] != '' else arg[1])
               for arg in _argument_prog.findall(pattern)]
         )
 
     def get_routes():
         # pyramid routes
         for route in mapper.get_routes():
             if not route.name.startswith('__'):
                 yield _extract_route_information(route)
 
     if asbool(registry.settings.get('generate_js_files', 'false')):
         static_path = AssetResolver().resolve('rhodecode:public').abspath()
         jsroutes = get_routes()
         jsroutes_file_content = generate_jsroutes_content(jsroutes)
         jsroutes_file_path = os.path.join(
             static_path, 'js', 'rhodecode', 'routes.js')
 
         try:
             with io.open(jsroutes_file_path, 'w', encoding='utf-8') as f:
                 f.write(jsroutes_file_content)
         except Exception:
             log.exception('Failed to write routes.js into %s', jsroutes_file_path)
 
 
 class Subscriber(object):
     """
     Base class for subscribers to the pyramid event system.
     """
     def __call__(self, event):
         self.run(event)
 
     def run(self, event):
         raise NotImplementedError('Subclass has to implement this.')
 
 
 class AsyncSubscriber(Subscriber):
     """
     Subscriber that handles the execution of events in a separate task to not
     block the execution of the code which triggers the event. It puts the
     received events into a queue from which the worker process takes them in
     order.
     """
     def __init__(self):
         self._stop = False
         self._eventq = queue.Queue()
         self._worker = self.create_worker()
         self._worker.start()
 
     def __call__(self, event):
         self._eventq.put(event)
 
     def create_worker(self):
         worker = Thread(target=self.do_work)
         worker.daemon = True
         return worker
 
     def stop_worker(self):
         self._stop = False
         self._eventq.put(None)
         self._worker.join()
 
     def do_work(self):
         while not self._stop:
             event = self._eventq.get()
             if event is not None:
                 self.run(event)
 
 
 class AsyncSubprocessSubscriber(AsyncSubscriber):
     """
     Subscriber that uses the subprocess module to execute a command if an
     event is received. Events are handled asynchronously::
 
         subscriber = AsyncSubprocessSubscriber('ls -la', timeout=10)
         subscriber(dummyEvent) # running __call__(event)
 
     """
 
     def __init__(self, cmd, timeout=None):
         if not isinstance(cmd, (list, tuple)):
             cmd = shlex.split(cmd)
         super(AsyncSubprocessSubscriber, self).__init__()
         self._cmd = cmd
         self._timeout = timeout
 
     def run(self, event):
         cmd = self._cmd
         timeout = self._timeout
         log.debug('Executing command %s.', cmd)
 
         try:
             output = subprocess.check_output(
                 cmd, timeout=timeout, stderr=subprocess.STDOUT)
             log.debug('Command finished %s', cmd)
             if output:
                 log.debug('Command output: %s', output)
         except subprocess.TimeoutExpired as e:
             log.exception('Timeout while executing command.')
             if e.output:
                 log.error('Command output: %s', e.output)
         except subprocess.CalledProcessError as e:
             log.exception('Error while executing command.')
             if e.output:
                 log.error('Command output: %s', e.output)
         except Exception:
             log.exception(
                 'Exception while executing command %s.', cmd)

rhodecode/templates/admin/admin_audit_log_entry.mako

@@ -1,115 +1,115 b''
 ## -*- coding: utf-8 -*-
 <%inherit file="/base/base.mako"/>
 <%namespace name="base" file="/base/base.mako"/>
 
 <%def name="title()">
     ${_('Admin audit log entry')}
     %if c.rhodecode_name:
         &middot; ${h.branding(c.rhodecode_name)}
     %endif
 </%def>
 
 <%def name="breadcrumbs_links()"></%def>
 
 <%def name="menu_bar_nav()">
     ${self.menu_items(active='admin')}
 </%def>
 
 <%def name="menu_bar_subnav()">
     ${self.admin_menu(active='audit_logs')}
 </%def>
 
 <%def name="main()">
 <div class="box">
 
     <div class="title">
         ${_('Audit long entry')} ${c.audit_log_entry.entry_id}
     </div>
 
     <div class="table">
         <div id="user_log">
             <table class="rctable audit-log">
                 <tr>
                     <td>
                     ${_('User')}:
                     </td>
                     <td>
                         %if c.audit_log_entry.user is not None:
                           ${base.gravatar_with_user(c.audit_log_entry.user.email)}
                         %else:
                           ${c.audit_log_entry.username}
                         %endif
                     </td>
                 </tr>
                 <tr>
                     <td>
                     ${_('Date')}:
                     </td>
                     <td>
                         ${h.format_date(c.audit_log_entry.action_date)}
                     </td>
                 </tr>
                 <tr>
                     <td>
                     ${_('IP')}:
                     </td>
                     <td>
                         ${c.audit_log_entry.user_ip}
                     </td>
                 </tr>
 
                 <tr>
                     <td>
                     ${_('Action')}:
                     </td>
                     <td>
                         % if c.audit_log_entry.version == c.audit_log_entry.VERSION_1:
                             ${h.action_parser(request, l)[0]()}
                         % else:
                             ${h.literal(c.audit_log_entry.action)}
                         % endif
 
                         <div class="journal_action_params">
                             % if c.audit_log_entry.version == c.audit_log_entry.VERSION_1:
                                 ${h.literal(h.action_parser(request, l)[1]())}
                             % endif
                         </div>
                     </td>
                 </tr>
                 <tr>
                     <td>
                     ${_('Action Data')}:
                     </td>
                     <td class="td-journalaction">
                         % if c.audit_log_entry.version == c.audit_log_entry.VERSION_2:
                             <div>
-                                <pre>${h.json.dumps(c.audit_log_entry.action_data, indent=4, sort_keys=True)}</pre>
+                                <pre>${h.formatted_str_json(c.audit_log_entry.action_data)}</pre>
                             </div>
                         % else:
                             <pre title="${_('data not available for v1 entries type')}">-</pre>
                         % endif
                     </td>
                 </tr>
                 <tr>
                     <td>
                     ${_('Repository')}:
                     </td>
                     <td class="td-journalaction">
                         %if c.audit_log_entry.repository is not None:
                           ${h.link_to(c.audit_log_entry.repository.repo_name, h.route_path('repo_summary',repo_name=c.audit_log_entry.repository.repo_name))}
                         %else:
                           ${c.audit_log_entry.repository_name or '-'}
                         %endif
                     </td>
                 </tr>
 
             </table>
 
         </div>
     </div>
 </div>
 
 <script>
 $('#j_filter').autoGrowInput();
 </script>
 </%def>

rhodecode/templates/admin/admin_log_base.mako

@@ -1,69 +1,69 b''
 <%namespace name="base" file="/base/base.mako"/>
 
 %if c.audit_logs:
 <table class="rctable admin_log">
     <tr>
         <th>${_('Uid')}</th>
         <th>${_('Username')}</th>
         <th>${_('Action')}</th>
         <th>${_('Action Data')}</th>
         <th>${_('Repository')}</th>
         <th>${_('Date')}</th>
         <th>${_('IP')}</th>
     </tr>
 
     %for cnt,l in enumerate(c.audit_logs):
     <tr class="parity${cnt%2}">
         <td class="td-col">
             <a href="${h.route_path('admin_audit_log_entry', audit_log_id=l.entry_id)}">${l.entry_id}</a>
         </td>
         <td class="td-user">
             %if l.user is not None:
               ${base.gravatar_with_user(l.user.email)}
             %else:
               ${l.username}
             %endif
         </td>
         <td class="td-journalaction">
             % if l.version == l.VERSION_1:
                 ${h.action_parser(request, l)[0]()}
             % else:
                 ${h.literal(l.action)}
             % endif
 
             <div class="journal_action_params">
                 % if l.version == l.VERSION_1:
                     ${h.literal(h.action_parser(request, l)[1]())}
                 % endif
             </div>
         </td>
         <td>
             % if l.version == l.VERSION_2:
                 <a href="#" onclick="$('#entry-'+${l.user_log_id}).toggle();return false">${_('toggle')}</a>
                 <div id="entry-${l.user_log_id}" style="display: none">
-                    <pre>${h.json.dumps(l.action_data, indent=4, sort_keys=True)}</pre>
+                    <pre>${h.formatted_str_json(l.action_data)}</pre>
                 </div>
             % else:
                 <pre title="${_('data not available for v1 entries type')}">-</pre>
             % endif
         </td>
         <td class="td-componentname">
             %if l.repository is not None:
               ${h.link_to(l.repository.repo_name, h.route_path('repo_summary',repo_name=l.repository.repo_name))}
             %else:
               ${l.repository_name}
             %endif
         </td>
 
         <td class="td-time">${h.format_date(l.action_date)}</td>
         <td class="td-ip">${l.user_ip}</td>
     </tr>
     %endfor
 </table>
 
 <div class="pagination-wh pagination-left">
 ${c.audit_logs.render()}
 </div>
 %else:
     ${_('No actions yet')}
 %endif
  No newline at end of file

rhodecode/templates/base/issue_tracker_settings.mako

@@ -1,328 +1,328 b''
 ## snippet for displaying issue tracker settings
 ## usage:
 ##    <%namespace name="its" file="/base/issue_tracker_settings.mako"/>
 ##    ${its.issue_tracker_settings_table(patterns, form_url, delete_url)}
 ##    ${its.issue_tracker_settings_test(test_url)}
 
 <%def name="issue_tracker_settings_table(patterns, form_url, delete_url)">
 <%
 # Name/desc, pattern, issue prefix
 examples = [
     (
     ' ',
     ' ',
     ' ',
     ' '
     ),
 
     (
     'Tickets with #123 (Redmine etc)',
     '(?<![a-zA-Z0-9_/]{1,10}-?)(#)(?P<issue_id>\d+)',
     'https://myissueserver.com/${repo}/issue/${issue_id}',
     ''
     ),
 
     (
     'Redmine - Alternative',
     '(?:issue-)(\d+)',
     'https://myissueserver.com/redmine/issue/${id}',
     ''
     ),
 
     (
     'Redmine - Wiki',
     '(?:wiki-)([a-zA-Z0-9]+)',
     'http://example.org/projects/${repo_name}/wiki/${id}',
     'wiki-'
     ),
 
     (
     'JIRA - All tickets',
     # official JIRA ticket pattern
     '(?<![a-zA-Z0-9_/#]-?)(?P<issue_id>[A-Z]{1,6}-(?:[1-9][0-9]{0,7}))',
     'https://myjira.com/browse/${issue_id}',
     ''
     ),
 
     (
     'JIRA - Single project (JRA-XXXXXXXX)',
     '(?<![a-zA-Z0-9_/#]-?)(?P<issue_id>JRA-(?:[1-9][0-9]{0,7}))',
     'https://myjira.com/${issue_id}',
     ''
     ),
 
     (
     'Confluence WIKI',
     '(?:conf-)([A-Z0-9]+)',
     'https://example.atlassian.net/display/wiki/${id}/${repo_name}',
     'CONF-',
     ),
 
     (
     'Pivotal Tracker',
     '(?:pivot-)(?P<project_id>\d+)-(?P<story>\d+)',
     'https://www.pivotaltracker.com/s/projects/${project_id}/stories/${story}',
     'PIV-',
     ),
 
     (
     'Trello',
     '(?:trello-)(?P<card_id>[a-zA-Z0-9]+)',
     'https://trello.com/example.com/${card_id}',
     'TRELLO-',
     ),
 ]
 %>
 
     <table class="rctable issuetracker">
       <tr>
           <th>${_('Description')}</th>
           <th>${_('Pattern')}</th>
           <th>${_('Url')}</th>
           <th>${_('Extra Prefix')}</th>
           <th ></th>
       </tr>
         % for name, pat, url, pref in examples:
         <tr class="it-examples" style="${'' if loop.index == 0 else 'display:none'}">
           <td class="td-issue-tracker-name issue-tracker-example">${name}</td>
           <td class="td-regex issue-tracker-example">${pat}</td>
           <td class="td-url issue-tracker-example">${url}</td>
           <td class="td-prefix issue-tracker-example">${pref}</td>
           <td>
               % if loop.index == 0:
               <a href="#showMore" onclick="$('.it-examples').toggle(); return false">${_('show examples')}</a>
               % else:
-              <a href="#copyToInput" onclick="copyToInput(this, '${h.json.dumps(name)}', '${h.json.dumps(pat)}', '${h.json.dumps(url)}', '${h.json.dumps(pref)}'); return false">copy to input</a>
+              <a href="#copyToInput" onclick="copyToInput(this, '${h.str_json(name)}', '${h.str_json(pat)}', '${h.str_json(url)}', '${h.str_json(pref)}'); return false">copy to input</a>
               % endif
           </td>
         </tr>
         % endfor
 
       %for uid, entry in patterns:
         <tr id="entry_${uid}">
                 <td class="td-issue-tracker-name issuetracker_desc">
                     <span class="entry">
                         ${entry.desc}
                     </span>
                     <span class="edit">
                         ${h.text('new_pattern_description_'+uid, class_='medium-inline', value=entry.desc or '')}
                     </span>
                 </td>
                 <td class="td-issue-tracker-regex issuetracker_pat">
                     <span class="entry">
                         ${entry.pat}
                     </span>
                     <span class="edit">
                         ${h.text('new_pattern_pattern_'+uid, class_='medium-inline', value=entry.pat or '')}
                     </span>
                 </td>
                 <td class="td-url issuetracker_url">
                     <span class="entry">
                         ${entry.url}
                     </span>
                     <span class="edit">
                         ${h.text('new_pattern_url_'+uid, class_='medium-inline', value=entry.url or '')}
                     </span>
                 </td>
                 <td class="td-prefix issuetracker_pref">
                     <span class="entry">
                         ${entry.pref}
                     </span>
                     <span class="edit">
                         ${h.text('new_pattern_prefix_'+uid, class_='medium-inline', value=entry.pref or '')}
                     </span>
                 </td>
                 <td class="td-action">
                     <div  class="grid_edit">
                         <span class="entry">
                             <a class="edit_issuetracker_entry" href="">${_('Edit')}</a>
                         </span>
                         <span class="edit">
                             <input id="uid_${uid}" name="uid" type="hidden" value="${uid}">
                         </span>
                     </div>
                     <div  class="grid_delete">
                         <span class="entry">
                             <a class="btn btn-link btn-danger delete_issuetracker_entry" data-desc="${entry.desc}" data-uid="${uid}">
                                 ${_('Delete')}
                             </a>
                         </span>
                         <span class="edit">
                             <a class="btn btn-link btn-danger edit_issuetracker_cancel" data-uid="${uid}">${_('Cancel')}</a>
                         </span>
                    </div>
                 </td>
         </tr>
       %endfor
       <tr id="last-row"></tr>
     </table>
     <p>
       <a id="add_pattern" class="link">
           ${_('Add new')}
       </a>
     </p>
 
     <script type="text/javascript">
         var newEntryLabel = $('label[for="new_entry"]');
 
         var resetEntry = function() {
           newEntryLabel.text("${_('New Entry')}:");
         };
 
         var delete_pattern = function(entry) {
           if (confirm("${_('Confirm to remove this pattern:')} "+$(entry).data('desc'))) {
             $.ajax({
               type: "POST",
               url: "${delete_url}",
               data: {
                 'csrf_token': CSRF_TOKEN,
                 'uid':$(entry).data('uid')
               },
               success: function(){
                 window.location.reload();
               },
               error: function(data, textStatus, errorThrown){
                   alert("Error while deleting entry.\nError code {0} ({1}). URL: {2}".format(data.status,data.statusText,$(entry)[0].url));
               }
             });
           }
         };
 
         $('.delete_issuetracker_entry').on('click', function(e){
           e.preventDefault();
           delete_pattern(this);
         });
 
         $('.edit_issuetracker_entry').on('click', function(e){
             e.preventDefault();
             $(this).parents('tr').addClass('editopen');
         });
 
         $('.edit_issuetracker_cancel').on('click', function(e){
             e.preventDefault();
             $(this).parents('tr').removeClass('editopen');
             // Reset to original value
             var uid = $(this).data('uid');
             $('#'+uid+' input').each(function(e) {
                 this.value = this.defaultValue;
             });
         });
 
         $('input#reset').on('click', function(e) {
             resetEntry();
         });
 
         $('#add_pattern').on('click', function(e) {
             addNewPatternInput();
         });
 
         var copied = false;
         copyToInput = function (elem, name, pat, url, pref) {
             if (copied === false) {
                 addNewPatternInput();
                 copied = true;
             }
             $(elem).hide();
             var load = function(text){
                 return text.replace(/["]/g, "")
             };
             $('#description_1').val(load(name));
             $('#pattern_1').val(load(pat));
             $('#url_1').val(load(url));
             $('#prefix_1').val(load(pref));
 
         }
 
     </script>    
 </%def>
 
 <%def name="issue_tracker_new_row()">
   <table id="add-row-tmpl" style="display: none;">
     <tbody>
     <tr class="new_pattern">
         <td class="td-issue-tracker-name issuetracker_desc">
           <span class="entry">
                 <input class="medium-inline" id="description_##UUID##" name="new_pattern_description_##UUID##" value="##DESCRIPTION##" type="text">
           </span>
         </td>
         <td class="td-issue-tracker-regex issuetracker_pat">
           <span class="entry">
                     <input class="medium-inline" id="pattern_##UUID##" name="new_pattern_pattern_##UUID##" placeholder="Pattern" 
                     value="##PATTERN##" type="text">
           </span>
         </td>
         <td class="td-url issuetracker_url">
           <span class="entry">
                   <input class="medium-inline" id="url_##UUID##" name="new_pattern_url_##UUID##" placeholder="Url" value="##URL##" type="text">
           </span>
         </td>
         <td class="td-prefix issuetracker_pref">
             <span class="entry">
                   <input class="medium-inline" id="prefix_##UUID##" name="new_pattern_prefix_##UUID##" placeholder="Prefix" value="##PREFIX##" type="text"> 
             </span>
         </td>
         <td class="td-action">
         </td>
         <input id="uid_##UUID##" name="uid_##UUID##" type="hidden" value="">
     </tr>
     </tbody>
   </table>
 </%def>
 
 <%def name="issue_tracker_settings_test(test_url)">
     <div class="form-vertical">
         <div class="fields">
             <div class="field">
                 <div class='textarea-full'>
                     <textarea id="test_pattern_data" rows="12">
 This is an example text for testing issue tracker patterns.
 This commit fixes ticket #451 and ticket #910, reference for JRA-401.
 The following tickets will get mentioned:
 #123
 #456 and PROJ-101
 JRA-123 and #123
 PROJ-456
 
 [my artifact](http://something.com/JRA-1234-build.zip)
 
 - #1001
 - JRA-998
 
 Open a pull request !101 to contribute!
 Added tag v1.3.0 for commit 0f3b629be725
 
 Add a test pattern here and hit preview to see the link.
                     </textarea>
                 </div>
             </div>
         </div>
         <div class="test_pattern_preview">
             <div id="test_pattern" class="btn btn-small" >${_('Preview')}</div>
             <p>${_('Test Pattern Preview')}</p>
             <div id="test_pattern_result" style="white-space: pre-wrap"></div>
         </div>
     </div>
 
     <script type="text/javascript">
         $('#test_pattern').on('click', function(e) {
           $.ajax({
               type: "POST",
               url: "${test_url}",
               data: {
                 'test_text': $('#test_pattern_data').val(),
                 'csrf_token': CSRF_TOKEN
               },
               success: function(data){
                   $('#test_pattern_result').html(data);
                   tooltipActivate();
               },
               error: function(jqXHR, textStatus, errorThrown){
                   $('#test_pattern_result').html('Error: ' + errorThrown);
               }
           });
           $('#test_pattern_result').show();
         });
     </script>
 </%def>
 
 

rhodecode/templates/base/root.mako

@@ -1,166 +1,166 b''
 ## -*- coding: utf-8 -*-
 <!DOCTYPE html>
 
 <%
 c.template_context['repo_name'] = getattr(c, 'repo_name', '')
 go_import_header = ''
 if hasattr(c, 'rhodecode_db_repo'):
     c.template_context['repo_type'] = c.rhodecode_db_repo.repo_type
     c.template_context['repo_landing_commit'] = c.rhodecode_db_repo.landing_ref_name
     c.template_context['repo_id'] = c.rhodecode_db_repo.repo_id
     c.template_context['repo_view_type'] = h.get_repo_view_type(request)
 
 if getattr(c, 'repo_group', None):
     c.template_context['repo_group_id'] = c.repo_group.group_id
     c.template_context['repo_group_name'] = c.repo_group.group_name
 
 if getattr(c, 'rhodecode_user', None) and c.rhodecode_user.user_id:
     c.template_context['rhodecode_user']['user_id'] = c.rhodecode_user.user_id
     c.template_context['rhodecode_user']['username'] = c.rhodecode_user.username
     c.template_context['rhodecode_user']['email'] = c.rhodecode_user.email
     c.template_context['rhodecode_user']['notification_status'] = c.rhodecode_user.get_instance().user_data.get('notification_status', True)
     c.template_context['rhodecode_user']['first_name'] = c.rhodecode_user.first_name
     c.template_context['rhodecode_user']['last_name'] = c.rhodecode_user.last_name
 
 c.template_context['visual']['default_renderer'] = h.get_visual_attr(c, 'default_renderer')
 c.template_context['default_user'] = {
     'username': h.DEFAULT_USER,
     'user_id': 1
 }
 c.template_context['search_context'] = {
     'repo_group_id': c.template_context.get('repo_group_id'),
     'repo_group_name': c.template_context.get('repo_group_name'),
     'repo_id': c.template_context.get('repo_id'),
     'repo_name': c.template_context.get('repo_name'),
     'repo_view_type': c.template_context.get('repo_view_type'),
 }
 
 c.template_context['attachment_store'] = {
     'max_file_size_mb': 10,
     'image_ext': ["png", "jpg", "gif", "jpeg"]
 }
 
 %>
 <html xmlns="http://www.w3.org/1999/xhtml">
     <head>
         <title>${self.title()}</title>
         <meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
 
         ${h.go_import_header(request, getattr(c, 'rhodecode_db_repo', None))}
 
         % if 'safari' in (request.user_agent or '').lower():
             <meta name="referrer" content="origin">
         % else:
             <meta name="referrer" content="origin-when-cross-origin">
         % endif
 
         <%def name="robots()">
             <meta name="robots" content="index, nofollow"/>
         </%def>
         ${self.robots()}
         <link rel="icon" href="${h.asset('images/favicon.ico', ver=c.rhodecode_version_hash)}" sizes="16x16 32x32" type="image/png" />
         <script src="${h.asset('js/vendors/webcomponentsjs/custom-elements-es5-adapter.js', ver=c.rhodecode_version_hash)}"></script>
         <script src="${h.asset('js/vendors/webcomponentsjs/webcomponents-bundle.js', ver=c.rhodecode_version_hash)}"></script>
 
         ## CSS definitions
         <%def name="css()">
             <link rel="stylesheet" type="text/css" href="${h.asset('css/style.css', ver=c.rhodecode_version_hash)}" media="screen"/>
             ## EXTRA FOR CSS
             ${self.css_extra()}
         </%def>
         ## CSS EXTRA - optionally inject some extra CSS stuff needed for specific websites
         <%def name="css_extra()">
         </%def>
 
         ${self.css()}
 
         ## JAVASCRIPT
         <%def name="js()">
 
             <script src="${h.asset('js/rhodecode/i18n/%s.js' % c.language, ver=c.rhodecode_version_hash)}"></script>
             <script type="text/javascript">
             // register templateContext to pass template variables to JS
-            var templateContext = ${h.json.dumps(c.template_context)|n};
+            var templateContext = ${h.str_json(c.template_context)|n};
 
             var APPLICATION_URL = "${h.route_path('home').rstrip('/')}";
             var APPLICATION_PLUGINS = [];
             var ASSET_URL = "${h.asset('')}";
             var DEFAULT_RENDERER = "${h.get_visual_attr(c, 'default_renderer')}";
             var CSRF_TOKEN = "${getattr(c, 'csrf_token', '')}";
 
             var APPENLIGHT = {
               enabled: ${'true' if getattr(c, 'appenlight_enabled', False) else 'false'},
               key: '${getattr(c, "appenlight_api_public_key", "")}',
               % if getattr(c, 'appenlight_server_url', None):
                   serverUrl: '${getattr(c, "appenlight_server_url", "")}',
               % endif
               requestInfo: {
               % if getattr(c, 'rhodecode_user', None):
                   ip: '${c.rhodecode_user.ip_addr}',
                   username: '${c.rhodecode_user.username}'
               % endif
               },
               tags: {
                   rhodecode_version: '${c.rhodecode_version}',
                   rhodecode_edition: '${c.rhodecode_edition}'
               }
             };
 
             </script>
             <%include file="/base/plugins_base.mako"/>
             <!--[if lt IE 9]>
             <script language="javascript" type="text/javascript" src="${h.asset('js/src/excanvas.min.js')}"></script>
             <![endif]-->
             <script language="javascript" type="text/javascript" src="${h.asset('js/rhodecode/routes.js', ver=c.rhodecode_version_hash)}"></script>
             <script> var alertMessagePayloads = ${h.flash.json_alerts(request=request)|n}; </script>
             ## avoide escaping the %N
             <script language="javascript" type="text/javascript" src="${h.asset('js/scripts.min.js', ver=c.rhodecode_version_hash)}"></script>
             <script>CodeMirror.modeURL = "${h.asset('') + 'js/mode/%N/%N.js?ver='+c.rhodecode_version_hash}";</script>
 
 
             ## JAVASCRIPT EXTRA - optionally inject some extra JS for specificed templates
             ${self.js_extra()}
 
             <script type="text/javascript">
             Rhodecode = (function() {
                 function _Rhodecode() {
                   this.comments = new CommentsController();
                 }
                 return new _Rhodecode();
             })();
 
             $(document).ready(function(){
               show_more_event();
               timeagoActivate();
               tooltipActivate();
               clipboardActivate();
             })
             </script>
 
         </%def>
 
         ## JAVASCRIPT EXTRA - optionally inject some extra JS for specificed templates
         <%def name="js_extra()"></%def>
         ${self.js()}
 
         <%def name="head_extra()"></%def>
         ${self.head_extra()}
         ## extra stuff
         %if c.pre_code:
             ${c.pre_code|n}
         %endif
     </head>
     <body id="body">
         <noscript>
             <div class="noscript-error">
                 ${_('Please enable JavaScript to use RhodeCode Enterprise')}
             </div>
         </noscript>
 
       ${next.body()}
       %if c.post_code:
         ${c.post_code|n}
       %endif
     <rhodecode-app></rhodecode-app>
     </body>
 </html>

rhodecode/templates/base/sidebar.mako

@@ -1,168 +1,168 b''
 ## snippet for sidebar elements
 ## usage:
 ##    <%namespace name="sidebar" file="/base/sidebar.mako"/>
 ##    ${sidebar.comments_table()}
 <%namespace name="base" file="/base/base.mako"/>
 
 <%def name="comments_table(comments, counter_num, todo_comments=False, draft_comments=False, existing_ids=None, is_pr=True)">
     <%
         if todo_comments:
             cls_ = 'todos-content-table'
             def sorter(entry):
                 user_id = entry.author.user_id
                 resolved = '1' if entry.resolved else '0'
                 if user_id == c.rhodecode_user.user_id:
                     # own comments first
                     user_id = 0
                 return '{}'.format(str(entry.comment_id).zfill(10000))
         elif draft_comments:
             cls_ = 'drafts-content-table'
             def sorter(entry):
                 return '{}'.format(str(entry.comment_id).zfill(10000))
         else:
             cls_ = 'comments-content-table'
             def sorter(entry):
                 return '{}'.format(str(entry.comment_id).zfill(10000))
 
         existing_ids = existing_ids or []
 
     %>
 
     <table class="todo-table ${cls_}" data-total-count="${len(comments)}" data-counter="${counter_num}">
 
      % for loop_obj, comment_obj in h.looper(reversed(sorted(comments, key=sorter))):
          <%
             display = ''
             _cls = ''
             ## Extra precaution to not show drafts in the sidebar for todo/comments
             if comment_obj.draft and not draft_comments:
                 continue
          %>
 
 
          <%
              comment_ver_index = comment_obj.get_index_version(getattr(c, 'versions', []))
              prev_comment_ver_index = 0
              if loop_obj.previous:
                 prev_comment_ver_index = loop_obj.previous.get_index_version(getattr(c, 'versions', []))
 
              ver_info = None
              if getattr(c, 'versions', []):
                 ver_info = c.versions[comment_ver_index-1] if comment_ver_index else None
          %>
          <% hidden_at_ver = comment_obj.outdated_at_version_js(c.at_version_num) %>
          <% is_from_old_ver = comment_obj.older_than_version_js(c.at_version_num) %>
          <%
          if (prev_comment_ver_index > comment_ver_index):
            comments_ver_divider = comment_ver_index
          else:
             comments_ver_divider = None
          %>
 
           % if todo_comments:
               % if comment_obj.resolved:
                   <% _cls = 'resolved-todo' %>
                   <% display = 'none' %>
               % endif
           % else:
               ## SKIP TODOs we display them in other area
               % if comment_obj.is_todo and not comment_obj.draft:
                   <% display = 'none' %>
               % endif
               ## Skip outdated comments
               % if comment_obj.outdated:
                   <% display = 'none' %>
                   <% _cls = 'hidden-comment' %>
               % endif
           % endif
 
           % if not todo_comments and comments_ver_divider:
           <tr class="old-comments-marker">
               <td colspan="3">
                   % if ver_info:
                     <code>v${comments_ver_divider} ${h.age_component(ver_info.created_on, time_is_local=True, tooltip=False)}</code>
                   % else:
                     <code>v${comments_ver_divider}</code>
                   % endif
               </td>
           </tr>
 
           % endif
 
           <tr class="${_cls}" style="display: ${display};" data-sidebar-comment-id="${comment_obj.comment_id}">
               % if draft_comments:
                 <td style="width: 15px;">
                 ${h.checkbox('submit_draft', id=None, value=comment_obj.comment_id)}
                 </td>
               % endif
               <td class="td-todo-number">
                   <%
                     version_info = ''
                     if is_pr:
                         version_info = (' made in older version (v{})'.format(comment_ver_index) if is_from_old_ver == 'true' else ' made in this version')
                   %>
                   ## new comments, since refresh
                   % if existing_ids and comment_obj.comment_id not in existing_ids:
                       <div class="tooltip" style="position: absolute; left: 8px; color: #682668" title="New comment">
                           !
                       </div>
                   % endif
 
                   <%
-                      data = h.json.dumps({
+                      data = h.str_json({
                             'comment_id': comment_obj.comment_id,
                             'version_info': version_info,
                             'file_name': comment_obj.f_path,
                             'line_no': comment_obj.line_no,
                             'outdated': comment_obj.outdated,
                             'inline': comment_obj.is_inline,
                             'is_todo': comment_obj.is_todo,
                             'created_on': h.format_date(comment_obj.created_on),
                             'datetime': '{}{}'.format(comment_obj.created_on, h.get_timezone(comment_obj.created_on, time_is_local=True)),
                             'review_status': (comment_obj.review_status or '')
                       })
 
                       icon = ''
 
                       if comment_obj.outdated:
                           icon += ' icon-comment-toggle'
                       elif comment_obj.is_inline:
                           icon += ' icon-code'
                       else:
                           icon += ' icon-comment'
 
                       if comment_obj.draft:
                           if comment_obj.is_todo:
                               icon = 'icon-flag-filled icon-draft'
                           else:
                               icon = 'icon-comment icon-draft'
 
                   %>
 
                   <i id="commentHovercard${comment_obj.comment_id}"
                      class="${icon} tooltip-hovercard"
                      data-hovercard-url="javascript:sidebarComment(${comment_obj.comment_id})"
                      data-comment-json-b64='${h.b64(data)}'>
                   </i>
 
               </td>
 
               <td class="td-todo-gravatar">
                   ${base.gravatar(comment_obj.author.email, 16, user=comment_obj.author, tooltip=True, extra_class=['no-margin'])}
               </td>
               <td class="todo-comment-text-wrapper">
                   <div class="todo-comment-text ${('todo-resolved' if comment_obj.resolved else '')}">
                       <a class="${('todo-resolved' if comment_obj.resolved else '')} permalink"
                          href="#comment-${comment_obj.comment_id}"
                          onclick="return Rhodecode.comments.scrollToComment($('#comment-${comment_obj.comment_id}'), 0, ${hidden_at_ver})">
 
                          ${h.chop_at_smart(comment_obj.text, '\n', suffix_if_chopped='...')}
                       </a>
                   </div>
               </td>
           </tr>
      % endfor
 
     </table>
 
 </%def>
  No newline at end of file

rhodecode/templates/changeset/changeset_file_comment.mako

@@ -1,557 +1,557 b''
 ## -*- coding: utf-8 -*-
 ## usage:
 ## <%namespace name="comment" file="/changeset/changeset_file_comment.mako"/>
 ## ${comment.comment_block(comment)}
 ##
 <%namespace name="base" file="/base/base.mako"/>
 
 <%!
     from rhodecode.lib import html_filters
 %>
 
 
 <%def name="comment_block(comment, inline=False, active_pattern_entries=None, is_new=False)">
 
     <%
       from rhodecode.model.comment import CommentsModel
       comment_model = CommentsModel()
 
       comment_ver = comment.get_index_version(getattr(c, 'versions', []))
       latest_ver = len(getattr(c, 'versions', []))
       visible_for_user = True
       if comment.draft:
         visible_for_user = comment.user_id == c.rhodecode_user.user_id
     %>
 
   % if inline:
       <% outdated_at_ver = comment.outdated_at_version(c.at_version_num) %>
   % else:
       <% outdated_at_ver = comment.older_than_version(c.at_version_num) %>
   % endif
 
   % if visible_for_user:
   <div class="comment
              ${'comment-inline' if inline else 'comment-general'}
              ${'comment-outdated' if outdated_at_ver else 'comment-current'}"
        id="comment-${comment.comment_id}"
        line="${comment.line_no}"
        data-comment-id="${comment.comment_id}"
        data-comment-type="${comment.comment_type}"
-       data-comment-draft=${h.json.dumps(comment.draft)}
+       data-comment-draft=${h.str_json(comment.draft)}
        data-comment-renderer="${comment.renderer}"
        data-comment-text="${comment.text | html_filters.base64,n}"
        data-comment-f-path="${comment.f_path}"
        data-comment-line-no="${comment.line_no}"
-       data-comment-inline=${h.json.dumps(inline)}
+       data-comment-inline=${h.str_json(inline)}
        style="${'display: none;' if outdated_at_ver else ''}">
 
       <div class="meta">
           <div class="comment-type-label">
                % if comment.draft:
                <div class="tooltip comment-draft" title="${_('Draft comments are only visible to the author until submitted')}.">
                 DRAFT
                </div>
                % elif is_new:
                <div class="tooltip comment-new" title="${_('This comment was added while you browsed this page')}.">
                 NEW
                </div>
                % endif
 
               <div class="comment-label ${comment.comment_type or 'note'}" id="comment-label-${comment.comment_id}">
 
               ## TODO COMMENT
               % if comment.comment_type == 'todo':
                   % if comment.resolved:
                       <div class="resolved tooltip" title="${_('Resolved by comment #{}').format(comment.resolved.comment_id)}">
                           <i class="icon-flag-filled"></i>
                           <a href="#comment-${comment.resolved.comment_id}">${comment.comment_type}</a>
                       </div>
                   % else:
                       <div class="resolved tooltip" style="display: none">
                           <span>${comment.comment_type}</span>
                       </div>
                       <div class="resolve tooltip" onclick="return Rhodecode.comments.createResolutionComment(${comment.comment_id});" title="${_('Click to create resolution comment.')}">
                         <i class="icon-flag-filled"></i>
                         ${comment.comment_type}
                       </div>
                   % endif
               ## NOTE COMMENT
               % else:
                   ## RESOLVED NOTE
                   % if comment.resolved_comment:
                     <div class="tooltip" title="${_('This comment resolves TODO #{}').format(comment.resolved_comment.comment_id)}">
                         fix
-                        <a href="#comment-${comment.resolved_comment.comment_id}" onclick="Rhodecode.comments.scrollToComment($('#comment-${comment.resolved_comment.comment_id}'), 0, ${h.json.dumps(comment.resolved_comment.outdated)})">
+                        <a href="#comment-${comment.resolved_comment.comment_id}" onclick="Rhodecode.comments.scrollToComment($('#comment-${comment.resolved_comment.comment_id}'), 0, ${h.str_json(comment.resolved_comment.outdated)})">
                             <span style="text-decoration: line-through">#${comment.resolved_comment.comment_id}</span>
                         </a>
                     </div>
                   ## STATUS CHANGE NOTE
                   % elif not comment.is_inline and comment.status_change:
                     <%
                         if comment.pull_request:
                             status_change_title = 'Status of review for pull request !{}'.format(comment.pull_request.pull_request_id)
                         else:
                             status_change_title = 'Status of review for commit {}'.format(h.short_id(comment.commit_id))
                     %>
 
                     <i class="icon-circle review-status-${comment.review_status}"></i>
                     <div class="changeset-status-lbl tooltip" title="${status_change_title}">
                          ${comment.review_status_lbl}
                     </div>
                   % else:
                     <div>
                         <i class="icon-comment"></i>
                         ${(comment.comment_type or 'note')}
                     </div>
                   % endif
               % endif
 
               </div>
           </div>
           ## NOTE 0 and .. => because we disable it for now until UI ready
           % if 0 and comment.status_change:
           <div class="pull-left">
               <span  class="tag authortag tooltip" title="${_('Status from pull request.')}">
                   <a href="${h.route_path('pullrequest_show',repo_name=comment.pull_request.target_repo.repo_name,pull_request_id=comment.pull_request.pull_request_id)}">
                       ${'!{}'.format(comment.pull_request.pull_request_id)}
                   </a>
               </span>
           </div>
           % endif
           ## Since only author can see drafts, we don't show it
           % if not comment.draft:
           <div class="author ${'author-inline' if inline else 'author-general'}">
               ${base.gravatar_with_user(comment.author.email, 16, tooltip=True)}
           </div>
           % endif
 
           <div class="date">
               ${h.age_component(comment.modified_at, time_is_local=True)}
           </div>
 
           % if comment.pull_request and comment.pull_request.author.user_id == comment.author.user_id:
             <span class="tag authortag tooltip" title="${_('Pull request author')}">
             ${_('author')}
             </span>
           % endif
 
           <%
           comment_version_selector = 'comment_versions_{}'.format(comment.comment_id)
           %>
 
           % if comment.history:
               <div class="date">
 
                  <input id="${comment_version_selector}" name="${comment_version_selector}"
                         type="hidden"
                         data-last-version="${comment.history[-1].version}">
 
                  <script type="text/javascript">
 
                     var preLoadVersionData = [
                        % for comment_history in comment.history:
                             {
                                 id: ${comment_history.comment_history_id},
                                 text: 'v${comment_history.version}',
                                 action: function () {
                                     Rhodecode.comments.showVersion(
                                         "${comment.comment_id}",
                                         "${comment_history.comment_history_id}"
                                     )
                                 },
                                 comment_version: "${comment_history.version}",
                                 comment_author_username: "${comment_history.author.username}",
                                 comment_author_gravatar: "${h.gravatar_url(comment_history.author.email, 16)}",
                                 comment_created_on: '${h.age_component(comment_history.created_on, time_is_local=True)}',
                             },
                        % endfor
                         ]
                     initVersionSelector("#${comment_version_selector}", {results: preLoadVersionData});
 
                   </script>
 
               </div>
           % else:
               <div class="date" style="display: none">
                 <input id="${comment_version_selector}" name="${comment_version_selector}"
                        type="hidden"
                        data-last-version="0">
               </div>
           %endif
 
           <div class="comment-links-block">
 
             % if inline:
                     <a class="pr-version-inline" href="${request.current_route_path(_query=dict(version=comment.pull_request_version_id), _anchor='comment-{}'.format(comment.comment_id))}">
                     % if outdated_at_ver:
                         <strong class="comment-outdated-label">outdated</strong> <code class="tooltip pr-version-num" title="${_('Outdated comment from pull request version v{0}, latest v{1}').format(comment_ver, latest_ver)}">${'v{}'.format(comment_ver)}</code>
                         <code class="action-divider">|</code>
                     % elif comment_ver:
                         <code class="tooltip pr-version-num" title="${_('Comment from pull request version v{0}, latest v{1}').format(comment_ver, latest_ver)}">${'v{}'.format(comment_ver)}</code>
                         <code class="action-divider">|</code>
                     % endif
                     </a>
             % else:
                 % if comment_ver:
 
                       % if comment.outdated:
                         <a class="pr-version"
                            href="?version=${comment.pull_request_version_id}#comment-${comment.comment_id}"
                         >
                             ${_('Outdated comment from pull request version v{0}, latest v{1}').format(comment_ver, latest_ver)}
                         </a>
                         <code class="action-divider">|</code>
                       % else:
                         <a class="tooltip pr-version"
                            title="${_('Comment from pull request version v{0}, latest v{1}').format(comment_ver, latest_ver)}"
                            href="${h.route_path('pullrequest_show',repo_name=comment.pull_request.target_repo.repo_name,pull_request_id=comment.pull_request.pull_request_id, version=comment.pull_request_version_id)}"
                         >
                             <code class="pr-version-num">${'v{}'.format(comment_ver)}</code>
                         </a>
                         <code class="action-divider">|</code>
                       % endif
 
                 % endif
             % endif
 
             <details class="details-reset details-inline-block">
               <summary class="noselect"><i class="icon-options cursor-pointer"></i></summary>
               <details-menu class="details-dropdown">
 
                 <div class="dropdown-item">
                     ${_('Comment')} #${comment.comment_id}
                     <span class="pull-right icon-clipboard clipboard-action" data-clipboard-text="${comment_model.get_url(comment,request, permalink=True, anchor='comment-{}'.format(comment.comment_id))}" title="${_('Copy permalink')}"></span>
                 </div>
 
                 ## show delete comment if it's not a PR (regular comments) or it's PR that is not closed
                 ## only super-admin, repo admin OR comment owner can delete, also hide delete if currently viewed comment is outdated
                 %if not outdated_at_ver and (not comment.pull_request or (comment.pull_request and not comment.pull_request.is_closed())):
                    ## permissions to delete
                    %if comment.immutable is False and (c.is_super_admin or h.HasRepoPermissionAny('repository.admin')(c.repo_name) or comment.author.user_id == c.rhodecode_user.user_id):
                        <div class="dropdown-divider"></div>
                        <div class="dropdown-item">
                         <a onclick="return Rhodecode.comments.editComment(this, '${comment.line_no}', '${comment.f_path}');" class="btn btn-link btn-sm edit-comment">${_('Edit')}</a>
                        </div>
                        <div class="dropdown-item">
                         <a onclick="return Rhodecode.comments.deleteComment(this);" class="btn btn-link btn-sm btn-danger delete-comment">${_('Delete')}</a>
                        </div>
                        ## Only available in EE edition
                        % if comment.draft and c.rhodecode_edition_id == 'EE':
                        <div class="dropdown-item">
                         <a onclick="return Rhodecode.comments.finalizeDrafts([${comment.comment_id}]);" class="btn btn-link btn-sm finalize-draft-comment">${_('Submit draft')}</a>
                        </div>
                        % endif
                    %else:
                       <div class="dropdown-divider"></div>
                       <div class="dropdown-item">
                         <a class="tooltip edit-comment link-disabled" disabled="disabled" title="${_('Action unavailable')}">${_('Edit')}</a>
                       </div>
                       <div class="dropdown-item">
                         <a class="tooltip edit-comment link-disabled" disabled="disabled" title="${_('Action unavailable')}">${_('Delete')}</a>
                       </div>
                    %endif
                 %else:
                     <div class="dropdown-divider"></div>
                     <div class="dropdown-item">
                       <a class="tooltip edit-comment link-disabled" disabled="disabled" title="${_('Action unavailable')}">${_('Edit')}</a>
                     </div>
                     <div class="dropdown-item">
                       <a class="tooltip edit-comment link-disabled" disabled="disabled" title="${_('Action unavailable')}">${_('Delete')}</a>
                     </div>
                 %endif
               </details-menu>
             </details>
 
             <code class="action-divider">|</code>
             % if outdated_at_ver:
                 <a onclick="return Rhodecode.comments.prevOutdatedComment(this);" class="tooltip prev-comment" title="${_('Jump to the previous outdated comment')}"> <i class="icon-angle-left"></i> </a>
                 <a onclick="return Rhodecode.comments.nextOutdatedComment(this);" class="tooltip next-comment" title="${_('Jump to the next outdated comment')}"> <i class="icon-angle-right"></i></a>
             % else:
                 <a onclick="return Rhodecode.comments.prevComment(this);" class="tooltip prev-comment" title="${_('Jump to the previous comment')}"> <i class="icon-angle-left"></i></a>
                 <a onclick="return Rhodecode.comments.nextComment(this);" class="tooltip next-comment" title="${_('Jump to the next comment')}"> <i class="icon-angle-right"></i></a>
             % endif
 
           </div>
       </div>
       <div class="text">
           ${h.render(comment.text, renderer=comment.renderer, mentions=True, repo_name=getattr(c, 'repo_name', None), active_pattern_entries=active_pattern_entries)}
       </div>
 
   </div>
   % endif
 </%def>
 
 ## generate main comments
 <%def name="generate_comments(comments, include_pull_request=False, is_pull_request=False)">
   <%
     active_pattern_entries = h.get_active_pattern_entries(getattr(c, 'repo_name', None))
   %>
 
   <div class="general-comments" id="comments">
     %for comment in comments:
         <div id="comment-tr-${comment.comment_id}">
           ## only render comments that are not from pull request, or from
           ## pull request and a status change
           %if not comment.pull_request or (comment.pull_request and comment.status_change) or include_pull_request:
           ${comment_block(comment, active_pattern_entries=active_pattern_entries)}
           %endif
         </div>
     %endfor
     ## to anchor ajax comments
     <div id="injected_page_comments"></div>
   </div>
 </%def>
 
 
 <%def name="comments(post_url, cur_status, is_pull_request=False, is_compare=False, change_status=True, form_extras=None)">
 
 <div class="comments">
     <%
       if is_pull_request:
         placeholder = _('Leave a comment on this Pull Request.')
       elif is_compare:
         placeholder = _('Leave a comment on {} commits in this range.').format(len(form_extras))
       else:
         placeholder = _('Leave a comment on this Commit.')
     %>
 
     % if c.rhodecode_user.username != h.DEFAULT_USER:
     <div class="js-template" id="cb-comment-general-form-template">
         ## template generated for injection
         ${comment_form(form_type='general', review_statuses=c.commit_statuses, form_extras=form_extras)}
     </div>
 
     <div id="cb-comment-general-form-placeholder" class="comment-form ac">
         ## inject form here
     </div>
     <script type="text/javascript">
         var resolvesCommentId = null;
         var generalCommentForm = Rhodecode.comments.createGeneralComment(
             'general', "${placeholder}", resolvesCommentId);
 
         // set custom success callback on rangeCommit
         % if is_compare:
             generalCommentForm.setHandleFormSubmit(function(o) {
                 var self = generalCommentForm;
 
                 var text = self.cm.getValue();
                 var status = self.getCommentStatus();
                 var commentType = self.getCommentType();
                 var isDraft = self.getDraftState();
 
                 if (text === "" && !status) {
                     return;
                 }
 
                 // we can pick which commits we want to make the comment by
                 // selecting them via click on preview pane, this will alter the hidden inputs
                 var cherryPicked = $('#changeset_compare_view_content .compare_select.hl').length > 0;
 
                 var commitIds = [];
                 $('#changeset_compare_view_content .compare_select').each(function(el) {
                     var commitId = this.id.replace('row-', '');
                     if ($(this).hasClass('hl') || !cherryPicked) {
                         $("input[data-commit-id='{0}']".format(commitId)).val(commitId);
                         commitIds.push(commitId);
                     } else {
                         $("input[data-commit-id='{0}']".format(commitId)).val('')
                     }
                 });
 
                 self.setActionButtonsDisabled(true);
                 self.cm.setOption("readOnly", true);
                 var postData = {
                     'text': text,
                     'changeset_status': status,
                     'comment_type': commentType,
                     'draft': isDraft,
                     'commit_ids': commitIds,
                     'csrf_token': CSRF_TOKEN
                 };
 
                 var submitSuccessCallback = function(o) {
                     location.reload(true);
                 };
                 var submitFailCallback = function(){
                     self.resetCommentFormState(text)
                 };
                 self.submitAjaxPOST(
                     self.submitUrl, postData, submitSuccessCallback, submitFailCallback);
             });
         % endif
 
     </script>
     % else:
     ## form state when not logged in
     <div class="comment-form ac">
 
         <div class="comment-area">
             <div class="comment-area-header">
                 <ul class="nav-links clearfix">
                     <li class="active">
                         <a class="disabled" href="#edit-btn" disabled="disabled" onclick="return false">${_('Write')}</a>
                     </li>
                     <li class="">
                         <a class="disabled" href="#preview-btn" disabled="disabled" onclick="return false">${_('Preview')}</a>
                     </li>
                 </ul>
             </div>
 
             <div class="comment-area-write" style="display: block;">
                 <div id="edit-container">
                     <div style="padding: 20px 0px 0px 0;">
                       ${_('You need to be logged in to leave comments.')}
                       <a href="${h.route_path('login', _query={'came_from': h.current_route_path(request)})}">${_('Login now')}</a>
                     </div>
                 </div>
                 <div id="preview-container" class="clearfix" style="display: none;">
                     <div id="preview-box" class="preview-box"></div>
                 </div>
             </div>
 
             <div class="comment-area-footer">
                 <div class="toolbar">
                     <div class="toolbar-text">
                     </div>
                 </div>
             </div>
         </div>
 
         <div class="comment-footer">
         </div>
 
     </div>
     % endif
 
     <script type="text/javascript">
         bindToggleButtons();
     </script>
 </div>
 </%def>
 
 
 <%def name="comment_form(form_type, form_id='', lineno_id='{1}', review_statuses=None, form_extras=None)">
 
   ## comment injected based on assumption that user is logged in
   <form ${('id="{}"'.format(form_id) if form_id else '') |n} action="#" method="GET">
 
     <div class="comment-area">
         <div class="comment-area-header">
             <div class="pull-left">
                 <ul class="nav-links clearfix">
                     <li class="active">
                         <a href="#edit-btn" tabindex="-1" id="edit-btn_${lineno_id}">${_('Write')}</a>
                     </li>
                     <li class="">
                         <a href="#preview-btn" tabindex="-1" id="preview-btn_${lineno_id}">${_('Preview')}</a>
                     </li>
                 </ul>
             </div>
             <div class="pull-right">
                 <span class="comment-area-text">${_('Mark as')}:</span>
                 <select class="comment-type" id="comment_type_${lineno_id}" name="comment_type">
                     % for val in c.visual.comment_types:
                         <option value="${val}">${val.upper()}</option>
                     % endfor
                 </select>
             </div>
         </div>
 
         <div class="comment-area-write" style="display: block;">
             <div id="edit-container_${lineno_id}" style="margin-top: -1px">
                 <textarea id="text_${lineno_id}" name="text" class="comment-block-ta ac-input"></textarea>
             </div>
             <div id="preview-container_${lineno_id}" class="clearfix" style="display: none;">
                 <div id="preview-box_${lineno_id}" class="preview-box"></div>
             </div>
         </div>
 
         <div class="comment-area-footer comment-attachment-uploader">
             <div class="toolbar">
 
                 <div class="comment-attachment-text">
                     <div class="dropzone-text">
                         ${_("Drag'n Drop files here or")} <span class="link pick-attachment">${_('Choose your files')}</span>.<br>
                     </div>
                     <div class="dropzone-upload" style="display:none">
                         <i class="icon-spin animate-spin"></i> ${_('uploading...')}
                     </div>
                 </div>
 
                 ## comments dropzone template, empty on purpose
                 <div style="display: none" class="comment-attachment-uploader-template">
                     <div class="dz-file-preview" style="margin: 0">
                         <div class="dz-error-message"></div>
                     </div>
                 </div>
 
             </div>
         </div>
     </div>
 
     <div class="comment-footer">
 
         ## inject extra inputs into the form
         % if form_extras and isinstance(form_extras, (list, tuple)):
             <div id="comment_form_extras">
                 % for form_ex_el in form_extras:
                     ${form_ex_el|n}
                 % endfor
             </div>
         % endif
 
         <div class="action-buttons">
             % if form_type != 'inline':
                 <div class="action-buttons-extra"></div>
             % endif
 
             <input class="btn btn-success comment-button-input submit-comment-action" id="save_${lineno_id}" name="save" type="submit" value="${_('Add comment')}" data-is-draft=false onclick="$(this).addClass('submitter')">
 
             % if form_type == 'inline':
                 % if c.rhodecode_edition_id == 'EE':
                     ## Disable the button for CE, the "real" validation is in the backend code anyway
                     <input class="btn btn-draft comment-button-input submit-draft-action" id="save_draft_${lineno_id}" name="save_draft" type="submit" value="${_('Add draft')}" data-is-draft=true onclick="$(this).addClass('submitter')">
                 % else:
                     <input class="btn btn-draft comment-button-input submit-draft-action disabled" disabled="disabled" type="submit"  value="${_('Add draft')}" onclick="return false;" title="Draft comments only available in EE edition of RhodeCode">
                 % endif
             % endif
 
             % if review_statuses:
             <div class="comment-status-box">
               <select id="change_status_${lineno_id}" name="changeset_status">
                   <option></option> ## Placeholder
                   % for status, lbl in review_statuses:
                   <option value="${status}" data-status="${status}">${lbl}</option>
                       %if is_pull_request and change_status and status in ('approved', 'rejected'):
                           <option value="${status}_closed" data-status="${status}">${lbl} & ${_('Closed')}</option>
                       %endif
                   % endfor
               </select>
             </div>
             % endif
 
             ## inline for has a file, and line-number together with cancel hide button.
             % if form_type == 'inline':
                 <input type="hidden" name="f_path" value="{0}">
                 <input type="hidden" name="line" value="${lineno_id}">
                 <span style="opacity: 0.7" class="cursor-pointer cb-comment-cancel" onclick="return Rhodecode.comments.cancelComment(this);">
                 ${_('dismiss')}
                 </span>
             % endif
         </div>
 
         <div class="toolbar-text">
             <% renderer_url = '<a href="%s">%s</a>' % (h.route_url('%s_help' % c.visual.default_renderer), c.visual.default_renderer.upper()) %>
             <span>${_('{} is supported.').format(renderer_url)|n}
 
             <i class="icon-info-circled tooltip-hovercard"
                data-hovercard-alt="ALT"
                data-hovercard-url="javascript:commentHelp('${c.visual.default_renderer.upper()}')"
-               data-comment-json-b64='${h.b64(h.json.dumps({}))}'></i>
+               data-comment-json-b64='${h.b64(h.str_json({}))}'></i>
             </span>
         </div>
     </div>
 
   </form>
 
 </%def>
  No newline at end of file

rhodecode/templates/codeblocks/diffs.mako

@@ -1,1404 +1,1404 b''
 <%namespace name="base" file="/base/base.mako"/>
 <%namespace name="commentblock" file="/changeset/changeset_file_comment.mako"/>
 
 <%def name="diff_line_anchor(commit, filename, line, type)"><%
 return '%s_%s_%i' % (h.md5_safe(commit+filename), type, line)
 %></%def>
 
 <%def name="action_class(action)">
 <%
     return {
         '-': 'cb-deletion',
         '+': 'cb-addition',
         ' ': 'cb-context',
         }.get(action, 'cb-empty')
 %>
 </%def>
 
 <%def name="op_class(op_id)">
 <%
     return {
         DEL_FILENODE: 'deletion', # file deleted
         BIN_FILENODE: 'warning' # binary diff hidden
     }.get(op_id, 'addition')
 %>
 </%def>
 
 
 
 <%def name="render_diffset(diffset, commit=None,
 
     # collapse all file diff entries when there are more than this amount of files in the diff
     collapse_when_files_over=20,
 
     # collapse lines in the diff when more than this amount of lines changed in the file diff
     lines_changed_limit=500,
 
     # add a ruler at to the output
     ruler_at_chars=0,
 
     # show inline comments
     use_comments=False,
 
     # disable new comments
     disable_new_comments=False,
 
     # special file-comments that were deleted in previous versions
     # it's used for showing outdated comments for deleted files in a PR
     deleted_files_comments=None,
 
     # for cache purpose
     inline_comments=None,
 
     # additional menu for PRs
     pull_request_menu=None,
 
     # show/hide todo next to comments
     show_todos=True,
 
 )">
 
 <%
     diffset_container_id = h.md5(diffset.target_ref)
     collapse_all = len(diffset.files) > collapse_when_files_over
     active_pattern_entries = h.get_active_pattern_entries(getattr(c, 'repo_name', None))
     from rhodecode.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
         MOD_FILENODE, RENAMED_FILENODE, CHMOD_FILENODE, BIN_FILENODE, COPIED_FILENODE
 %>
 
 %if use_comments:
 
 ## Template for injecting comments
 <div id="cb-comments-inline-container-template" class="js-template">
   ${inline_comments_container([])}
 </div>
 
 <div class="js-template" id="cb-comment-inline-form-template">
     <div class="comment-inline-form ac">
     %if not c.rhodecode_user.is_default:
         ## render template for inline comments
         ${commentblock.comment_form(form_type='inline')}
     %endif
     </div>
 </div>
 
 %endif
 
 %if c.user_session_attrs["diffmode"] == 'sideside':
 <style>
 .wrapper {
     max-width: 1600px !important;
 }
 </style>
 %endif
 
 %if ruler_at_chars:
 <style>
 .diff table.cb .cb-content:after {
     content: "";
     border-left: 1px solid blue;
     position: absolute;
     top: 0;
     height: 18px;
     opacity: .2;
     z-index: 10;
     //## +5 to account for diff action (+/-)
     left: ${ruler_at_chars + 5}ch;
 </style>
 %endif
 
 <div class="diffset ${disable_new_comments and 'diffset-comments-disabled'}">
 
     <div style="height: 20px; line-height: 20px">
         ## expand/collapse action
         <div class="pull-left">
             <a class="${'collapsed' if collapse_all else ''}" href="#expand-files" onclick="toggleExpand(this, '${diffset_container_id}'); return false">
             % if collapse_all:
                 <i class="icon-plus-squared-alt icon-no-margin"></i>${_('Expand all files')}
             % else:
                 <i class="icon-minus-squared-alt icon-no-margin"></i>${_('Collapse all files')}
             % endif
             </a>
 
         </div>
 
         ## todos
         % if show_todos and getattr(c, 'at_version', None):
         <div class="pull-right">
             <i class="icon-flag-filled" style="color: #949494">TODOs:</i>
              ${_('not available in this view')}
         </div>
         % elif show_todos:
         <div class="pull-right">
             <div class="comments-number" style="padding-left: 10px">
                 % if hasattr(c, 'unresolved_comments') and hasattr(c, 'resolved_comments'):
                     <i class="icon-flag-filled" style="color: #949494">TODOs:</i>
                     % if c.unresolved_comments:
                         <a href="#show-todos" onclick="$('#todo-box').toggle(); return false">
                             ${_('{} unresolved').format(len(c.unresolved_comments))}
                         </a>
                     % else:
                         ${_('0 unresolved')}
                     % endif
 
                     ${_('{} Resolved').format(len(c.resolved_comments))}
                 % endif
             </div>
         </div>
         % endif
 
 ##         ## comments
 ##         <div class="pull-right">
 ##             <div class="comments-number" style="padding-left: 10px">
 ##                 % if hasattr(c, 'comments') and hasattr(c, 'inline_cnt'):
 ##                     <i class="icon-comment" style="color: #949494">COMMENTS:</i>
 ##                     % if c.comments:
 ##                         <a href="#comments">${_ungettext("{} General", "{} General", len(c.comments)).format(len(c.comments))}</a>,
 ##                     % else:
 ##                         ${_('0 General')}
 ##                     % endif
 ##
 ##                     % if c.inline_cnt:
 ##                         <a href="#" onclick="return Rhodecode.comments.nextComment();"
 ##                            id="inline-comments-counter">${_ungettext("{} Inline", "{} Inline", c.inline_cnt).format(c.inline_cnt)}
 ##                         </a>
 ##                     % else:
 ##                         ${_('0 Inline')}
 ##                     % endif
 ##                 % endif
 ##
 ##                 % if pull_request_menu:
 ##                     <%
 ##                     outdated_comm_count_ver = pull_request_menu['outdated_comm_count_ver']
 ##                     %>
 ##
 ##                     % if outdated_comm_count_ver:
 ##                         <a href="#" onclick="showOutdated(); Rhodecode.comments.nextOutdatedComment(); return false;">
 ##                             (${_("{} Outdated").format(outdated_comm_count_ver)})
 ##                         </a>
 ##                         <a href="#" class="showOutdatedComments" onclick="showOutdated(this); return false;"> | ${_('show outdated')}</a>
 ##                         <a href="#" class="hideOutdatedComments" style="display: none" onclick="hideOutdated(this); return false;"> | ${_('hide outdated')}</a>
 ##                     % else:
 ##                         (${_("{} Outdated").format(outdated_comm_count_ver)})
 ##                     % endif
 ##
 ##                 % endif
 ##
 ##             </div>
 ##         </div>
 
     </div>
 
     % if diffset.limited_diff:
         <div class="diffset-heading ${(diffset.limited_diff and 'diffset-heading-warning' or '')}">
             <h2 class="clearinner">
                 ${_('The requested changes are too big and content was truncated.')}
                 <a href="${h.current_route_path(request, fulldiff=1)}" onclick="return confirm('${_("Showing a big diff might take some time and resources, continue?")}')">${_('Show full diff')}</a>
             </h2>
         </div>
     % endif
 
     <div id="todo-box">
         % if hasattr(c, 'unresolved_comments') and c.unresolved_comments:
             % for co in c.unresolved_comments:
                 <a class="permalink" href="#comment-${co.comment_id}"
                    onclick="Rhodecode.comments.scrollToComment($('#comment-${co.comment_id}'))">
                     <i class="icon-flag-filled-red"></i>
                     ${co.comment_id}</a>${('' if loop.last else ',')}
             % endfor
         % endif
     </div>
     %if diffset.has_hidden_changes:
         <p class="empty_data">${_('Some changes may be hidden')}</p>
     %elif not diffset.files:
         <p class="empty_data">${_('No files')}</p>
     %endif
 
     <div class="filediffs">
 
     ## initial value could be marked as False later on
     <% over_lines_changed_limit = False %>
     %for i, filediff in enumerate(diffset.files):
 
         %if filediff.source_file_path and filediff.target_file_path:
             %if filediff.source_file_path != filediff.target_file_path:
                  ## file was renamed, or copied
                 %if RENAMED_FILENODE in filediff.patch['stats']['ops']:
                     <%
                         final_file_name = h.literal(u'{} <i class="icon-angle-left"></i> <del>{}</del>'.format(filediff.target_file_path, filediff.source_file_path))
                         final_path = filediff.target_file_path
                     %>
                 %elif COPIED_FILENODE in filediff.patch['stats']['ops']:
                     <%
                         final_file_name = h.literal(u'{} <i class="icon-angle-left"></i> {}'.format(filediff.target_file_path, filediff.source_file_path))
                         final_path = filediff.target_file_path
                     %>
                 %endif
             %else:
                 ## file was modified
                 <%
                     final_file_name = filediff.source_file_path
                     final_path = final_file_name
                 %>
             %endif
         %else:
             %if filediff.source_file_path:
                 ## file was deleted
                 <%
                     final_file_name = filediff.source_file_path
                     final_path =  final_file_name
                 %>
             %else:
                 ## file was added
                 <%
                     final_file_name = filediff.target_file_path
                     final_path = final_file_name
                 %>
             %endif
         %endif
 
         <%
         lines_changed = filediff.patch['stats']['added'] + filediff.patch['stats']['deleted']
         over_lines_changed_limit = lines_changed > lines_changed_limit
         %>
         ## anchor with support of sticky header
         <div class="anchor" id="a_${h.FID(filediff.raw_id, filediff.patch['filename'])}"></div>
 
         <input ${(collapse_all and 'checked' or '')} class="filediff-collapse-state collapse-${diffset_container_id}" id="filediff-collapse-${id(filediff)}" type="checkbox" onchange="updateSticky();">
         <div
             class="filediff"
             data-f-path="${filediff.patch['filename']}"
             data-anchor-id="${h.FID(filediff.raw_id, filediff.patch['filename'])}"
         >
         <label for="filediff-collapse-${id(filediff)}" class="filediff-heading">
             <%
                 file_comments = (get_inline_comments(inline_comments, filediff.patch['filename']) or {}).values()
                 total_file_comments = [_c for _c in h.itertools.chain.from_iterable(file_comments) if not (_c.outdated or _c.draft)]
             %>
             <div class="filediff-collapse-indicator icon-"></div>
 
             ## Comments/Options PILL
             <span class="pill-group pull-right">
                 <span class="pill" op="comments">
                     <i class="icon-comment"></i> ${len(total_file_comments)}
                 </span>
 
                 <details class="details-reset details-inline-block">
                   <summary class="noselect">
                       <i class="pill icon-options cursor-pointer" op="options"></i>
                   </summary>
                   <details-menu class="details-dropdown">
 
                     <div class="dropdown-item">
                         <span>${final_path}</span>
                         <span class="pull-right icon-clipboard clipboard-action" data-clipboard-text="${final_path}" title="Copy file path"></span>
                     </div>
 
                     <div class="dropdown-divider"></div>
 
                    <div class="dropdown-item">
                         <% permalink = request.current_route_url(_anchor='a_{}'.format(h.FID(filediff.raw_id, filediff.patch['filename']))) %>
                         <a href="${permalink}">¶ permalink</a>
                         <span class="pull-right icon-clipboard clipboard-action" data-clipboard-text="${permalink}" title="Copy permalink"></span>
                    </div>
 
 
                   </details-menu>
                 </details>
 
             </span>
 
             ${diff_ops(final_file_name, filediff)}
 
         </label>
 
         ${diff_menu(filediff, use_comments=use_comments)}
         <table id="file-${h.safeid(h.safe_unicode(filediff.patch['filename']))}" data-f-path="${filediff.patch['filename']}" data-anchor-id="${h.FID(filediff.raw_id, filediff.patch['filename'])}" class="code-visible-block cb cb-diff-${c.user_session_attrs["diffmode"]} code-highlight ${(over_lines_changed_limit and 'cb-collapsed' or '')}">
 
         ## new/deleted/empty content case
         % if not filediff.hunks:
             ## Comment container, on "fakes" hunk that contains all data to render comments
             ${render_hunk_lines(filediff, c.user_session_attrs["diffmode"], filediff.hunk_ops, use_comments=use_comments, inline_comments=inline_comments, active_pattern_entries=active_pattern_entries)}
         % endif
 
         %if filediff.limited_diff:
                 <tr class="cb-warning cb-collapser">
                     <td class="cb-text" ${(c.user_session_attrs["diffmode"] == 'unified' and 'colspan=4' or 'colspan=6')}>
                         ${_('The requested commit or file is too big and content was truncated.')} <a href="${h.current_route_path(request, fulldiff=1)}" onclick="return confirm('${_("Showing a big diff might take some time and resources, continue?")}')">${_('Show full diff')}</a>
                     </td>
                 </tr>
         %else:
             %if over_lines_changed_limit:
                 <tr class="cb-warning cb-collapser">
                     <td class="cb-text" ${(c.user_session_attrs["diffmode"] == 'unified' and 'colspan=4' or 'colspan=6')}>
                         ${_('This diff has been collapsed as it changes many lines, (%i lines changed)' % lines_changed)}
                         <a href="#" class="cb-expand"
                            onclick="$(this).closest('table').removeClass('cb-collapsed'); updateSticky(); return false;">${_('Show them')}
                         </a>
                         <a href="#" class="cb-collapse"
                            onclick="$(this).closest('table').addClass('cb-collapsed'); updateSticky(); return false;">${_('Hide them')}
                         </a>
                     </td>
                 </tr>
             %endif
         %endif
 
         % for hunk in filediff.hunks:
             <tr class="cb-hunk">
                 <td ${(c.user_session_attrs["diffmode"] == 'unified' and 'colspan=3' or '')}>
                     ## TODO: dan: add ajax loading of more context here
                     ## <a href="#">
                         <i class="icon-more"></i>
                     ## </a>
                 </td>
                 <td ${(c.user_session_attrs["diffmode"] == 'sideside' and 'colspan=5' or '')}>
                     @@
                     -${hunk.source_start},${hunk.source_length}
                     +${hunk.target_start},${hunk.target_length}
                     ${hunk.section_header}
                 </td>
             </tr>
 
             ${render_hunk_lines(filediff, c.user_session_attrs["diffmode"], hunk, use_comments=use_comments, inline_comments=inline_comments, active_pattern_entries=active_pattern_entries)}
         % endfor
 
         <% unmatched_comments = (inline_comments or {}).get(filediff.patch['filename'], {}) %>
 
         ## outdated comments that do not fit into currently displayed lines
         % for lineno, comments in unmatched_comments.items():
 
             %if c.user_session_attrs["diffmode"] == 'unified':
                 % if loop.index == 0:
                 <tr class="cb-hunk">
                     <td colspan="3"></td>
                     <td>
                         <div>
                         ${_('Unmatched/outdated inline comments below')}
                         </div>
                     </td>
                 </tr>
                 % endif
                 <tr class="cb-line">
                     <td class="cb-data cb-context"></td>
                     <td class="cb-lineno cb-context"></td>
                     <td class="cb-lineno cb-context"></td>
                     <td class="cb-content cb-context">
                         ${inline_comments_container(comments, active_pattern_entries=active_pattern_entries)}
                     </td>
                 </tr>
             %elif c.user_session_attrs["diffmode"] == 'sideside':
                 % if loop.index == 0:
                 <tr class="cb-comment-info">
                     <td colspan="2"></td>
                     <td class="cb-line">
                         <div>
                         ${_('Unmatched/outdated inline comments below')}
                         </div>
                     </td>
                     <td colspan="2"></td>
                     <td class="cb-line">
                         <div>
                         ${_('Unmatched/outdated comments below')}
                         </div>
                     </td>
                 </tr>
                 % endif
                 <tr class="cb-line">
                     <td class="cb-data cb-context"></td>
                     <td class="cb-lineno cb-context"></td>
                     <td class="cb-content cb-context">
                         % if lineno.startswith('o'):
                             ${inline_comments_container(comments, active_pattern_entries=active_pattern_entries)}
                         % endif
                     </td>
 
                     <td class="cb-data cb-context"></td>
                     <td class="cb-lineno cb-context"></td>
                     <td class="cb-content cb-context">
                         % if lineno.startswith('n'):
                             ${inline_comments_container(comments, active_pattern_entries=active_pattern_entries)}
                         % endif
                     </td>
                 </tr>
             %endif
 
         % endfor
 
             </table>
         </div>
     %endfor
 
     ## outdated comments that are made for a file that has been deleted
     % for filename, comments_dict in (deleted_files_comments or {}).items():
 
         <%
             display_state = 'display: none'
             open_comments_in_file = [x for x in comments_dict['comments'] if x.outdated is False]
             if open_comments_in_file:
                 display_state = ''
             fid = str(id(filename))
         %>
         <div class="filediffs filediff-outdated" style="${display_state}">
             <input ${(collapse_all and 'checked' or '')} class="filediff-collapse-state collapse-${diffset_container_id}" id="filediff-collapse-${id(filename)}" type="checkbox" onchange="updateSticky();">
             <div class="filediff" data-f-path="${filename}"  id="a_${h.FID(fid, filename)}">
                 <label for="filediff-collapse-${id(filename)}" class="filediff-heading">
                     <div class="filediff-collapse-indicator icon-"></div>
 
                     <span class="pill">
                         ## file was deleted
                         ${filename}
                     </span>
                     <span class="pill-group pull-left" >
                         ## file op, doesn't need translation
                         <span class="pill" op="removed">unresolved comments</span>
                     </span>
                     <a class="pill filediff-anchor" href="#a_${h.FID(fid, filename)}">¶</a>
                     <span class="pill-group pull-right">
                         <span class="pill" op="deleted">
                             % if comments_dict['stats'] >0:
                                 -${comments_dict['stats']}
                             % else:
                                  ${comments_dict['stats']}
                             % endif
                         </span>
                     </span>
                 </label>
 
                 <table class="cb cb-diff-${c.user_session_attrs["diffmode"]} code-highlight ${(over_lines_changed_limit and 'cb-collapsed' or '')}">
                     <tr>
                         % if c.user_session_attrs["diffmode"] == 'unified':
                         <td></td>
                         %endif
 
                         <td></td>
                         <td class="cb-text cb-${op_class(BIN_FILENODE)}" ${(c.user_session_attrs["diffmode"] == 'unified' and 'colspan=4' or 'colspan=5')}>
                         <strong>${_('This file was removed from diff during updates to this pull-request.')}</strong><br/>
                         ${_('There are still outdated/unresolved comments attached to it.')}
                         </td>
                     </tr>
                     %if c.user_session_attrs["diffmode"] == 'unified':
                     <tr class="cb-line">
                         <td class="cb-data cb-context"></td>
                         <td class="cb-lineno cb-context"></td>
                         <td class="cb-lineno cb-context"></td>
                         <td class="cb-content cb-context">
                             ${inline_comments_container(comments_dict['comments'], active_pattern_entries=active_pattern_entries)}
                         </td>
                     </tr>
                     %elif c.user_session_attrs["diffmode"] == 'sideside':
                     <tr class="cb-line">
                         <td class="cb-data cb-context"></td>
                         <td class="cb-lineno cb-context"></td>
                         <td class="cb-content cb-context"></td>
 
                         <td class="cb-data cb-context"></td>
                         <td class="cb-lineno cb-context"></td>
                         <td class="cb-content cb-context">
                             ${inline_comments_container(comments_dict['comments'], active_pattern_entries=active_pattern_entries)}
                         </td>
                     </tr>
                     %endif
                 </table>
             </div>
         </div>
     % endfor
 
 </div>
 </div>
 </%def>
 
 <%def name="diff_ops(file_name, filediff)">
     <%
     from rhodecode.lib.diffs import NEW_FILENODE, DEL_FILENODE, \
         MOD_FILENODE, RENAMED_FILENODE, CHMOD_FILENODE, BIN_FILENODE, COPIED_FILENODE
     %>
     <span class="pill">
         <i class="icon-file-text"></i>
         ${file_name}
     </span>
 
     <span class="pill-group pull-right">
 
         ## ops pills
         %if filediff.limited_diff:
         <span class="pill tooltip" op="limited" title="The stats for this diff are not complete">limited diff</span>
         %endif
 
         %if NEW_FILENODE in filediff.patch['stats']['ops']:
         <span class="pill" op="created">created</span>
             %if filediff['target_mode'].startswith('120'):
         <span class="pill" op="symlink">symlink</span>
             %else:
         <span class="pill" op="mode">${nice_mode(filediff['target_mode'])}</span>
             %endif
         %endif
 
         %if RENAMED_FILENODE in filediff.patch['stats']['ops']:
         <span class="pill" op="renamed">renamed</span>
         %endif
 
         %if COPIED_FILENODE in filediff.patch['stats']['ops']:
         <span class="pill" op="copied">copied</span>
         %endif
 
         %if DEL_FILENODE in filediff.patch['stats']['ops']:
         <span class="pill" op="removed">removed</span>
         %endif
 
         %if CHMOD_FILENODE in filediff.patch['stats']['ops']:
         <span class="pill" op="mode">
             ${nice_mode(filediff['source_mode'])} ➡ ${nice_mode(filediff['target_mode'])}
         </span>
         %endif
 
         %if BIN_FILENODE in filediff.patch['stats']['ops']:
         <span class="pill" op="binary">binary</span>
             %if MOD_FILENODE in filediff.patch['stats']['ops']:
         <span class="pill" op="modified">modified</span>
             %endif
         %endif
 
         <span class="pill" op="added">${('+' if filediff.patch['stats']['added'] else '')}${filediff.patch['stats']['added']}</span>
         <span class="pill" op="deleted">${((h.safe_int(filediff.patch['stats']['deleted']) or 0) * -1)}</span>
 
     </span>
 
 </%def>
 
 <%def name="nice_mode(filemode)">
     ${(filemode.startswith('100') and filemode[3:] or filemode)}
 </%def>
 
 <%def name="diff_menu(filediff, use_comments=False)">
     <div class="filediff-menu">
 
     %if filediff.diffset.source_ref:
 
     ## FILE BEFORE CHANGES
     %if filediff.operation in ['D', 'M']:
         <a
             class="tooltip"
             href="${h.route_path('repo_files',repo_name=filediff.diffset.target_repo_name,commit_id=filediff.diffset.source_ref,f_path=filediff.source_file_path)}"
             title="${h.tooltip(_('Show file at commit: %(commit_id)s') % {'commit_id': filediff.diffset.source_ref[:12]})}"
         >
             ${_('Show file before')}
         </a> |
     %else:
         <span
             class="tooltip"
             title="${h.tooltip(_('File not present at commit: %(commit_id)s') % {'commit_id': filediff.diffset.source_ref[:12]})}"
         >
             ${_('Show file before')}
         </span> |
     %endif
 
     ## FILE AFTER CHANGES
     %if filediff.operation in ['A', 'M']:
         <a
             class="tooltip"
             href="${h.route_path('repo_files',repo_name=filediff.diffset.source_repo_name,commit_id=filediff.diffset.target_ref,f_path=filediff.target_file_path)}"
             title="${h.tooltip(_('Show file at commit: %(commit_id)s') % {'commit_id': filediff.diffset.target_ref[:12]})}"
         >
             ${_('Show file after')}
         </a>
     %else:
         <span
             class="tooltip"
             title="${h.tooltip(_('File not present at commit: %(commit_id)s') % {'commit_id': filediff.diffset.target_ref[:12]})}"
             >
             ${_('Show file after')}
         </span>
     %endif
 
     % if use_comments:
     |
     <a href="#" onclick="Rhodecode.comments.toggleDiffComments(this);return toggleElement(this)"
                 data-toggle-on="${_('Hide comments')}"
                 data-toggle-off="${_('Show comments')}">
         <span class="hide-comment-button">${_('Hide comments')}</span>
     </a>
     % endif
 
     %endif
 
     </div>
 </%def>
 
 
 <%def name="inline_comments_container(comments, active_pattern_entries=None, line_no='', f_path='')">
 
 <div class="inline-comments">
     %for comment in comments:
         ${commentblock.comment_block(comment, inline=True, active_pattern_entries=active_pattern_entries)}
     %endfor
 
     <%
         extra_class = ''
         extra_style = ''
 
         if comments and comments[-1].outdated_at_version(c.at_version_num):
             extra_class = ' comment-outdated'
             extra_style = 'display: none;'
 
     %>
 
     <div class="reply-thread-container-wrapper${extra_class}" style="${extra_style}">
         <div class="reply-thread-container${extra_class}">
             <div class="reply-thread-gravatar">
                 % if c.rhodecode_user.username != h.DEFAULT_USER:
                 ${base.gravatar(c.rhodecode_user.email, 20, tooltip=True, user=c.rhodecode_user)}
                 % endif
             </div>
 
             <div class="reply-thread-reply-button">
                 % if c.rhodecode_user.username != h.DEFAULT_USER:
                 ## initial reply button, some JS logic can append here a FORM to leave a first comment.
                 <button class="cb-comment-add-button" onclick="return Rhodecode.comments.createComment(this, '${f_path}', '${line_no}', null)">Reply...</button>
                 % endif
             </div>
             ##% endif
             <div class="reply-thread-last"></div>
         </div>
     </div>
 </div>
 
 </%def>
 
 <%!
 
 def get_inline_comments(comments, filename):
     if hasattr(filename, 'unicode_path'):
         filename = filename.unicode_path
 
     if not isinstance(filename, str):
         return None
 
     if comments and filename in comments:
         return comments[filename]
 
     return None
 
 def get_comments_for(diff_type, comments, filename, line_version, line_number):
     if hasattr(filename, 'unicode_path'):
         filename = filename.unicode_path
 
     if not isinstance(filename, str):
         return None
 
     file_comments = get_inline_comments(comments, filename)
     if file_comments is None:
         return None
 
     line_key = '{}{}'.format(line_version, line_number) ## e.g o37, n12
     if line_key in file_comments:
         data = file_comments.pop(line_key)
         return data
 %>
 
 <%def name="render_hunk_lines_sideside(filediff, hunk, use_comments=False, inline_comments=None, active_pattern_entries=None)">
 
     <% chunk_count = 1 %>
     %for loop_obj, item in h.looper(hunk.sideside):
     <%
     line = item
     i = loop_obj.index
     prev_line = loop_obj.previous
     old_line_anchor, new_line_anchor = None, None
 
     if line.original.lineno:
         old_line_anchor = diff_line_anchor(filediff.raw_id, hunk.source_file_path, line.original.lineno, 'o')
     if line.modified.lineno:
         new_line_anchor = diff_line_anchor(filediff.raw_id, hunk.target_file_path, line.modified.lineno, 'n')
 
     line_action = line.modified.action or line.original.action
     prev_line_action = prev_line and (prev_line.modified.action or prev_line.original.action)
     %>
 
     <tr class="cb-line">
         <td class="cb-data ${action_class(line.original.action)}"
             data-line-no="${line.original.lineno}"
             >
 
             <% line_old_comments, line_old_comments_no_drafts = None, None %>
             %if line.original.get_comment_args:
                 <%
                     line_old_comments = get_comments_for('side-by-side', inline_comments, *line.original.get_comment_args)
                     line_old_comments_no_drafts = [c for c in line_old_comments if not c.draft] if line_old_comments else []
                     has_outdated = any([x.outdated for x in line_old_comments_no_drafts])
                 %>
             %endif
             %if line_old_comments_no_drafts:
                 % if has_outdated:
                     <i class="tooltip toggle-comment-action icon-comment-toggle" title="${_('Comments including outdated: {}. Click here to toggle them.').format(len(line_old_comments_no_drafts))}" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                 % else:
                     <i class="tooltip toggle-comment-action icon-comment" title="${_('Comments: {}. Click to toggle them.').format(len(line_old_comments_no_drafts))}" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                 % endif
             %endif
         </td>
         <td class="cb-lineno ${action_class(line.original.action)}"
             data-line-no="${line.original.lineno}"
             %if old_line_anchor:
             id="${old_line_anchor}"
             %endif
         >
             %if line.original.lineno:
             <a name="${old_line_anchor}" href="#${old_line_anchor}">${line.original.lineno}</a>
             %endif
         </td>
 
         <% line_no = 'o{}'.format(line.original.lineno) %>
         <td class="cb-content ${action_class(line.original.action)}"
             data-line-no="${line_no}"
             >
             %if use_comments and line.original.lineno:
             ${render_add_comment_button(line_no=line_no, f_path=filediff.patch['filename'])}
             %endif
             <span class="cb-code"><span class="cb-action ${action_class(line.original.action)}"></span>${line.original.content or '' | n}</span>
 
             %if use_comments and line.original.lineno and line_old_comments:
                 ${inline_comments_container(line_old_comments, active_pattern_entries=active_pattern_entries, line_no=line_no, f_path=filediff.patch['filename'])}
             %endif
 
         </td>
         <td class="cb-data ${action_class(line.modified.action)}"
             data-line-no="${line.modified.lineno}"
             >
             <div>
 
             <% line_new_comments, line_new_comments_no_drafts = None, None %>
             %if line.modified.get_comment_args:
                 <%
                     line_new_comments = get_comments_for('side-by-side', inline_comments, *line.modified.get_comment_args)
                     line_new_comments_no_drafts = [c for c in line_new_comments if not c.draft] if line_new_comments else []
                     has_outdated = any([x.outdated for x in line_new_comments_no_drafts])
                 %>
             %endif
 
             %if line_new_comments_no_drafts:
                 % if has_outdated:
                     <i class="tooltip toggle-comment-action icon-comment-toggle" title="${_('Comments including outdated: {}. Click here to toggle them.').format(len(line_new_comments_no_drafts))}" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                 % else:
                     <i class="tooltip toggle-comment-action icon-comment" title="${_('Comments: {}. Click to toggle them.').format(len(line_new_comments_no_drafts))}" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                 % endif
             %endif
             </div>
         </td>
         <td class="cb-lineno ${action_class(line.modified.action)}"
             data-line-no="${line.modified.lineno}"
             %if new_line_anchor:
             id="${new_line_anchor}"
             %endif
             >
             %if line.modified.lineno:
                 <a name="${new_line_anchor}" href="#${new_line_anchor}">${line.modified.lineno}</a>
             %endif
         </td>
 
         <% line_no = 'n{}'.format(line.modified.lineno) %>
         <td class="cb-content ${action_class(line.modified.action)}"
             data-line-no="${line_no}"
             >
             %if use_comments and line.modified.lineno:
             ${render_add_comment_button(line_no=line_no, f_path=filediff.patch['filename'])}
             %endif
             <span class="cb-code"><span class="cb-action ${action_class(line.modified.action)}"></span>${line.modified.content or '' | n}</span>
                 % if line_action in ['+', '-'] and prev_line_action not in ['+', '-']:
                 <div class="nav-chunk" style="visibility: hidden">
                     <i class="icon-eye" title="viewing diff hunk-${hunk.index}-${chunk_count}"></i>
                 </div>
                 <% chunk_count +=1 %>
                 % endif
             %if use_comments and line.modified.lineno and line_new_comments:
             ${inline_comments_container(line_new_comments, active_pattern_entries=active_pattern_entries, line_no=line_no, f_path=filediff.patch['filename'])}
             %endif
 
         </td>
     </tr>
     %endfor
 </%def>
 
 
 <%def name="render_hunk_lines_unified(filediff, hunk, use_comments=False,  inline_comments=None, active_pattern_entries=None)">
     %for old_line_no, new_line_no, action, content, comments_args in hunk.unified:
 
     <%
     old_line_anchor, new_line_anchor = None, None
     if old_line_no:
         old_line_anchor = diff_line_anchor(filediff.raw_id, hunk.source_file_path, old_line_no, 'o')
     if new_line_no:
         new_line_anchor = diff_line_anchor(filediff.raw_id, hunk.target_file_path, new_line_no, 'n')
     %>
     <tr class="cb-line">
         <td class="cb-data ${action_class(action)}">
             <div>
 
                 <% comments, comments_no_drafts = None, None %>
                 %if comments_args:
                     <%
                         comments = get_comments_for('unified', inline_comments, *comments_args)
                         comments_no_drafts = [c for c in line_new_comments if not c.draft] if line_new_comments else []
                         has_outdated = any([x.outdated for x in comments_no_drafts])
                     %>
                 %endif
 
                 % if comments_no_drafts:
                     % if has_outdated:
                         <i class="tooltip toggle-comment-action icon-comment-toggle" title="${_('Comments including outdated: {}. Click here to toggle them.').format(len(comments_no_drafts))}" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                     % else:
                         <i class="tooltip toggle-comment-action icon-comment" title="${_('Comments: {}. Click to toggle them.').format(len(comments_no_drafts))}" onclick="return Rhodecode.comments.toggleLineComments(this)"></i>
                     % endif
                 % endif
             </div>
         </td>
         <td class="cb-lineno ${action_class(action)}"
             data-line-no="${old_line_no}"
             %if old_line_anchor:
             id="${old_line_anchor}"
             %endif
         >
             %if old_line_anchor:
             <a name="${old_line_anchor}" href="#${old_line_anchor}">${old_line_no}</a>
             %endif
         </td>
         <td class="cb-lineno ${action_class(action)}"
             data-line-no="${new_line_no}"
             %if new_line_anchor:
             id="${new_line_anchor}"
             %endif
         >
             %if new_line_anchor:
             <a name="${new_line_anchor}" href="#${new_line_anchor}">${new_line_no}</a>
             %endif
         </td>
         <% line_no = '{}{}'.format(new_line_no and 'n' or 'o', new_line_no or old_line_no) %>
         <td class="cb-content ${action_class(action)}"
             data-line-no="${line_no}"
             >
             %if use_comments:
             ${render_add_comment_button(line_no=line_no, f_path=filediff.patch['filename'])}
             %endif
             <span class="cb-code"><span class="cb-action ${action_class(action)}"></span> ${content or '' | n}</span>
             %if use_comments and comments:
             ${inline_comments_container(comments, active_pattern_entries=active_pattern_entries, line_no=line_no, f_path=filediff.patch['filename'])}
             %endif
         </td>
     </tr>
     %endfor
 </%def>
 
 
 <%def name="render_hunk_lines(filediff, diff_mode, hunk, use_comments, inline_comments, active_pattern_entries)">
     % if diff_mode == 'unified':
         ${render_hunk_lines_unified(filediff, hunk, use_comments=use_comments, inline_comments=inline_comments, active_pattern_entries=active_pattern_entries)}
     % elif diff_mode == 'sideside':
         ${render_hunk_lines_sideside(filediff, hunk, use_comments=use_comments, inline_comments=inline_comments, active_pattern_entries=active_pattern_entries)}
     % else:
         <tr class="cb-line">
             <td>unknown diff mode</td>
         </tr>
     % endif
 </%def>file changes
 
 
 <%def name="render_add_comment_button(line_no='', f_path='')">
 % if not c.rhodecode_user.is_default:
 <button class="btn btn-small btn-primary cb-comment-box-opener" onclick="return Rhodecode.comments.createComment(this, '${f_path}', '${line_no}', null)">
     <span><i class="icon-comment"></i></span>
 </button>
 % endif
 </%def>
 
 <%def name="render_diffset_menu(diffset, range_diff_on=None, commit=None, pull_request_menu=None)">
     <% diffset_container_id = h.md5(diffset.target_ref) %>
 
     <div id="diff-file-sticky" class="diffset-menu clearinner">
         ## auto adjustable
         <div class="sidebar__inner">
             <div class="sidebar__bar">
             <div class="pull-right">
 
                 <div class="btn-group" style="margin-right: 5px;">
                     <a class="tooltip btn" onclick="scrollDown();return false" title="${_('Scroll to page bottom')}">
                         <i class="icon-arrow_down"></i>
                     </a>
                     <a class="tooltip btn" onclick="scrollUp();return false" title="${_('Scroll to page top')}">
                         <i class="icon-arrow_up"></i>
                     </a>
                 </div>
 
                 <div class="btn-group">
                     <a class="btn tooltip toggle-wide-diff" href="#toggle-wide-diff" onclick="toggleWideDiff(this); return false" title="${h.tooltip(_('Toggle wide diff'))}">
                     <i class="icon-wide-mode"></i>
                     </a>
                 </div>
                 <div class="btn-group">
 
                 <a
                   class="btn ${(c.user_session_attrs["diffmode"] == 'sideside' and 'btn-active')} tooltip"
                   title="${h.tooltip(_('View diff as side by side'))}"
                   href="${h.current_route_path(request, diffmode='sideside')}">
                     <span>${_('Side by Side')}</span>
                 </a>
 
                 <a
                   class="btn ${(c.user_session_attrs["diffmode"] == 'unified' and 'btn-active')} tooltip"
                   title="${h.tooltip(_('View diff as unified'))}" href="${h.current_route_path(request, diffmode='unified')}">
                     <span>${_('Unified')}</span>
                 </a>
 
                 % if range_diff_on is True:
                     <a
                       title="${_('Turn off: Show the diff as commit range')}"
                       class="btn btn-primary"
                       href="${h.current_route_path(request, **{"range-diff":"0"})}">
                         <span>${_('Range Diff')}</span>
                    </a>
                 % elif range_diff_on is False:
                     <a
                       title="${_('Show the diff as commit range')}"
                       class="btn"
                       href="${h.current_route_path(request, **{"range-diff":"1"})}">
                         <span>${_('Range Diff')}</span>
                    </a>
                 % endif
             </div>
                 <div class="btn-group">
 
                 <details class="details-reset details-inline-block">
                   <summary class="noselect btn">
                         <i class="icon-options cursor-pointer" op="options"></i>
                   </summary>
 
                   <div>
                   <details-menu class="details-dropdown" style="top: 35px;">
 
                     <div class="dropdown-item">
                         <div style="padding: 2px 0px">
                         % if request.GET.get('ignorews', '') == '1':
                             <a href="${h.current_route_path(request, ignorews=0)}">${_('Show whitespace changes')}</a>
                         % else:
                             <a href="${h.current_route_path(request, ignorews=1)}">${_('Hide whitespace changes')}</a>
                         % endif
                         </div>
                     </div>
 
                    <div class="dropdown-item">
                        <div style="padding: 2px 0px">
                         % if request.GET.get('fullcontext', '') == '1':
                             <a href="${h.current_route_path(request, fullcontext=0)}">${_('Hide full context diff')}</a>
                         % else:
                             <a href="${h.current_route_path(request, fullcontext=1)}">${_('Show full context diff')}</a>
                         % endif
                         </div>
                    </div>
 
                   </details-menu>
                   </div>
                 </details>
 
                 </div>
         </div>
             <div class="pull-left">
             <div class="btn-group">
               <div class="pull-left">
                 ${h.hidden('file_filter_{}'.format(diffset_container_id))}
               </div>
 
             </div>
             </div>
         </div>
         <div class="fpath-placeholder pull-left">
             <i class="icon-file-text"></i>
             <strong class="fpath-placeholder-text">
             Context file:
             </strong>
         </div>
         <div class="pull-right noselect">
             %if commit:
                 <span>
                     <code>${h.show_id(commit)}</code>
                 </span>
             %elif pull_request_menu and pull_request_menu.get('pull_request'):
                 <span>
                     <code>!${pull_request_menu['pull_request'].pull_request_id}</code>
                 </span>
             %endif
             % if commit or pull_request_menu:
                 <span class="tooltip" title="Navigate to previous or next change inside files." id="diff_nav">Loading diff...:</span>
                 <span class="cursor-pointer" onclick="scrollToPrevChunk(); return false">
                     <i class="icon-angle-up"></i>
                 </span>
                 <span class="cursor-pointer" onclick="scrollToNextChunk(); return false">
                     <i class="icon-angle-down"></i>
                 </span>
             % endif
         </div>
         <div class="sidebar_inner_shadow"></div>
         </div>
     </div>
 
     % if diffset:
         %if diffset.limited_diff:
             <% file_placeholder = _ungettext('%(num)s file changed', '%(num)s files changed', diffset.changed_files) % {'num': diffset.changed_files} %>
         %else:
             <% file_placeholder = h.literal(_ungettext('%(num)s file changed: <span class="op-added">%(linesadd)s inserted</span>, <span class="op-deleted">%(linesdel)s deleted</span>', '%(num)s files changed: <span class="op-added">%(linesadd)s inserted</span>, <span class="op-deleted">%(linesdel)s deleted</span>',
             diffset.changed_files) % {'num': diffset.changed_files, 'linesadd': diffset.lines_added, 'linesdel': diffset.lines_deleted}) %>
 
         %endif
         ## case on range-diff placeholder needs to be updated
         % if range_diff_on is True:
             <% file_placeholder = _('Disabled on range diff') %>
         % endif
 
         <script type="text/javascript">
         var feedFilesOptions = function (query, initialData) {
             var data = {results: []};
             var isQuery = typeof query.term !== 'undefined';
 
             var section = _gettext('Changed files');
             var filteredData = [];
 
             //filter results
             $.each(initialData.results, function (idx, value) {
 
                 if (!isQuery || query.term.length === 0 || value.text.toUpperCase().indexOf(query.term.toUpperCase()) >= 0) {
                     filteredData.push({
                         'id': this.id,
                         'text': this.text,
                         "ops": this.ops,
                     })
                 }
 
             });
 
             data.results = filteredData;
 
             query.callback(data);
         };
 
         var selectionFormatter = function(data, escapeMarkup) {
             var container = '<div class="filelist" style="padding-right:100px">{0}</div>';
             var tmpl = '<div><strong>{0}</strong></div>'.format(escapeMarkup(data['text']));
             var pill = '<div class="pill-group" style="position: absolute; top:7px; right: 0">' +
                         '<span class="pill" op="added">{0}</span>' +
                         '<span class="pill" op="deleted">{1}</span>' +
                        '</div>'
                     ;
             var added = data['ops']['added'];
             if (added === 0) {
                 // don't show +0
                 added = 0;
             } else {
                 added = '+' + added;
             }
 
             var deleted = -1*data['ops']['deleted'];
 
             tmpl += pill.format(added, deleted);
             return container.format(tmpl);
         };
         var formatFileResult = function(result, container, query, escapeMarkup) {
             return selectionFormatter(result, escapeMarkup);
         };
 
         var formatSelection = function (data, container) {
             return '${file_placeholder}'
         };
 
         if (window.preloadFileFilterData === undefined) {
             window.preloadFileFilterData = {}
         }
 
         preloadFileFilterData["${diffset_container_id}"] = {
             results: [
                 % for filediff in diffset.files:
                     {id:"a_${h.FID(filediff.raw_id, filediff.patch['filename'])}",
                      text:"${filediff.patch['filename']}",
-                     ops:${h.json.dumps(filediff.patch['stats'])|n}}${('' if loop.last else ',')}
+                     ops:${h.str_json(filediff.patch['stats'])|n}}${('' if loop.last else ',')}
                 % endfor
             ]
         };
 
         var diffFileFilterId = "#file_filter_" + "${diffset_container_id}";
         var diffFileFilter = $(diffFileFilterId).select2({
             'dropdownAutoWidth': true,
             'width': 'auto',
 
             containerCssClass: "drop-menu",
             dropdownCssClass: "drop-menu-dropdown",
             data: preloadFileFilterData["${diffset_container_id}"],
             query: function(query) {
                 feedFilesOptions(query, preloadFileFilterData["${diffset_container_id}"]);
             },
             initSelection: function(element, callback) {
               callback({'init': true});
             },
             formatResult: formatFileResult,
             formatSelection: formatSelection
         });
 
         % if range_diff_on is True:
             diffFileFilter.select2("enable", false);
         % endif
 
         $(diffFileFilterId).on('select2-selecting', function (e) {
             var idSelector = e.choice.id;
 
             // expand the container if we quick-select the field
             $('#'+idSelector).next().prop('checked', false);
             // hide the mast as we later do preventDefault()
             $("#select2-drop-mask").click();
 
             window.location.hash = '#'+idSelector;
             updateSticky();
 
             e.preventDefault();
         });
 
         diffNavText = 'diff navigation:'
 
         getCurrentChunk = function () {
 
             var chunksAll = $('.nav-chunk').filter(function () {
                 return $(this).parents('.filediff').prev().get(0).checked !== true
             })
             var chunkSelected = $('.nav-chunk.selected');
             var initial = false;
 
             if (chunkSelected.length === 0) {
                 // no initial chunk selected, we pick first
                 chunkSelected = $(chunksAll.get(0));
                 var initial = true;
             }
 
             return {
                 'all': chunksAll,
                 'selected': chunkSelected,
                 'initial': initial,
             }
         }
 
         animateDiffNavText = function () {
             var $diffNav = $('#diff_nav')
 
             var callback = function () {
                 $diffNav.animate({'opacity': 1.00}, 200)
             };
             $diffNav.animate({'opacity': 0.15}, 200, callback);
         }
 
         scrollToChunk = function (moveBy) {
             var chunk = getCurrentChunk();
             var all = chunk.all
             var selected = chunk.selected
 
             var curPos = all.index(selected);
             var newPos = curPos;
             if (!chunk.initial) {
                 var newPos = curPos + moveBy;
             }
 
             var curElem = all.get(newPos);
 
             if (curElem === undefined) {
                 // end or back
                 $('#diff_nav').html('no next diff element:')
                 animateDiffNavText()
                 return
             } else if (newPos < 0) {
                 $('#diff_nav').html('no previous diff element:')
                 animateDiffNavText()
                 return
             } else {
                 $('#diff_nav').html(diffNavText)
             }
 
             curElem = $(curElem)
             var offset = 100;
             $(window).scrollTop(curElem.position().top - offset);
 
             //clear selection
             all.removeClass('selected')
             curElem.addClass('selected')
         }
 
         scrollToPrevChunk = function () {
             scrollToChunk(-1)
         }
         scrollToNextChunk = function () {
             scrollToChunk(1)
         }
 
         </script>
     % endif
 
     <script type="text/javascript">
         $('#diff_nav').html('loading diff...') // wait until whole page is loaded
 
         $(document).ready(function () {
 
             var contextPrefix = _gettext('Context file: ');
             ## sticky sidebar
             var sidebarElement = document.getElementById('diff-file-sticky');
             sidebar = new StickySidebar(sidebarElement, {
                   topSpacing: 0,
                   bottomSpacing: 0,
                   innerWrapperSelector: '.sidebar__inner'
             });
             sidebarElement.addEventListener('affixed.static.stickySidebar', function () {
                 // reset our file so it's not holding new value
                 $('.fpath-placeholder-text').html(contextPrefix + ' - ')
             });
 
             updateSticky = function () {
                 sidebar.updateSticky();
                 Waypoint.refreshAll();
             };
 
             var animateText = function (fPath, anchorId) {
                 fPath = Select2.util.escapeMarkup(fPath);
                 $('.fpath-placeholder-text').html(contextPrefix + '<a href="#a_' + anchorId + '">' + fPath + '</a>')
             };
 
             ## dynamic file waypoints
             var setFPathInfo = function(fPath, anchorId){
                 animateText(fPath, anchorId)
             };
 
             var codeBlock = $('.filediff');
 
             // forward waypoint
             codeBlock.waypoint(
                 function(direction) {
                     if (direction === "down"){
                         setFPathInfo($(this.element).data('fPath'), $(this.element).data('anchorId'))
                     }
                 }, {
                     offset: function () {
                         return 70;
                     },
                     context: '.fpath-placeholder'
                 }
             );
 
             // backward waypoint
             codeBlock.waypoint(
                 function(direction) {
                     if (direction === "up"){
                         setFPathInfo($(this.element).data('fPath'), $(this.element).data('anchorId'))
                     }
                 }, {
                     offset: function () {
                         return -this.element.clientHeight + 90;
                     },
                     context: '.fpath-placeholder'
                 }
             );
 
             toggleWideDiff = function (el) {
                 updateSticky();
                 var wide = Rhodecode.comments.toggleWideMode(this);
                 storeUserSessionAttr('rc_user_session_attr.wide_diff_mode', wide);
                 if (wide === true) {
                     $(el).addClass('btn-active');
                 } else {
                     $(el).removeClass('btn-active');
                 }
                 return null;
             };
 
             toggleExpand = function (el, diffsetEl) {
                 var el = $(el);
                 if (el.hasClass('collapsed')) {
                     $('.filediff-collapse-state.collapse-{0}'.format(diffsetEl)).prop('checked', false);
                     el.removeClass('collapsed');
                     el.html(
                         '<i class="icon-minus-squared-alt icon-no-margin"></i>' +
                         _gettext('Collapse all files'));
                 }
                 else {
                     $('.filediff-collapse-state.collapse-{0}'.format(diffsetEl)).prop('checked', true);
                     el.addClass('collapsed');
                     el.html(
                         '<i class="icon-plus-squared-alt icon-no-margin"></i>' +
                         _gettext('Expand all files'));
                 }
                 updateSticky()
             };
 
             toggleCommitExpand = function (el) {
                 var $el = $(el);
                 var commits = $el.data('toggleCommitsCnt');
                 var collapseMsg = _ngettext('Collapse {0} commit', 'Collapse {0} commits', commits).format(commits);
                 var expandMsg = _ngettext('Expand {0} commit', 'Expand {0} commits', commits).format(commits);
 
                 if ($el.hasClass('collapsed')) {
                     $('.compare_select').show();
                     $('.compare_select_hidden').hide();
 
                     $el.removeClass('collapsed');
                     $el.html(
                         '<i class="icon-minus-squared-alt icon-no-margin"></i>' +
                        collapseMsg);
                 }
                 else {
                     $('.compare_select').hide();
                     $('.compare_select_hidden').show();
                     $el.addClass('collapsed');
                     $el.html(
                         '<i class="icon-plus-squared-alt icon-no-margin"></i>' +
                         expandMsg);
                 }
                 updateSticky();
             };
 
             // get stored diff mode and pre-enable it
             if (templateContext.session_attrs.wide_diff_mode === "true") {
                 Rhodecode.comments.toggleWideMode(null);
                 $('.toggle-wide-diff').addClass('btn-active');
                 updateSticky();
             }
 
             // DIFF NAV //
 
             // element to detect scroll direction of
             var $window = $(window);
 
             // initialize last scroll position
             var lastScrollY = $window.scrollTop();
 
             $window.on('resize scrollstop', {latency: 350}, function () {
                 var visibleChunks = $('.nav-chunk').withinviewport({top: 75});
 
                 // get current scroll position
                 var currentScrollY = $window.scrollTop();
 
                 // determine current scroll direction
                 if (currentScrollY > lastScrollY) {
                     var y = 'down'
                 } else if (currentScrollY !== lastScrollY) {
                     var y = 'up';
                 }
 
                 var pos = -1; // by default we use last element in viewport
                 if (y === 'down') {
                     pos = -1;
                 } else if (y === 'up') {
                     pos = 0;
                 }
 
                 if (visibleChunks.length > 0) {
                     $('.nav-chunk').removeClass('selected');
                     $(visibleChunks.get(pos)).addClass('selected');
                 }
 
                 // update last scroll position to current position
                 lastScrollY = currentScrollY;
 
             });
             $('#diff_nav').html(diffNavText);
 
         });
     </script>
 
 </%def>

rhodecode/templates/pullrequests/pullrequest_merge_checks.mako

@@ -1,82 +1,82 b''
 
 <div class="pull-request-wrap">
 
     % if c.pr_merge_possible:
         <h2 class="merge-status">
             <span class="merge-icon success"><i class="icon-ok"></i></span>
             ${_('This pull request can be merged automatically.')}
         </h2>
     % else:
         <h2 class="merge-status">
             <span class="merge-icon warning"><i class="icon-false"></i></span>
             ${_('Merge is not currently possible because of below failed checks.')}
         </h2>
     % endif
 
     % if c.pr_merge_errors.items():
     <ul>
         % for pr_check_key, pr_check_details in c.pr_merge_errors.items():
             <% pr_check_type = pr_check_details['error_type'] %>
             <li>
                 <div class="merge-message ${pr_check_type}" data-role="merge-message">
                     <span style="white-space: pre-line">- ${pr_check_details['message']}</span>
                     % if pr_check_key == 'todo':
                         % for co in pr_check_details['details']:
-                            <a class="permalink" href="#comment-${co.comment_id}" onclick="Rhodecode.comments.scrollToComment($('#comment-${co.comment_id}'), 0, ${h.json.dumps(co.outdated)})"> #${co.comment_id}</a>${'' if loop.last else ','}
+                            <a class="permalink" href="#comment-${co.comment_id}" onclick="Rhodecode.comments.scrollToComment($('#comment-${co.comment_id}'), 0, ${h.str_json(co.outdated)})"> #${co.comment_id}</a>${'' if loop.last else ','}
                         % endfor
                     % endif
                 </div>
             </li>
         % endfor
     </ul>
     % endif
 
     <div class="pull-request-merge-actions">
         % if c.allowed_to_merge:
           ## Merge info, show only if all errors are taken care of
           % if not c.pr_merge_errors and c.pr_merge_info:
               <div class="pull-request-merge-info">
                   <ul>
                     % for pr_merge_key, pr_merge_details in c.pr_merge_info.items():
                     <li>
                         - ${pr_merge_details['message']}
                     </li>
                     % endfor
                   </ul>
               </div>
           % endif
 
         <div>
           ${h.secure_form(h.route_path('pullrequest_merge', repo_name=c.repo_name, pull_request_id=c.pull_request.pull_request_id), id='merge_pull_request_form', request=request)}
           <% merge_disabled = ' disabled' if c.pr_merge_possible is False else '' %>
 
           % if c.allowed_to_close:
             ## close PR action, injected later next to COMMENT button
             % if c.pull_request_review_status == c.REVIEW_STATUS_APPROVED:
             <a id="close-pull-request-action" class="btn btn-approved-status" href="#close-as-approved" onclick="closePullRequest('${c.REVIEW_STATUS_APPROVED}'); return false;">
                 ${_('Close with status {}').format(h.commit_status_lbl(c.REVIEW_STATUS_APPROVED))}
             </a>
             % else:
             <a id="close-pull-request-action" class="btn btn-rejected-status" href="#close-as-rejected" onclick="closePullRequest('${c.REVIEW_STATUS_REJECTED}'); return false;">
                 ${_('Close with status {}').format(h.commit_status_lbl(c.REVIEW_STATUS_REJECTED))}
             </a>
             % endif
           % endif
 
           <input type="submit" id="merge_pull_request" value="${_('Merge and close Pull Request')}" class="btn${merge_disabled}"${merge_disabled}>
           ${h.end_form()}
 
           <div class="pull-request-merge-refresh">
               <a href="#refreshChecks" onclick="refreshMergeChecks(); return false;">${_('refresh checks')}</a>
           </div>
 
         </div>
         % elif c.rhodecode_user.username != h.DEFAULT_USER:
             <a class="btn" href="#" onclick="refreshMergeChecks(); return false;">${_('refresh checks')}</a>
             <input type="submit" value="${_('Merge and close Pull Request')}" class="btn disabled" disabled="disabled" title="${_('You are not allowed to merge this pull request.')}">
         % else:
           <input type="submit" value="${_('Login to Merge this Pull Request')}" class="btn disabled" disabled="disabled">
         % endif
     </div>
 
 </div>

rhodecode/tests/lib/middleware/test_simplegit.py

@@ -1,140 +1,139 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
 import urllib.parse
 import mock
-import simplejson as json
 
 from rhodecode.lib.vcs.backends.base import Config
 from rhodecode.tests.lib.middleware import mock_scm_app
 import rhodecode.lib.middleware.simplegit as simplegit
 
 
 def get_environ(url, request_method):
     """Construct a minimum WSGI environ based on the URL."""
     parsed_url = urllib.parse.urlparse(url)
     environ = {
         'PATH_INFO': parsed_url.path,
         'QUERY_STRING': parsed_url.query,
         'REQUEST_METHOD': request_method,
     }
 
     return environ
 
 
 @pytest.mark.parametrize(
     'url, expected_action, request_method',
     [
         ('/foo/bar/info/refs?service=git-upload-pack', 'pull', 'GET'),
         ('/foo/bar/info/refs?service=git-receive-pack', 'push', 'GET'),
         ('/foo/bar/git-upload-pack', 'pull', 'GET'),
         ('/foo/bar/git-receive-pack', 'push', 'GET'),
         # Edge case: missing data for info/refs
         ('/foo/info/refs?service=', 'pull', 'GET'),
         ('/foo/info/refs', 'pull', 'GET'),
         # Edge case: git command comes with service argument
         ('/foo/git-upload-pack?service=git-receive-pack', 'pull', 'GET'),
         ('/foo/git-receive-pack?service=git-upload-pack', 'push', 'GET'),
         # Edge case: repo name conflicts with git commands
         ('/git-receive-pack/git-upload-pack', 'pull', 'GET'),
         ('/git-receive-pack/git-receive-pack', 'push', 'GET'),
         ('/git-upload-pack/git-upload-pack', 'pull', 'GET'),
         ('/git-upload-pack/git-receive-pack', 'push', 'GET'),
         ('/foo/git-receive-pack', 'push', 'GET'),
         # Edge case: not a smart protocol url
         ('/foo/bar', 'pull', 'GET'),
         # GIT LFS cases, batch
         ('/foo/bar/info/lfs/objects/batch', 'push', 'GET'),
         ('/foo/bar/info/lfs/objects/batch', 'pull', 'POST'),
         # GIT LFS oid, dl/upl
         ('/foo/bar/info/lfs/abcdeabcde', 'pull', 'GET'),
         ('/foo/bar/info/lfs/abcdeabcde', 'push', 'PUT'),
         ('/foo/bar/info/lfs/abcdeabcde', 'push', 'POST'),
         # Edge case: repo name conflicts with git commands
         ('/info/lfs/info/lfs/objects/batch', 'push', 'GET'),
         ('/info/lfs/info/lfs/objects/batch', 'pull', 'POST'),
 
     ])
 def test_get_action(url, expected_action, request_method, baseapp, request_stub):
     app = simplegit.SimpleGit(config={'auth_ret_code': '', 'base_path': ''},
                               registry=request_stub.registry)
     assert expected_action == app._get_action(get_environ(url, request_method))
 
 
 @pytest.mark.parametrize(
     'url, expected_repo_name, request_method',
     [
         ('/foo/info/refs?service=git-upload-pack', 'foo', 'GET'),
         ('/foo/bar/info/refs?service=git-receive-pack', 'foo/bar', 'GET'),
         ('/foo/git-upload-pack', 'foo', 'GET'),
         ('/foo/git-receive-pack', 'foo', 'GET'),
         ('/foo/bar/git-upload-pack', 'foo/bar', 'GET'),
         ('/foo/bar/git-receive-pack', 'foo/bar', 'GET'),
 
         # GIT LFS cases, batch
         ('/foo/bar/info/lfs/objects/batch', 'foo/bar', 'GET'),
         ('/example-git/info/lfs/objects/batch', 'example-git', 'POST'),
         # GIT LFS oid, dl/upl
         ('/foo/info/lfs/abcdeabcde', 'foo', 'GET'),
         ('/foo/bar/info/lfs/abcdeabcde', 'foo/bar', 'PUT'),
         ('/my-git-repo/info/lfs/abcdeabcde', 'my-git-repo', 'POST'),
         # Edge case: repo name conflicts with git commands
         ('/info/lfs/info/lfs/objects/batch', 'info/lfs', 'GET'),
         ('/info/lfs/info/lfs/objects/batch', 'info/lfs', 'POST'),
 
     ])
 def test_get_repository_name(url, expected_repo_name, request_method, baseapp, request_stub):
     app = simplegit.SimpleGit(config={'auth_ret_code': '', 'base_path': ''},
                               registry=request_stub.registry)
     assert expected_repo_name == app._get_repository_name(
         get_environ(url, request_method))
 
 
 def test_get_config(user_util, baseapp, request_stub):
     repo = user_util.create_repo(repo_type='git')
     app = simplegit.SimpleGit(config={'auth_ret_code': '', 'base_path': ''},
                               registry=request_stub.registry)
     extras = {'foo': 'FOO', 'bar': 'BAR'}
 
     # We copy the extras as the method below will change the contents.
     git_config = app._create_config(dict(extras), repo_name=repo.repo_name)
 
     expected_config = dict(extras)
     expected_config.update({
         'git_update_server_info': False,
         'git_lfs_enabled': False,
         'git_lfs_store_path': git_config['git_lfs_store_path'],
         'git_lfs_http_scheme': 'http'
     })
 
     assert git_config == expected_config
 
 
 def test_create_wsgi_app_uses_scm_app_from_simplevcs(baseapp, request_stub):
     config = {
         'auth_ret_code': '',
         'base_path': '',
         'vcs.scm_app_implementation':
             'rhodecode.tests.lib.middleware.mock_scm_app',
     }
     app = simplegit.SimpleGit(config=config, registry=request_stub.registry)
     wsgi_app = app._create_wsgi_app('/tmp/test', 'test_repo', {})
     assert wsgi_app is mock_scm_app.mock_git_wsgi