libs: removed utf8 markers

rhodecode/lib/__init__.py

@@ -1,23 +1,22 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 RhodeCode libs
 """

rhodecode/lib/action_parser.py

@@ -1,357 +1,356 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 from webhelpers2.html.builder import literal
 from webhelpers2.html.tags import link_to
 
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.lib.vcs.backends.base import BaseCommit
 from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError
 
 
 log = logging.getLogger(__name__)
 
 
 def action_parser(request, user_log, feed=False, parse_cs=False):
     """
     This helper will action_map the specified string action into translated
     fancy names with icons and links
 
     :param user_log: user log instance
     :param feed: use output for feeds (no html and fancy icons)
     :param parse_cs: parse Changesets into VCS instances
     """
     if user_log.version == 'v2':
         ap = AuditLogParser(request, user_log)
         return ap.callbacks()
     else:
         # old style
         ap = ActionParser(request, user_log, feed=False, parse_commits=False)
         return ap.callbacks()
 
 
 class ActionParser(object):
 
     commits_limit = 3  # display this amount always
     commits_top_limit = 50  # show up to this amount of commits hidden
 
     def __init__(self, request, user_log, feed=False, parse_commits=False):
         self.user_log = user_log
         self.feed = feed
         self.parse_commits = parse_commits
         self.request = request
 
         self.action = user_log.action
         self.action_params = ' '
         x = self.action.split(':', 1)
         if len(x) > 1:
             self.action, self.action_params = x
 
     def callbacks(self):
         action_str = self.action_map.get(self.action, self.action)
         if self.feed:
             action = action_str[0].replace('[', '').replace(']', '')
         else:
             action = action_str[0]\
                 .replace('[', '<span class="journal_highlight">')\
                 .replace(']', '</span>')
 
         action_params_func = _no_params_func
         if callable(action_str[1]):
             action_params_func = action_str[1]
 
         # returned callbacks we need to call to get
         return [
             lambda: literal(action), action_params_func,
             self.action_parser_icon]
 
     @property
     def action_map(self):
         _ = self.request.translate
         # action : translated str, callback(extractor), icon
         action_map = {
             'user_deleted_repo': (
                 _('[deleted] repository'),
                 None, 'icon-trash'),
             'user_created_repo': (
                 _('[created] repository'),
                 None, 'icon-plus icon-plus-colored'),
             'user_created_fork': (
                 _('[created] repository as fork'),
                 None, 'icon-code-fork'),
             'user_forked_repo': (
                 _('[forked] repository'),
                 self.get_fork_name, 'icon-code-fork'),
             'user_updated_repo': (
                 _('[updated] repository'),
                 None, 'icon-pencil icon-pencil-colored'),
             'user_downloaded_archive': (
                 _('[downloaded] archive from repository'),
                 self.get_archive_name, 'icon-download-alt'),
             'admin_deleted_repo': (
                 _('[delete] repository'),
                 None, 'icon-trash'),
             'admin_created_repo': (
                 _('[created] repository'),
                 None, 'icon-plus icon-plus-colored'),
             'admin_forked_repo': (
                 _('[forked] repository'),
                 None, 'icon-code-fork icon-fork-colored'),
             'admin_updated_repo': (
                 _('[updated] repository'),
                 None, 'icon-pencil icon-pencil-colored'),
             'admin_created_user': (
                 _('[created] user'),
                 self.get_user_name, 'icon-user icon-user-colored'),
             'admin_updated_user': (
                 _('[updated] user'),
                 self.get_user_name, 'icon-user icon-user-colored'),
             'admin_created_users_group': (
                 _('[created] user group'),
                 self.get_users_group, 'icon-pencil icon-pencil-colored'),
             'admin_updated_users_group': (
                 _('[updated] user group'),
                 self.get_users_group, 'icon-pencil icon-pencil-colored'),
             'user_commented_revision': (
                 _('[commented] on commit in repository'),
                 self.get_cs_links, 'icon-comment icon-comment-colored'),
             'user_commented_pull_request': (
                 _('[commented] on pull request for'),
                 self.get_pull_request, 'icon-comment icon-comment-colored'),
             'user_closed_pull_request': (
                 _('[closed] pull request for'),
                 self.get_pull_request, 'icon-check'),
             'user_merged_pull_request': (
                 _('[merged] pull request for'),
                 self.get_pull_request, 'icon-check'),
             'push': (
                 _('[pushed] into'),
                 self.get_cs_links, 'icon-arrow-up'),
             'push_local': (
                 _('[committed via RhodeCode] into repository'),
                 self.get_cs_links, 'icon-pencil icon-pencil-colored'),
             'push_remote': (
                 _('[pulled from remote] into repository'),
                 self.get_cs_links, 'icon-arrow-up'),
             'pull': (
                 _('[pulled] from'),
                 None, 'icon-arrow-down'),
             'started_following_repo': (
                 _('[started following] repository'),
                 None, 'icon-heart icon-heart-colored'),
             'stopped_following_repo': (
                 _('[stopped following] repository'),
                 None, 'icon-heart-empty icon-heart-colored'),
         }
         return action_map
 
     def get_fork_name(self):
         from rhodecode.lib import helpers as h
         _ = self.request.translate
         repo_name = self.action_params
         _url = h.route_path('repo_summary', repo_name=repo_name)
         return _('fork name %s') % link_to(self.action_params, _url)
 
     def get_user_name(self):
         user_name = self.action_params
         return user_name
 
     def get_users_group(self):
         group_name = self.action_params
         return group_name
 
     def get_pull_request(self):
         from rhodecode.lib import helpers as h
         _ = self.request.translate
         pull_request_id = self.action_params
         if self.is_deleted():
             repo_name = self.user_log.repository_name
         else:
             repo_name = self.user_log.repository.repo_name
         return link_to(
             _('Pull request #%s') % pull_request_id,
             h.route_path('pullrequest_show', repo_name=repo_name,
                          pull_request_id=pull_request_id))
 
     def get_archive_name(self):
         archive_name = self.action_params
         return archive_name
 
     def action_parser_icon(self):
         tmpl = """<i class="%s" alt="%s"></i>"""
         ico = self.action_map.get(self.action, ['', '', ''])[2]
         return literal(tmpl % (ico, self.action))
 
     def get_cs_links(self):
         from rhodecode.lib import helpers as h
         _ = self.request.translate
         if self.is_deleted():
             return self.action_params
 
         repo_name = self.user_log.repository.repo_name
         commit_ids = self.action_params.split(',')
         commits = self.get_commits(commit_ids)
 
         link_generator = (
             self.lnk(commit, repo_name)
             for commit in commits[:self.commits_limit])
         commit_links = [" " + ', '.join(link_generator)]
         _op1, _name1 = _get_op(commit_ids[0])
         _op2, _name2 = _get_op(commit_ids[-1])
 
         commit_id_range = '%s...%s' % (_name1, _name2)
 
         compare_view = (
             ' <div class="compare_view tooltip" title="%s">'
             '<a href="%s">%s</a> </div>' % (
                 _('Show all combined commits %s->%s') % (
                     commit_ids[0][:12], commit_ids[-1][:12]
                 ),
                 h.route_path(
                     'repo_commit', repo_name=repo_name,
                     commit_id=commit_id_range), _('compare view')
             )
         )
 
         if len(commit_ids) > self.commits_limit:
             more_count = len(commit_ids) - self.commits_limit
             commit_links.append(
                 _(' and %(num)s more commits') % {'num': more_count}
             )
 
         if len(commits) > 1:
             commit_links.append(compare_view)
         return ''.join(commit_links)
 
     def get_commits(self, commit_ids):
         commits = []
         if not [v for v in commit_ids if v != '']:
             return commits
 
         repo = None
         if self.parse_commits:
             repo = self.user_log.repository.scm_instance()
 
         for commit_id in commit_ids[:self.commits_top_limit]:
             _op, _name = _get_op(commit_id)
 
             # we want parsed commits, or new log store format is bad
             if self.parse_commits:
                 try:
                     commit = repo.get_commit(commit_id=commit_id)
                     commits.append(commit)
                 except CommitDoesNotExistError:
                     log.error(
                         'cannot find commit id %s in this repository',
                         commit_id)
                     commits.append(commit_id)
                     continue
             else:
                 fake_commit = AttributeDict({
                     'short_id': commit_id[:12],
                     'raw_id': commit_id,
                     'message': '',
                     'op': _op,
                     'ref_name': _name
                 })
                 commits.append(fake_commit)
 
         return commits
 
     def lnk(self, commit_or_id, repo_name):
         from rhodecode.lib.helpers import tooltip
         from rhodecode.lib import helpers as h
         _ = self.request.translate
         title = ''
         lazy_cs = True
         if isinstance(commit_or_id, (BaseCommit, AttributeDict)):
             lazy_cs = True
             if (getattr(commit_or_id, 'op', None) and
                     getattr(commit_or_id, 'ref_name', None)):
                 lazy_cs = False
                 lbl = '?'
                 if commit_or_id.op == 'delete_branch':
                     lbl = '%s' % _('Deleted branch: %s') % commit_or_id.ref_name
                     title = ''
                 elif commit_or_id.op == 'tag':
                     lbl = '%s' % _('Created tag: %s') % commit_or_id.ref_name
                     title = ''
                 _url = '#'
 
             else:
                 lbl = '%s' % (commit_or_id.short_id[:8])
                 _url = h.route_path('repo_commit', repo_name=repo_name,
                                     commit_id=commit_or_id.raw_id)
                 title = tooltip(commit_or_id.message)
         else:
             # commit cannot be found/striped/removed etc.
             lbl = ('%s' % commit_or_id)[:12]
             _url = '#'
             title = _('Commit not found')
         if self.parse_commits:
             return link_to(lbl, _url, title=title, class_='tooltip')
         return link_to(lbl, _url, raw_id=commit_or_id.raw_id, repo_name=repo_name,
                        class_='lazy-cs' if lazy_cs else '')
 
     def is_deleted(self):
         return self.user_log.repository is None
 
 
 class AuditLogParser(object):
     def __init__(self, request, audit_log_entry):
         self.audit_log_entry = audit_log_entry
         self.request = request
 
     def get_icon(self, action):
         return 'icon-rhodecode'
 
     def callbacks(self):
         action_str = self.audit_log_entry.action
 
         def callback():
             # returned callbacks we need to call to get
             action = action_str \
                 .replace('[', '<span class="journal_highlight">')\
                 .replace(']', '</span>')
             return literal(action)
 
         def icon():
             tmpl = """<i class="%s" alt="%s"></i>"""
             ico = self.get_icon(action_str)
             return literal(tmpl % (ico, action_str))
 
         action_params_func = _no_params_func
 
         return [
             callback, action_params_func, icon]
 
 
 def _no_params_func():
     return ""
 
 
 def _get_op(commit_id):
     _op = None
     _name = commit_id
     if len(commit_id.split('=>')) == 2:
         _op, _name = commit_id.split('=>')
     return _op, _name

rhodecode/lib/audit_logger.py

@@ -1,305 +1,305 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2017-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import datetime
 
 from rhodecode.lib.jsonalchemy import JsonRaw
 from rhodecode.model import meta
 from rhodecode.model.db import User, UserLog, Repository
 
 
 log = logging.getLogger(__name__)
 
 # action as key, and expected action_data as value
 ACTIONS_V1 = {
     'user.login.success': {'user_agent': ''},
     'user.login.failure': {'user_agent': ''},
     'user.logout': {'user_agent': ''},
     'user.register': {},
     'user.password.reset_request': {},
     'user.push': {'user_agent': '', 'commit_ids': []},
     'user.pull': {'user_agent': ''},
 
     'user.create': {'data': {}},
     'user.delete': {'old_data': {}},
     'user.edit': {'old_data': {}},
     'user.edit.permissions': {},
     'user.edit.ip.add': {'ip': {}, 'user': {}},
     'user.edit.ip.delete': {'ip': {}, 'user': {}},
     'user.edit.token.add': {'token': {}, 'user': {}},
     'user.edit.token.delete': {'token': {}, 'user': {}},
     'user.edit.email.add': {'email': ''},
     'user.edit.email.delete': {'email': ''},
     'user.edit.ssh_key.add': {'token': {}, 'user': {}},
     'user.edit.ssh_key.delete': {'token': {}, 'user': {}},
     'user.edit.password_reset.enabled': {},
     'user.edit.password_reset.disabled': {},
 
     'user_group.create': {'data': {}},
     'user_group.delete': {'old_data': {}},
     'user_group.edit': {'old_data': {}},
     'user_group.edit.permissions': {},
     'user_group.edit.member.add': {'user': {}},
     'user_group.edit.member.delete': {'user': {}},
 
     'repo.create': {'data': {}},
     'repo.fork': {'data': {}},
     'repo.edit': {'old_data': {}},
     'repo.edit.permissions': {},
     'repo.edit.permissions.branch': {},
     'repo.archive': {'old_data': {}},
     'repo.delete': {'old_data': {}},
 
     'repo.archive.download': {'user_agent': '', 'archive_name': '',
                               'archive_spec': '', 'archive_cached': ''},
 
     'repo.permissions.branch_rule.create': {},
     'repo.permissions.branch_rule.edit': {},
     'repo.permissions.branch_rule.delete': {},
 
     'repo.pull_request.create': '',
     'repo.pull_request.edit': '',
     'repo.pull_request.delete': '',
     'repo.pull_request.close': '',
     'repo.pull_request.merge': '',
     'repo.pull_request.vote': '',
     'repo.pull_request.comment.create': '',
     'repo.pull_request.comment.edit': '',
     'repo.pull_request.comment.delete': '',
 
     'repo.pull_request.reviewer.add': '',
     'repo.pull_request.reviewer.delete': '',
 
     'repo.pull_request.observer.add': '',
     'repo.pull_request.observer.delete': '',
 
     'repo.commit.strip': {'commit_id': ''},
     'repo.commit.comment.create': {'data': {}},
     'repo.commit.comment.delete': {'data': {}},
     'repo.commit.comment.edit': {'data': {}},
     'repo.commit.vote': '',
 
     'repo.artifact.add': '',
     'repo.artifact.delete': '',
 
     'repo_group.create': {'data': {}},
     'repo_group.edit': {'old_data': {}},
     'repo_group.edit.permissions': {},
     'repo_group.delete': {'old_data': {}},
 }
 
 ACTIONS = ACTIONS_V1
 
 SOURCE_WEB = 'source_web'
 SOURCE_API = 'source_api'
 
 
 class UserWrap(object):
     """
     Fake object used to imitate AuthUser
     """
 
     def __init__(self, user_id=None, username=None, ip_addr=None):
         self.user_id = user_id
         self.username = username
         self.ip_addr = ip_addr
 
 
 class RepoWrap(object):
     """
     Fake object used to imitate RepoObject that audit logger requires
     """
 
     def __init__(self, repo_id=None, repo_name=None):
         self.repo_id = repo_id
         self.repo_name = repo_name
 
 
 def _store_log(action_name, action_data, user_id, username, user_data,
                ip_address, repository_id, repository_name):
     user_log = UserLog()
     user_log.version = UserLog.VERSION_2
 
     user_log.action = action_name
     user_log.action_data = action_data or JsonRaw('{}')
 
     user_log.user_ip = ip_address
 
     user_log.user_id = user_id
     user_log.username = username
     user_log.user_data = user_data or JsonRaw('{}')
 
     user_log.repository_id = repository_id
     user_log.repository_name = repository_name
 
     user_log.action_date = datetime.datetime.now()
 
     return user_log
 
 
 def store_web(*args, **kwargs):
     action_data = {}
     org_action_data = kwargs.pop('action_data', {})
     action_data.update(org_action_data)
     action_data['source'] = SOURCE_WEB
     kwargs['action_data'] = action_data
 
     return store(*args, **kwargs)
 
 
 def store_api(*args, **kwargs):
     action_data = {}
     org_action_data = kwargs.pop('action_data', {})
     action_data.update(org_action_data)
     action_data['source'] = SOURCE_API
     kwargs['action_data'] = action_data
 
     return store(*args, **kwargs)
 
 
 def store(action, user, action_data=None, user_data=None, ip_addr=None,
           repo=None, sa_session=None, commit=False):
     """
     Audit logger for various actions made by users, typically this
     results in a call such::
 
         from rhodecode.lib import audit_logger
 
         audit_logger.store(
             'repo.edit', user=self._rhodecode_user)
         audit_logger.store(
             'repo.delete', action_data={'data': repo_data},
             user=audit_logger.UserWrap(username='itried-login', ip_addr='8.8.8.8'))
 
         # repo action
         audit_logger.store(
             'repo.delete',
             user=audit_logger.UserWrap(username='itried-login', ip_addr='8.8.8.8'),
             repo=audit_logger.RepoWrap(repo_name='some-repo'))
 
         # repo action, when we know and have the repository object already
         audit_logger.store(
             'repo.delete', action_data={'source': audit_logger.SOURCE_WEB, },
             user=self._rhodecode_user,
             repo=repo_object)
 
         # alternative wrapper to the above
         audit_logger.store_web(
             'repo.delete', action_data={},
             user=self._rhodecode_user,
             repo=repo_object)
 
         # without an user ?
         audit_logger.store(
             'user.login.failure',
             user=audit_logger.UserWrap(
                     username=self.request.params.get('username'),
                     ip_addr=self.request.remote_addr))
 
     """
     from rhodecode.lib.utils2 import safe_unicode
     from rhodecode.lib.auth import AuthUser
 
     action_spec = ACTIONS.get(action, None)
     if action_spec is None:
         raise ValueError('Action `{}` is not supported'.format(action))
 
     if not sa_session:
         sa_session = meta.Session()
 
     try:
         username = getattr(user, 'username', None)
         if not username:
             pass
 
         user_id = getattr(user, 'user_id', None)
         if not user_id:
             # maybe we have username ? Try to figure user_id from username
             if username:
                 user_id = getattr(
                     User.get_by_username(username), 'user_id', None)
 
         ip_addr = ip_addr or getattr(user, 'ip_addr', None)
         if not ip_addr:
             pass
 
         if not user_data:
             # try to get this from the auth user
             if isinstance(user, AuthUser):
                 user_data = {
                     'username': user.username,
                     'email': user.email,
                 }
 
         repository_name = getattr(repo, 'repo_name', None)
         repository_id = getattr(repo, 'repo_id', None)
         if not repository_id:
             # maybe we have repo_name ? Try to figure repo_id from repo_name
             if repository_name:
                 repository_id = getattr(
                     Repository.get_by_repo_name(repository_name), 'repo_id', None)
 
         action_name = safe_unicode(action)
         ip_address = safe_unicode(ip_addr)
 
         with sa_session.no_autoflush:
 
             user_log = _store_log(
                 action_name=action_name,
                 action_data=action_data or {},
                 user_id=user_id,
                 username=username,
                 user_data=user_data or {},
                 ip_address=ip_address,
                 repository_id=repository_id,
                 repository_name=repository_name
             )
 
             sa_session.add(user_log)
             if commit:
                 sa_session.commit()
             entry_id = user_log.entry_id or ''
 
             update_user_last_activity(sa_session, user_id)
 
             if commit:
                 sa_session.commit()
 
         log.info('AUDIT[%s]: Logging action: `%s` by user:id:%s[%s] ip:%s',
                  entry_id, action_name, user_id, username, ip_address,
                  extra={"entry_id": entry_id, "action": action_name,
                         "user_id": user_id, "ip": ip_address})
 
     except Exception:
         log.exception('AUDIT: failed to store audit log')
 
 
 def update_user_last_activity(sa_session, user_id):
     _last_activity = datetime.datetime.now()
     try:
         sa_session.query(User).filter(User.user_id == user_id).update(
             {"last_activity": _last_activity})
         log.debug(
             'updated user `%s` last activity to:%s', user_id, _last_activity)
     except Exception:
         log.exception("Failed last activity update for user_id: %s", user_id)
         sa_session.rollback()
 

rhodecode/lib/auth.py

@@ -1,2533 +1,2532 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 authentication and permission libraries
 """
 
 import os
 
 import colander
 import time
 import collections
 import fnmatch
 import itertools
 import logging
 import random
 import traceback
 from functools import wraps
 
 import ipaddress
 
 from pyramid.httpexceptions import HTTPForbidden, HTTPFound, HTTPNotFound
 from sqlalchemy.orm.exc import ObjectDeletedError
 from sqlalchemy.orm import joinedload
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 import rhodecode
 from rhodecode.model import meta
 from rhodecode.model.meta import Session
 from rhodecode.model.user import UserModel
 from rhodecode.model.db import (
     false, User, Repository, Permission, UserToPerm, UserGroupToPerm, UserGroupMember,
     UserIpMap, UserApiKeys, RepoGroup, UserGroup, UserNotice)
 from rhodecode.lib import rc_cache
 from rhodecode.lib.utils import (
     get_repo_slug, get_repo_group_slug, get_user_group_slug)
 from rhodecode.lib.type_utils import aslist
 from rhodecode.lib.hash_utils import sha1, sha256, md5
 from rhodecode.lib.str_utils import ascii_bytes, safe_str, safe_int, safe_bytes
 from rhodecode.lib.caching_query import FromCache
 
 
 if rhodecode.is_unix:
     import bcrypt
 
 log = logging.getLogger(__name__)
 
 csrf_token_key = "csrf_token"
 
 
 class PasswordGenerator(object):
     """
     This is a simple class for generating password from different sets of
     characters
     usage::
         passwd_gen = PasswordGenerator()
         #print 8-letter password containing only big and small letters
             of alphabet
         passwd_gen.gen_password(8, passwd_gen.ALPHABETS_BIG_SMALL)
     """
     ALPHABETS_NUM = r'''1234567890'''
     ALPHABETS_SMALL = r'''qwertyuiopasdfghjklzxcvbnm'''
     ALPHABETS_BIG = r'''QWERTYUIOPASDFGHJKLZXCVBNM'''
     ALPHABETS_SPECIAL = r'''`-=[]\;',./~!@#$%^&*()_+{}|:"<>?'''
     ALPHABETS_FULL = ALPHABETS_BIG + ALPHABETS_SMALL \
         + ALPHABETS_NUM + ALPHABETS_SPECIAL
     ALPHABETS_ALPHANUM = ALPHABETS_BIG + ALPHABETS_SMALL + ALPHABETS_NUM
     ALPHABETS_BIG_SMALL = ALPHABETS_BIG + ALPHABETS_SMALL
     ALPHABETS_ALPHANUM_BIG = ALPHABETS_BIG + ALPHABETS_NUM
     ALPHABETS_ALPHANUM_SMALL = ALPHABETS_SMALL + ALPHABETS_NUM
 
     def __init__(self, passwd=''):
         self.passwd = passwd
 
     def gen_password(self, length, type_=None):
         if type_ is None:
             type_ = self.ALPHABETS_FULL
         self.passwd = ''.join([random.choice(type_) for _ in range(length)])
         return self.passwd
 
 
 class _RhodeCodeCryptoBase(object):
     ENC_PREF = None
 
     def hash_create(self, str_):
         """
         hash the string using
 
         :param str_: password to hash
         """
         raise NotImplementedError
 
     def hash_check_with_upgrade(self, password, hashed):
         """
         Returns tuple in which first element is boolean that states that
         given password matches it's hashed version, and the second is new hash
         of the password, in case this password should be migrated to new
         cipher.
         """
         checked_hash = self.hash_check(password, hashed)
         return checked_hash, None
 
     def hash_check(self, password, hashed):
         """
         Checks matching password with it's hashed value.
 
         :param password: password
         :param hashed: password in hashed form
         """
         raise NotImplementedError
 
     def _assert_bytes(self, value):
         """
         Passing in an `unicode` object can lead to hard to detect issues
         if passwords contain non-ascii characters.  Doing a type check
         during runtime, so that such mistakes are detected early on.
         """
         if not isinstance(value, str):
             raise TypeError(
                 "Bytestring required as input, got %r." % (value, ))
 
 
 class _RhodeCodeCryptoBCrypt(_RhodeCodeCryptoBase):
     ENC_PREF = ('$2a$10', '$2b$10')
 
     def hash_create(self, str_):
         self._assert_bytes(str_)
         return bcrypt.hashpw(str_, bcrypt.gensalt(10))
 
     def hash_check_with_upgrade(self, password, hashed):
         """
         Returns tuple in which first element is boolean that states that
         given password matches it's hashed version, and the second is new hash
         of the password, in case this password should be migrated to new
         cipher.
 
         This implements special upgrade logic which works like that:
          - check if the given password == bcrypted hash, if yes then we
            properly used password and it was already in bcrypt. Proceed
            without any changes
          - if bcrypt hash check is not working try with sha256. If hash compare
            is ok, it means we using correct but old hashed password. indicate
            hash change and proceed
         """
 
         new_hash = None
 
         # regular pw check
         password_match_bcrypt = self.hash_check(password, hashed)
 
         # now we want to know if the password was maybe from sha256
         # basically calling _RhodeCodeCryptoSha256().hash_check()
         if not password_match_bcrypt:
             if _RhodeCodeCryptoSha256().hash_check(password, hashed):
                 new_hash = self.hash_create(password)  # make new bcrypt hash
                 password_match_bcrypt = True
 
         return password_match_bcrypt, new_hash
 
     def hash_check(self, password, hashed):
         """
         Checks matching password with it's hashed value.
 
         :param password: password
         :param hashed: password in hashed form
         """
         self._assert_bytes(password)
         try:
             return bcrypt.hashpw(password, hashed) == hashed
         except ValueError as e:
             # we're having a invalid salt here probably, we should not crash
             # just return with False as it would be a wrong password.
             log.debug('Failed to check password hash using bcrypt %s',
                       safe_str(e))
 
         return False
 
 
 class _RhodeCodeCryptoSha256(_RhodeCodeCryptoBase):
     ENC_PREF = '_'
 
     def hash_create(self, str_):
         self._assert_bytes(str_)
         return sha256(str_)
 
     def hash_check(self, password, hashed):
         """
         Checks matching password with it's hashed value.
 
         :param password: password
         :param hashed: password in hashed form
         """
         self._assert_bytes(password)
         return sha256(password) == hashed
 
 
 class _RhodeCodeCryptoTest(_RhodeCodeCryptoBase):
     ENC_PREF = '_'
 
     def hash_create(self, str_):
         self._assert_bytes(str_)
         return sha1(str_)
 
     def hash_check(self, password, hashed):
         """
         Checks matching password with it's hashed value.
 
         :param password: password
         :param hashed: password in hashed form
         """
         self._assert_bytes(password)
         return sha1(password) == hashed
 
 
 def crypto_backend():
     """
     Return the matching crypto backend.
 
     Selection is based on if we run tests or not, we pick sha1-test backend to run
     tests faster since BCRYPT is expensive to calculate
     """
     if rhodecode.is_test:
         RhodeCodeCrypto = _RhodeCodeCryptoTest()
     else:
         RhodeCodeCrypto = _RhodeCodeCryptoBCrypt()
 
     return RhodeCodeCrypto
 
 
 def get_crypt_password(password):
     """
     Create the hash of `password` with the active crypto backend.
 
     :param password: The cleartext password.
     :type password: unicode
     """
     password = safe_str(password)
     return crypto_backend().hash_create(password)
 
 
 def check_password(password, hashed):
     """
     Check if the value in `password` matches the hash in `hashed`.
 
     :param password: The cleartext password.
     :type password: unicode
 
     :param hashed: The expected hashed version of the password.
     :type hashed: The hash has to be passed in in text representation.
     """
     password = safe_str(password)
     return crypto_backend().hash_check(password, hashed)
 
 
 def generate_auth_token(data, salt=None):
     """
     Generates API KEY from given string
     """
 
     if salt is None:
         salt = os.urandom(16)
     return sha1(data + salt)
 
 
 def get_came_from(request):
     """
     get query_string+path from request sanitized after removing auth_token
     """
     _req = request
 
     path = _req.path
     if 'auth_token' in _req.GET:
         # sanitize the request and remove auth_token for redirection
         _req.GET.pop('auth_token')
     qs = _req.query_string
     if qs:
         path += '?' + qs
 
     return path
 
 
 class CookieStoreWrapper(object):
 
     def __init__(self, cookie_store):
         self.cookie_store = cookie_store
 
     def __repr__(self):
         return 'CookieStore<%s>' % (self.cookie_store)
 
     def get(self, key, other=None):
         if isinstance(self.cookie_store, dict):
             return self.cookie_store.get(key, other)
         elif isinstance(self.cookie_store, AuthUser):
             return self.cookie_store.__dict__.get(key, other)
 
 
 def _cached_perms_data(user_id, scope, user_is_admin,
                        user_inherit_default_permissions, explicit, algo,
                        calculate_super_admin):
 
     permissions = PermissionCalculator(
         user_id, scope, user_is_admin, user_inherit_default_permissions,
         explicit, algo, calculate_super_admin)
     return permissions.calculate()
 
 
 class PermOrigin(object):
     SUPER_ADMIN = 'superadmin'
     ARCHIVED = 'archived'
 
     REPO_USER = 'user:%s'
     REPO_USERGROUP = 'usergroup:%s'
     REPO_OWNER = 'repo.owner'
     REPO_DEFAULT = 'repo.default'
     REPO_DEFAULT_NO_INHERIT = 'repo.default.no.inherit'
     REPO_PRIVATE = 'repo.private'
 
     REPOGROUP_USER = 'user:%s'
     REPOGROUP_USERGROUP = 'usergroup:%s'
     REPOGROUP_OWNER = 'group.owner'
     REPOGROUP_DEFAULT = 'group.default'
     REPOGROUP_DEFAULT_NO_INHERIT = 'group.default.no.inherit'
 
     USERGROUP_USER = 'user:%s'
     USERGROUP_USERGROUP = 'usergroup:%s'
     USERGROUP_OWNER = 'usergroup.owner'
     USERGROUP_DEFAULT = 'usergroup.default'
     USERGROUP_DEFAULT_NO_INHERIT = 'usergroup.default.no.inherit'
 
 
 class PermOriginDict(dict):
     """
     A special dict used for tracking permissions along with their origins.
 
     `__setitem__` has been overridden to expect a tuple(perm, origin)
     `__getitem__` will return only the perm
     `.perm_origin_stack` will return the stack of (perm, origin) set per key
 
     >>> perms = PermOriginDict()
     >>> perms['resource'] = 'read', 'default', 1
     >>> perms['resource']
     'read'
     >>> perms['resource'] = 'write', 'admin', 2
     >>> perms['resource']
     'write'
     >>> perms.perm_origin_stack
     {'resource': [('read', 'default', 1), ('write', 'admin', 2)]}
     """
 
     def __init__(self, *args, **kw):
         dict.__init__(self, *args, **kw)
         self.perm_origin_stack = collections.OrderedDict()
 
     def __setitem__(self, key, perm_origin_obj_id):
         # set (most likely via pickle) key:val pair without tuple
         if not isinstance(perm_origin_obj_id, tuple):
             perm = perm_origin_obj_id
             dict.__setitem__(self, key, perm)
         else:
             # unpack if we create a key from tuple
             (perm, origin, obj_id) = perm_origin_obj_id
             self.perm_origin_stack.setdefault(key, []).append((perm, origin, obj_id))
             dict.__setitem__(self, key, perm)
 
 
 class BranchPermOriginDict(dict):
     """
     Dedicated branch permissions dict, with tracking of patterns and origins.
 
     >>> perms = BranchPermOriginDict()
     >>> perms['resource'] = '*pattern', 'read', 'default'
     >>> perms['resource']
     {'*pattern': 'read'}
     >>> perms['resource'] = '*pattern', 'write', 'admin'
     >>> perms['resource']
     {'*pattern': 'write'}
     >>> perms.perm_origin_stack
     {'resource': {'*pattern': [('read', 'default'), ('write', 'admin')]}}
     """
     def __init__(self, *args, **kw):
         dict.__init__(self, *args, **kw)
         self.perm_origin_stack = collections.OrderedDict()
 
     def __setitem__(self, key, pattern_perm_origin):
         # set (most likely via pickle) key:val pair without tuple
         if not isinstance(pattern_perm_origin, tuple):
             pattern_perm = pattern_perm_origin
             dict.__setitem__(self, key, pattern_perm)
 
         else:
             (pattern_perm, origin) = pattern_perm_origin
             # we're passing in the dict, so we save the the stack
             for pattern, perm in pattern_perm.items():
                 self.perm_origin_stack.setdefault(key, {})\
                     .setdefault(pattern, []).append((perm, origin))
 
             dict.__setitem__(self, key, pattern_perm)
 
 
 class PermissionCalculator(object):
 
     def __init__(
             self, user_id, scope, user_is_admin,
             user_inherit_default_permissions, explicit, algo,
             calculate_super_admin_as_user=False):
 
         self.user_id = user_id
         self.user_is_admin = user_is_admin
         self.inherit_default_permissions = user_inherit_default_permissions
         self.explicit = explicit
         self.algo = algo
         self.calculate_super_admin_as_user = calculate_super_admin_as_user
 
         scope = scope or {}
         self.scope_repo_id = scope.get('repo_id')
         self.scope_repo_group_id = scope.get('repo_group_id')
         self.scope_user_group_id = scope.get('user_group_id')
 
         self.default_user_id = User.get_default_user(cache=True).user_id
 
         self.permissions_repositories = PermOriginDict()
         self.permissions_repository_groups = PermOriginDict()
         self.permissions_user_groups = PermOriginDict()
         self.permissions_repository_branches = BranchPermOriginDict()
         self.permissions_global = set()
 
         self.default_repo_perms = Permission.get_default_repo_perms(
             self.default_user_id, self.scope_repo_id)
         self.default_repo_groups_perms = Permission.get_default_group_perms(
             self.default_user_id, self.scope_repo_group_id)
         self.default_user_group_perms = \
             Permission.get_default_user_group_perms(
                 self.default_user_id, self.scope_user_group_id)
 
         # default branch perms
         self.default_branch_repo_perms = \
             Permission.get_default_repo_branch_perms(
                 self.default_user_id, self.scope_repo_id)
 
     def calculate(self):
         if self.user_is_admin and not self.calculate_super_admin_as_user:
             return self._calculate_super_admin_permissions()
 
         self._calculate_global_default_permissions()
         self._calculate_global_permissions()
         self._calculate_default_permissions()
         self._calculate_repository_permissions()
         self._calculate_repository_branch_permissions()
         self._calculate_repository_group_permissions()
         self._calculate_user_group_permissions()
         return self._permission_structure()
 
     def _calculate_super_admin_permissions(self):
         """
         super-admin user have all default rights for repositories
         and groups set to admin
         """
         self.permissions_global.add('hg.admin')
         self.permissions_global.add('hg.create.write_on_repogroup.true')
 
         # repositories
         for perm in self.default_repo_perms:
             r_k = perm.UserRepoToPerm.repository.repo_name
             obj_id = perm.UserRepoToPerm.repository.repo_id
             archived = perm.UserRepoToPerm.repository.archived
             p = 'repository.admin'
             self.permissions_repositories[r_k] = p, PermOrigin.SUPER_ADMIN, obj_id
             # special case for archived repositories, which we block still even for
             # super admins
             if archived:
                 p = 'repository.read'
                 self.permissions_repositories[r_k] = p, PermOrigin.ARCHIVED, obj_id
 
         # repository groups
         for perm in self.default_repo_groups_perms:
             rg_k = perm.UserRepoGroupToPerm.group.group_name
             obj_id = perm.UserRepoGroupToPerm.group.group_id
             p = 'group.admin'
             self.permissions_repository_groups[rg_k] = p, PermOrigin.SUPER_ADMIN, obj_id
 
         # user groups
         for perm in self.default_user_group_perms:
             u_k = perm.UserUserGroupToPerm.user_group.users_group_name
             obj_id = perm.UserUserGroupToPerm.user_group.users_group_id
             p = 'usergroup.admin'
             self.permissions_user_groups[u_k] = p, PermOrigin.SUPER_ADMIN, obj_id
 
         # branch permissions
         # since super-admin also can have custom rule permissions
         # we *always* need to calculate those inherited from default, and also explicit
         self._calculate_default_permissions_repository_branches(
             user_inherit_object_permissions=False)
         self._calculate_repository_branch_permissions()
 
         return self._permission_structure()
 
     def _calculate_global_default_permissions(self):
         """
         global permissions taken from the default user
         """
         default_global_perms = UserToPerm.query()\
             .filter(UserToPerm.user_id == self.default_user_id)\
             .options(joinedload(UserToPerm.permission))
 
         for perm in default_global_perms:
             self.permissions_global.add(perm.permission.permission_name)
 
         if self.user_is_admin:
             self.permissions_global.add('hg.admin')
             self.permissions_global.add('hg.create.write_on_repogroup.true')
 
     def _calculate_global_permissions(self):
         """
         Set global system permissions with user permissions or permissions
         taken from the user groups of the current user.
 
         The permissions include repo creating, repo group creating, forking
         etc.
         """
 
         # now we read the defined permissions and overwrite what we have set
         # before those can be configured from groups or users explicitly.
 
         # In case we want to extend this list we should make sure
         # this is in sync with User.DEFAULT_USER_PERMISSIONS definitions
         from rhodecode.model.permission import PermissionModel
 
         _configurable = frozenset([
             PermissionModel.FORKING_DISABLED, PermissionModel.FORKING_ENABLED,
             'hg.create.none', 'hg.create.repository',
             'hg.usergroup.create.false', 'hg.usergroup.create.true',
             'hg.repogroup.create.false', 'hg.repogroup.create.true',
             'hg.create.write_on_repogroup.false', 'hg.create.write_on_repogroup.true',
             'hg.inherit_default_perms.false', 'hg.inherit_default_perms.true'
         ])
 
         # USER GROUPS comes first user group global permissions
         user_perms_from_users_groups = Session().query(UserGroupToPerm)\
             .options(joinedload(UserGroupToPerm.permission))\
             .join((UserGroupMember, UserGroupToPerm.users_group_id ==
                    UserGroupMember.users_group_id))\
             .filter(UserGroupMember.user_id == self.user_id)\
             .order_by(UserGroupToPerm.users_group_id)\
             .all()
 
         # need to group here by groups since user can be in more than
         # one group, so we get all groups
         _explicit_grouped_perms = [
             [x, list(y)] for x, y in
             itertools.groupby(user_perms_from_users_groups,
                               lambda _x: _x.users_group)]
 
         for gr, perms in _explicit_grouped_perms:
             # since user can be in multiple groups iterate over them and
             # select the lowest permissions first (more explicit)
             # TODO(marcink): do this^^
 
             # group doesn't inherit default permissions so we actually set them
             if not gr.inherit_default_permissions:
                 # NEED TO IGNORE all previously set configurable permissions
                 # and replace them with explicitly set from this user
                 # group permissions
                 self.permissions_global = self.permissions_global.difference(
                     _configurable)
                 for perm in perms:
                     self.permissions_global.add(perm.permission.permission_name)
 
         # user explicit global permissions
         user_perms = Session().query(UserToPerm)\
             .options(joinedload(UserToPerm.permission))\
             .filter(UserToPerm.user_id == self.user_id).all()
 
         if not self.inherit_default_permissions:
             # NEED TO IGNORE all configurable permissions and
             # replace them with explicitly set from this user permissions
             self.permissions_global = self.permissions_global.difference(
                 _configurable)
             for perm in user_perms:
                 self.permissions_global.add(perm.permission.permission_name)
 
     def _calculate_default_permissions_repositories(self, user_inherit_object_permissions):
         for perm in self.default_repo_perms:
             r_k = perm.UserRepoToPerm.repository.repo_name
             obj_id = perm.UserRepoToPerm.repository.repo_id
             archived = perm.UserRepoToPerm.repository.archived
             p = perm.Permission.permission_name
             o = PermOrigin.REPO_DEFAULT
             self.permissions_repositories[r_k] = p, o, obj_id
 
             # if we decide this user isn't inheriting permissions from
             # default user we set him to .none so only explicit
             # permissions work
             if not user_inherit_object_permissions:
                 p = 'repository.none'
                 o = PermOrigin.REPO_DEFAULT_NO_INHERIT
                 self.permissions_repositories[r_k] = p, o, obj_id
 
             if perm.Repository.private and not (
                     perm.Repository.user_id == self.user_id):
                 # disable defaults for private repos,
                 p = 'repository.none'
                 o = PermOrigin.REPO_PRIVATE
                 self.permissions_repositories[r_k] = p, o, obj_id
 
             elif perm.Repository.user_id == self.user_id:
                 # set admin if owner
                 p = 'repository.admin'
                 o = PermOrigin.REPO_OWNER
                 self.permissions_repositories[r_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'repository.admin'
                 o = PermOrigin.SUPER_ADMIN
                 self.permissions_repositories[r_k] = p, o, obj_id
 
             # finally in case of archived repositories, we downgrade  higher
             # permissions to read
             if archived:
                 current_perm = self.permissions_repositories[r_k]
                 if current_perm in ['repository.write', 'repository.admin']:
                     p = 'repository.read'
                     o = PermOrigin.ARCHIVED
                     self.permissions_repositories[r_k] = p, o, obj_id
 
     def _calculate_default_permissions_repository_branches(self, user_inherit_object_permissions):
         for perm in self.default_branch_repo_perms:
 
             r_k = perm.UserRepoToPerm.repository.repo_name
             p = perm.Permission.permission_name
             pattern = perm.UserToRepoBranchPermission.branch_pattern
             o = PermOrigin.REPO_USER % perm.UserRepoToPerm.user.username
 
             if not self.explicit:
                 cur_perm = self.permissions_repository_branches.get(r_k)
                 if cur_perm:
                     cur_perm = cur_perm[pattern]
                 cur_perm = cur_perm or 'branch.none'
 
                 p = self._choose_permission(p, cur_perm)
 
             # NOTE(marcink): register all pattern/perm instances in this
             # special dict that aggregates entries
             self.permissions_repository_branches[r_k] = {pattern: p}, o
 
     def _calculate_default_permissions_repository_groups(self, user_inherit_object_permissions):
         for perm in self.default_repo_groups_perms:
             rg_k = perm.UserRepoGroupToPerm.group.group_name
             obj_id = perm.UserRepoGroupToPerm.group.group_id
             p = perm.Permission.permission_name
             o = PermOrigin.REPOGROUP_DEFAULT
             self.permissions_repository_groups[rg_k] = p, o, obj_id
 
             # if we decide this user isn't inheriting permissions from default
             # user we set him to .none so only explicit permissions work
             if not user_inherit_object_permissions:
                 p = 'group.none'
                 o = PermOrigin.REPOGROUP_DEFAULT_NO_INHERIT
                 self.permissions_repository_groups[rg_k] = p, o, obj_id
 
             if perm.RepoGroup.user_id == self.user_id:
                 # set admin if owner
                 p = 'group.admin'
                 o = PermOrigin.REPOGROUP_OWNER
                 self.permissions_repository_groups[rg_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'group.admin'
                 o = PermOrigin.SUPER_ADMIN
                 self.permissions_repository_groups[rg_k] = p, o, obj_id
 
     def _calculate_default_permissions_user_groups(self, user_inherit_object_permissions):
         for perm in self.default_user_group_perms:
             u_k = perm.UserUserGroupToPerm.user_group.users_group_name
             obj_id = perm.UserUserGroupToPerm.user_group.users_group_id
             p = perm.Permission.permission_name
             o = PermOrigin.USERGROUP_DEFAULT
             self.permissions_user_groups[u_k] = p, o, obj_id
 
             # if we decide this user isn't inheriting permissions from default
             # user we set him to .none so only explicit permissions work
             if not user_inherit_object_permissions:
                 p = 'usergroup.none'
                 o = PermOrigin.USERGROUP_DEFAULT_NO_INHERIT
                 self.permissions_user_groups[u_k] = p, o, obj_id
 
             if perm.UserGroup.user_id == self.user_id:
                 # set admin if owner
                 p = 'usergroup.admin'
                 o = PermOrigin.USERGROUP_OWNER
                 self.permissions_user_groups[u_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'usergroup.admin'
                 o = PermOrigin.SUPER_ADMIN
                 self.permissions_user_groups[u_k] = p, o, obj_id
 
     def _calculate_default_permissions(self):
         """
         Set default user permissions for repositories, repository branches,
         repository groups, user groups taken from the default user.
 
         Calculate inheritance of object permissions based on what we have now
         in GLOBAL permissions. We check if .false is in GLOBAL since this is
         explicitly set. Inherit is the opposite of .false being there.
 
         .. note::
 
            the syntax is little bit odd but what we need to check here is
            the opposite of .false permission being in the list so even for
            inconsistent state when both .true/.false is there
            .false is more important
 
         """
         user_inherit_object_permissions = not ('hg.inherit_default_perms.false'
                                                in self.permissions_global)
 
         # default permissions inherited from `default` user permissions
         self._calculate_default_permissions_repositories(
             user_inherit_object_permissions)
 
         self._calculate_default_permissions_repository_branches(
             user_inherit_object_permissions)
 
         self._calculate_default_permissions_repository_groups(
             user_inherit_object_permissions)
 
         self._calculate_default_permissions_user_groups(
             user_inherit_object_permissions)
 
     def _calculate_repository_permissions(self):
         """
         Repository access permissions for the current user.
 
         Check if the user is part of user groups for this repository and
         fill in the permission from it. `_choose_permission` decides of which
         permission should be selected based on selected method.
         """
 
         # user group for repositories permissions
         user_repo_perms_from_user_group = Permission\
             .get_default_repo_perms_from_user_group(
                 self.user_id, self.scope_repo_id)
 
         multiple_counter = collections.defaultdict(int)
         for perm in user_repo_perms_from_user_group:
             r_k = perm.UserGroupRepoToPerm.repository.repo_name
             obj_id = perm.UserGroupRepoToPerm.repository.repo_id
             multiple_counter[r_k] += 1
             p = perm.Permission.permission_name
             o = PermOrigin.REPO_USERGROUP % perm.UserGroupRepoToPerm\
                 .users_group.users_group_name
 
             if multiple_counter[r_k] > 1:
                 cur_perm = self.permissions_repositories[r_k]
                 p = self._choose_permission(p, cur_perm)
 
             self.permissions_repositories[r_k] = p, o, obj_id
 
             if perm.Repository.user_id == self.user_id:
                 # set admin if owner
                 p = 'repository.admin'
                 o = PermOrigin.REPO_OWNER
                 self.permissions_repositories[r_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'repository.admin'
                 o = PermOrigin.SUPER_ADMIN
                 self.permissions_repositories[r_k] = p, o, obj_id
 
         # user explicit permissions for repositories, overrides any specified
         # by the group permission
         user_repo_perms = Permission.get_default_repo_perms(
             self.user_id, self.scope_repo_id)
         for perm in user_repo_perms:
             r_k = perm.UserRepoToPerm.repository.repo_name
             obj_id = perm.UserRepoToPerm.repository.repo_id
             archived = perm.UserRepoToPerm.repository.archived
             p = perm.Permission.permission_name
             o = PermOrigin.REPO_USER % perm.UserRepoToPerm.user.username
 
             if not self.explicit:
                 cur_perm = self.permissions_repositories.get(
                     r_k, 'repository.none')
                 p = self._choose_permission(p, cur_perm)
 
             self.permissions_repositories[r_k] = p, o, obj_id
 
             if perm.Repository.user_id == self.user_id:
                 # set admin if owner
                 p = 'repository.admin'
                 o = PermOrigin.REPO_OWNER
                 self.permissions_repositories[r_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'repository.admin'
                 o = PermOrigin.SUPER_ADMIN
                 self.permissions_repositories[r_k] = p, o, obj_id
 
             # finally in case of archived repositories, we downgrade  higher
             # permissions to read
             if archived:
                 current_perm = self.permissions_repositories[r_k]
                 if current_perm in ['repository.write', 'repository.admin']:
                     p = 'repository.read'
                     o = PermOrigin.ARCHIVED
                     self.permissions_repositories[r_k] = p, o, obj_id
 
     def _calculate_repository_branch_permissions(self):
         # user group for repositories permissions
         user_repo_branch_perms_from_user_group = Permission\
             .get_default_repo_branch_perms_from_user_group(
                 self.user_id, self.scope_repo_id)
 
         multiple_counter = collections.defaultdict(int)
         for perm in user_repo_branch_perms_from_user_group:
             r_k = perm.UserGroupRepoToPerm.repository.repo_name
             p = perm.Permission.permission_name
             pattern = perm.UserGroupToRepoBranchPermission.branch_pattern
             o = PermOrigin.REPO_USERGROUP % perm.UserGroupRepoToPerm\
                 .users_group.users_group_name
 
             multiple_counter[r_k] += 1
             if multiple_counter[r_k] > 1:
                 cur_perm = self.permissions_repository_branches[r_k][pattern]
                 p = self._choose_permission(p, cur_perm)
 
             self.permissions_repository_branches[r_k] = {pattern: p}, o
 
         # user explicit branch permissions for repositories, overrides
         # any specified by the group permission
         user_repo_branch_perms = Permission.get_default_repo_branch_perms(
             self.user_id, self.scope_repo_id)
 
         for perm in user_repo_branch_perms:
 
             r_k = perm.UserRepoToPerm.repository.repo_name
             p = perm.Permission.permission_name
             pattern = perm.UserToRepoBranchPermission.branch_pattern
             o = PermOrigin.REPO_USER % perm.UserRepoToPerm.user.username
 
             if not self.explicit:
                 cur_perm = self.permissions_repository_branches.get(r_k)
                 if cur_perm:
                     cur_perm = cur_perm[pattern]
                 cur_perm = cur_perm or 'branch.none'
                 p = self._choose_permission(p, cur_perm)
 
             # NOTE(marcink): register all pattern/perm instances in this
             # special dict that aggregates entries
             self.permissions_repository_branches[r_k] = {pattern: p}, o
 
     def _calculate_repository_group_permissions(self):
         """
         Repository group permissions for the current user.
 
         Check if the user is part of user groups for repository groups and
         fill in the permissions from it. `_choose_permission` decides of which
         permission should be selected based on selected method.
         """
         # user group for repo groups permissions
         user_repo_group_perms_from_user_group = Permission\
             .get_default_group_perms_from_user_group(
                 self.user_id, self.scope_repo_group_id)
 
         multiple_counter = collections.defaultdict(int)
         for perm in user_repo_group_perms_from_user_group:
             rg_k = perm.UserGroupRepoGroupToPerm.group.group_name
             obj_id = perm.UserGroupRepoGroupToPerm.group.group_id
             multiple_counter[rg_k] += 1
             o = PermOrigin.REPOGROUP_USERGROUP % perm.UserGroupRepoGroupToPerm\
                 .users_group.users_group_name
             p = perm.Permission.permission_name
 
             if multiple_counter[rg_k] > 1:
                 cur_perm = self.permissions_repository_groups[rg_k]
                 p = self._choose_permission(p, cur_perm)
             self.permissions_repository_groups[rg_k] = p, o, obj_id
 
             if perm.RepoGroup.user_id == self.user_id:
                 # set admin if owner, even for member of other user group
                 p = 'group.admin'
                 o = PermOrigin.REPOGROUP_OWNER
                 self.permissions_repository_groups[rg_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'group.admin'
                 o = PermOrigin.SUPER_ADMIN
                 self.permissions_repository_groups[rg_k] = p, o, obj_id
 
         # user explicit permissions for repository groups
         user_repo_groups_perms = Permission.get_default_group_perms(
             self.user_id, self.scope_repo_group_id)
         for perm in user_repo_groups_perms:
             rg_k = perm.UserRepoGroupToPerm.group.group_name
             obj_id = perm.UserRepoGroupToPerm.group.group_id
             o = PermOrigin.REPOGROUP_USER % perm.UserRepoGroupToPerm\
                 .user.username
             p = perm.Permission.permission_name
 
             if not self.explicit:
                 cur_perm = self.permissions_repository_groups.get(rg_k, 'group.none')
                 p = self._choose_permission(p, cur_perm)
 
             self.permissions_repository_groups[rg_k] = p, o, obj_id
 
             if perm.RepoGroup.user_id == self.user_id:
                 # set admin if owner
                 p = 'group.admin'
                 o = PermOrigin.REPOGROUP_OWNER
                 self.permissions_repository_groups[rg_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'group.admin'
                 o = PermOrigin.SUPER_ADMIN
                 self.permissions_repository_groups[rg_k] = p, o, obj_id
 
     def _calculate_user_group_permissions(self):
         """
         User group permissions for the current user.
         """
         # user group for user group permissions
         user_group_from_user_group = Permission\
             .get_default_user_group_perms_from_user_group(
                 self.user_id, self.scope_user_group_id)
 
         multiple_counter = collections.defaultdict(int)
         for perm in user_group_from_user_group:
             ug_k = perm.UserGroupUserGroupToPerm.target_user_group.users_group_name
             obj_id = perm.UserGroupUserGroupToPerm.target_user_group.users_group_id
             multiple_counter[ug_k] += 1
             o = PermOrigin.USERGROUP_USERGROUP % perm.UserGroupUserGroupToPerm\
                 .user_group.users_group_name
             p = perm.Permission.permission_name
 
             if multiple_counter[ug_k] > 1:
                 cur_perm = self.permissions_user_groups[ug_k]
                 p = self._choose_permission(p, cur_perm)
 
             self.permissions_user_groups[ug_k] = p, o, obj_id
 
             if perm.UserGroup.user_id == self.user_id:
                 # set admin if owner, even for member of other user group
                 p = 'usergroup.admin'
                 o = PermOrigin.USERGROUP_OWNER
                 self.permissions_user_groups[ug_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'usergroup.admin'
                 o = PermOrigin.SUPER_ADMIN
                 self.permissions_user_groups[ug_k] = p, o, obj_id
 
         # user explicit permission for user groups
         user_user_groups_perms = Permission.get_default_user_group_perms(
             self.user_id, self.scope_user_group_id)
         for perm in user_user_groups_perms:
             ug_k = perm.UserUserGroupToPerm.user_group.users_group_name
             obj_id = perm.UserUserGroupToPerm.user_group.users_group_id
             o = PermOrigin.USERGROUP_USER % perm.UserUserGroupToPerm\
                 .user.username
             p = perm.Permission.permission_name
 
             if not self.explicit:
                 cur_perm = self.permissions_user_groups.get(ug_k, 'usergroup.none')
                 p = self._choose_permission(p, cur_perm)
 
             self.permissions_user_groups[ug_k] = p, o, obj_id
 
             if perm.UserGroup.user_id == self.user_id:
                 # set admin if owner
                 p = 'usergroup.admin'
                 o = PermOrigin.USERGROUP_OWNER
                 self.permissions_user_groups[ug_k] = p, o, obj_id
 
             if self.user_is_admin:
                 p = 'usergroup.admin'
                 o = PermOrigin.SUPER_ADMIN
                 self.permissions_user_groups[ug_k] = p, o, obj_id
 
     def _choose_permission(self, new_perm, cur_perm):
         new_perm_val = Permission.PERM_WEIGHTS[new_perm]
         cur_perm_val = Permission.PERM_WEIGHTS[cur_perm]
         if self.algo == 'higherwin':
             if new_perm_val > cur_perm_val:
                 return new_perm
             return cur_perm
         elif self.algo == 'lowerwin':
             if new_perm_val < cur_perm_val:
                 return new_perm
             return cur_perm
 
     def _permission_structure(self):
         return {
             'global': self.permissions_global,
             'repositories': self.permissions_repositories,
             'repository_branches': self.permissions_repository_branches,
             'repositories_groups': self.permissions_repository_groups,
             'user_groups': self.permissions_user_groups,
         }
 
 
 def allowed_auth_token_access(view_name, auth_token, whitelist=None):
     """
     Check if given controller_name is in whitelist of auth token access
     """
     if not whitelist:
         from rhodecode import CONFIG
         whitelist = aslist(
             CONFIG.get('api_access_controllers_whitelist'), sep=',')
     # backward compat translation
     compat = {
         # old controller, new VIEW
         'ChangesetController:*': 'RepoCommitsView:*',
         'ChangesetController:changeset_patch': 'RepoCommitsView:repo_commit_patch',
         'ChangesetController:changeset_raw': 'RepoCommitsView:repo_commit_raw',
         'FilesController:raw': 'RepoCommitsView:repo_commit_raw',
         'FilesController:archivefile': 'RepoFilesView:repo_archivefile',
         'GistsController:*': 'GistView:*',
     }
 
     log.debug(
         'Allowed views for AUTH TOKEN access: %s', whitelist)
     auth_token_access_valid = False
 
     for entry in whitelist:
         token_match = True
         if entry in compat:
             # translate from old Controllers to Pyramid Views
             entry = compat[entry]
 
         if '@' in entry:
             # specific AuthToken
             entry, allowed_token = entry.split('@', 1)
             token_match = auth_token == allowed_token
 
         if fnmatch.fnmatch(view_name, entry) and token_match:
             auth_token_access_valid = True
             break
 
     if auth_token_access_valid:
         log.debug('view: `%s` matches entry in whitelist: %s',
                   view_name, whitelist)
 
     else:
         msg = ('view: `%s` does *NOT* match any entry in whitelist: %s'
                % (view_name, whitelist))
         if auth_token:
             # if we use auth token key and don't have access it's a warning
             log.warning(msg)
         else:
             log.debug(msg)
 
     return auth_token_access_valid
 
 
 class AuthUser(object):
     """
     A simple object that handles all attributes of user in RhodeCode
 
     It does lookup based on API key,given user, or user present in session
     Then it fills all required information for such user. It also checks if
     anonymous access is enabled and if so, it returns default user as logged in
     """
     GLOBAL_PERMS = [x[0] for x in Permission.PERMS]
     repo_read_perms = ['repository.read', 'repository.admin', 'repository.write']
     repo_group_read_perms = ['group.read', 'group.write', 'group.admin']
     user_group_read_perms = ['usergroup.read', 'usergroup.write', 'usergroup.admin']
 
     def __init__(self, user_id=None, api_key=None, username=None, ip_addr=None):
 
         self.user_id = user_id
         self._api_key = api_key
 
         self.api_key = None
         self.username = username
         self.ip_addr = ip_addr
         self.name = ''
         self.lastname = ''
         self.first_name = ''
         self.last_name = ''
         self.email = ''
         self.is_authenticated = False
         self.admin = False
         self.inherit_default_permissions = False
         self.password = ''
 
         self.anonymous_user = None  # propagated on propagate_data
         self.propagate_data()
         self._instance = None
         self._permissions_scoped_cache = {}  # used to bind scoped calculation
 
     @LazyProperty
     def permissions(self):
         return self.get_perms(user=self, cache=None)
 
     @LazyProperty
     def permissions_safe(self):
         """
         Filtered permissions excluding not allowed repositories
         """
         perms = self.get_perms(user=self, cache=None)
 
         perms['repositories'] = {
             k: v for k, v in perms['repositories'].items()
             if v != 'repository.none'}
         perms['repositories_groups'] = {
             k: v for k, v in perms['repositories_groups'].items()
             if v != 'group.none'}
         perms['user_groups'] = {
             k: v for k, v in perms['user_groups'].items()
             if v != 'usergroup.none'}
         perms['repository_branches'] = {
             k: v for k, v in perms['repository_branches'].items()
             if v != 'branch.none'}
         return perms
 
     @LazyProperty
     def permissions_full_details(self):
         return self.get_perms(
             user=self, cache=None, calculate_super_admin=True)
 
     def permissions_with_scope(self, scope):
         """
         Call the get_perms function with scoped data. The scope in that function
         narrows the SQL calls to the given ID of objects resulting in fetching
         Just particular permission we want to obtain. If scope is an empty dict
         then it basically narrows the scope to GLOBAL permissions only.
 
         :param scope: dict
         """
         if 'repo_name' in scope:
             obj = Repository.get_by_repo_name(scope['repo_name'])
             if obj:
                 scope['repo_id'] = obj.repo_id
         _scope = collections.OrderedDict()
         _scope['repo_id'] = -1
         _scope['user_group_id'] = -1
         _scope['repo_group_id'] = -1
 
         for k in sorted(scope.keys()):
             _scope[k] = scope[k]
 
         # store in cache to mimic how the @LazyProperty works,
         # the difference here is that we use the unique key calculated
         # from params and values
         return self.get_perms(user=self, cache=None, scope=_scope)
 
     def get_instance(self):
         return User.get(self.user_id)
 
     def propagate_data(self):
         """
         Fills in user data and propagates values to this instance. Maps fetched
         user attributes to this class instance attributes
         """
         log.debug('AuthUser: starting data propagation for new potential user')
         user_model = UserModel()
         anon_user = self.anonymous_user = User.get_default_user(cache=True)
         is_user_loaded = False
 
         # lookup by userid
         if self.user_id is not None and self.user_id != anon_user.user_id:
             log.debug('Trying Auth User lookup by USER ID: `%s`', self.user_id)
             is_user_loaded = user_model.fill_data(self, user_id=self.user_id)
 
         # try go get user by api key
         elif self._api_key and self._api_key != anon_user.api_key:
             log.debug('Trying Auth User lookup by API KEY: `...%s`', self._api_key[-4:])
             is_user_loaded = user_model.fill_data(self, api_key=self._api_key)
 
         # lookup by username
         elif self.username:
             log.debug('Trying Auth User lookup by USER NAME: `%s`', self.username)
             is_user_loaded = user_model.fill_data(self, username=self.username)
         else:
             log.debug('No data in %s that could been used to log in', self)
 
         if not is_user_loaded:
             log.debug(
                 'Failed to load user. Fallback to default user %s', anon_user)
             # if we cannot authenticate user try anonymous
             if anon_user.active:
                 log.debug('default user is active, using it as a session user')
                 user_model.fill_data(self, user_id=anon_user.user_id)
                 # then we set this user is logged in
                 self.is_authenticated = True
             else:
                 log.debug('default user is NOT active')
                 # in case of disabled anonymous user we reset some of the
                 # parameters so such user is "corrupted", skipping the fill_data
                 for attr in ['user_id', 'username', 'admin', 'active']:
                     setattr(self, attr, None)
                 self.is_authenticated = False
 
         if not self.username:
             self.username = 'None'
 
         log.debug('AuthUser: propagated user is now %s', self)
 
     def get_perms(self, user, scope=None, explicit=True, algo='higherwin',
                   calculate_super_admin=False, cache=None):
         """
         Fills user permission attribute with permissions taken from database
         works for permissions given for repositories, and for permissions that
         are granted to groups
 
         :param user: instance of User object from database
         :param explicit: In case there are permissions both for user and a group
             that user is part of, explicit flag will defiine if user will
             explicitly override permissions from group, if it's False it will
             make decision based on the algo
         :param algo: algorithm to decide what permission should be choose if
             it's multiple defined, eg user in two different groups. It also
             decides if explicit flag is turned off how to specify the permission
             for case when user is in a group + have defined separate permission
         :param calculate_super_admin: calculate permissions for super-admin in the
             same way as for regular user without speedups
         :param cache: Use caching for calculation, None = let the cache backend decide
         """
         user_id = user.user_id
         user_is_admin = user.is_admin
 
         # inheritance of global permissions like create repo/fork repo etc
         user_inherit_default_permissions = user.inherit_default_permissions
 
         cache_seconds = safe_int(
             rhodecode.CONFIG.get('rc_cache.cache_perms.expiration_time'))
 
         if cache is None:
             # let the backend cache decide
             cache_on = cache_seconds > 0
         else:
             cache_on = cache
 
         log.debug(
             'Computing PERMISSION tree for user %s scope `%s` '
             'with caching: %s[TTL: %ss]', user, scope, cache_on, cache_seconds or 0)
 
         cache_namespace_uid = 'cache_user_auth.{}'.format(user_id)
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                                condition=cache_on)
         def compute_perm_tree(cache_name, cache_ver,
                 user_id, scope, user_is_admin,user_inherit_default_permissions,
                 explicit, algo, calculate_super_admin):
             return _cached_perms_data(
                 user_id, scope, user_is_admin, user_inherit_default_permissions,
                 explicit, algo, calculate_super_admin)
 
         start = time.time()
         result = compute_perm_tree(
             'permissions', 'v1', user_id, scope, user_is_admin,
              user_inherit_default_permissions, explicit, algo,
              calculate_super_admin)
 
         result_repr = []
         for k in result:
             result_repr.append((k, len(result[k])))
         total = time.time() - start
         log.debug('PERMISSION tree for user %s computed in %.4fs: %s',
                   user, total, result_repr)
 
         return result
 
     @property
     def is_default(self):
         return self.username == User.DEFAULT_USER
 
     @property
     def is_admin(self):
         return self.admin
 
     @property
     def is_user_object(self):
         return self.user_id is not None
 
     @property
     def repositories_admin(self):
         """
         Returns list of repositories you're an admin of
         """
         return [
             x[0] for x in self.permissions['repositories'].items()
             if x[1] == 'repository.admin']
 
     @property
     def repository_groups_admin(self):
         """
         Returns list of repository groups you're an admin of
         """
         return [
             x[0] for x in self.permissions['repositories_groups'].items()
             if x[1] == 'group.admin']
 
     @property
     def user_groups_admin(self):
         """
         Returns list of user groups you're an admin of
         """
         return [
             x[0] for x in self.permissions['user_groups'].items()
             if x[1] == 'usergroup.admin']
 
     def repo_acl_ids_from_stack(self, perms=None, prefix_filter=None, cache=False):
         if not perms:
             perms = AuthUser.repo_read_perms
         allowed_ids = []
         for k, stack_data in self.permissions['repositories'].perm_origin_stack.items():
             perm, origin, obj_id = stack_data[-1]  # last item is the current permission
             if prefix_filter and not k.startswith(prefix_filter):
                 continue
             if perm in perms:
                 allowed_ids.append(obj_id)
         return allowed_ids
 
     def repo_acl_ids(self, perms=None, name_filter=None, cache=False):
         """
         Returns list of repository ids that user have access to based on given
         perms. The cache flag should be only used in cases that are used for
         display purposes, NOT IN ANY CASE for permission checks.
         """
         from rhodecode.model.scm import RepoList
         if not perms:
             perms = AuthUser.repo_read_perms
 
         if not isinstance(perms, list):
             raise ValueError('perms parameter must be a list got {} instead'.format(perms))
 
         def _cached_repo_acl(perm_def, _name_filter):
             qry = Repository.query()
             if _name_filter:
                 ilike_expression = '%{}%'.format(_name_filter)
                 qry = qry.filter(
                     Repository.repo_name.ilike(ilike_expression))
 
             return [x.repo_id for x in
                     RepoList(qry, perm_set=perm_def, extra_kwargs={'user': self})]
 
         log.debug('Computing REPO ACL IDS user %s', self)
 
         cache_namespace_uid = 'cache_user_repo_acl_ids.{}'.format(self.user_id)
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache)
         def compute_repo_acl_ids(cache_ver, user_id, perm_def, _name_filter):
             return _cached_repo_acl(perm_def, _name_filter)
 
         start = time.time()
         result = compute_repo_acl_ids('v1', self.user_id, perms, name_filter)
         total = time.time() - start
         log.debug('REPO ACL IDS for user %s computed in %.4fs', self, total)
 
         return result
 
     def repo_group_acl_ids_from_stack(self, perms=None, prefix_filter=None, cache=False):
         if not perms:
             perms = AuthUser.repo_group_read_perms
         allowed_ids = []
         for k, stack_data in self.permissions['repositories_groups'].perm_origin_stack.items():
             perm, origin, obj_id = stack_data[-1]  # last item is the current permission
             if prefix_filter and not k.startswith(prefix_filter):
                 continue
             if perm in perms:
                 allowed_ids.append(obj_id)
         return allowed_ids
 
     def repo_group_acl_ids(self, perms=None, name_filter=None, cache=False):
         """
         Returns list of repository group ids that user have access to based on given
         perms. The cache flag should be only used in cases that are used for
         display purposes, NOT IN ANY CASE for permission checks.
         """
         from rhodecode.model.scm import RepoGroupList
         if not perms:
             perms = AuthUser.repo_group_read_perms
 
         if not isinstance(perms, list):
             raise ValueError('perms parameter must be a list got {} instead'.format(perms))
 
         def _cached_repo_group_acl(perm_def, _name_filter):
             qry = RepoGroup.query()
             if _name_filter:
                 ilike_expression = '%{}%'.format(_name_filter)
                 qry = qry.filter(
                     RepoGroup.group_name.ilike(ilike_expression))
 
             return [x.group_id for x in
                     RepoGroupList(qry, perm_set=perm_def, extra_kwargs={'user': self})]
 
         log.debug('Computing REPO GROUP ACL IDS user %s', self)
 
         cache_namespace_uid = 'cache_user_repo_group_acl_ids.{}'.format(self.user_id)
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache)
         def compute_repo_group_acl_ids(cache_ver, user_id, perm_def, _name_filter):
             return _cached_repo_group_acl(perm_def, _name_filter)
 
         start = time.time()
         result = compute_repo_group_acl_ids('v1', self.user_id, perms, name_filter)
         total = time.time() - start
         log.debug('REPO GROUP ACL IDS for user %s computed in %.4fs', self, total)
 
         return result
 
     def user_group_acl_ids_from_stack(self, perms=None, cache=False):
         if not perms:
             perms = AuthUser.user_group_read_perms
         allowed_ids = []
         for k, stack_data in self.permissions['user_groups'].perm_origin_stack.items():
             perm, origin, obj_id = stack_data[-1]  # last item is the current permission
             if perm in perms:
                 allowed_ids.append(obj_id)
         return allowed_ids
 
     def user_group_acl_ids(self, perms=None, name_filter=None, cache=False):
         """
         Returns list of user group ids that user have access to based on given
         perms. The cache flag should be only used in cases that are used for
         display purposes, NOT IN ANY CASE for permission checks.
         """
         from rhodecode.model.scm import UserGroupList
         if not perms:
             perms = AuthUser.user_group_read_perms
 
         if not isinstance(perms, list):
             raise ValueError('perms parameter must be a list got {} instead'.format(perms))
 
         def _cached_user_group_acl(perm_def, _name_filter):
             qry = UserGroup.query()
             if _name_filter:
                 ilike_expression = '%{}%'.format(_name_filter)
                 qry = qry.filter(
                     UserGroup.users_group_name.ilike(ilike_expression))
 
             return [x.users_group_id for x in
                     UserGroupList(qry, perm_set=perm_def, extra_kwargs={'user': self})]
 
         log.debug('Computing USER GROUP ACL IDS user %s', self)
 
         cache_namespace_uid = 'cache_user_user_group_acl_ids.{}'.format(self.user_id)
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache)
         def compute_user_group_acl_ids(cache_ver, user_id, perm_def, _name_filter):
             return _cached_user_group_acl(perm_def, _name_filter)
 
         start = time.time()
         result = compute_user_group_acl_ids('v1', self.user_id, perms, name_filter)
         total = time.time() - start
         log.debug('USER GROUP ACL IDS for user %s computed in %.4fs', self, total)
 
         return result
 
     @property
     def ip_allowed(self):
         """
         Checks if ip_addr used in constructor is allowed from defined list of
         allowed ip_addresses for user
 
         :returns: boolean, True if ip is in allowed ip range
         """
         # check IP
         inherit = self.inherit_default_permissions
         return AuthUser.check_ip_allowed(self.user_id, self.ip_addr,
                                          inherit_from_default=inherit)
 
     @property
     def personal_repo_group(self):
         return RepoGroup.get_user_personal_repo_group(self.user_id)
 
     @LazyProperty
     def feed_token(self):
         return self.get_instance().feed_token
 
     @LazyProperty
     def artifact_token(self):
         return self.get_instance().artifact_token
 
     @classmethod
     def check_ip_allowed(cls, user_id, ip_addr, inherit_from_default):
         allowed_ips = AuthUser.get_allowed_ips(
             user_id, cache=True, inherit_from_default=inherit_from_default)
         if check_ip_access(source_ip=ip_addr, allowed_ips=allowed_ips):
             log.debug('IP:%s for user %s is in range of %s',
                       ip_addr, user_id, allowed_ips)
             return True
         else:
             log.info('Access for IP:%s forbidden for user %s, '
                      'not in %s', ip_addr, user_id, allowed_ips,
                      extra={"ip": ip_addr, "user_id": user_id})
             return False
 
     def get_branch_permissions(self, repo_name, perms=None):
         perms = perms or self.permissions_with_scope({'repo_name': repo_name})
         branch_perms = perms.get('repository_branches', {})
         if not branch_perms:
             return {}
         repo_branch_perms = branch_perms.get(repo_name)
         return repo_branch_perms or {}
 
     def get_rule_and_branch_permission(self, repo_name, branch_name):
         """
         Check if this AuthUser has defined any permissions for branches. If any of
         the rules match in order, we return the matching permissions
         """
 
         rule = default_perm = ''
 
         repo_branch_perms = self.get_branch_permissions(repo_name=repo_name)
         if not repo_branch_perms:
             return rule, default_perm
 
         # now calculate the permissions
         for pattern, branch_perm in repo_branch_perms.items():
             if fnmatch.fnmatch(branch_name, pattern):
                 rule = '`{}`=>{}'.format(pattern, branch_perm)
                 return rule, branch_perm
 
         return rule, default_perm
 
     def get_notice_messages(self):
 
         notice_level = 'notice-error'
         notice_messages = []
         if self.is_default:
             return [], notice_level
 
         notices = UserNotice.query()\
             .filter(UserNotice.user_id == self.user_id)\
             .filter(UserNotice.notice_read == false())\
             .all()
 
         try:
             for entry in notices:
 
                 msg = {
                     'msg_id': entry.user_notice_id,
                     'level': entry.notification_level,
                     'subject': entry.notice_subject,
                     'body': entry.notice_body,
                 }
                 notice_messages.append(msg)
 
             log.debug('Got user %s %s messages', self, len(notice_messages))
 
             levels = [x['level'] for x in notice_messages]
             notice_level = 'notice-error' if 'error' in levels else 'notice-warning'
         except Exception:
             pass
 
         return notice_messages, notice_level
 
     def __repr__(self):
         return self.repr_user(self.user_id, self.username, self.ip_addr, self.is_authenticated)
 
     def set_authenticated(self, authenticated=True):
         if self.user_id != self.anonymous_user.user_id:
             self.is_authenticated = authenticated
 
     def get_cookie_store(self):
         return {
             'username': self.username,
             'password': md5(safe_bytes(self.password or '')),
             'user_id': self.user_id,
             'is_authenticated': self.is_authenticated
         }
 
     @classmethod
     def repr_user(cls, user_id=0, username='ANONYMOUS', ip='0.0.0.0', is_authenticated=False):
         tmpl = "<AuthUser('id:{}[{}] ip:{} auth:{}')>"
         return tmpl.format(user_id, username, ip, is_authenticated)
 
     @classmethod
     def from_cookie_store(cls, cookie_store):
         """
         Creates AuthUser from a cookie store
 
         :param cls:
         :param cookie_store:
         """
         user_id = cookie_store.get('user_id')
         username = cookie_store.get('username')
         api_key = cookie_store.get('api_key')
         return AuthUser(user_id, api_key, username)
 
     @classmethod
     def get_allowed_ips(cls, user_id, cache=False, inherit_from_default=False):
         _set = set()
 
         if inherit_from_default:
             def_user_id = User.get_default_user(cache=True).user_id
             default_ips = UserIpMap.query().filter(UserIpMap.user_id == def_user_id)
             if cache:
                 default_ips = default_ips.options(
                     FromCache("sql_cache_short", "get_user_ips_default"))
 
             # populate from default user
             for ip in default_ips:
                 try:
                     _set.add(ip.ip_addr)
                 except ObjectDeletedError:
                     # since we use heavy caching sometimes it happens that
                     # we get deleted objects here, we just skip them
                     pass
 
         # NOTE:(marcink) we don't want to load any rules for empty
         # user_id which is the case of access of non logged users when anonymous
         # access is disabled
         user_ips = []
         if user_id:
             user_ips = UserIpMap.query().filter(UserIpMap.user_id == user_id)
             if cache:
                 user_ips = user_ips.options(
                     FromCache("sql_cache_short", "get_user_ips_%s" % user_id))
 
         for ip in user_ips:
             try:
                 _set.add(ip.ip_addr)
             except ObjectDeletedError:
                 # since we use heavy caching sometimes it happens that we get
                 # deleted objects here, we just skip them
                 pass
         return _set or {ip for ip in ['0.0.0.0/0', '::/0']}
 
 
 def set_available_permissions(settings):
     """
     This function will propagate pyramid settings with all available defined
     permission given in db. We don't want to check each time from db for new
     permissions since adding a new permission also requires application restart
     ie. to decorate new views with the newly created permission
 
     :param settings: current pyramid registry.settings
 
     """
     log.debug('auth: getting information about all available permissions')
     try:
         sa = meta.Session
         all_perms = sa.query(Permission).all()
         settings.setdefault('available_permissions',
                             [x.permission_name for x in all_perms])
         log.debug('auth: set available permissions')
     except Exception:
         log.exception('Failed to fetch permissions from the database.')
         raise
 
 
 def get_csrf_token(session, force_new=False, save_if_missing=True):
     """
     Return the current authentication token, creating one if one doesn't
     already exist and the save_if_missing flag is present.
 
     :param session: pass in the pyramid session, else we use the global ones
     :param force_new: force to re-generate the token and store it in session
     :param save_if_missing: save the newly generated token if it's missing in
         session
     """
     # NOTE(marcink): probably should be replaced with below one from pyramid 1.9
     # from pyramid.csrf import get_csrf_token
 
     if (csrf_token_key not in session and save_if_missing) or force_new:
         token = sha1(ascii_bytes(str(random.getrandbits(128))))
         session[csrf_token_key] = token
         if hasattr(session, 'save'):
             session.save()
     return session.get(csrf_token_key)
 
 
 def get_request(perm_class_instance):
     from pyramid.threadlocal import get_current_request
     pyramid_request = get_current_request()
     return pyramid_request
 
 
 # CHECK DECORATORS
 class CSRFRequired(object):
     """
     Decorator for authenticating a form
 
     This decorator uses an authorization token stored in the client's
     session for prevention of certain Cross-site request forgery (CSRF)
     attacks (See
     http://en.wikipedia.org/wiki/Cross-site_request_forgery for more
     information).
 
     For use with the ``secure_form`` helper functions.
 
     """
     def __init__(self, token=csrf_token_key, header='X-CSRF-Token', except_methods=None):
         self.token = token
         self.header = header
         self.except_methods = except_methods or []
 
     def __call__(self, func):
         return get_cython_compat_decorator(self.__wrapper, func)
 
     def _get_csrf(self, _request):
         return _request.POST.get(self.token, _request.headers.get(self.header))
 
     def check_csrf(self, _request, cur_token):
         supplied_token = self._get_csrf(_request)
         return supplied_token and supplied_token == cur_token
 
     def _get_request(self):
         return get_request(self)
 
     def __wrapper(self, func, *fargs, **fkwargs):
         request = self._get_request()
 
         if request.method in self.except_methods:
             return func(*fargs, **fkwargs)
 
         cur_token = get_csrf_token(request.session, save_if_missing=False)
         if self.check_csrf(request, cur_token):
             if request.POST.get(self.token):
                 del request.POST[self.token]
             return func(*fargs, **fkwargs)
         else:
             reason = 'token-missing'
             supplied_token = self._get_csrf(request)
             if supplied_token and cur_token != supplied_token:
                 reason = 'token-mismatch [%s:%s]' % (
                     cur_token or ''[:6], supplied_token or ''[:6])
 
             csrf_message = \
                 ("Cross-site request forgery detected, request denied. See "
                  "http://en.wikipedia.org/wiki/Cross-site_request_forgery for "
                  "more information.")
         log.warn('Cross-site request forgery detected, request %r DENIED: %s '
                  'REMOTE_ADDR:%s, HEADERS:%s' % (
                      request, reason, request.remote_addr, request.headers))
 
         raise HTTPForbidden(explanation=csrf_message)
 
 
 class LoginRequired(object):
     """
     Must be logged in to execute this function else
     redirect to login page
 
     :param api_access: if enabled this checks only for valid auth token
         and grants access based on valid token
     """
     def __init__(self, auth_token_access=None):
         self.auth_token_access = auth_token_access
         if self.auth_token_access:
             valid_type = set(auth_token_access).intersection(set(UserApiKeys.ROLES))
             if not valid_type:
                 raise ValueError('auth_token_access must be on of {}, got {}'.format(
                     UserApiKeys.ROLES, auth_token_access))
 
     def __call__(self, func):
         return get_cython_compat_decorator(self.__wrapper, func)
 
     def _get_request(self):
         return get_request(self)
 
     def __wrapper(self, func, *fargs, **fkwargs):
         from rhodecode.lib import helpers as h
         cls = fargs[0]
         user = cls._rhodecode_user
         request = self._get_request()
         _ = request.translate
 
         loc = "%s:%s" % (cls.__class__.__name__, func.__name__)
         log.debug('Starting login restriction checks for user: %s', user)
         # check if our IP is allowed
         ip_access_valid = True
         if not user.ip_allowed:
             h.flash(h.literal(_('IP {} not allowed'.format(user.ip_addr))),
                     category='warning')
             ip_access_valid = False
 
         # we used stored token that is extract from GET or URL param (if any)
         _auth_token = request.user_auth_token
 
         # check if we used an AUTH_TOKEN and it's a valid one
         # defined white-list of controllers which API access will be enabled
         whitelist = None
         if self.auth_token_access:
             # since this location is allowed by @LoginRequired decorator it's our
             # only whitelist
             whitelist = [loc]
         auth_token_access_valid = allowed_auth_token_access(
             loc, whitelist=whitelist, auth_token=_auth_token)
 
         # explicit controller is enabled or API is in our whitelist
         if auth_token_access_valid:
             log.debug('Checking AUTH TOKEN access for %s', cls)
             db_user = user.get_instance()
 
             if db_user:
                 if self.auth_token_access:
                     roles = self.auth_token_access
                 else:
                     roles = [UserApiKeys.ROLE_HTTP]
                 log.debug('AUTH TOKEN: checking auth for user %s and roles %s',
                           db_user, roles)
                 token_match = db_user.authenticate_by_token(
                     _auth_token, roles=roles)
             else:
                 log.debug('Unable to fetch db instance for auth user: %s', user)
                 token_match = False
 
             if _auth_token and token_match:
                 auth_token_access_valid = True
                 log.debug('AUTH TOKEN ****%s is VALID', _auth_token[-4:])
             else:
                 auth_token_access_valid = False
                 if not _auth_token:
                     log.debug("AUTH TOKEN *NOT* present in request")
                 else:
                     log.warning("AUTH TOKEN ****%s *NOT* valid", _auth_token[-4:])
 
         log.debug('Checking if %s is authenticated @ %s', user.username, loc)
         reason = 'RHODECODE_AUTH' if user.is_authenticated \
             else 'AUTH_TOKEN_AUTH'
 
         if ip_access_valid and (
                 user.is_authenticated or auth_token_access_valid):
             log.info('user %s authenticating with:%s IS authenticated on func %s',
                      user, reason, loc)
 
             return func(*fargs, **fkwargs)
         else:
             log.warning(
                 'user %s authenticating with:%s NOT authenticated on '
                 'func: %s: IP_ACCESS:%s AUTH_TOKEN_ACCESS:%s',
                 user, reason, loc, ip_access_valid, auth_token_access_valid)
             # we preserve the get PARAM
             came_from = get_came_from(request)
 
             log.debug('redirecting to login page with %s', came_from)
             raise HTTPFound(
                 h.route_path('login', _query={'came_from': came_from}))
 
 
 class NotAnonymous(object):
     """
     Must be logged in to execute this function else
     redirect to login page
     """
 
     def __call__(self, func):
         return get_cython_compat_decorator(self.__wrapper, func)
 
     def _get_request(self):
         return get_request(self)
 
     def __wrapper(self, func, *fargs, **fkwargs):
         import rhodecode.lib.helpers as h
         cls = fargs[0]
         self.user = cls._rhodecode_user
         request = self._get_request()
         _ = request.translate
         log.debug('Checking if user is not anonymous @%s', cls)
 
         anonymous = self.user.username == User.DEFAULT_USER
 
         if anonymous:
             came_from = get_came_from(request)
             h.flash(_('You need to be a registered user to '
                       'perform this action'),
                     category='warning')
             raise HTTPFound(
                 h.route_path('login', _query={'came_from': came_from}))
         else:
             return func(*fargs, **fkwargs)
 
 
 class PermsDecorator(object):
     """
     Base class for controller decorators, we extract the current user from
     the class itself, which has it stored in base controllers
     """
 
     def __init__(self, *required_perms):
         self.required_perms = set(required_perms)
 
     def __call__(self, func):
         return get_cython_compat_decorator(self.__wrapper, func)
 
     def _get_request(self):
         return get_request(self)
 
     def __wrapper(self, func, *fargs, **fkwargs):
         import rhodecode.lib.helpers as h
         cls = fargs[0]
         _user = cls._rhodecode_user
         request = self._get_request()
         _ = request.translate
 
         log.debug('checking %s permissions %s for %s %s',
                   self.__class__.__name__, self.required_perms, cls, _user)
 
         if self.check_permissions(_user):
             log.debug('Permission granted for %s %s', cls, _user)
             return func(*fargs, **fkwargs)
 
         else:
             log.debug('Permission denied for %s %s', cls, _user)
             anonymous = _user.username == User.DEFAULT_USER
 
             if anonymous:
                 came_from = get_came_from(self._get_request())
                 h.flash(_('You need to be signed in to view this page'),
                         category='warning')
                 raise HTTPFound(
                     h.route_path('login', _query={'came_from': came_from}))
 
             else:
                 # redirect with 404 to prevent resource discovery
                 raise HTTPNotFound()
 
     def check_permissions(self, user):
         """Dummy function for overriding"""
         raise NotImplementedError(
             'You have to write this function in child class')
 
 
 class HasPermissionAllDecorator(PermsDecorator):
     """
     Checks for access permission for all given predicates. All of them
     have to be meet in order to fulfill the request
     """
 
     def check_permissions(self, user):
         perms = user.permissions_with_scope({})
         if self.required_perms.issubset(perms['global']):
             return True
         return False
 
 
 class HasPermissionAnyDecorator(PermsDecorator):
     """
     Checks for access permission for any of given predicates. In order to
     fulfill the request any of predicates must be meet
     """
 
     def check_permissions(self, user):
         perms = user.permissions_with_scope({})
         if self.required_perms.intersection(perms['global']):
             return True
         return False
 
 
 class HasRepoPermissionAllDecorator(PermsDecorator):
     """
     Checks for access permission for all given predicates for specific
     repository. All of them have to be meet in order to fulfill the request
     """
     def _get_repo_name(self):
         _request = self._get_request()
         return get_repo_slug(_request)
 
     def check_permissions(self, user):
         perms = user.permissions
         repo_name = self._get_repo_name()
 
         try:
             user_perms = {perms['repositories'][repo_name]}
         except KeyError:
             log.debug('cannot locate repo with name: `%s` in permissions defs',
                       repo_name)
             return False
 
         log.debug('checking `%s` permissions for repo `%s`',
                   user_perms, repo_name)
         if self.required_perms.issubset(user_perms):
             return True
         return False
 
 
 class HasRepoPermissionAnyDecorator(PermsDecorator):
     """
     Checks for access permission for any of given predicates for specific
     repository. In order to fulfill the request any of predicates must be meet
     """
     def _get_repo_name(self):
         _request = self._get_request()
         return get_repo_slug(_request)
 
     def check_permissions(self, user):
         perms = user.permissions
         repo_name = self._get_repo_name()
 
         try:
             user_perms = {perms['repositories'][repo_name]}
         except KeyError:
             log.debug(
                 'cannot locate repo with name: `%s` in permissions defs',
                 repo_name)
             return False
 
         log.debug('checking `%s` permissions for repo `%s`',
                   user_perms, repo_name)
         if self.required_perms.intersection(user_perms):
             return True
         return False
 
 
 class HasRepoGroupPermissionAllDecorator(PermsDecorator):
     """
     Checks for access permission for all given predicates for specific
     repository group. All of them have to be meet in order to
     fulfill the request
     """
     def _get_repo_group_name(self):
         _request = self._get_request()
         return get_repo_group_slug(_request)
 
     def check_permissions(self, user):
         perms = user.permissions
         group_name = self._get_repo_group_name()
         try:
             user_perms = {perms['repositories_groups'][group_name]}
         except KeyError:
             log.debug(
                 'cannot locate repo group with name: `%s` in permissions defs',
                 group_name)
             return False
 
         log.debug('checking `%s` permissions for repo group `%s`',
                   user_perms, group_name)
         if self.required_perms.issubset(user_perms):
             return True
         return False
 
 
 class HasRepoGroupPermissionAnyDecorator(PermsDecorator):
     """
     Checks for access permission for any of given predicates for specific
     repository group. In order to fulfill the request any
     of predicates must be met
     """
     def _get_repo_group_name(self):
         _request = self._get_request()
         return get_repo_group_slug(_request)
 
     def check_permissions(self, user):
         perms = user.permissions
         group_name = self._get_repo_group_name()
 
         try:
             user_perms = {perms['repositories_groups'][group_name]}
         except KeyError:
             log.debug(
                 'cannot locate repo group with name: `%s` in permissions defs',
                 group_name)
             return False
 
         log.debug('checking `%s` permissions for repo group `%s`',
                   user_perms, group_name)
         if self.required_perms.intersection(user_perms):
             return True
         return False
 
 
 class HasUserGroupPermissionAllDecorator(PermsDecorator):
     """
     Checks for access permission for all given predicates for specific
     user group. All of them have to be meet in order to fulfill the request
     """
     def _get_user_group_name(self):
         _request = self._get_request()
         return get_user_group_slug(_request)
 
     def check_permissions(self, user):
         perms = user.permissions
         group_name = self._get_user_group_name()
         try:
             user_perms = {perms['user_groups'][group_name]}
         except KeyError:
             return False
 
         if self.required_perms.issubset(user_perms):
             return True
         return False
 
 
 class HasUserGroupPermissionAnyDecorator(PermsDecorator):
     """
     Checks for access permission for any of given predicates for specific
     user group. In order to fulfill the request any of predicates must be meet
     """
     def _get_user_group_name(self):
         _request = self._get_request()
         return get_user_group_slug(_request)
 
     def check_permissions(self, user):
         perms = user.permissions
         group_name = self._get_user_group_name()
         try:
             user_perms = {perms['user_groups'][group_name]}
         except KeyError:
             return False
 
         if self.required_perms.intersection(user_perms):
             return True
         return False
 
 
 # CHECK FUNCTIONS
 class PermsFunction(object):
     """Base function for other check functions"""
 
     def __init__(self, *perms):
         self.required_perms = set(perms)
         self.repo_name = None
         self.repo_group_name = None
         self.user_group_name = None
 
     def __bool__(self):
         import inspect
         frame = inspect.currentframe()
         stack_trace = traceback.format_stack(frame)
         log.error('Checking bool value on a class instance of perm '
                   'function is not allowed: %s', ''.join(stack_trace))
         # rather than throwing errors, here we always return False so if by
         # accident someone checks truth for just an instance it will always end
         # up in returning False
         return False
     __nonzero__ = __bool__
 
     def __call__(self, check_location='', user=None):
         if not user:
             log.debug('Using user attribute from global request')
             request = self._get_request()
             user = request.user
 
         # init auth user if not already given
         if not isinstance(user, AuthUser):
             log.debug('Wrapping user %s into AuthUser', user)
             user = AuthUser(user.user_id)
 
         cls_name = self.__class__.__name__
         check_scope = self._get_check_scope(cls_name)
         check_location = check_location or 'unspecified location'
 
         log.debug('checking cls:%s %s usr:%s %s @ %s', cls_name,
                   self.required_perms, user, check_scope, check_location)
         if not user:
             log.warning('Empty user given for permission check')
             return False
 
         if self.check_permissions(user):
             log.debug('Permission to repo:`%s` GRANTED for user:`%s` @ %s',
                       check_scope, user, check_location)
             return True
 
         else:
             log.debug('Permission to repo:`%s` DENIED for user:`%s` @ %s',
                       check_scope, user, check_location)
             return False
 
     def _get_request(self):
         return get_request(self)
 
     def _get_check_scope(self, cls_name):
         return {
             'HasPermissionAll':          'GLOBAL',
             'HasPermissionAny':          'GLOBAL',
             'HasRepoPermissionAll':      'repo:%s' % self.repo_name,
             'HasRepoPermissionAny':      'repo:%s' % self.repo_name,
             'HasRepoGroupPermissionAll': 'repo_group:%s' % self.repo_group_name,
             'HasRepoGroupPermissionAny': 'repo_group:%s' % self.repo_group_name,
             'HasUserGroupPermissionAll': 'user_group:%s' % self.user_group_name,
             'HasUserGroupPermissionAny': 'user_group:%s' % self.user_group_name,
         }.get(cls_name, '?:%s' % cls_name)
 
     def check_permissions(self, user):
         """Dummy function for overriding"""
         raise Exception('You have to write this function in child class')
 
 
 class HasPermissionAll(PermsFunction):
     def check_permissions(self, user):
         perms = user.permissions_with_scope({})
         if self.required_perms.issubset(perms.get('global')):
             return True
         return False
 
 
 class HasPermissionAny(PermsFunction):
     def check_permissions(self, user):
         perms = user.permissions_with_scope({})
         if self.required_perms.intersection(perms.get('global')):
             return True
         return False
 
 
 class HasRepoPermissionAll(PermsFunction):
     def __call__(self, repo_name=None, check_location='', user=None):
         self.repo_name = repo_name
         return super(HasRepoPermissionAll, self).__call__(check_location, user)
 
     def _get_repo_name(self):
         if not self.repo_name:
             _request = self._get_request()
             self.repo_name = get_repo_slug(_request)
         return self.repo_name
 
     def check_permissions(self, user):
         self.repo_name = self._get_repo_name()
         perms = user.permissions
         try:
             user_perms = {perms['repositories'][self.repo_name]}
         except KeyError:
             return False
         if self.required_perms.issubset(user_perms):
             return True
         return False
 
 
 class HasRepoPermissionAny(PermsFunction):
     def __call__(self, repo_name=None, check_location='', user=None):
         self.repo_name = repo_name
         return super(HasRepoPermissionAny, self).__call__(check_location, user)
 
     def _get_repo_name(self):
         if not self.repo_name:
             _request = self._get_request()
             self.repo_name = get_repo_slug(_request)
         return self.repo_name
 
     def check_permissions(self, user):
         self.repo_name = self._get_repo_name()
         perms = user.permissions
         try:
             user_perms = {perms['repositories'][self.repo_name]}
         except KeyError:
             return False
         if self.required_perms.intersection(user_perms):
             return True
         return False
 
 
 class HasRepoGroupPermissionAny(PermsFunction):
     def __call__(self, group_name=None, check_location='', user=None):
         self.repo_group_name = group_name
         return super(HasRepoGroupPermissionAny, self).__call__(check_location, user)
 
     def check_permissions(self, user):
         perms = user.permissions
         try:
             user_perms = {perms['repositories_groups'][self.repo_group_name]}
         except KeyError:
             return False
         if self.required_perms.intersection(user_perms):
             return True
         return False
 
 
 class HasRepoGroupPermissionAll(PermsFunction):
     def __call__(self, group_name=None, check_location='', user=None):
         self.repo_group_name = group_name
         return super(HasRepoGroupPermissionAll, self).__call__(check_location, user)
 
     def check_permissions(self, user):
         perms = user.permissions
         try:
             user_perms = {perms['repositories_groups'][self.repo_group_name]}
         except KeyError:
             return False
         if self.required_perms.issubset(user_perms):
             return True
         return False
 
 
 class HasUserGroupPermissionAny(PermsFunction):
     def __call__(self, user_group_name=None, check_location='', user=None):
         self.user_group_name = user_group_name
         return super(HasUserGroupPermissionAny, self).__call__(check_location, user)
 
     def check_permissions(self, user):
         perms = user.permissions
         try:
             user_perms = {perms['user_groups'][self.user_group_name]}
         except KeyError:
             return False
         if self.required_perms.intersection(user_perms):
             return True
         return False
 
 
 class HasUserGroupPermissionAll(PermsFunction):
     def __call__(self, user_group_name=None, check_location='', user=None):
         self.user_group_name = user_group_name
         return super(HasUserGroupPermissionAll, self).__call__(check_location, user)
 
     def check_permissions(self, user):
         perms = user.permissions
         try:
             user_perms = {perms['user_groups'][self.user_group_name]}
         except KeyError:
             return False
         if self.required_perms.issubset(user_perms):
             return True
         return False
 
 
 # SPECIAL VERSION TO HANDLE MIDDLEWARE AUTH
 class HasPermissionAnyMiddleware(object):
     def __init__(self, *perms):
         self.required_perms = set(perms)
 
     def __call__(self, auth_user, repo_name):
         # # repo_name MUST be unicode, since we handle keys in permission
         # # dict by unicode
         #TODO: verify
         # repo_name = safe_unicode(repo_name)
 
         log.debug(
             'Checking VCS protocol permissions %s for user:%s repo:`%s`',
             self.required_perms, auth_user, repo_name)
 
         if self.check_permissions(auth_user, repo_name):
             log.debug('Permission to repo:`%s` GRANTED for user:%s @ %s',
                       repo_name, auth_user, 'PermissionMiddleware')
             return True
 
         else:
             log.debug('Permission to repo:`%s` DENIED for user:%s @ %s',
                       repo_name, auth_user, 'PermissionMiddleware')
             return False
 
     def check_permissions(self, user, repo_name):
         perms = user.permissions_with_scope({'repo_name': repo_name})
 
         try:
             user_perms = {perms['repositories'][repo_name]}
         except Exception:
             log.exception('Error while accessing user permissions')
             return False
 
         if self.required_perms.intersection(user_perms):
             return True
         return False
 
 
 # SPECIAL VERSION TO HANDLE API AUTH
 class _BaseApiPerm(object):
     def __init__(self, *perms):
         self.required_perms = set(perms)
 
     def __call__(self, check_location=None, user=None, repo_name=None,
                  group_name=None, user_group_name=None):
         cls_name = self.__class__.__name__
         check_scope = 'global:%s' % (self.required_perms,)
         if repo_name:
             check_scope += ', repo_name:%s' % (repo_name,)
 
         if group_name:
             check_scope += ', repo_group_name:%s' % (group_name,)
 
         if user_group_name:
             check_scope += ', user_group_name:%s' % (user_group_name,)
 
         log.debug('checking cls:%s %s %s @ %s',
                   cls_name, self.required_perms, check_scope, check_location)
         if not user:
             log.debug('Empty User passed into arguments')
             return False
 
         # process user
         if not isinstance(user, AuthUser):
             user = AuthUser(user.user_id)
         if not check_location:
             check_location = 'unspecified'
         if self.check_permissions(user.permissions, repo_name, group_name,
                                   user_group_name):
             log.debug('Permission to repo:`%s` GRANTED for user:`%s` @ %s',
                       check_scope, user, check_location)
             return True
 
         else:
             log.debug('Permission to repo:`%s` DENIED for user:`%s` @ %s',
                       check_scope, user, check_location)
             return False
 
     def check_permissions(self, perm_defs, repo_name=None, group_name=None,
                           user_group_name=None):
         """
         implement in child class should return True if permissions are ok,
         False otherwise
 
         :param perm_defs: dict with permission definitions
         :param repo_name: repo name
         """
         raise NotImplementedError()
 
 
 class HasPermissionAllApi(_BaseApiPerm):
     def check_permissions(self, perm_defs, repo_name=None, group_name=None,
                           user_group_name=None):
         if self.required_perms.issubset(perm_defs.get('global')):
             return True
         return False
 
 
 class HasPermissionAnyApi(_BaseApiPerm):
     def check_permissions(self, perm_defs, repo_name=None, group_name=None,
                           user_group_name=None):
         if self.required_perms.intersection(perm_defs.get('global')):
             return True
         return False
 
 
 class HasRepoPermissionAllApi(_BaseApiPerm):
     def check_permissions(self, perm_defs, repo_name=None, group_name=None,
                           user_group_name=None):
         try:
             _user_perms = {perm_defs['repositories'][repo_name]}
         except KeyError:
             log.warning(traceback.format_exc())
             return False
         if self.required_perms.issubset(_user_perms):
             return True
         return False
 
 
 class HasRepoPermissionAnyApi(_BaseApiPerm):
     def check_permissions(self, perm_defs, repo_name=None, group_name=None,
                           user_group_name=None):
         try:
             _user_perms = {perm_defs['repositories'][repo_name]}
         except KeyError:
             log.warning(traceback.format_exc())
             return False
         if self.required_perms.intersection(_user_perms):
             return True
         return False
 
 
 class HasRepoGroupPermissionAnyApi(_BaseApiPerm):
     def check_permissions(self, perm_defs, repo_name=None, group_name=None,
                           user_group_name=None):
         try:
             _user_perms = {perm_defs['repositories_groups'][group_name]}
         except KeyError:
             log.warning(traceback.format_exc())
             return False
         if self.required_perms.intersection(_user_perms):
             return True
         return False
 
 
 class HasRepoGroupPermissionAllApi(_BaseApiPerm):
     def check_permissions(self, perm_defs, repo_name=None, group_name=None,
                           user_group_name=None):
         try:
             _user_perms = {perm_defs['repositories_groups'][group_name]}
         except KeyError:
             log.warning(traceback.format_exc())
             return False
         if self.required_perms.issubset(_user_perms):
             return True
         return False
 
 
 class HasUserGroupPermissionAnyApi(_BaseApiPerm):
     def check_permissions(self, perm_defs, repo_name=None, group_name=None,
                           user_group_name=None):
         try:
             _user_perms = {perm_defs['user_groups'][user_group_name]}
         except KeyError:
             log.warning(traceback.format_exc())
             return False
         if self.required_perms.intersection(_user_perms):
             return True
         return False
 
 
 def check_ip_access(source_ip, allowed_ips=None):
     """
     Checks if source_ip is a subnet of any of allowed_ips.
 
     :param source_ip:
     :param allowed_ips: list of allowed ips together with mask
     """
     log.debug('checking if ip:%s is subnet of %s', source_ip, allowed_ips)
     source_ip_address = ipaddress.ip_address(source_ip)
     if isinstance(allowed_ips, (tuple, list, set)):
         for ip in allowed_ips:
             #TODO: verify
             #ip = safe_unicode(ip)
             try:
                 network_address = ipaddress.ip_network(ip, strict=False)
                 if source_ip_address in network_address:
                     log.debug('IP %s is network %s', source_ip_address, network_address)
                     return True
                 # for any case we cannot determine the IP, don't crash just
                 # skip it and log as error, we want to say forbidden still when
                 # sending bad IP
             except Exception:
                 log.error(traceback.format_exc())
                 continue
     return False
 
 
 def get_cython_compat_decorator(wrapper, func):
     """
     Creates a cython compatible decorator. The previously used
     decorator.decorator() function seems to be incompatible with cython.
 
     :param wrapper: __wrapper method of the decorator class
     :param func: decorated function
     """
     @wraps(func)
     def local_wrapper(*args, **kwds):
         return wrapper(func, *args, **kwds)
     local_wrapper.__wrapped__ = func
     return local_wrapper
 
 

rhodecode/lib/base.py

@@ -1,611 +1,610 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 The base Controller API
 Provides the BaseController class for subclassing. And usage in different
 controllers
 """
 
 import logging
 import socket
 
 import markupsafe
 import ipaddress
 
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpexceptions import HTTPUnauthorized, HTTPForbidden, get_exception
 from paste.httpheaders import WWW_AUTHENTICATE, AUTHORIZATION
 
 import rhodecode
 from rhodecode.authentication.base import VCS_TYPE
 from rhodecode.lib import auth, utils2
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import AuthUser, CookieStoreWrapper
 from rhodecode.lib.exceptions import UserCreationError
 from rhodecode.lib.utils import (password_changed, get_enabled_hook_classes)
 from rhodecode.lib.utils2 import (
     str2bool, safe_unicode, AttributeDict, safe_int, sha1, aslist, safe_str)
 from rhodecode.model.db import Repository, User, ChangesetComment, UserBookmark
 from rhodecode.model.notification import NotificationModel
 from rhodecode.model.settings import VcsSettingsModel, SettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def _filter_proxy(ip):
     """
     Passed in IP addresses in HEADERS can be in a special format of multiple
     ips. Those comma separated IPs are passed from various proxies in the
     chain of request processing. The left-most being the original client.
     We only care about the first IP which came from the org. client.
 
     :param ip: ip string from headers
     """
     if ',' in ip:
         _ips = ip.split(',')
         _first_ip = _ips[0].strip()
         log.debug('Got multiple IPs %s, using %s', ','.join(_ips), _first_ip)
         return _first_ip
     return ip
 
 
 def _filter_port(ip):
     """
     Removes a port from ip, there are 4 main cases to handle here.
     - ipv4 eg. 127.0.0.1
     - ipv6 eg. ::1
     - ipv4+port eg. 127.0.0.1:8080
     - ipv6+port eg. [::1]:8080
 
     :param ip:
     """
     def is_ipv6(ip_addr):
         if hasattr(socket, 'inet_pton'):
             try:
                 socket.inet_pton(socket.AF_INET6, ip_addr)
             except socket.error:
                 return False
         else:
             # fallback to ipaddress
             try:
                 ipaddress.IPv6Address(safe_str(ip_addr))
             except Exception:
                 return False
         return True
 
     if ':' not in ip:  # must be ipv4 pure ip
         return ip
 
     if '[' in ip and ']' in ip:  # ipv6 with port
         return ip.split(']')[0][1:].lower()
 
     # must be ipv6 or ipv4 with port
     if is_ipv6(ip):
         return ip
     else:
         ip, _port = ip.split(':')[:2]  # means ipv4+port
         return ip
 
 
 def get_ip_addr(environ):
     proxy_key = 'HTTP_X_REAL_IP'
     proxy_key2 = 'HTTP_X_FORWARDED_FOR'
     def_key = 'REMOTE_ADDR'
     _filters = lambda x: _filter_port(_filter_proxy(x))
 
     ip = environ.get(proxy_key)
     if ip:
         return _filters(ip)
 
     ip = environ.get(proxy_key2)
     if ip:
         return _filters(ip)
 
     ip = environ.get(def_key, '0.0.0.0')
     return _filters(ip)
 
 
 def get_server_ip_addr(environ, log_errors=True):
     hostname = environ.get('SERVER_NAME')
     try:
         return socket.gethostbyname(hostname)
     except Exception as e:
         if log_errors:
             # in some cases this lookup is not possible, and we don't want to
             # make it an exception in logs
             log.exception('Could not retrieve server ip address: %s', e)
         return hostname
 
 
 def get_server_port(environ):
     return environ.get('SERVER_PORT')
 
 
 def get_access_path(environ):
     path = environ.get('PATH_INFO')
     org_req = environ.get('pylons.original_request')
     if org_req:
         path = org_req.environ.get('PATH_INFO')
     return path
 
 
 def get_user_agent(environ):
     return environ.get('HTTP_USER_AGENT')
 
 
 def vcs_operation_context(
         environ, repo_name, username, action, scm, check_locking=True,
         is_shadow_repo=False, check_branch_perms=False, detect_force_push=False):
     """
     Generate the context for a vcs operation, e.g. push or pull.
 
     This context is passed over the layers so that hooks triggered by the
     vcs operation know details like the user, the user's IP address etc.
 
     :param check_locking: Allows to switch of the computation of the locking
         data. This serves mainly the need of the simplevcs middleware to be
         able to disable this for certain operations.
 
     """
     # Tri-state value: False: unlock, None: nothing, True: lock
     make_lock = None
     locked_by = [None, None, None]
     is_anonymous = username == User.DEFAULT_USER
     user = User.get_by_username(username)
     if not is_anonymous and check_locking:
         log.debug('Checking locking on repository "%s"', repo_name)
         repo = Repository.get_by_repo_name(repo_name)
         make_lock, __, locked_by = repo.get_locking_state(
             action, user.user_id)
     user_id = user.user_id
     settings_model = VcsSettingsModel(repo=repo_name)
     ui_settings = settings_model.get_ui_settings()
 
     # NOTE(marcink): This should be also in sync with
     # rhodecode/apps/ssh_support/lib/backends/base.py:update_environment scm_data
     store = [x for x in ui_settings if x.key == '/']
     repo_store = ''
     if store:
         repo_store = store[0].value
 
     scm_data = {
         'ip': get_ip_addr(environ),
         'username': username,
         'user_id': user_id,
         'action': action,
         'repository': repo_name,
         'scm': scm,
         'config': rhodecode.CONFIG['__file__'],
         'repo_store': repo_store,
         'make_lock': make_lock,
         'locked_by': locked_by,
         'server_url': utils2.get_server_url(environ),
         'user_agent': get_user_agent(environ),
         'hooks': get_enabled_hook_classes(ui_settings),
         'is_shadow_repo': is_shadow_repo,
         'detect_force_push': detect_force_push,
         'check_branch_perms': check_branch_perms,
     }
     return scm_data
 
 
 class BasicAuth(AuthBasicAuthenticator):
 
     def __init__(self, realm, authfunc, registry, auth_http_code=None,
                  initial_call_detection=False, acl_repo_name=None, rc_realm=''):
         self.realm = realm
         self.rc_realm = rc_realm
         self.initial_call = initial_call_detection
         self.authfunc = authfunc
         self.registry = registry
         self.acl_repo_name = acl_repo_name
         self._rc_auth_http_code = auth_http_code
 
     def _get_response_from_code(self, http_code):
         try:
             return get_exception(safe_int(http_code))
         except Exception:
             log.exception('Failed to fetch response for code %s', http_code)
             return HTTPForbidden
 
     def get_rc_realm(self):
         return safe_str(self.rc_realm)
 
     def build_authentication(self):
         head = WWW_AUTHENTICATE.tuples('Basic realm="%s"' % self.realm)
         if self._rc_auth_http_code and not self.initial_call:
             # return alternative HTTP code if alternative http return code
             # is specified in RhodeCode config, but ONLY if it's not the
             # FIRST call
             custom_response_klass = self._get_response_from_code(
                 self._rc_auth_http_code)
             return custom_response_klass(headers=head)
         return HTTPUnauthorized(headers=head)
 
     def authenticate(self, environ):
         authorization = AUTHORIZATION(environ)
         if not authorization:
             return self.build_authentication()
         (authmeth, auth) = authorization.split(' ', 1)
         if 'basic' != authmeth.lower():
             return self.build_authentication()
         auth = auth.strip().decode('base64')
         _parts = auth.split(':', 1)
         if len(_parts) == 2:
             username, password = _parts
             auth_data = self.authfunc(
                     username, password, environ, VCS_TYPE,
                     registry=self.registry, acl_repo_name=self.acl_repo_name)
             if auth_data:
                 return {'username': username, 'auth_data': auth_data}
             if username and password:
                 # we mark that we actually executed authentication once, at
                 # that point we can use the alternative auth code
                 self.initial_call = False
 
         return self.build_authentication()
 
     __call__ = authenticate
 
 
 def calculate_version_hash(config):
     return sha1(
         config.get('beaker.session.secret', '') +
         rhodecode.__version__)[:8]
 
 
 def get_current_lang(request):
     # NOTE(marcink): remove after pyramid move
     try:
         return translation.get_lang()[0]
     except:
         pass
 
     return getattr(request, '_LOCALE_', request.locale_name)
 
 
 def attach_context_attributes(context, request, user_id=None, is_api=None):
     """
     Attach variables into template context called `c`.
     """
     config = request.registry.settings
 
     rc_config = SettingsModel().get_all_settings(cache=True, from_request=False)
     context.rc_config = rc_config
     context.rhodecode_version = rhodecode.__version__
     context.rhodecode_edition = config.get('rhodecode.edition')
     context.rhodecode_edition_id = config.get('rhodecode.edition_id')
     # unique secret + version does not leak the version but keep consistency
     context.rhodecode_version_hash = calculate_version_hash(config)
 
     # Default language set for the incoming request
     context.language = get_current_lang(request)
 
     # Visual options
     context.visual = AttributeDict({})
 
     # DB stored Visual Items
     context.visual.show_public_icon = str2bool(
         rc_config.get('rhodecode_show_public_icon'))
     context.visual.show_private_icon = str2bool(
         rc_config.get('rhodecode_show_private_icon'))
     context.visual.stylify_metatags = str2bool(
         rc_config.get('rhodecode_stylify_metatags'))
     context.visual.dashboard_items = safe_int(
         rc_config.get('rhodecode_dashboard_items', 100))
     context.visual.admin_grid_items = safe_int(
         rc_config.get('rhodecode_admin_grid_items', 100))
     context.visual.show_revision_number = str2bool(
         rc_config.get('rhodecode_show_revision_number', True))
     context.visual.show_sha_length = safe_int(
         rc_config.get('rhodecode_show_sha_length', 100))
     context.visual.repository_fields = str2bool(
         rc_config.get('rhodecode_repository_fields'))
     context.visual.show_version = str2bool(
         rc_config.get('rhodecode_show_version'))
     context.visual.use_gravatar = str2bool(
         rc_config.get('rhodecode_use_gravatar'))
     context.visual.gravatar_url = rc_config.get('rhodecode_gravatar_url')
     context.visual.default_renderer = rc_config.get(
         'rhodecode_markup_renderer', 'rst')
     context.visual.comment_types = ChangesetComment.COMMENT_TYPES
     context.visual.rhodecode_support_url = \
         rc_config.get('rhodecode_support_url') or h.route_url('rhodecode_support')
 
     context.visual.affected_files_cut_off = 60
 
     context.pre_code = rc_config.get('rhodecode_pre_code')
     context.post_code = rc_config.get('rhodecode_post_code')
     context.rhodecode_name = rc_config.get('rhodecode_title')
     context.default_encodings = aslist(config.get('default_encoding'), sep=',')
     # if we have specified default_encoding in the request, it has more
     # priority
     if request.GET.get('default_encoding'):
         context.default_encodings.insert(0, request.GET.get('default_encoding'))
     context.clone_uri_tmpl = rc_config.get('rhodecode_clone_uri_tmpl')
     context.clone_uri_id_tmpl = rc_config.get('rhodecode_clone_uri_id_tmpl')
     context.clone_uri_ssh_tmpl = rc_config.get('rhodecode_clone_uri_ssh_tmpl')
 
     # INI stored
     context.labs_active = str2bool(
         config.get('labs_settings_active', 'false'))
     context.ssh_enabled = str2bool(
         config.get('ssh.generate_authorized_keyfile', 'false'))
     context.ssh_key_generator_enabled = str2bool(
         config.get('ssh.enable_ui_key_generator', 'true'))
 
     context.visual.allow_repo_location_change = str2bool(
         config.get('allow_repo_location_change', True))
     context.visual.allow_custom_hooks_settings = str2bool(
         config.get('allow_custom_hooks_settings', True))
     context.debug_style = str2bool(config.get('debug_style', False))
 
     context.rhodecode_instanceid = config.get('instance_id')
 
     context.visual.cut_off_limit_diff = safe_int(
         config.get('cut_off_limit_diff'))
     context.visual.cut_off_limit_file = safe_int(
         config.get('cut_off_limit_file'))
 
     context.license = AttributeDict({})
     context.license.hide_license_info = str2bool(
         config.get('license.hide_license_info', False))
 
     # AppEnlight
     context.appenlight_enabled = config.get('appenlight', False)
     context.appenlight_api_public_key = config.get(
         'appenlight.api_public_key', '')
     context.appenlight_server_url = config.get('appenlight.server_url', '')
 
     diffmode = {
         "unified": "unified",
         "sideside": "sideside"
     }.get(request.GET.get('diffmode'))
 
     if is_api is not None:
         is_api = hasattr(request, 'rpc_user')
     session_attrs = {
         # defaults
         "clone_url_format": "http",
         "diffmode": "sideside",
         "license_fingerprint": request.session.get('license_fingerprint')
     }
 
     if not is_api:
         # don't access pyramid session for API calls
         if diffmode and diffmode != request.session.get('rc_user_session_attr.diffmode'):
             request.session['rc_user_session_attr.diffmode'] = diffmode
 
         # session settings per user
 
         for k, v in request.session.items():
             pref = 'rc_user_session_attr.'
             if k and k.startswith(pref):
                 k = k[len(pref):]
                 session_attrs[k] = v
 
     context.user_session_attrs = session_attrs
 
     # JS template context
     context.template_context = {
         'repo_name': None,
         'repo_type': None,
         'repo_landing_commit': None,
         'rhodecode_user': {
             'username': None,
             'email': None,
             'notification_status': False
         },
         'session_attrs': session_attrs,
         'visual': {
             'default_renderer': None
         },
         'commit_data': {
             'commit_id': None
         },
         'pull_request_data': {'pull_request_id': None},
         'timeago': {
             'refresh_time': 120 * 1000,
             'cutoff_limit': 1000 * 60 * 60 * 24 * 7
         },
         'pyramid_dispatch': {
 
         },
         'extra': {'plugins': {}}
     }
     # END CONFIG VARS
     if is_api:
         csrf_token = None
     else:
         csrf_token = auth.get_csrf_token(session=request.session)
 
     context.csrf_token = csrf_token
     context.backends = rhodecode.BACKENDS.keys()
 
     unread_count = 0
     user_bookmark_list = []
     if user_id:
         unread_count = NotificationModel().get_unread_cnt_for_user(user_id)
         user_bookmark_list = UserBookmark.get_bookmarks_for_user(user_id)
     context.unread_notifications = unread_count
     context.bookmark_items = user_bookmark_list
 
     # web case
     if hasattr(request, 'user'):
         context.auth_user = request.user
         context.rhodecode_user = request.user
 
     # api case
     if hasattr(request, 'rpc_user'):
         context.auth_user = request.rpc_user
         context.rhodecode_user = request.rpc_user
 
     # attach the whole call context to the request
     request.set_call_context(context)
 
 
 def get_auth_user(request):
     environ = request.environ
     session = request.session
 
     ip_addr = get_ip_addr(environ)
 
     # make sure that we update permissions each time we call controller
     _auth_token = (
             # ?auth_token=XXX
             request.GET.get('auth_token', '')
             # ?api_key=XXX !LEGACY
             or request.GET.get('api_key', '')
             # or headers....
             or request.headers.get('X-Rc-Auth-Token', '')
     )
     if not _auth_token and request.matchdict:
         url_auth_token = request.matchdict.get('_auth_token')
         _auth_token = url_auth_token
         if _auth_token:
             log.debug('Using URL extracted auth token `...%s`', _auth_token[-4:])
 
     if _auth_token:
         # when using API_KEY we assume user exists, and
         # doesn't need auth based on cookies.
         auth_user = AuthUser(api_key=_auth_token, ip_addr=ip_addr)
         authenticated = False
     else:
         cookie_store = CookieStoreWrapper(session.get('rhodecode_user'))
         try:
             auth_user = AuthUser(user_id=cookie_store.get('user_id', None),
                                  ip_addr=ip_addr)
         except UserCreationError as e:
             h.flash(e, 'error')
             # container auth or other auth functions that create users
             # on the fly can throw this exception signaling that there's
             # issue with user creation, explanation should be provided
             # in Exception itself. We then create a simple blank
             # AuthUser
             auth_user = AuthUser(ip_addr=ip_addr)
 
         # in case someone changes a password for user it triggers session
         # flush and forces a re-login
         if password_changed(auth_user, session):
             session.invalidate()
             cookie_store = CookieStoreWrapper(session.get('rhodecode_user'))
             auth_user = AuthUser(ip_addr=ip_addr)
 
         authenticated = cookie_store.get('is_authenticated')
 
     if not auth_user.is_authenticated and auth_user.is_user_object:
         # user is not authenticated and not empty
         auth_user.set_authenticated(authenticated)
 
     return auth_user, _auth_token
 
 
 def h_filter(s):
     """
     Custom filter for Mako templates. Mako by standard uses `markupsafe.escape`
     we wrap this with additional functionality that converts None to empty
     strings
     """
     if s is None:
         return markupsafe.Markup()
     return markupsafe.escape(s)
 
 
 def add_events_routes(config):
     """
     Adds routing that can be used in events. Because some events are triggered
     outside of pyramid context, we need to bootstrap request with some
     routing registered
     """
 
     from rhodecode.apps._base import ADMIN_PREFIX
 
     config.add_route(name='home', pattern='/')
     config.add_route(name='main_page_repos_data', pattern='/_home_repos')
     config.add_route(name='main_page_repo_groups_data', pattern='/_home_repo_groups')
 
     config.add_route(name='login', pattern=ADMIN_PREFIX + '/login')
     config.add_route(name='logout', pattern=ADMIN_PREFIX + '/logout')
     config.add_route(name='repo_summary', pattern='/{repo_name}')
     config.add_route(name='repo_summary_explicit', pattern='/{repo_name}/summary')
     config.add_route(name='repo_group_home', pattern='/{repo_group_name}')
 
     config.add_route(name='pullrequest_show',
                      pattern='/{repo_name}/pull-request/{pull_request_id}')
     config.add_route(name='pull_requests_global',
                      pattern='/pull-request/{pull_request_id}')
 
     config.add_route(name='repo_commit',
                      pattern='/{repo_name}/changeset/{commit_id}')
     config.add_route(name='repo_files',
                      pattern='/{repo_name}/files/{commit_id}/{f_path}')
 
     config.add_route(name='hovercard_user',
                      pattern='/_hovercard/user/{user_id}')
 
     config.add_route(name='hovercard_user_group',
                      pattern='/_hovercard/user_group/{user_group_id}')
 
     config.add_route(name='hovercard_pull_request',
                      pattern='/_hovercard/pull_request/{pull_request_id}')
 
     config.add_route(name='hovercard_repo_commit',
                      pattern='/_hovercard/commit/{repo_name}/{commit_id}')
 
 
 def bootstrap_config(request, registry_name='RcTestRegistry'):
     import pyramid.testing
     registry = pyramid.testing.Registry(registry_name)
 
     config = pyramid.testing.setUp(registry=registry, request=request)
 
     # allow pyramid lookup in testing
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
 
     add_events_routes(config)
 
     return config
 
 
 def bootstrap_request(**kwargs):
     """
     Returns a thin version of Request Object that is used in non-web context like testing/celery
     """
 
     import pyramid.testing
     from rhodecode.lib.request import ThinRequest as _ThinRequest
 
     class ThinRequest(_ThinRequest):
         application_url = kwargs.pop('application_url', 'http://example.com')
         host = kwargs.pop('host', 'example.com:80')
         domain = kwargs.pop('domain', 'example.com')
 
     class ThinSession(pyramid.testing.DummySession):
         def save(*arg, **kw):
             pass
 
     request = ThinRequest(**kwargs)
     request.session = ThinSession()
 
     return request
 

rhodecode/lib/caching_query.py

@@ -1,249 +1,248 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """caching_query.py
 
 Represent functions and classes
 which allow the usage of Dogpile caching with SQLAlchemy.
 Introduces a query option called FromCache.
 
 .. versionchanged:: 1.4  the caching approach has been altered to work
    based on a session event.
 
 
 The three new concepts introduced here are:
 
  * ORMCache - an extension for an ORM :class:`.Session`
    retrieves results in/from dogpile.cache.
  * FromCache - a query option that establishes caching
    parameters on a Query
  * RelationshipCache - a variant of FromCache which is specific
    to a query invoked during a lazy load.
 
 The rest of what's here are standard SQLAlchemy and
 dogpile.cache constructs.
 
 """
 from dogpile.cache.api import NO_VALUE
 
 from sqlalchemy import event
 from sqlalchemy.orm import loading
 from sqlalchemy.orm.interfaces import UserDefinedOption
 
 
 DEFAULT_REGION = "sql_cache_short"
 
 
 class ORMCache:
 
     """An add-on for an ORM :class:`.Session` optionally loads full results
     from a dogpile cache region.
 
     cache = ORMCache(regions={})
     cache.listen_on_session(Session)
 
     """
 
     def __init__(self, regions):
         self.cache_regions = regions or self._get_region()
         self._statement_cache = {}
 
     @classmethod
     def _get_region(cls):
         from rhodecode.lib.rc_cache import region_meta
         return region_meta.dogpile_cache_regions
 
     def listen_on_session(self, session_factory):
         event.listen(session_factory, "do_orm_execute", self._do_orm_execute)
 
     def _do_orm_execute(self, orm_context):
 
         for opt in orm_context.user_defined_options:
             if isinstance(opt, RelationshipCache):
                 opt = opt._process_orm_context(orm_context)
                 if opt is None:
                     continue
 
             if isinstance(opt, FromCache):
                 dogpile_region = self.cache_regions[opt.region]
 
                 if opt.cache_key:
                     our_cache_key = f'SQL_CACHE_{opt.cache_key}'
                 else:
                     our_cache_key = opt._generate_cache_key(
                         orm_context.statement, orm_context.parameters, self
                     )
 
                 if opt.ignore_expiration:
                     cached_value = dogpile_region.get(
                         our_cache_key,
                         expiration_time=opt.expiration_time,
                         ignore_expiration=opt.ignore_expiration,
                     )
                 else:
 
                     def createfunc():
                         return orm_context.invoke_statement().freeze()
 
                     cached_value = dogpile_region.get_or_create(
                         our_cache_key,
                         createfunc,
                         expiration_time=opt.expiration_time,
                     )
 
                 if cached_value is NO_VALUE:
                     # keyerror?   this is bigger than a keyerror...
                     raise KeyError()
 
                 orm_result = loading.merge_frozen_result(
                     orm_context.session,
                     orm_context.statement,
                     cached_value,
                     load=False,
                 )
                 return orm_result()
 
         else:
             return None
 
     def invalidate(self, statement, parameters, opt):
         """Invalidate the cache value represented by a statement."""
 
         statement = statement.__clause_element__()
 
         dogpile_region = self.cache_regions[opt.region]
 
         cache_key = opt._generate_cache_key(statement, parameters, self)
 
         dogpile_region.delete(cache_key)
 
 
 class FromCache(UserDefinedOption):
     """Specifies that a Query should load results from a cache."""
 
     propagate_to_loaders = False
 
     def __init__(
         self,
         region=DEFAULT_REGION,
         cache_key=None,
         expiration_time=None,
         ignore_expiration=False,
     ):
         """Construct a new FromCache.
 
         :param region: the cache region.  Should be a
          region configured in the dictionary of dogpile
          regions.
 
         :param cache_key: optional.  A string cache key
          that will serve as the key to the query.   Use this
          if your query has a huge amount of parameters (such
          as when using in_()) which correspond more simply to
          some other identifier.
 
         """
         self.region = region
         self.cache_key = cache_key
         self.expiration_time = expiration_time
         self.ignore_expiration = ignore_expiration
 
     # this is not needed as of SQLAlchemy 1.4.28;
     # UserDefinedOption classes no longer participate in the SQL
     # compilation cache key
     def _gen_cache_key(self, anon_map, bindparams):
         return None
 
     def _generate_cache_key(self, statement, parameters, orm_cache):
         """generate a cache key with which to key the results of a statement.
 
         This leverages the use of the SQL compilation cache key which is
         repurposed as a SQL results key.
 
         """
         statement_cache_key = statement._generate_cache_key()
 
         key = statement_cache_key.to_offline_string(
             orm_cache._statement_cache, statement, parameters
         ) + repr(self.cache_key)
         # print("here's our key...%s" % key)
         return key
 
 
 class RelationshipCache(FromCache):
     """Specifies that a Query as called within a "lazy load"
     should load results from a cache."""
 
     propagate_to_loaders = True
 
     def __init__(
         self,
         attribute,
         region=DEFAULT_REGION,
         cache_key=None,
         expiration_time=None,
         ignore_expiration=False,
     ):
         """Construct a new RelationshipCache.
 
         :param attribute: A Class.attribute which
          indicates a particular class relationship() whose
          lazy loader should be pulled from the cache.
 
         :param region: name of the cache region.
 
         :param cache_key: optional.  A string cache key
          that will serve as the key to the query, bypassing
          the usual means of forming a key from the Query itself.
 
         """
         self.region = region
         self.cache_key = cache_key
         self.expiration_time = expiration_time
         self.ignore_expiration = ignore_expiration
         self._relationship_options = {
             (attribute.property.parent.class_, attribute.property.key): self
         }
 
     def _process_orm_context(self, orm_context):
         current_path = orm_context.loader_strategy_path
 
         if current_path:
             mapper, prop = current_path[-2:]
             key = prop.key
 
             for cls in mapper.class_.__mro__:
                 if (cls, key) in self._relationship_options:
                     relationship_option = self._relationship_options[
                         (cls, key)
                     ]
                     return relationship_option
 
     def and_(self, option):
         """Chain another RelationshipCache option to this one.
 
         While many RelationshipCache objects can be specified on a single
         Query separately, chaining them together allows for a more efficient
         lookup during load.
 
         """
         self._relationship_options.update(option._relationship_options)
         return self
 

rhodecode/lib/celerylib/__init__.py

@@ -1,95 +1,94 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import socket
 import logging
 
 import rhodecode
 from zope.cachedescriptors.property import Lazy as LazyProperty
 from rhodecode.lib.celerylib.loader import (
     celery_app, RequestContextTask, get_logger)
 from rhodecode.lib.statsd_client import StatsdClient
 
 async_task = celery_app.task
 
 
 log = logging.getLogger(__name__)
 
 
 class ResultWrapper(object):
     def __init__(self, task):
         self.task = task
 
     @LazyProperty
     def result(self):
         return self.task
 
 
 def run_task(task, *args, **kwargs):
     import celery
     log.debug('Got task `%s` for execution, celery mode enabled:%s', task, rhodecode.CELERY_ENABLED)
     if task is None:
         raise ValueError('Got non-existing task for execution')
 
     exec_mode = 'sync'
     allow_async = True
 
     # if we're already in a celery task, don't allow async execution again
     # e.g task within task
     in_task = celery.current_task
     if in_task:
         log.debug('This task in in context of another task: %s, not allowing another async execution', in_task)
         allow_async = False
         if kwargs.pop('allow_subtask', False):
             log.debug('Forced async by allow_async=True flag')
             allow_async = True
 
     t = None
     if rhodecode.CELERY_ENABLED and allow_async:
 
         try:
             t = task.apply_async(args=args, kwargs=kwargs)
             log.debug('executing task %s:%s in async mode', t.task_id, task)
         except socket.error as e:
             if isinstance(e, IOError) and e.errno == 111:
                 log.error('Unable to connect to celeryd `%s`. Sync execution', e)
             else:
                 log.exception("Exception while connecting to celeryd.")
         except KeyError as e:
             log.error('Unable to connect to celeryd `%s`. Sync execution', e)
         except Exception as e:
             log.exception(
                 "Exception while trying to run task asynchronous. "
                 "Fallback to sync execution.")
 
     else:
         log.debug('executing task %s:%s in sync mode', 'TASK', task)
         statsd = StatsdClient.statsd
         if statsd:
             task_repr = getattr(task, 'name', task)
             statsd.incr('rhodecode_celery_task_total', tags=[
                 'task:{}'.format(task_repr),
                 'mode:sync'
             ])
 
     # we got async task, return it after statsd call
     if t:
         return t
     return ResultWrapper(task(*args, **kwargs))

rhodecode/lib/celerylib/loader.py

@@ -1,356 +1,355 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 """
 Celery loader, run with::
 
     celery worker \
         --task-events \
         --beat \
         --autoscale=20,2 \
         --max-tasks-per-child 1 \
         --app rhodecode.lib.celerylib.loader \
         --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler \
         --loglevel DEBUG --ini=.dev/dev.ini
 """
 import os
 import logging
 import importlib
 
 from celery import Celery
 from celery import signals
 from celery import Task
 from celery import exceptions  # pragma: no cover
 from kombu.serialization import register
 
 import rhodecode
 
 from rhodecode.lib.statsd_client import StatsdClient
 from rhodecode.lib.celerylib.utils import parse_ini_vars, ping_db
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.pyramid_utils import bootstrap, setup_logging
 from rhodecode.lib.utils2 import str2bool
 from rhodecode.model import meta
 
 
 register('json_ext', json.dumps, json.loads,
          content_type='application/x-json-ext',
          content_encoding='utf-8')
 
 log = logging.getLogger('celery.rhodecode.loader')
 
 
 imports = ['rhodecode.lib.celerylib.tasks']
 
 try:
     # try if we have EE tasks available
     importlib.import_module('rc_ee')
     imports.append('rc_ee.lib.celerylib.tasks')
 except ImportError:
     pass
 
 
 base_celery_config = {
     'result_backend': 'rpc://',
     'result_expires': 60 * 60 * 24,
     'result_persistent': True,
     'imports': imports,
     'worker_max_tasks_per_child': 20,
     'accept_content': ['json_ext', 'json'],
     'task_serializer': 'json_ext',
     'result_serializer': 'json_ext',
     'worker_hijack_root_logger': False,
     'database_table_names': {
         'task': 'beat_taskmeta',
         'group': 'beat_groupmeta',
     }
 }
 
 
 def add_preload_arguments(parser):
     parser.add_argument(
         '--ini', default=None,
         help='Path to ini configuration file.'
     )
     parser.add_argument(
         '--ini-var', default=None,
         help='Comma separated list of key=value to pass to ini.'
     )
 
 
 def get_logger(obj):
     custom_log = logging.getLogger(
         'rhodecode.task.{}'.format(obj.__class__.__name__))
 
     if rhodecode.CELERY_ENABLED:
         try:
             custom_log = obj.get_logger()
         except Exception:
             pass
 
     return custom_log
 
 
 # init main celery app
 celery_app = Celery()
 celery_app.user_options['preload'].add(add_preload_arguments)
 
 
 @signals.setup_logging.connect
 def setup_logging_callback(**kwargs):
     ini_file = celery_app.conf['RC_INI_FILE']
     setup_logging(ini_file)
 
 
 @signals.user_preload_options.connect
 def on_preload_parsed(options, **kwargs):
 
     ini_file = options['ini']
     ini_vars = options['ini_var']
 
     if ini_file is None:
         print('You must provide the --ini argument to start celery')
         exit(-1)
 
     options = None
     if ini_vars is not None:
         options = parse_ini_vars(ini_vars)
 
     celery_app.conf['RC_INI_FILE'] = ini_file
     celery_app.conf['RC_INI_OPTIONS'] = options
     setup_logging(ini_file)
 
 
 def _init_celery(app_type=''):
     from rhodecode.config.middleware import get_celery_config
 
     log.debug('Bootstrapping RhodeCode application for %s...', app_type)
 
     ini_file = celery_app.conf['RC_INI_FILE']
     options = celery_app.conf['RC_INI_OPTIONS']
 
     env = None
     try:
         env = bootstrap(ini_file, options=options)
     except Exception:
         log.exception('Failed to bootstrap RhodeCode APP')
 
     if not env:
         raise EnvironmentError(
             'Failed to load pyramid ENV. '
             'Probably there is another error present that prevents from running pyramid app')
 
     log.debug('Got Pyramid ENV: %s', env)
 
     celery_settings = get_celery_config(env['registry'].settings)
 
     setup_celery_app(
         app=env['app'], root=env['root'], request=env['request'],
         registry=env['registry'], closer=env['closer'],
         celery_settings=celery_settings)
 
 
 @signals.celeryd_init.connect
 def on_celeryd_init(sender=None, conf=None, **kwargs):
     _init_celery('celery worker')
 
     # fix the global flag even if it's disabled via .ini file because this
     # is a worker code that doesn't need this to be disabled.
     rhodecode.CELERY_ENABLED = True
 
 
 @signals.beat_init.connect
 def on_beat_init(sender=None, conf=None, **kwargs):
     _init_celery('celery beat')
 
 
 @signals.task_prerun.connect
 def task_prerun_signal(task_id, task, args, **kwargs):
     ping_db()
     statsd = StatsdClient.statsd
     if statsd:
         task_repr = getattr(task, 'name', task)
         statsd.incr('rhodecode_celery_task_total', tags=[
             'task:{}'.format(task_repr),
             'mode:async'
         ])
 
 
 @signals.task_success.connect
 def task_success_signal(result, **kwargs):
     meta.Session.commit()
     closer = celery_app.conf['PYRAMID_CLOSER']
     if closer:
         closer()
 
 
 
 @signals.task_retry.connect
 def task_retry_signal(
         request, reason, einfo, **kwargs):
     meta.Session.remove()
     closer = celery_app.conf['PYRAMID_CLOSER']
     if closer:
         closer()
 
 
 @signals.task_failure.connect
 def task_failure_signal(
         task_id, exception, args, kwargs, traceback, einfo, **kargs):
 
     log.error('Task: %s failed !! exc_info: %s', task_id, einfo)
     from rhodecode.lib.exc_tracking import store_exception
     from rhodecode.lib.statsd_client import StatsdClient
 
     meta.Session.remove()
 
     # simulate sys.exc_info()
     exc_info = (einfo.type, einfo.exception, einfo.tb)
     store_exception(id(exc_info), exc_info, prefix='rhodecode-celery')
     statsd = StatsdClient.statsd
     if statsd:
         exc_type = "{}.{}".format(einfo.__class__.__module__, einfo.__class__.__name__)
         statsd.incr('rhodecode_exception_total',
                     tags=["exc_source:celery", "type:{}".format(exc_type)])
 
     closer = celery_app.conf['PYRAMID_CLOSER']
     if closer:
         closer()
 
 
 @signals.task_revoked.connect
 def task_revoked_signal(
         request, terminated, signum, expired, **kwargs):
     closer = celery_app.conf['PYRAMID_CLOSER']
     if closer:
         closer()
 
 
 class UNSET(object):
     pass
 
 
 _unset = UNSET()
 
 
 def set_celery_conf(app=_unset, root=_unset, request=_unset, registry=_unset, closer=_unset):
 
     if request is not UNSET:
         celery_app.conf.update({'PYRAMID_REQUEST': request})
 
     if registry is not UNSET:
         celery_app.conf.update({'PYRAMID_REGISTRY': registry})
 
 
 def setup_celery_app(app, root, request, registry, closer, celery_settings):
     log.debug('Got custom celery conf: %s', celery_settings)
     celery_config = base_celery_config
     celery_config.update({
         # store celerybeat scheduler db where the .ini file is
         'beat_schedule_filename': registry.settings['celerybeat-schedule.path'],
     })
 
     celery_config.update(celery_settings)
     celery_app.config_from_object(celery_config)
 
     celery_app.conf.update({'PYRAMID_APP': app})
     celery_app.conf.update({'PYRAMID_ROOT': root})
     celery_app.conf.update({'PYRAMID_REQUEST': request})
     celery_app.conf.update({'PYRAMID_REGISTRY': registry})
     celery_app.conf.update({'PYRAMID_CLOSER': closer})
 
 
 def configure_celery(config, celery_settings):
     """
     Helper that is called from our application creation logic. It gives
     connection info into running webapp and allows execution of tasks from
     RhodeCode itself
     """
     # store some globals into rhodecode
     rhodecode.CELERY_ENABLED = str2bool(
         config.registry.settings.get('use_celery'))
     if rhodecode.CELERY_ENABLED:
         log.info('Configuring celery based on `%s` settings', celery_settings)
         setup_celery_app(
             app=None, root=None, request=None, registry=config.registry,
             closer=None, celery_settings=celery_settings)
 
 
 def maybe_prepare_env(req):
     environ = {}
     try:
         environ.update({
             'PATH_INFO': req.environ['PATH_INFO'],
             'SCRIPT_NAME': req.environ['SCRIPT_NAME'],
             'HTTP_HOST': req.environ.get('HTTP_HOST', req.environ['SERVER_NAME']),
             'SERVER_NAME': req.environ['SERVER_NAME'],
             'SERVER_PORT': req.environ['SERVER_PORT'],
             'wsgi.url_scheme': req.environ['wsgi.url_scheme'],
         })
     except Exception:
         pass
 
     return environ
 
 
 class RequestContextTask(Task):
     """
     This is a celery task which will create a rhodecode app instance context
     for the task, patch pyramid with the original request
     that created the task and also add the user to the context.
     """
 
     def apply_async(self, args=None, kwargs=None, task_id=None, producer=None,
                     link=None, link_error=None, shadow=None, **options):
         """ queue the job to run (we are in web request context here) """
         from rhodecode.lib.base import get_ip_addr
 
         req = self.app.conf['PYRAMID_REQUEST']
         if not req:
             raise ValueError('celery_app.conf is having empty PYRAMID_REQUEST key')
 
         log.debug('Running Task with class: %s. Request Class: %s',
                   self.__class__, req.__class__)
 
         user_id = 0
 
         # web case
         if hasattr(req, 'user'):
             user_id = req.user.user_id
 
         # api case
         elif hasattr(req, 'rpc_user'):
             user_id = req.rpc_user.user_id
 
         # we hook into kwargs since it is the only way to pass our data to
         # the celery worker
         environ = maybe_prepare_env(req)
         options['headers'] = options.get('headers', {})
         options['headers'].update({
             'rhodecode_proxy_data': {
                 'environ': environ,
                 'auth_user': {
                     'ip_addr': get_ip_addr(req.environ),
                     'user_id': user_id
                 },
             }
         })
 
         return super(RequestContextTask, self).apply_async(
             args, kwargs, task_id, producer, link, link_error, shadow, **options)

rhodecode/lib/celerylib/scheduler.py

@@ -1,60 +1,59 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import logging
 import importlib
 
 from celery.beat import (
     PersistentScheduler, ScheduleEntry as CeleryScheduleEntry)
 
 log = logging.getLogger(__name__)
 
 
 class FileScheduleEntry(CeleryScheduleEntry):
     def __init__(self, name=None, task=None, last_run_at=None,
                  total_run_count=None, schedule=None, args=(), kwargs=None,
                  options=None, relative=False, app=None, **_kwargs):
         kwargs = kwargs or {}
         options = options or {}
 
         # because our custom loader passes in some variables that the original
         # function doesn't expect, we have this thin wrapper
 
         super(FileScheduleEntry, self).__init__(
             name=name, task=task, last_run_at=last_run_at,
             total_run_count=total_run_count, schedule=schedule, args=args,
             kwargs=kwargs, options=options, relative=relative, app=app)
 
 
 class FileScheduler(PersistentScheduler):
     """CE base scheduler"""
     Entry = FileScheduleEntry
 
     def setup_schedule(self):
         log.info("setup_schedule called")
         super(FileScheduler, self).setup_schedule()
 
 
 try:
     # try if we have EE scheduler available
     module = importlib.import_module('rc_ee.lib.celerylib.scheduler')
     RcScheduler = module.RcScheduler
 except ImportError:
     # fallback to CE scheduler
     RcScheduler = FileScheduler

rhodecode/lib/celerylib/tasks.py

@@ -1,414 +1,414 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 RhodeCode task modules, containing all task that suppose to be run
 by celery daemon
 """
 
 import os
 import time
 
 from pyramid_mailer.mailer import Mailer
 from pyramid_mailer.message import Message
 from email.utils import formatdate
 
 import rhodecode
 from rhodecode.lib import audit_logger
 from rhodecode.lib.celerylib import get_logger, async_task, RequestContextTask, run_task
 from rhodecode.lib import hooks_base
 from rhodecode.lib.utils2 import safe_int, str2bool, aslist
 from rhodecode.lib.statsd_client import StatsdClient
 from rhodecode.model.db import (
     Session, IntegrityError, true, Repository, RepoGroup, User)
 from rhodecode.model.permission import PermissionModel
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def send_email(recipients, subject, body='', html_body='', email_config=None,
                extra_headers=None):
     """
     Sends an email with defined parameters from the .ini files.
 
     :param recipients: list of recipients, it this is empty the defined email
         address from field 'email_to' is used instead
     :param subject: subject of the mail
     :param body: body of the mail
     :param html_body: html version of body
     :param email_config: specify custom configuration for mailer
     :param extra_headers: specify custom headers
     """
     log = get_logger(send_email)
 
     email_config = email_config or rhodecode.CONFIG
 
     mail_server = email_config.get('smtp_server') or None
     if mail_server is None:
         log.error("SMTP server information missing. Sending email failed. "
                   "Make sure that `smtp_server` variable is configured "
                   "inside the .ini file")
         return False
 
     subject = "%s %s" % (email_config.get('email_prefix', ''), subject)
 
     if recipients:
         if isinstance(recipients, str):
             recipients = recipients.split(',')
     else:
         # if recipients are not defined we send to email_config + all admins
         admins = []
         for u in User.query().filter(User.admin == true()).all():
             if u.email:
                 admins.append(u.email)
         recipients = []
         config_email = email_config.get('email_to')
         if config_email:
             recipients += [config_email]
         recipients += admins
 
     # translate our LEGACY config into the one that pyramid_mailer supports
     email_conf = dict(
         host=mail_server,
         port=email_config.get('smtp_port', 25),
         username=email_config.get('smtp_username'),
         password=email_config.get('smtp_password'),
 
         tls=str2bool(email_config.get('smtp_use_tls')),
         ssl=str2bool(email_config.get('smtp_use_ssl')),
 
         # SSL key file
         # keyfile='',
 
         # SSL certificate file
         # certfile='',
 
         # Location of maildir
         # queue_path='',
 
         default_sender=email_config.get('app_email_from', '[e-mail]'),
 
         debug=str2bool(email_config.get('smtp_debug')),
         # /usr/sbin/sendmail	Sendmail executable
         # sendmail_app='',
 
         # {sendmail_app} -t -i -f {sender}	Template for sendmail execution
         # sendmail_template='',
     )
 
     if extra_headers is None:
         extra_headers = {}
 
     extra_headers.setdefault('Date', formatdate(time.time()))
 
     if 'thread_ids' in extra_headers:
         thread_ids = extra_headers.pop('thread_ids')
         extra_headers['References'] = ' '.join('<{}>'.format(t) for t in thread_ids)
 
     try:
         mailer = Mailer(**email_conf)
 
         message = Message(subject=subject,
                           sender=email_conf['default_sender'],
                           recipients=recipients,
                           body=body, html=html_body,
                           extra_headers=extra_headers)
         mailer.send_immediately(message)
         statsd = StatsdClient.statsd
         if statsd:
             statsd.incr('rhodecode_email_sent_total')
 
     except Exception:
         log.exception('Mail sending failed')
         return False
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def create_repo(form_data, cur_user):
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.user import UserModel
     from rhodecode.model.scm import ScmModel
     from rhodecode.model.settings import SettingsModel
 
     log = get_logger(create_repo)
 
     cur_user = UserModel()._get_user(cur_user)
     owner = cur_user
 
     repo_name = form_data['repo_name']
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     description = form_data['repo_description']
     private = form_data['repo_private']
     clone_uri = form_data.get('clone_uri')
     repo_group = safe_int(form_data['repo_group'])
     copy_fork_permissions = form_data.get('copy_permissions')
     copy_group_permissions = form_data.get('repo_copy_permissions')
     fork_of = form_data.get('fork_parent_id')
     state = form_data.get('repo_state', Repository.STATE_PENDING)
 
     # repo creation defaults, private and repo_type are filled in form
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     enable_statistics = form_data.get(
         'enable_statistics', defs.get('repo_enable_statistics'))
     enable_locking = form_data.get(
         'enable_locking', defs.get('repo_enable_locking'))
     enable_downloads = form_data.get(
         'enable_downloads', defs.get('repo_enable_downloads'))
 
     # set landing rev based on default branches for SCM
     landing_ref, _label = ScmModel.backend_landing_ref(repo_type)
 
     try:
         RepoModel()._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_ref,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions,
             copy_group_permissions=copy_group_permissions,
             enable_statistics=enable_statistics,
             enable_locking=enable_locking,
             enable_downloads=enable_downloads,
             state=state
         )
         Session().commit()
 
         # now create this repo on Filesystem
         RepoModel()._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel()._get_repo_group(repo_group),
             clone_uri=clone_uri,
         )
         repo = Repository.get_by_repo_name(repo_name_full)
         hooks_base.create_repository(created_by=owner.username, **repo.get_dict())
 
         # update repo commit caches initially
         repo.update_commit_cache()
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
         repo_id = repo.repo_id
         repo_data = repo.get_api_data()
 
         audit_logger.store(
             'repo.create', action_data={'data': repo_data},
             user=cur_user,
             repo=audit_logger.RepoWrap(repo_name=repo_name, repo_id=repo_id))
 
         Session().commit()
 
         PermissionModel().trigger_permission_flush()
 
     except Exception as e:
         log.warning('Exception occurred when creating repository, '
                     'doing cleanup...', exc_info=True)
         if isinstance(e, IntegrityError):
             Session().rollback()
 
         # rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             Session().commit()
             RepoModel()._delete_filesystem_repo(repo)
         log.info('Cleanup of repo %s finished', repo_name_full)
         raise
 
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def create_repo_fork(form_data, cur_user):
     """
     Creates a fork of repository using internal VCS methods
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.user import UserModel
 
     log = get_logger(create_repo_fork)
 
     cur_user = UserModel()._get_user(cur_user)
     owner = cur_user
 
     repo_name = form_data['repo_name']  # fork in this case
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     description = form_data['description']
     private = form_data['private']
     clone_uri = form_data.get('clone_uri')
     repo_group = safe_int(form_data['repo_group'])
     landing_ref = form_data['landing_rev']
     copy_fork_permissions = form_data.get('copy_permissions')
     fork_id = safe_int(form_data.get('fork_parent_id'))
 
     try:
         fork_of = RepoModel()._get_repo(fork_id)
         RepoModel()._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_ref,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions
         )
 
         Session().commit()
 
         base_path = Repository.base_path()
         source_repo_path = os.path.join(base_path, fork_of.repo_name)
 
         # now create this repo on Filesystem
         RepoModel()._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel()._get_repo_group(repo_group),
             clone_uri=source_repo_path,
         )
         repo = Repository.get_by_repo_name(repo_name_full)
         hooks_base.create_repository(created_by=owner.username, **repo.get_dict())
 
         # update repo commit caches initially
         config = repo._config
         config.set('extensions', 'largefiles', '')
         repo.update_commit_cache(config=config)
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
 
         repo_id = repo.repo_id
         repo_data = repo.get_api_data()
         audit_logger.store(
             'repo.fork', action_data={'data': repo_data},
             user=cur_user,
             repo=audit_logger.RepoWrap(repo_name=repo_name, repo_id=repo_id))
 
         Session().commit()
     except Exception as e:
         log.warning('Exception occurred when forking repository, '
                     'doing cleanup...', exc_info=True)
         if isinstance(e, IntegrityError):
             Session().rollback()
 
         # rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             Session().commit()
             RepoModel()._delete_filesystem_repo(repo)
         log.info('Cleanup of repo %s finished', repo_name_full)
         raise
 
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def repo_maintenance(repoid):
     from rhodecode.lib import repo_maintenance as repo_maintenance_lib
     log = get_logger(repo_maintenance)
     repo = Repository.get_by_id_or_repo_name(repoid)
     if repo:
         maintenance = repo_maintenance_lib.RepoMaintenance()
         tasks = maintenance.get_tasks_for_repo(repo)
         log.debug('Executing %s tasks on repo `%s`', tasks, repoid)
         executed_types = maintenance.execute(repo)
         log.debug('Got execution results %s', executed_types)
     else:
         log.debug('Repo `%s` not found or without a clone_url', repoid)
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def check_for_update(send_email_notification=True, email_recipients=None):
     from rhodecode.model.update import UpdateModel
     from rhodecode.model.notification import EmailNotificationModel
 
     log = get_logger(check_for_update)
     update_url = UpdateModel().get_update_url()
     cur_ver = rhodecode.__version__
 
     try:
         data = UpdateModel().get_update_data(update_url)
 
         current_ver = UpdateModel().get_stored_version(fallback=cur_ver)
         latest_ver = data['versions'][0]['version']
         UpdateModel().store_version(latest_ver)
 
         if send_email_notification:
             log.debug('Send email notification is enabled. '
                       'Current RhodeCode version: %s, latest known: %s', current_ver, latest_ver)
             if UpdateModel().is_outdated(current_ver, latest_ver):
 
                 email_kwargs = {
                     'current_ver': current_ver,
                     'latest_ver': latest_ver,
                 }
 
                 (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
                     EmailNotificationModel.TYPE_UPDATE_AVAILABLE, **email_kwargs)
 
                 email_recipients = aslist(email_recipients, sep=',') or \
                                    [user.email for user in User.get_all_super_admins()]
                 run_task(send_email, email_recipients, subject,
                          email_body_plaintext, email_body)
 
     except Exception:
         log.exception('Failed to check for update')
         raise
 
 
 def sync_last_update_for_objects(*args, **kwargs):
     skip_repos = kwargs.get('skip_repos')
     if not skip_repos:
         repos = Repository.query() \
             .order_by(Repository.group_id.asc())
 
         for repo in repos:
             repo.update_commit_cache()
 
     skip_groups = kwargs.get('skip_groups')
     if not skip_groups:
         repo_groups = RepoGroup.query() \
             .filter(RepoGroup.group_parent_id == None)
 
         for root_gr in repo_groups:
             for repo_gr in reversed(root_gr.recursive_groups()):
                 repo_gr.update_commit_cache()
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def sync_last_update(*args, **kwargs):
     sync_last_update_for_objects(*args, **kwargs)
 
 
 @async_task(ignore_result=False)
 def beat_check(*args, **kwargs):
     log = get_logger(beat_check)
     log.info('%r: Got args: %r and kwargs %r', beat_check, args, kwargs)
     return time.time()

rhodecode/lib/celerylib/utils.py

@@ -1,141 +1,140 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import json
 import logging
 import datetime
 import time
 
 from functools import partial
 
 import configparser
 from celery.result import AsyncResult
 import celery.loaders.base
 import celery.schedules
 
 log = logging.getLogger(__name__)
 
 
 def get_task_id(task):
     task_id = None
     if isinstance(task, AsyncResult):
         task_id = task.task_id
 
     return task_id
 
 
 def crontab(value):
     return celery.schedules.crontab(**value)
 
 
 def timedelta(value):
     return datetime.timedelta(**value)
 
 
 def safe_json(get, section, key):
     value = ''
     try:
         value = get(key)
         json_value = json.loads(value)
     except ValueError:
         msg = 'The %s=%s is not valid json in section %s' % (
             key, value, section
         )
         raise ValueError(msg)
 
     return json_value
 
 
 def raw_2_schedule(schedule_value, schedule_type):
     schedule_type_map = {
         'crontab': crontab,
         'timedelta': timedelta,
         'integer': int
     }
     scheduler_cls = schedule_type_map.get(schedule_type)
 
     if scheduler_cls is None:
         raise ValueError(
             'schedule type %s in section is invalid' % (
                 schedule_type,
             )
         )
     try:
         schedule = scheduler_cls(schedule_value)
     except TypeError:
         log.exception('Failed to compose a schedule from value: %r', schedule_value)
         schedule = None
     return schedule
 
 
 def get_beat_config(parser, section):
 
     get = partial(parser.get, section)
     has_option = partial(parser.has_option, section)
 
     schedule_type = get('type')
     schedule_value = safe_json(get, section, 'schedule')
 
     config = {
         'schedule_type': schedule_type,
         'schedule_value': schedule_value,
         'task': get('task'),
     }
     schedule = raw_2_schedule(schedule_value, schedule_type)
     if schedule:
         config['schedule'] = schedule
 
     if has_option('args'):
         config['args'] = safe_json(get, section, 'args')
 
     if has_option('kwargs'):
         config['kwargs'] = safe_json(get, section, 'kwargs')
 
     if has_option('force_update'):
         config['force_update'] = get('force_update')
 
     return config
 
 
 def parse_ini_vars(ini_vars):
     options = {}
     for pairs in ini_vars.split(','):
         key, value = pairs.split('=')
         options[key] = value
     return options
 
 
 def ping_db():
     from rhodecode.model import meta
     from rhodecode.model.db import DbMigrateVersion
     log.info('Testing DB connection...')
 
     for test in range(10):
         try:
             scalar = DbMigrateVersion.query().scalar()
             log.debug('DB PING %s@%s', scalar, scalar.version)
             break
         except Exception:
             retry = 1
             log.debug('DB not ready, next try in %ss', retry)
             time.sleep(retry)
         finally:
             meta.Session.remove()

rhodecode/lib/channelstream.py

@@ -1,372 +1,372 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import itsdangerous
 import logging
 import requests
 import datetime
 
 from dogpile.util.readwrite_lock import ReadWriteMutex
 from pyramid.threadlocal import get_current_registry
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.auth import HasRepoPermissionAny
 from rhodecode.lib.ext_json import json
 from rhodecode.model.db import User
 from rhodecode.lib.str_utils import ascii_str
 from rhodecode.lib.hash_utils import sha1_safe
 
 log = logging.getLogger(__name__)
 
 LOCK = ReadWriteMutex()
 
 USER_STATE_PUBLIC_KEYS = [
     'id', 'username', 'first_name', 'last_name',
     'icon_link', 'display_name', 'display_link']
 
 
 class ChannelstreamException(Exception):
     pass
 
 
 class ChannelstreamConnectionException(ChannelstreamException):
     pass
 
 
 class ChannelstreamPermissionException(ChannelstreamException):
     pass
 
 
 def get_channelstream_server_url(config, endpoint):
     return 'http://{}{}'.format(config['server'], endpoint)
 
 
 def channelstream_request(config, payload, endpoint, raise_exc=True):
     signer = itsdangerous.TimestampSigner(config['secret'])
     sig_for_server = signer.sign(endpoint)
     secret_headers = {'x-channelstream-secret': sig_for_server,
                       'x-channelstream-endpoint': endpoint,
                       'Content-Type': 'application/json'}
     req_url = get_channelstream_server_url(config, endpoint)
 
     log.debug('Sending a channelstream request to endpoint: `%s`', req_url)
     response = None
     try:
         response = requests.post(req_url, data=json.dumps(payload),
                                  headers=secret_headers).json()
     except requests.ConnectionError:
         log.exception('ConnectionError occurred for endpoint %s', req_url)
         if raise_exc:
             raise ChannelstreamConnectionException(req_url)
     except Exception:
         log.exception('Exception related to Channelstream happened')
         if raise_exc:
             raise ChannelstreamConnectionException()
     log.debug('Got channelstream response: %s', response)
     return response
 
 
 def get_user_data(user_id):
     user = User.get(user_id)
     return {
         'id': user.user_id,
         'username': user.username,
         'first_name': user.first_name,
         'last_name': user.last_name,
         'icon_link': h.gravatar_url(user.email, 60),
         'display_name': h.person(user, 'username_or_name_or_email'),
         'display_link': h.link_to_user(user),
         'notifications': user.user_data.get('notification_status', True)
     }
 
 
 def broadcast_validator(channel_name):
     """ checks if user can access the broadcast channel """
     if channel_name == 'broadcast':
         return True
 
 
 def repo_validator(channel_name):
     """ checks if user can access the broadcast channel """
     channel_prefix = '/repo$'
     if channel_name.startswith(channel_prefix):
         elements = channel_name[len(channel_prefix):].split('$')
         repo_name = elements[0]
         can_access = HasRepoPermissionAny(
             'repository.read',
             'repository.write',
             'repository.admin')(repo_name)
         log.debug(
             'permission check for %s channel resulted in %s',
             repo_name, can_access)
         if can_access:
             return True
     return False
 
 
 def check_channel_permissions(channels, plugin_validators, should_raise=True):
     valid_channels = []
 
     validators = [broadcast_validator, repo_validator]
     if plugin_validators:
         validators.extend(plugin_validators)
     for channel_name in channels:
         is_valid = False
         for validator in validators:
             if validator(channel_name):
                 is_valid = True
                 break
         if is_valid:
             valid_channels.append(channel_name)
         else:
             if should_raise:
                 raise ChannelstreamPermissionException()
     return valid_channels
 
 
 def get_channels_info(self, channels):
     payload = {'channels': channels}
     # gather persistence info
     return channelstream_request(self._config(), payload, '/info')
 
 
 def parse_channels_info(info_result, include_channel_info=None):
     """
     Returns data that contains only secure information that can be
     presented to clients
     """
     include_channel_info = include_channel_info or []
 
     user_state_dict = {}
     for userinfo in info_result['users']:
         user_state_dict[userinfo['user']] = {
             k: v for k, v in list(userinfo['state'].items())
             if k in USER_STATE_PUBLIC_KEYS
             }
 
     channels_info = {}
 
     for c_name, c_info in list(info_result['channels'].items()):
         if c_name not in include_channel_info:
             continue
         connected_list = []
         for username in c_info['users']:
             connected_list.append({
                 'user': username,
                 'state': user_state_dict[username]
             })
         channels_info[c_name] = {'users': connected_list,
                                  'history': c_info['history']}
 
     return channels_info
 
 
 def log_filepath(history_location, channel_name):
 
     channel_hash = ascii_str(sha1_safe(channel_name))
     filename = f'{channel_hash}.log'
     filepath = os.path.join(history_location, filename)
     return filepath
 
 
 def read_history(history_location, channel_name):
     filepath = log_filepath(history_location, channel_name)
     if not os.path.exists(filepath):
         return []
     history_lines_limit = -100
     history = []
     with open(filepath, 'rb') as f:
         for line in f.readlines()[history_lines_limit:]:
             try:
                 history.append(json.loads(line))
             except Exception:
                 log.exception('Failed to load history')
     return history
 
 
 def update_history_from_logs(config, channels, payload):
     history_location = config.get('history.location')
     for channel in channels:
         history = read_history(history_location, channel)
         payload['channels_info'][channel]['history'] = history
 
 
 def write_history(config, message):
     """ writes a message to a base64encoded filename """
     history_location = config.get('history.location')
     if not os.path.exists(history_location):
         return
     try:
         LOCK.acquire_write_lock()
         filepath = log_filepath(history_location, message['channel'])
         json_message = json.dumps(message)
         with open(filepath, 'ab') as f:
             f.write(json_message)
             f.write('\n')
     finally:
         LOCK.release_write_lock()
 
 
 def get_connection_validators(registry):
     validators = []
     for k, config in list(registry.rhodecode_plugins.items()):
         validator = config.get('channelstream', {}).get('connect_validator')
         if validator:
             validators.append(validator)
     return validators
 
 
 def get_channelstream_config(registry=None):
     if not registry:
         registry = get_current_registry()
 
     rhodecode_plugins = getattr(registry, 'rhodecode_plugins', {})
     channelstream_config = rhodecode_plugins.get('channelstream', {})
     return channelstream_config
 
 
 def post_message(channel, message, username, registry=None):
     channelstream_config = get_channelstream_config(registry)
     if not channelstream_config.get('enabled'):
         return
 
     message_obj = message
     if isinstance(message, str):
         message_obj = {
             'message': message,
             'level': 'success',
             'topic': '/notifications'
         }
 
     log.debug('Channelstream: sending notification to channel %s', channel)
     payload = {
         'type': 'message',
         'timestamp': datetime.datetime.utcnow(),
         'user': 'system',
         'exclude_users': [username],
         'channel': channel,
         'message': message_obj
     }
 
     try:
         return channelstream_request(
             channelstream_config, [payload], '/message',
             raise_exc=False)
     except ChannelstreamException:
         log.exception('Failed to send channelstream data')
         raise
 
 
 def _reload_link(label):
     return (
         '<a onclick="window.location.reload()">'
         '<strong>{}</strong>'
         '</a>'.format(label)
     )
 
 
 def pr_channel(pull_request):
     repo_name = pull_request.target_repo.repo_name
     pull_request_id = pull_request.pull_request_id
     channel = '/repo${}$/pr/{}'.format(repo_name, pull_request_id)
     log.debug('Getting pull-request channelstream broadcast channel: %s', channel)
     return channel
 
 
 def comment_channel(repo_name, commit_obj=None, pull_request_obj=None):
     channel = None
     if commit_obj:
         channel = '/repo${}$/commit/{}'.format(
             repo_name, commit_obj.raw_id
         )
     elif pull_request_obj:
         channel = '/repo${}$/pr/{}'.format(
             repo_name, pull_request_obj.pull_request_id
         )
     log.debug('Getting comment channelstream broadcast channel: %s', channel)
 
     return channel
 
 
 def pr_update_channelstream_push(request, pr_broadcast_channel, user, msg, **kwargs):
     """
     Channel push on pull request update
     """
     if not pr_broadcast_channel:
         return
 
     _ = request.translate
 
     message = '{} {}'.format(
         msg,
         _reload_link(_(' Reload page to load changes')))
 
     message_obj = {
         'message': message,
         'level': 'success',
         'topic': '/notifications'
     }
 
     post_message(
         pr_broadcast_channel, message_obj, user.username,
         registry=request.registry)
 
 
 def comment_channelstream_push(request, comment_broadcast_channel, user, msg, **kwargs):
     """
     Channelstream push on comment action, on commit, or pull-request
     """
     if not comment_broadcast_channel:
         return
 
     _ = request.translate
 
     comment_data = kwargs.pop('comment_data', {})
     user_data = kwargs.pop('user_data', {})
     comment_id = list(comment_data.keys())[0] if comment_data else ''
 
     message = '<strong>{}</strong> {} #{}'.format(
         user.username,
         msg,
         comment_id,
     )
 
     message_obj = {
         'message': message,
         'level': 'success',
         'topic': '/notifications'
     }
 
     post_message(
         comment_broadcast_channel, message_obj, user.username,
         registry=request.registry)
 
     message_obj = {
         'message': None,
         'user': user.username,
         'comment_id': comment_id,
         'comment_data': comment_data,
         'user_data': user_data,
         'topic': '/comment'
     }
     post_message(
         comment_broadcast_channel, message_obj, user.username,
         registry=request.registry)

rhodecode/lib/codeblocks.py

@@ -1,797 +1,797 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import difflib
 from itertools import groupby
 
 from pygments import lex
 from pygments.formatters.html import _get_ttype_class as pygment_token_class
 from pygments.lexers.special import TextLexer, Token
 from pygments.lexers import get_lexer_by_name
 
 from rhodecode.lib.helpers import (
     get_lexer_for_filenode, html_escape, get_custom_lexer)
 from rhodecode.lib.utils2 import AttributeDict, StrictAttributeDict, safe_unicode
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib.vcs.exceptions import VCSError, NodeDoesNotExistError
 from rhodecode.lib.diff_match_patch import diff_match_patch
 from rhodecode.lib.diffs import LimitedDiffContainer, DEL_FILENODE, BIN_FILENODE
 
 
 plain_text_lexer = get_lexer_by_name(
     'text', stripall=False, stripnl=False, ensurenl=False)
 
 
 log = logging.getLogger(__name__)
 
 
 def filenode_as_lines_tokens(filenode, lexer=None):
     org_lexer = lexer
     lexer = lexer or get_lexer_for_filenode(filenode)
     log.debug('Generating file node pygment tokens for %s, %s, org_lexer:%s',
               lexer, filenode, org_lexer)
     content = filenode.content
     tokens = tokenize_string(content, lexer)
     lines = split_token_stream(tokens, content)
     rv = list(lines)
     return rv
 
 
 def tokenize_string(content, lexer):
     """
     Use pygments to tokenize some content based on a lexer
     ensuring all original new lines and whitespace is preserved
     """
 
     lexer.stripall = False
     lexer.stripnl = False
     lexer.ensurenl = False
 
     if isinstance(lexer, TextLexer):
         lexed = [(Token.Text, content)]
     else:
         lexed = lex(content, lexer)
 
     for token_type, token_text in lexed:
         yield pygment_token_class(token_type), token_text
 
 
 def split_token_stream(tokens, content):
     """
     Take a list of (TokenType, text) tuples and split them by a string
 
     split_token_stream([(TEXT, 'some\ntext'), (TEXT, 'more\n')])
     [(TEXT, 'some'), (TEXT, 'text'),
      (TEXT, 'more'), (TEXT, 'text')]
     """
 
     token_buffer = []
     for token_class, token_text in tokens:
         parts = token_text.split('\n')
         for part in parts[:-1]:
             token_buffer.append((token_class, part))
             yield token_buffer
             token_buffer = []
 
         token_buffer.append((token_class, parts[-1]))
 
     if token_buffer:
         yield token_buffer
     elif content:
         # this is a special case, we have the content, but tokenization didn't produce
         # any results. THis can happen if know file extensions like .css have some bogus
         # unicode content without any newline characters
         yield [(pygment_token_class(Token.Text), content)]
 
 
 def filenode_as_annotated_lines_tokens(filenode):
     """
     Take a file node and return a list of annotations => lines, if no annotation
     is found, it will be None.
 
     eg:
 
     [
         (annotation1, [
             (1, line1_tokens_list),
             (2, line2_tokens_list),
         ]),
         (annotation2, [
             (3, line1_tokens_list),
         ]),
         (None, [
             (4, line1_tokens_list),
         ]),
         (annotation1, [
             (5, line1_tokens_list),
             (6, line2_tokens_list),
         ])
     ]
     """
 
     commit_cache = {}  # cache commit_getter lookups
 
     def _get_annotation(commit_id, commit_getter):
         if commit_id not in commit_cache:
             commit_cache[commit_id] = commit_getter()
         return commit_cache[commit_id]
 
     annotation_lookup = {
         line_no: _get_annotation(commit_id, commit_getter)
         for line_no, commit_id, commit_getter, line_content
         in filenode.annotate
     }
 
     annotations_lines = ((annotation_lookup.get(line_no), line_no, tokens)
                           for line_no, tokens
                           in enumerate(filenode_as_lines_tokens(filenode), 1))
 
     grouped_annotations_lines = groupby(annotations_lines, lambda x: x[0])
 
     for annotation, group in grouped_annotations_lines:
         yield (
             annotation, [(line_no, tokens)
                           for (_, line_no, tokens) in group]
         )
 
 
 def render_tokenstream(tokenstream):
     result = []
     for token_class, token_ops_texts in rollup_tokenstream(tokenstream):
 
         if token_class:
             result.append('<span class="%s">' % token_class)
         else:
             result.append('<span>')
 
         for op_tag, token_text in token_ops_texts:
 
             if op_tag:
                 result.append('<%s>' % op_tag)
 
             # NOTE(marcink): in some cases of mixed encodings, we might run into
             # troubles in the html_escape, in this case we say unicode force on token_text
             # that would ensure "correct" data even with the cost of rendered
             try:
                 escaped_text = html_escape(token_text)
             except TypeError:
                 escaped_text = html_escape(safe_unicode(token_text))
 
             # TODO: dan: investigate showing hidden characters like space/nl/tab
             # escaped_text = escaped_text.replace(' ', '<sp> </sp>')
             # escaped_text = escaped_text.replace('\n', '<nl>\n</nl>')
             # escaped_text = escaped_text.replace('\t', '<tab>\t</tab>')
 
             result.append(escaped_text)
 
             if op_tag:
                 result.append('</%s>' % op_tag)
 
         result.append('</span>')
 
     html = ''.join(result)
     return html
 
 
 def rollup_tokenstream(tokenstream):
     """
     Group a token stream of the format:
 
         ('class', 'op', 'text')
     or
         ('class', 'text')
 
     into
 
         [('class1',
             [('op1', 'text'),
              ('op2', 'text')]),
          ('class2',
             [('op3', 'text')])]
 
     This is used to get the minimal tags necessary when
     rendering to html eg for a token stream ie.
 
     <span class="A"><ins>he</ins>llo</span>
     vs
     <span class="A"><ins>he</ins></span><span class="A">llo</span>
 
     If a 2 tuple is passed in, the output op will be an empty string.
 
     eg:
 
     >>> rollup_tokenstream([('classA', '',      'h'),
                             ('classA', 'del',   'ell'),
                             ('classA', '',      'o'),
                             ('classB', '',      ' '),
                             ('classA', '',      'the'),
                             ('classA', '',      're'),
                             ])
 
     [('classA', [('', 'h'), ('del', 'ell'), ('', 'o')],
      ('classB', [('', ' ')],
      ('classA', [('', 'there')]]
 
     """
     if tokenstream and len(tokenstream[0]) == 2:
         tokenstream = ((t[0], '', t[1]) for t in tokenstream)
 
     result = []
     for token_class, op_list in groupby(tokenstream, lambda t: t[0]):
         ops = []
         for token_op, token_text_list in groupby(op_list, lambda o: o[1]):
             text_buffer = []
             for t_class, t_op, t_text in token_text_list:
                 text_buffer.append(t_text)
             ops.append((token_op, ''.join(text_buffer)))
         result.append((token_class, ops))
     return result
 
 
 def tokens_diff(old_tokens, new_tokens, use_diff_match_patch=True):
     """
     Converts a list of (token_class, token_text) tuples to a list of
     (token_class, token_op, token_text) tuples where token_op is one of
     ('ins', 'del', '')
 
     :param old_tokens: list of (token_class, token_text) tuples of old line
     :param new_tokens: list of (token_class, token_text) tuples of new line
     :param use_diff_match_patch: boolean, will use google's diff match patch
         library which has options to 'smooth' out the character by character
         differences making nicer ins/del blocks
     """
 
     old_tokens_result = []
     new_tokens_result = []
 
     similarity = difflib.SequenceMatcher(None,
         ''.join(token_text for token_class, token_text in old_tokens),
         ''.join(token_text for token_class, token_text in new_tokens)
     ).ratio()
 
     if similarity < 0.6: # return, the blocks are too different
         for token_class, token_text in old_tokens:
             old_tokens_result.append((token_class, '', token_text))
         for token_class, token_text in new_tokens:
             new_tokens_result.append((token_class, '', token_text))
         return old_tokens_result, new_tokens_result, similarity
 
     token_sequence_matcher = difflib.SequenceMatcher(None,
         [x[1] for x in old_tokens],
         [x[1] for x in new_tokens])
 
     for tag, o1, o2, n1, n2 in token_sequence_matcher.get_opcodes():
         # check the differences by token block types first to give a more
         # nicer "block" level replacement vs character diffs
 
         if tag == 'equal':
             for token_class, token_text in old_tokens[o1:o2]:
                 old_tokens_result.append((token_class, '', token_text))
             for token_class, token_text in new_tokens[n1:n2]:
                 new_tokens_result.append((token_class, '', token_text))
         elif tag == 'delete':
             for token_class, token_text in old_tokens[o1:o2]:
                 old_tokens_result.append((token_class, 'del', token_text))
         elif tag == 'insert':
             for token_class, token_text in new_tokens[n1:n2]:
                 new_tokens_result.append((token_class, 'ins', token_text))
         elif tag == 'replace':
             # if same type token blocks must be replaced, do a diff on the
             # characters in the token blocks to show individual changes
 
             old_char_tokens = []
             new_char_tokens = []
             for token_class, token_text in old_tokens[o1:o2]:
                 for char in token_text:
                     old_char_tokens.append((token_class, char))
 
             for token_class, token_text in new_tokens[n1:n2]:
                 for char in token_text:
                     new_char_tokens.append((token_class, char))
 
             old_string = ''.join([token_text for
                 token_class, token_text in old_char_tokens])
             new_string = ''.join([token_text for
                 token_class, token_text in new_char_tokens])
 
             char_sequence = difflib.SequenceMatcher(
                 None, old_string, new_string)
             copcodes = char_sequence.get_opcodes()
             obuffer, nbuffer = [], []
 
             if use_diff_match_patch:
                 dmp = diff_match_patch()
                 dmp.Diff_EditCost = 11  # TODO: dan: extract this to a setting
                 reps = dmp.diff_main(old_string, new_string)
                 dmp.diff_cleanupEfficiency(reps)
 
                 a, b = 0, 0
                 for op, rep in reps:
                     l = len(rep)
                     if op == 0:
                         for i, c in enumerate(rep):
                             obuffer.append((old_char_tokens[a+i][0], '', c))
                             nbuffer.append((new_char_tokens[b+i][0], '', c))
                         a += l
                         b += l
                     elif op == -1:
                         for i, c in enumerate(rep):
                             obuffer.append((old_char_tokens[a+i][0], 'del', c))
                         a += l
                     elif op == 1:
                         for i, c in enumerate(rep):
                             nbuffer.append((new_char_tokens[b+i][0], 'ins', c))
                         b += l
             else:
                 for ctag, co1, co2, cn1, cn2 in copcodes:
                     if ctag == 'equal':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, '', token_text))
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, '', token_text))
                     elif ctag == 'delete':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, 'del', token_text))
                     elif ctag == 'insert':
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, 'ins', token_text))
                     elif ctag == 'replace':
                         for token_class, token_text in old_char_tokens[co1:co2]:
                             obuffer.append((token_class, 'del', token_text))
                         for token_class, token_text in new_char_tokens[cn1:cn2]:
                             nbuffer.append((token_class, 'ins', token_text))
 
             old_tokens_result.extend(obuffer)
             new_tokens_result.extend(nbuffer)
 
     return old_tokens_result, new_tokens_result, similarity
 
 
 def diffset_node_getter(commit):
     def get_node(fname):
         try:
             return commit.get_node(fname)
         except NodeDoesNotExistError:
             return None
 
     return get_node
 
 
 class DiffSet(object):
     """
     An object for parsing the diff result from diffs.DiffProcessor and
     adding highlighting, side by side/unified renderings and line diffs
     """
 
     HL_REAL = 'REAL'  # highlights using original file, slow
     HL_FAST = 'FAST'  # highlights using just the line, fast but not correct
                       # in the case of multiline code
     HL_NONE = 'NONE'  # no highlighting, fastest
 
     def __init__(self, highlight_mode=HL_REAL, repo_name=None,
                  source_repo_name=None,
                  source_node_getter=lambda filename: None,
                  target_repo_name=None,
                  target_node_getter=lambda filename: None,
                  source_nodes=None, target_nodes=None,
                  # files over this size will use fast highlighting
                  max_file_size_limit=150 * 1024,
                  ):
 
         self.highlight_mode = highlight_mode
         self.highlighted_filenodes = {
             'before': {},
             'after': {}
         }
         self.source_node_getter = source_node_getter
         self.target_node_getter = target_node_getter
         self.source_nodes = source_nodes or {}
         self.target_nodes = target_nodes or {}
         self.repo_name = repo_name
         self.target_repo_name = target_repo_name or repo_name
         self.source_repo_name = source_repo_name or repo_name
         self.max_file_size_limit = max_file_size_limit
 
     def render_patchset(self, patchset, source_ref=None, target_ref=None):
         diffset = AttributeDict(dict(
             lines_added=0,
             lines_deleted=0,
             changed_files=0,
             files=[],
             file_stats={},
             limited_diff=isinstance(patchset, LimitedDiffContainer),
             repo_name=self.repo_name,
             target_repo_name=self.target_repo_name,
             source_repo_name=self.source_repo_name,
             source_ref=source_ref,
             target_ref=target_ref,
         ))
         for patch in patchset:
             diffset.file_stats[patch['filename']] = patch['stats']
             filediff = self.render_patch(patch)
             filediff.diffset = StrictAttributeDict(dict(
                 source_ref=diffset.source_ref,
                 target_ref=diffset.target_ref,
                 repo_name=diffset.repo_name,
                 source_repo_name=diffset.source_repo_name,
                 target_repo_name=diffset.target_repo_name,
             ))
             diffset.files.append(filediff)
             diffset.changed_files += 1
             if not patch['stats']['binary']:
                 diffset.lines_added += patch['stats']['added']
                 diffset.lines_deleted += patch['stats']['deleted']
 
         return diffset
 
     _lexer_cache = {}
 
     def _get_lexer_for_filename(self, filename, filenode=None):
         # cached because we might need to call it twice for source/target
         if filename not in self._lexer_cache:
             if filenode:
                 lexer = filenode.lexer
                 extension = filenode.extension
             else:
                 lexer = FileNode.get_lexer(filename=filename)
                 extension = filename.split('.')[-1]
 
             lexer = get_custom_lexer(extension) or lexer
             self._lexer_cache[filename] = lexer
         return self._lexer_cache[filename]
 
     def render_patch(self, patch):
         log.debug('rendering diff for %r', patch['filename'])
 
         source_filename = patch['original_filename']
         target_filename = patch['filename']
 
         source_lexer = plain_text_lexer
         target_lexer = plain_text_lexer
 
         if not patch['stats']['binary']:
             node_hl_mode = self.HL_NONE if patch['chunks'] == [] else None
             hl_mode = node_hl_mode or self.highlight_mode
 
             if hl_mode == self.HL_REAL:
                 if (source_filename and patch['operation'] in ('D', 'M')
                     and source_filename not in self.source_nodes):
                         self.source_nodes[source_filename] = (
                             self.source_node_getter(source_filename))
 
                 if (target_filename and patch['operation'] in ('A', 'M')
                     and target_filename not in self.target_nodes):
                         self.target_nodes[target_filename] = (
                             self.target_node_getter(target_filename))
 
             elif hl_mode == self.HL_FAST:
                 source_lexer = self._get_lexer_for_filename(source_filename)
                 target_lexer = self._get_lexer_for_filename(target_filename)
 
         source_file = self.source_nodes.get(source_filename, source_filename)
         target_file = self.target_nodes.get(target_filename, target_filename)
         raw_id_uid = ''
         if self.source_nodes.get(source_filename):
             raw_id_uid = self.source_nodes[source_filename].commit.raw_id
 
         if not raw_id_uid and self.target_nodes.get(target_filename):
             # in case this is a new file we only have it in target
             raw_id_uid = self.target_nodes[target_filename].commit.raw_id
 
         source_filenode, target_filenode = None, None
 
         # TODO: dan: FileNode.lexer works on the content of the file - which
         # can be slow - issue #4289 explains a lexer clean up - which once
         # done can allow caching a lexer for a filenode to avoid the file lookup
         if isinstance(source_file, FileNode):
             source_filenode = source_file
             #source_lexer = source_file.lexer
             source_lexer = self._get_lexer_for_filename(source_filename)
             source_file.lexer = source_lexer
 
         if isinstance(target_file, FileNode):
             target_filenode = target_file
             #target_lexer = target_file.lexer
             target_lexer = self._get_lexer_for_filename(target_filename)
             target_file.lexer = target_lexer
 
         source_file_path, target_file_path = None, None
 
         if source_filename != '/dev/null':
             source_file_path = source_filename
         if target_filename != '/dev/null':
             target_file_path = target_filename
 
         source_file_type = source_lexer.name
         target_file_type = target_lexer.name
 
         filediff = AttributeDict({
             'source_file_path': source_file_path,
             'target_file_path': target_file_path,
             'source_filenode': source_filenode,
             'target_filenode': target_filenode,
             'source_file_type': target_file_type,
             'target_file_type': source_file_type,
             'patch': {'filename': patch['filename'], 'stats': patch['stats']},
             'operation': patch['operation'],
             'source_mode': patch['stats']['old_mode'],
             'target_mode': patch['stats']['new_mode'],
             'limited_diff': patch['is_limited_diff'],
             'hunks': [],
             'hunk_ops': None,
             'diffset': self,
             'raw_id': raw_id_uid,
         })
 
         file_chunks = patch['chunks'][1:]
         for i, hunk in enumerate(file_chunks, 1):
             hunkbit = self.parse_hunk(hunk, source_file, target_file)
             hunkbit.source_file_path = source_file_path
             hunkbit.target_file_path = target_file_path
             hunkbit.index = i
             filediff.hunks.append(hunkbit)
 
         # Simulate hunk on OPS type line which doesn't really contain any diff
         # this allows commenting on those
         if not file_chunks:
             actions = []
             for op_id, op_text in filediff.patch['stats']['ops'].items():
                 if op_id == DEL_FILENODE:
                     actions.append('file was removed')
                 elif op_id == BIN_FILENODE:
                     actions.append('binary diff hidden')
                 else:
                     actions.append(safe_unicode(op_text))
             action_line = 'NO CONTENT: ' + \
                           ', '.join(actions) or 'UNDEFINED_ACTION'
 
             hunk_ops = {'source_length': 0, 'source_start': 0,
                         'lines': [
                             {'new_lineno': 0, 'old_lineno': 1,
                              'action': 'unmod-no-hl', 'line': action_line}
                         ],
                         'section_header': '', 'target_start': 1, 'target_length': 1}
 
             hunkbit = self.parse_hunk(hunk_ops, source_file, target_file)
             hunkbit.source_file_path = source_file_path
             hunkbit.target_file_path = target_file_path
             filediff.hunk_ops = hunkbit
         return filediff
 
     def parse_hunk(self, hunk, source_file, target_file):
         result = AttributeDict(dict(
             source_start=hunk['source_start'],
             source_length=hunk['source_length'],
             target_start=hunk['target_start'],
             target_length=hunk['target_length'],
             section_header=hunk['section_header'],
             lines=[],
         ))
         before, after = [], []
 
         for line in hunk['lines']:
             if line['action'] in ['unmod', 'unmod-no-hl']:
                 no_hl = line['action'] == 'unmod-no-hl'
                 result.lines.extend(
                     self.parse_lines(before, after, source_file, target_file, no_hl=no_hl))
                 after.append(line)
                 before.append(line)
             elif line['action'] == 'add':
                 after.append(line)
             elif line['action'] == 'del':
                 before.append(line)
             elif line['action'] == 'old-no-nl':
                 before.append(line)
             elif line['action'] == 'new-no-nl':
                 after.append(line)
 
         all_actions = [x['action'] for x in after] + [x['action'] for x in before]
         no_hl = {x for x in all_actions} == {'unmod-no-hl'}
         result.lines.extend(
             self.parse_lines(before, after, source_file, target_file, no_hl=no_hl))
         # NOTE(marcink): we must keep list() call here so we can cache the result...
         result.unified = list(self.as_unified(result.lines))
         result.sideside = result.lines
 
         return result
 
     def parse_lines(self, before_lines, after_lines, source_file, target_file,
                     no_hl=False):
         # TODO: dan: investigate doing the diff comparison and fast highlighting
         # on the entire before and after buffered block lines rather than by
         # line, this means we can get better 'fast' highlighting if the context
         # allows it - eg.
         # line 4: """
         # line 5: this gets highlighted as a string
         # line 6: """
 
         lines = []
 
         before_newline = AttributeDict()
         after_newline = AttributeDict()
         if before_lines and before_lines[-1]['action'] == 'old-no-nl':
             before_newline_line = before_lines.pop(-1)
             before_newline.content = '\n {}'.format(
                 render_tokenstream(
                     [(x[0], '', x[1])
                      for x in [('nonl', before_newline_line['line'])]]))
 
         if after_lines and after_lines[-1]['action'] == 'new-no-nl':
             after_newline_line = after_lines.pop(-1)
             after_newline.content = '\n {}'.format(
                 render_tokenstream(
                     [(x[0], '', x[1])
                      for x in [('nonl', after_newline_line['line'])]]))
 
         while before_lines or after_lines:
             before, after = None, None
             before_tokens, after_tokens = None, None
 
             if before_lines:
                 before = before_lines.pop(0)
             if after_lines:
                 after = after_lines.pop(0)
 
             original = AttributeDict()
             modified = AttributeDict()
 
             if before:
                 if before['action'] == 'old-no-nl':
                     before_tokens = [('nonl', before['line'])]
                 else:
                     before_tokens = self.get_line_tokens(
                         line_text=before['line'], line_number=before['old_lineno'],
                         input_file=source_file, no_hl=no_hl, source='before')
                 original.lineno = before['old_lineno']
                 original.content = before['line']
                 original.action = self.action_to_op(before['action'])
 
                 original.get_comment_args = (
                     source_file, 'o', before['old_lineno'])
 
             if after:
                 if after['action'] == 'new-no-nl':
                     after_tokens = [('nonl', after['line'])]
                 else:
                     after_tokens = self.get_line_tokens(
                         line_text=after['line'], line_number=after['new_lineno'],
                         input_file=target_file, no_hl=no_hl, source='after')
                 modified.lineno = after['new_lineno']
                 modified.content = after['line']
                 modified.action = self.action_to_op(after['action'])
 
                 modified.get_comment_args = (target_file, 'n', after['new_lineno'])
 
             # diff the lines
             if before_tokens and after_tokens:
                 o_tokens, m_tokens, similarity = tokens_diff(
                     before_tokens, after_tokens)
                 original.content = render_tokenstream(o_tokens)
                 modified.content = render_tokenstream(m_tokens)
             elif before_tokens:
                 original.content = render_tokenstream(
                     [(x[0], '', x[1]) for x in before_tokens])
             elif after_tokens:
                 modified.content = render_tokenstream(
                     [(x[0], '', x[1]) for x in after_tokens])
 
             if not before_lines and before_newline:
                 original.content += before_newline.content
                 before_newline = None
             if not after_lines and after_newline:
                 modified.content += after_newline.content
                 after_newline = None
 
             lines.append(AttributeDict({
                 'original': original,
                 'modified': modified,
             }))
 
         return lines
 
     def get_line_tokens(self, line_text, line_number, input_file=None, no_hl=False, source=''):
         filenode = None
         filename = None
 
         if isinstance(input_file, str):
             filename = input_file
         elif isinstance(input_file, FileNode):
             filenode = input_file
             filename = input_file.unicode_path
 
         hl_mode = self.HL_NONE if no_hl else self.highlight_mode
         if hl_mode == self.HL_REAL and filenode:
             lexer = self._get_lexer_for_filename(filename)
             file_size_allowed = input_file.size < self.max_file_size_limit
             if line_number and file_size_allowed:
                 return self.get_tokenized_filenode_line(input_file, line_number, lexer, source)
 
         if hl_mode in (self.HL_REAL, self.HL_FAST) and filename:
             lexer = self._get_lexer_for_filename(filename)
             return list(tokenize_string(line_text, lexer))
 
         return list(tokenize_string(line_text, plain_text_lexer))
 
     def get_tokenized_filenode_line(self, filenode, line_number, lexer=None, source=''):
 
         def tokenize(_filenode):
             self.highlighted_filenodes[source][filenode] = filenode_as_lines_tokens(filenode, lexer)
 
         if filenode not in self.highlighted_filenodes[source]:
             tokenize(filenode)
 
         try:
             return self.highlighted_filenodes[source][filenode][line_number - 1]
         except Exception:
             log.exception('diff rendering error')
             return [('', 'L{}: rhodecode diff rendering error'.format(line_number))]
 
     def action_to_op(self, action):
         return {
             'add': '+',
             'del': '-',
             'unmod': ' ',
             'unmod-no-hl': ' ',
             'old-no-nl': ' ',
             'new-no-nl': ' ',
         }.get(action, action)
 
     def as_unified(self, lines):
         """
         Return a generator that yields the lines of a diff in unified order
         """
         def generator():
             buf = []
             for line in lines:
 
                 if buf and not line.original or line.original.action == ' ':
                     for b in buf:
                         yield b
                     buf = []
 
                 if line.original:
                     if line.original.action == ' ':
                         yield (line.original.lineno, line.modified.lineno,
                                line.original.action, line.original.content,
                                line.original.get_comment_args)
                         continue
 
                     if line.original.action == '-':
                         yield (line.original.lineno, None,
                                line.original.action, line.original.content,
                                line.original.get_comment_args)
 
                     if line.modified.action == '+':
                         buf.append((
                             None, line.modified.lineno,
                             line.modified.action, line.modified.content,
                             line.modified.get_comment_args))
                         continue
 
                 if line.modified:
                     yield (None, line.modified.lineno,
                            line.modified.action, line.modified.content,
                            line.modified.get_comment_args)
 
             for b in buf:
                 yield b
 
         return generator()

rhodecode/lib/colander_utils.py

@@ -1,30 +1,30 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 def strip_whitespace(value):
     """
     Removes leading/trailing whitespace, newlines, and tabs from the value.
     Implements the `colander.interface.Preparer` interface.
     """
     if isinstance(value, str):
         return value.strip(' \t\n\r')
     else:
         return value

rhodecode/lib/colored_formatter.py

@@ -1,51 +1,50 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Legacy support for old logging formatter names.
 
 .. deprecated:: 3.2.0
 
    Got replaced by rhodecode.lib.logging_formatter. This module stays for a
    few versions to ease the migration of INI files.
 
 """
 
 import warnings
 
 from rhodecode.lib import logging_formatter
 
 
 def _deprecated_formatter(name):
     BaseFormatter = getattr(logging_formatter, name)
 
     class LegacyFormatter(BaseFormatter):
 
         def __init__(self, *args, **kwargs):
             warnings.warn(
                 "Use rhodecode.lib.logging_formatter.%s instead." % name,
                 DeprecationWarning)
             BaseFormatter.__init__(self, *args, **kwargs)
 
     return LegacyFormatter
 
 
 ColorFormatter = _deprecated_formatter('ColorFormatter')
 ColorFormatterSql = _deprecated_formatter('ColorFormatterSql')

rhodecode/lib/datelib.py

@@ -1,79 +1,78 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Provides utilities around date and time handling
 """
 
 import datetime
 import time
 
 
 def makedate():
     lt = time.localtime()
     if lt[8] == 1 and time.daylight:
         tz = time.altzone
     else:
         tz = time.timezone
     return time.mktime(lt), tz
 
 
 def utcdate_fromtimestamp(unixts, tzoffset=0):
     """
     Makes a local datetime object out of unix timestamp
 
     :param unixts:
     :param tzoffset:
     """
 
     return datetime.datetime.utcfromtimestamp(float(unixts))
 
 
 def date_astimestamp(value):
     """
     Convert a given `datetime.datetime` into a `float` like `time.time`
     """
     return time.mktime(value.timetuple()) + value.microsecond / 1E6
 
 
 def date_to_timestamp_plus_offset(value):
     """
     Convert a given `datetime.datetime` into a unix timestamp and offset.
     """
     # TODO: johbo: The time handling looks quite fragile here since we mix
     # system time zones with naive datetime instances.
     if value is None:
         value = time.time()
     elif isinstance(value, datetime.datetime):
         assert not is_aware(value), (
             "This code is not prepared to handle aware datetime instances")
         value = date_astimestamp(value)
     return (value, time.timezone)
 
 
 def is_aware(value):
     """
     Determines if a given datetime.time is aware.
 
     The logic is described in Python's docs:
     http://docs.python.org/library/datetime.html#datetime.tzinfo
     """
     return (value.tzinfo is not None
             and value.tzinfo.utcoffset(value) is not None)

rhodecode/lib/db_manage.py

@@ -1,680 +1,679 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Database creation, and setup module for RhodeCode Enterprise. Used for creation
 of database as well as for migration operations
 """
 
 import os
 import sys
 import time
 import uuid
 import logging
 import getpass
 from os.path import dirname as dn, join as jn
 
 from sqlalchemy.engine import create_engine
 
 from rhodecode import __dbversion__
 from rhodecode.model import init_model
 from rhodecode.model.user import UserModel
 from rhodecode.model.db import (
     User, Permission, RhodeCodeUi, RhodeCodeSetting, UserToPerm,
     DbMigrateVersion, RepoGroup, UserRepoGroupToPerm, CacheKey, Repository)
 from rhodecode.model.meta import Session, Base
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import SettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 def notify(msg):
     """
     Notification for migrations messages
     """
     ml = len(msg) + (4 * 2)
     print(('\n%s\n*** %s ***\n%s' % ('*' * ml, msg, '*' * ml)).upper())
 
 
 class DbManage(object):
 
     def __init__(self, log_sql, dbconf, root, tests=False,
                  SESSION=None, cli_args=None):
         self.dbname = dbconf.split('/')[-1]
         self.tests = tests
         self.root = root
         self.dburi = dbconf
         self.log_sql = log_sql
         self.cli_args = cli_args or {}
         self.init_db(SESSION=SESSION)
         self.ask_ok = self.get_ask_ok_func(self.cli_args.get('force_ask'))
 
     def db_exists(self):
         if not self.sa:
             self.init_db()
         try:
             self.sa.query(RhodeCodeUi)\
                 .filter(RhodeCodeUi.ui_key == '/')\
                 .scalar()
             return True
         except Exception:
             return False
         finally:
             self.sa.rollback()
 
     def get_ask_ok_func(self, param):
         if param not in [None]:
             # return a function lambda that has a default set to param
             return lambda *args, **kwargs: param
         else:
             from rhodecode.lib.utils import ask_ok
             return ask_ok
 
     def init_db(self, SESSION=None):
         if SESSION:
             self.sa = SESSION
         else:
             # init new sessions
             engine = create_engine(self.dburi, echo=self.log_sql)
             init_model(engine)
             self.sa = Session()
 
     def create_tables(self, override=False):
         """
         Create a auth database
         """
 
         log.info("Existing database with the same name is going to be destroyed.")
         log.info("Setup command will run DROP ALL command on that database.")
         if self.tests:
             destroy = True
         else:
             destroy = self.ask_ok('Are you sure that you want to destroy the old database? [y/n]')
         if not destroy:
             log.info('db tables bootstrap: Nothing done.')
             sys.exit(0)
         if destroy:
             Base.metadata.drop_all()
 
         checkfirst = not override
         Base.metadata.create_all(checkfirst=checkfirst)
         log.info('Created tables for %s', self.dbname)
 
     def set_db_version(self):
         ver = DbMigrateVersion()
         ver.version = __dbversion__
         ver.repository_id = 'rhodecode_db_migrations'
         ver.repository_path = 'versions'
         self.sa.add(ver)
         log.info('db version set to: %s', __dbversion__)
 
     def run_post_migration_tasks(self):
         """
         Run various tasks before actually doing migrations
         """
         # delete cache keys on each upgrade
         total = CacheKey.query().count()
         log.info("Deleting (%s) cache keys now...", total)
         CacheKey.delete_all_cache()
 
     def upgrade(self, version=None):
         """
         Upgrades given database schema to given revision following
         all needed steps, to perform the upgrade
 
         """
 
         from rhodecode.lib.dbmigrate.migrate.versioning import api
         from rhodecode.lib.dbmigrate.migrate.exceptions import \
             DatabaseNotControlledError
 
         if 'sqlite' in self.dburi:
             print(
                '********************** WARNING **********************\n'
                'Make sure your version of sqlite is at least 3.7.X.  \n'
                'Earlier versions are known to fail on some migrations\n'
                '*****************************************************\n')
 
         upgrade = self.ask_ok(
             'You are about to perform a database upgrade. Make '
             'sure you have backed up your database. '
             'Continue ? [y/n]')
         if not upgrade:
             log.info('No upgrade performed')
             sys.exit(0)
 
         repository_path = jn(dn(dn(dn(os.path.realpath(__file__)))),
                              'rhodecode/lib/dbmigrate')
         db_uri = self.dburi
 
         if version:
             DbMigrateVersion.set_version(version)
 
         try:
             curr_version = api.db_version(db_uri, repository_path)
             msg = ('Found current database db_uri under version '
                    'control with version {}'.format(curr_version))
 
         except (RuntimeError, DatabaseNotControlledError):
             curr_version = 1
             msg = ('Current database is not under version control. Setting '
                    'as version %s' % curr_version)
             api.version_control(db_uri, repository_path, curr_version)
 
         notify(msg)
 
 
         if curr_version == __dbversion__:
             log.info('This database is already at the newest version')
             sys.exit(0)
 
         upgrade_steps = range(curr_version + 1, __dbversion__ + 1)
         notify('attempting to upgrade database from '
                'version %s to version %s' % (curr_version, __dbversion__))
 
         # CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE
         _step = None
         for step in upgrade_steps:
             notify('performing upgrade step %s' % step)
             time.sleep(0.5)
 
             api.upgrade(db_uri, repository_path, step)
             self.sa.rollback()
             notify('schema upgrade for step %s completed' % (step,))
 
             _step = step
 
         self.run_post_migration_tasks()
         notify('upgrade to version %s successful' % _step)
 
     def fix_repo_paths(self):
         """
         Fixes an old RhodeCode version path into new one without a '*'
         """
 
         paths = self.sa.query(RhodeCodeUi)\
                 .filter(RhodeCodeUi.ui_key == '/')\
                 .scalar()
 
         paths.ui_value = paths.ui_value.replace('*', '')
 
         try:
             self.sa.add(paths)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_default_user(self):
         """
         Fixes an old default user with some 'nicer' default values,
         used mostly for anonymous access
         """
         def_user = self.sa.query(User)\
                 .filter(User.username == User.DEFAULT_USER)\
                 .one()
 
         def_user.name = 'Anonymous'
         def_user.lastname = 'User'
         def_user.email = User.DEFAULT_USER_EMAIL
 
         try:
             self.sa.add(def_user)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_settings(self):
         """
         Fixes rhodecode settings and adds ga_code key for google analytics
         """
 
         hgsettings3 = RhodeCodeSetting('ga_code', '')
 
         try:
             self.sa.add(hgsettings3)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def create_admin_and_prompt(self):
 
         # defaults
         defaults = self.cli_args
         username = defaults.get('username')
         password = defaults.get('password')
         email = defaults.get('email')
 
         if username is None:
             username = eval(input('Specify admin username:'))
         if password is None:
             password = self._get_admin_password()
             if not password:
                 # second try
                 password = self._get_admin_password()
                 if not password:
                     sys.exit()
         if email is None:
             email = eval(input('Specify admin email:'))
         api_key = self.cli_args.get('api_key')
         self.create_user(username, password, email, True,
                          strict_creation_check=False,
                          api_key=api_key)
 
     def _get_admin_password(self):
         password = getpass.getpass('Specify admin password '
                                    '(min 6 chars):')
         confirm = getpass.getpass('Confirm password:')
 
         if password != confirm:
             log.error('passwords mismatch')
             return False
         if len(password) < 6:
             log.error('password is too short - use at least 6 characters')
             return False
 
         return password
 
     def create_test_admin_and_users(self):
         log.info('creating admin and regular test users')
         from rhodecode.tests import TEST_USER_ADMIN_LOGIN, \
             TEST_USER_ADMIN_PASS, TEST_USER_ADMIN_EMAIL, \
             TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, \
             TEST_USER_REGULAR_EMAIL, TEST_USER_REGULAR2_LOGIN, \
             TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR2_EMAIL
 
         self.create_user(TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,
                          TEST_USER_ADMIN_EMAIL, True, api_key=True)
 
         self.create_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS,
                          TEST_USER_REGULAR_EMAIL, False, api_key=True)
 
         self.create_user(TEST_USER_REGULAR2_LOGIN, TEST_USER_REGULAR2_PASS,
                          TEST_USER_REGULAR2_EMAIL, False, api_key=True)
 
     def create_ui_settings(self, repo_store_path):
         """
         Creates ui settings, fills out hooks
         and disables dotencode
         """
         settings_model = SettingsModel(sa=self.sa)
         from rhodecode.lib.vcs.backends.hg import largefiles_store
         from rhodecode.lib.vcs.backends.git import lfs_store
 
         # Build HOOKS
         hooks = [
             (RhodeCodeUi.HOOK_REPO_SIZE, 'python:vcsserver.hooks.repo_size'),
 
             # HG
             (RhodeCodeUi.HOOK_PRE_PULL, 'python:vcsserver.hooks.pre_pull'),
             (RhodeCodeUi.HOOK_PULL, 'python:vcsserver.hooks.log_pull_action'),
             (RhodeCodeUi.HOOK_PRE_PUSH, 'python:vcsserver.hooks.pre_push'),
             (RhodeCodeUi.HOOK_PRETX_PUSH, 'python:vcsserver.hooks.pre_push'),
             (RhodeCodeUi.HOOK_PUSH, 'python:vcsserver.hooks.log_push_action'),
             (RhodeCodeUi.HOOK_PUSH_KEY, 'python:vcsserver.hooks.key_push'),
 
         ]
 
         for key, value in hooks:
             hook_obj = settings_model.get_ui_by_key(key)
             hooks2 = hook_obj if hook_obj else RhodeCodeUi()
             hooks2.ui_section = 'hooks'
             hooks2.ui_key = key
             hooks2.ui_value = value
             self.sa.add(hooks2)
 
         # enable largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'extensions'
         largefiles.ui_key = 'largefiles'
         largefiles.ui_value = ''
         self.sa.add(largefiles)
 
         # set default largefiles cache dir, defaults to
         # /repo_store_location/.cache/largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'largefiles'
         largefiles.ui_key = 'usercache'
         largefiles.ui_value = largefiles_store(repo_store_path)
 
         self.sa.add(largefiles)
 
         # set default lfs cache dir, defaults to
         # /repo_store_location/.cache/lfs_store
         lfsstore = RhodeCodeUi()
         lfsstore.ui_section = 'vcs_git_lfs'
         lfsstore.ui_key = 'store_location'
         lfsstore.ui_value = lfs_store(repo_store_path)
 
         self.sa.add(lfsstore)
 
         # enable hgsubversion disabled by default
         hgsubversion = RhodeCodeUi()
         hgsubversion.ui_section = 'extensions'
         hgsubversion.ui_key = 'hgsubversion'
         hgsubversion.ui_value = ''
         hgsubversion.ui_active = False
         self.sa.add(hgsubversion)
 
         # enable hgevolve disabled by default
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'extensions'
         hgevolve.ui_key = 'evolve'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'experimental'
         hgevolve.ui_key = 'evolution'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'experimental'
         hgevolve.ui_key = 'evolution.exchange'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'extensions'
         hgevolve.ui_key = 'topic'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         # enable hggit disabled by default
         hggit = RhodeCodeUi()
         hggit.ui_section = 'extensions'
         hggit.ui_key = 'hggit'
         hggit.ui_value = ''
         hggit.ui_active = False
         self.sa.add(hggit)
 
         # set svn branch defaults
         branches = ["/branches/*", "/trunk"]
         tags = ["/tags/*"]
 
         for branch in branches:
             settings_model.create_ui_section_value(
                 RhodeCodeUi.SVN_BRANCH_ID, branch)
 
         for tag in tags:
             settings_model.create_ui_section_value(RhodeCodeUi.SVN_TAG_ID, tag)
 
     def create_auth_plugin_options(self, skip_existing=False):
         """
         Create default auth plugin settings, and make it active
 
         :param skip_existing:
         """
         defaults = [
             ('auth_plugins',
              'egg:rhodecode-enterprise-ce#token,egg:rhodecode-enterprise-ce#rhodecode',
              'list'),
 
             ('auth_authtoken_enabled',
              'True',
              'bool'),
 
             ('auth_rhodecode_enabled',
              'True',
              'bool'),
         ]
         for k, v, t in defaults:
             if (skip_existing and
                     SettingsModel().get_setting_by_name(k) is not None):
                 log.debug('Skipping option %s', k)
                 continue
             setting = RhodeCodeSetting(k, v, t)
             self.sa.add(setting)
 
     def create_default_options(self, skip_existing=False):
         """Creates default settings"""
 
         for k, v, t in [
             ('default_repo_enable_locking',  False, 'bool'),
             ('default_repo_enable_downloads', False, 'bool'),
             ('default_repo_enable_statistics', False, 'bool'),
             ('default_repo_private', False, 'bool'),
             ('default_repo_type', 'hg', 'unicode')]:
 
             if (skip_existing and
                     SettingsModel().get_setting_by_name(k) is not None):
                 log.debug('Skipping option %s', k)
                 continue
             setting = RhodeCodeSetting(k, v, t)
             self.sa.add(setting)
 
     def fixup_groups(self):
         def_usr = User.get_default_user()
         for g in RepoGroup.query().all():
             g.group_name = g.get_new_name(g.name)
             self.sa.add(g)
             # get default perm
             default = UserRepoGroupToPerm.query()\
                 .filter(UserRepoGroupToPerm.group == g)\
                 .filter(UserRepoGroupToPerm.user == def_usr)\
                 .scalar()
 
             if default is None:
                 log.debug('missing default permission for group %s adding', g)
                 perm_obj = RepoGroupModel()._create_default_perms(g)
                 self.sa.add(perm_obj)
 
     def reset_permissions(self, username):
         """
         Resets permissions to default state, useful when old systems had
         bad permissions, we must clean them up
 
         :param username:
         """
         default_user = User.get_by_username(username)
         if not default_user:
             return
 
         u2p = UserToPerm.query()\
             .filter(UserToPerm.user == default_user).all()
         fixed = False
         if len(u2p) != len(Permission.DEFAULT_USER_PERMISSIONS):
             for p in u2p:
                 Session().delete(p)
             fixed = True
             self.populate_default_permissions()
         return fixed
 
     def config_prompt(self, test_repo_path='', retries=3):
         defaults = self.cli_args
         _path = defaults.get('repos_location')
         if retries == 3:
             log.info('Setting up repositories config')
 
         if _path is not None:
             path = _path
         elif not self.tests and not test_repo_path:
             path = eval(input(
                  'Enter a valid absolute path to store repositories. '
                  'All repositories in that path will be added automatically:'
             ))
         else:
             path = test_repo_path
         path_ok = True
 
         # check proper dir
         if not os.path.isdir(path):
             path_ok = False
             log.error('Given path %s is not a valid directory', path)
 
         elif not os.path.isabs(path):
             path_ok = False
             log.error('Given path %s is not an absolute path', path)
 
         # check if path is at least readable.
         if not os.access(path, os.R_OK):
             path_ok = False
             log.error('Given path %s is not readable', path)
 
         # check write access, warn user about non writeable paths
         elif not os.access(path, os.W_OK) and path_ok:
             log.warning('No write permission to given path %s', path)
 
             q = ('Given path %s is not writeable, do you want to '
                  'continue with read only mode ? [y/n]' % (path,))
             if not self.ask_ok(q):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         if retries == 0:
             sys.exit('max retries reached')
         if not path_ok:
             retries -= 1
             return self.config_prompt(test_repo_path, retries)
 
         real_path = os.path.normpath(os.path.realpath(path))
 
         if real_path != os.path.normpath(path):
             q = ('Path looks like a symlink, RhodeCode Enterprise will store '
                  'given path as %s ? [y/n]') % (real_path,)
             if not self.ask_ok(q):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         return real_path
 
     def create_settings(self, path):
 
         self.create_ui_settings(path)
 
         ui_config = [
             ('web', 'push_ssl', 'False'),
             ('web', 'allow_archive', 'gz zip bz2'),
             ('web', 'allow_push', '*'),
             ('web', 'baseurl', '/'),
             ('paths', '/', path),
             ('phases', 'publish', 'True')
         ]
         for section, key, value in ui_config:
             ui_conf = RhodeCodeUi()
             setattr(ui_conf, 'ui_section', section)
             setattr(ui_conf, 'ui_key', key)
             setattr(ui_conf, 'ui_value', value)
             self.sa.add(ui_conf)
 
         # rhodecode app settings
         settings = [
             ('realm', 'RhodeCode', 'unicode'),
             ('title', '', 'unicode'),
             ('pre_code', '', 'unicode'),
             ('post_code', '', 'unicode'),
 
             # Visual
             ('show_public_icon', True, 'bool'),
             ('show_private_icon', True, 'bool'),
             ('stylify_metatags', True, 'bool'),
             ('dashboard_items', 100, 'int'),
             ('admin_grid_items', 25, 'int'),
 
             ('markup_renderer', 'markdown', 'unicode'),
 
             ('repository_fields', True, 'bool'),
             ('show_version', True, 'bool'),
             ('show_revision_number', True, 'bool'),
             ('show_sha_length', 12, 'int'),
 
             ('use_gravatar', False, 'bool'),
             ('gravatar_url', User.DEFAULT_GRAVATAR_URL, 'unicode'),
 
             ('clone_uri_tmpl', Repository.DEFAULT_CLONE_URI, 'unicode'),
             ('clone_uri_id_tmpl', Repository.DEFAULT_CLONE_URI_ID, 'unicode'),
             ('clone_uri_ssh_tmpl', Repository.DEFAULT_CLONE_URI_SSH, 'unicode'),
             ('support_url', '', 'unicode'),
             ('update_url', RhodeCodeSetting.DEFAULT_UPDATE_URL, 'unicode'),
 
             # VCS Settings
             ('pr_merge_enabled', True, 'bool'),
             ('use_outdated_comments', True, 'bool'),
             ('diff_cache', True, 'bool'),
         ]
 
         for key, val, type_ in settings:
             sett = RhodeCodeSetting(key, val, type_)
             self.sa.add(sett)
 
         self.create_auth_plugin_options()
         self.create_default_options()
 
         log.info('created ui config')
 
     def create_user(self, username, password, email='', admin=False,
                     strict_creation_check=True, api_key=None):
         log.info('creating user `%s`', username)
         user = UserModel().create_or_update(
             username, password, email, firstname='RhodeCode', lastname='Admin',
             active=True, admin=admin, extern_type="rhodecode",
             strict_creation_check=strict_creation_check)
 
         if api_key:
             log.info('setting a new default auth token for user `%s`', username)
             UserModel().add_auth_token(
                 user=user, lifetime_minutes=-1,
                 role=UserModel.auth_token_role.ROLE_ALL,
                 description='BUILTIN TOKEN')
 
     def create_default_user(self):
         log.info('creating default user')
         # create default user for handling default permissions.
         user = UserModel().create_or_update(username=User.DEFAULT_USER,
                                             password=str(uuid.uuid1())[:20],
                                             email=User.DEFAULT_USER_EMAIL,
                                             firstname='Anonymous',
                                             lastname='User',
                                             strict_creation_check=False)
         # based on configuration options activate/de-activate this user which
         # controls anonymous access
         if self.cli_args.get('public_access') is False:
             log.info('Public access disabled')
             user.active = False
             Session().add(user)
             Session().commit()
 
     def create_permissions(self):
         """
         Creates all permissions defined in the system
         """
         # module.(access|create|change|delete)_[name]
         # module.(none|read|write|admin)
         log.info('creating permissions')
         PermissionModel(self.sa).create_permissions()
 
     def populate_default_permissions(self):
         """
         Populate default permissions. It will create only the default
         permissions that are missing, and not alter already defined ones
         """
         log.info('creating default user permissions')
         PermissionModel(self.sa).create_default_user_permissions(user=User.DEFAULT_USER)

rhodecode/lib/diffs.py

@@ -1,1271 +1,1271 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Set of diffing helpers, previously part of vcs
 """
 
 import os
 import re
 import bz2
 import gzip
 import time
 
 import collections
 import difflib
 import logging
 import pickle
 from itertools import tee
 
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.nodes import FileNode, SubModuleNode
 from rhodecode.lib.utils2 import safe_unicode, safe_str
 
 log = logging.getLogger(__name__)
 
 # define max context, a file with more than this numbers of lines is unusable
 # in browser anyway
 MAX_CONTEXT = 20 * 1024
 DEFAULT_CONTEXT = 3
 
 
 def get_diff_context(request):
     return MAX_CONTEXT if request.GET.get('fullcontext', '') == '1' else DEFAULT_CONTEXT
 
 
 def get_diff_whitespace_flag(request):
     return request.GET.get('ignorews', '') == '1'
 
 
 class OPS(object):
     ADD = 'A'
     MOD = 'M'
     DEL = 'D'
 
 
 def get_gitdiff(filenode_old, filenode_new, ignore_whitespace=True, context=3):
     """
     Returns git style diff between given ``filenode_old`` and ``filenode_new``.
 
     :param ignore_whitespace: ignore whitespaces in diff
     """
     # make sure we pass in default context
     context = context or 3
     # protect against IntOverflow when passing HUGE context
     if context > MAX_CONTEXT:
         context = MAX_CONTEXT
 
     submodules = [o for o in [filenode_new, filenode_old] if isinstance(o, SubModuleNode)]
     if submodules:
         return ''
 
     for filenode in (filenode_old, filenode_new):
         if not isinstance(filenode, FileNode):
             raise VCSError(
                 "Given object should be FileNode object, not %s"
                 % filenode.__class__)
 
     repo = filenode_new.commit.repository
     old_commit = filenode_old.commit or repo.EMPTY_COMMIT
     new_commit = filenode_new.commit
 
     vcs_gitdiff = repo.get_diff(
         old_commit, new_commit, filenode_new.path,
         ignore_whitespace, context, path1=filenode_old.path)
     return vcs_gitdiff
 
 NEW_FILENODE = 1
 DEL_FILENODE = 2
 MOD_FILENODE = 3
 RENAMED_FILENODE = 4
 COPIED_FILENODE = 5
 CHMOD_FILENODE = 6
 BIN_FILENODE = 7
 
 
 class LimitedDiffContainer(object):
 
     def __init__(self, diff_limit, cur_diff_size, diff):
         self.diff = diff
         self.diff_limit = diff_limit
         self.cur_diff_size = cur_diff_size
 
     def __getitem__(self, key):
         return self.diff.__getitem__(key)
 
     def __iter__(self):
         for l in self.diff:
             yield l
 
 
 class Action(object):
     """
     Contains constants for the action value of the lines in a parsed diff.
     """
 
     ADD = 'add'
     DELETE = 'del'
     UNMODIFIED = 'unmod'
 
     CONTEXT = 'context'
     OLD_NO_NL = 'old-no-nl'
     NEW_NO_NL = 'new-no-nl'
 
 
 class DiffProcessor(object):
     """
     Give it a unified or git diff and it returns a list of the files that were
     mentioned in the diff together with a dict of meta information that
     can be used to render it in a HTML template.
 
     .. note:: Unicode handling
 
        The original diffs are a byte sequence and can contain filenames
        in mixed encodings. This class generally returns `unicode` objects
        since the result is intended for presentation to the user.
 
     """
     _chunk_re = re.compile(r'^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)')
     _newline_marker = re.compile(r'^\\ No newline at end of file')
 
     # used for inline highlighter word split
     _token_re = re.compile(r'()(&gt;|&lt;|&amp;|\W+?)')
 
     # collapse ranges of commits over given number
     _collapse_commits_over = 5
 
     def __init__(self, diff, format='gitdiff', diff_limit=None,
                  file_limit=None, show_full_diff=True):
         """
         :param diff: A `Diff` object representing a diff from a vcs backend
         :param format: format of diff passed, `udiff` or `gitdiff`
         :param diff_limit: define the size of diff that is considered "big"
             based on that parameter cut off will be triggered, set to None
             to show full diff
         """
         self._diff = diff
         self._format = format
         self.adds = 0
         self.removes = 0
         # calculate diff size
         self.diff_limit = diff_limit
         self.file_limit = file_limit
         self.show_full_diff = show_full_diff
         self.cur_diff_size = 0
         self.parsed = False
         self.parsed_diff = []
 
         log.debug('Initialized DiffProcessor with %s mode', format)
         if format == 'gitdiff':
             self.differ = self._highlight_line_difflib
             self._parser = self._parse_gitdiff
         else:
             self.differ = self._highlight_line_udiff
             self._parser = self._new_parse_gitdiff
 
     def _copy_iterator(self):
         """
         make a fresh copy of generator, we should not iterate thru
         an original as it's needed for repeating operations on
         this instance of DiffProcessor
         """
         self.__udiff, iterator_copy = tee(self.__udiff)
         return iterator_copy
 
     def _escaper(self, string):
         """
         Escaper for diff escapes special chars and checks the diff limit
 
         :param string:
         """
         self.cur_diff_size += len(string)
 
         if not self.show_full_diff and (self.cur_diff_size > self.diff_limit):
             raise DiffLimitExceeded('Diff Limit Exceeded')
 
         return string \
             .replace('&', '&amp;')\
             .replace('<', '&lt;')\
             .replace('>', '&gt;')
 
     def _line_counter(self, l):
         """
         Checks each line and bumps total adds/removes for this diff
 
         :param l:
         """
         if l.startswith('+') and not l.startswith('+++'):
             self.adds += 1
         elif l.startswith('-') and not l.startswith('---'):
             self.removes += 1
         return safe_unicode(l)
 
     def _highlight_line_difflib(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
 
         if line['action'] == Action.DELETE:
             old, new = line, next_
         else:
             old, new = next_, line
 
         oldwords = self._token_re.split(old['line'])
         newwords = self._token_re.split(new['line'])
         sequence = difflib.SequenceMatcher(None, oldwords, newwords)
 
         oldfragments, newfragments = [], []
         for tag, i1, i2, j1, j2 in sequence.get_opcodes():
             oldfrag = ''.join(oldwords[i1:i2])
             newfrag = ''.join(newwords[j1:j2])
             if tag != 'equal':
                 if oldfrag:
                     oldfrag = '<del>%s</del>' % oldfrag
                 if newfrag:
                     newfrag = '<ins>%s</ins>' % newfrag
             oldfragments.append(oldfrag)
             newfragments.append(newfrag)
 
         old['line'] = "".join(oldfragments)
         new['line'] = "".join(newfragments)
 
     def _highlight_line_udiff(self, line, next_):
         """
         Highlight inline changes in both lines.
         """
         start = 0
         limit = min(len(line['line']), len(next_['line']))
         while start < limit and line['line'][start] == next_['line'][start]:
             start += 1
         end = -1
         limit -= start
         while -end <= limit and line['line'][end] == next_['line'][end]:
             end -= 1
         end += 1
         if start or end:
             def do(l):
                 last = end + len(l['line'])
                 if l['action'] == Action.ADD:
                     tag = 'ins'
                 else:
                     tag = 'del'
                 l['line'] = '%s<%s>%s</%s>%s' % (
                     l['line'][:start],
                     tag,
                     l['line'][start:last],
                     tag,
                     l['line'][last:]
                 )
             do(line)
             do(next_)
 
     def _clean_line(self, line, command):
         if command in ['+', '-', ' ']:
             # only modify the line if it's actually a diff thing
             line = line[1:]
         return line
 
     def _parse_gitdiff(self, inline_diff=True):
         _files = []
         diff_container = lambda arg: arg
 
         for chunk in self._diff.chunks():
             head = chunk.header
 
             diff = map(self._escaper, self.diff_splitter(chunk.diff))
             raw_diff = chunk.raw
             limited_diff = False
             exceeds_limit = False
 
             op = None
             stats = {
                 'added': 0,
                 'deleted': 0,
                 'binary': False,
                 'ops': {},
             }
 
             if head['deleted_file_mode']:
                 op = OPS.DEL
                 stats['binary'] = True
                 stats['ops'][DEL_FILENODE] = 'deleted file'
 
             elif head['new_file_mode']:
                 op = OPS.ADD
                 stats['binary'] = True
                 stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']
             else:  # modify operation, can be copy, rename or chmod
 
                 # CHMOD
                 if head['new_mode'] and head['old_mode']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][CHMOD_FILENODE] = (
                         'modified file chmod %s => %s' % (
                             head['old_mode'], head['new_mode']))
                 # RENAME
                 if head['rename_from'] != head['rename_to']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][RENAMED_FILENODE] = (
                         'file renamed from %s to %s' % (
                             head['rename_from'], head['rename_to']))
                 # COPY
                 if head.get('copy_from') and head.get('copy_to'):
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][COPIED_FILENODE] = (
                         'file copied from %s to %s' % (
                             head['copy_from'], head['copy_to']))
 
                 # If our new parsed headers didn't match anything fallback to
                 # old style detection
                 if op is None:
                     if not head['a_file'] and head['b_file']:
                         op = OPS.ADD
                         stats['binary'] = True
                         stats['ops'][NEW_FILENODE] = 'new file'
 
                     elif head['a_file'] and not head['b_file']:
                         op = OPS.DEL
                         stats['binary'] = True
                         stats['ops'][DEL_FILENODE] = 'deleted file'
 
                 # it's not ADD not DELETE
                 if op is None:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][MOD_FILENODE] = 'modified file'
 
             # a real non-binary diff
             if head['a_file'] or head['b_file']:
                 try:
                     raw_diff, chunks, _stats = self._parse_lines(diff)
                     stats['binary'] = False
                     stats['added'] = _stats[0]
                     stats['deleted'] = _stats[1]
                     # explicit mark that it's a modified file
                     if op == OPS.MOD:
                         stats['ops'][MOD_FILENODE] = 'modified file'
                     exceeds_limit = len(raw_diff) > self.file_limit
 
                     # changed from _escaper function so we validate size of
                     # each file instead of the whole diff
                     # diff will hide big files but still show small ones
                     # from my tests, big files are fairly safe to be parsed
                     # but the browser is the bottleneck
                     if not self.show_full_diff and exceeds_limit:
                         raise DiffLimitExceeded('File Limit Exceeded')
 
                 except DiffLimitExceeded:
                     diff_container = lambda _diff: \
                         LimitedDiffContainer(
                             self.diff_limit, self.cur_diff_size, _diff)
 
                     exceeds_limit = len(raw_diff) > self.file_limit
                     limited_diff = True
                     chunks = []
 
             else:  # GIT format binary patch, or possibly empty diff
                 if head['bin_patch']:
                     # we have operation already extracted, but we mark simply
                     # it's a diff we wont show for binary files
                     stats['ops'][BIN_FILENODE] = 'binary diff hidden'
                 chunks = []
 
             if chunks and not self.show_full_diff and op == OPS.DEL:
                 # if not full diff mode show deleted file contents
                 # TODO: anderson: if the view is not too big, there is no way
                 # to see the content of the file
                 chunks = []
 
             chunks.insert(0, [{
                                   'old_lineno': '',
                                   'new_lineno': '',
                                   'action': Action.CONTEXT,
                                   'line': msg,
                               } for _op, msg in stats['ops'].items()
                               if _op not in [MOD_FILENODE]])
 
             _files.append({
                 'filename': safe_unicode(head['b_path']),
                 'old_revision': head['a_blob_id'],
                 'new_revision': head['b_blob_id'],
                 'chunks': chunks,
                 'raw_diff': safe_unicode(raw_diff),
                 'operation': op,
                 'stats': stats,
                 'exceeds_limit': exceeds_limit,
                 'is_limited_diff': limited_diff,
             })
 
         sorter = lambda info: {OPS.ADD: 0, OPS.MOD: 1,
                                OPS.DEL: 2}.get(info['operation'])
 
         if not inline_diff:
             return diff_container(sorted(_files, key=sorter))
 
         # highlight inline changes
         for diff_data in _files:
             for chunk in diff_data['chunks']:
                 lineiter = iter(chunk)
                 try:
                     while 1:
                         line = next(lineiter)
                         if line['action'] not in (
                                 Action.UNMODIFIED, Action.CONTEXT):
                             nextline = next(lineiter)
                             if nextline['action'] in ['unmod', 'context'] or \
                                nextline['action'] == line['action']:
                                 continue
                             self.differ(line, nextline)
                 except StopIteration:
                     pass
 
         return diff_container(sorted(_files, key=sorter))
 
     def _check_large_diff(self):
         if self.diff_limit:
             log.debug('Checking if diff exceeds current diff_limit of %s', self.diff_limit)
         if not self.show_full_diff and (self.cur_diff_size > self.diff_limit):
             raise DiffLimitExceeded('Diff Limit `%s` Exceeded', self.diff_limit)
 
     # FIXME: NEWDIFFS: dan: this replaces _parse_gitdiff
     def _new_parse_gitdiff(self, inline_diff=True):
         _files = []
 
         # this can be overriden later to a LimitedDiffContainer type
         diff_container = lambda arg: arg
 
         for chunk in self._diff.chunks():
             head = chunk.header
             log.debug('parsing diff %r', head)
 
             raw_diff = chunk.raw
             limited_diff = False
             exceeds_limit = False
 
             op = None
             stats = {
                 'added': 0,
                 'deleted': 0,
                 'binary': False,
                 'old_mode': None,
                 'new_mode': None,
                 'ops': {},
             }
             if head['old_mode']:
                 stats['old_mode'] = head['old_mode']
             if head['new_mode']:
                 stats['new_mode'] = head['new_mode']
             if head['b_mode']:
                 stats['new_mode'] = head['b_mode']
 
             # delete file
             if head['deleted_file_mode']:
                 op = OPS.DEL
                 stats['binary'] = True
                 stats['ops'][DEL_FILENODE] = 'deleted file'
 
             # new file
             elif head['new_file_mode']:
                 op = OPS.ADD
                 stats['binary'] = True
                 stats['old_mode'] = None
                 stats['new_mode'] = head['new_file_mode']
                 stats['ops'][NEW_FILENODE] = 'new file %s' % head['new_file_mode']
 
             # modify operation, can be copy, rename or chmod
             else:
                 # CHMOD
                 if head['new_mode'] and head['old_mode']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][CHMOD_FILENODE] = (
                         'modified file chmod %s => %s' % (
                             head['old_mode'], head['new_mode']))
 
                 # RENAME
                 if head['rename_from'] != head['rename_to']:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['renamed'] = (head['rename_from'], head['rename_to'])
                     stats['ops'][RENAMED_FILENODE] = (
                         'file renamed from %s to %s' % (
                             head['rename_from'], head['rename_to']))
                 # COPY
                 if head.get('copy_from') and head.get('copy_to'):
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['copied'] = (head['copy_from'], head['copy_to'])
                     stats['ops'][COPIED_FILENODE] = (
                         'file copied from %s to %s' % (
                             head['copy_from'], head['copy_to']))
 
                 # If our new parsed headers didn't match anything fallback to
                 # old style detection
                 if op is None:
                     if not head['a_file'] and head['b_file']:
                         op = OPS.ADD
                         stats['binary'] = True
                         stats['new_file'] = True
                         stats['ops'][NEW_FILENODE] = 'new file'
 
                     elif head['a_file'] and not head['b_file']:
                         op = OPS.DEL
                         stats['binary'] = True
                         stats['ops'][DEL_FILENODE] = 'deleted file'
 
                 # it's not ADD not DELETE
                 if op is None:
                     op = OPS.MOD
                     stats['binary'] = True
                     stats['ops'][MOD_FILENODE] = 'modified file'
 
             # a real non-binary diff
             if head['a_file'] or head['b_file']:
                 # simulate splitlines, so we keep the line end part
                 diff = self.diff_splitter(chunk.diff)
 
                 # append each file to the diff size
                 raw_chunk_size = len(raw_diff)
 
                 exceeds_limit = raw_chunk_size > self.file_limit
                 self.cur_diff_size += raw_chunk_size
 
                 try:
                     # Check each file instead of the whole diff.
                     # Diff will hide big files but still show small ones.
                     # From the tests big files are fairly safe to be parsed
                     # but the browser is the bottleneck.
                     if not self.show_full_diff and exceeds_limit:
                         log.debug('File `%s` exceeds current file_limit of %s',
                                   safe_unicode(head['b_path']), self.file_limit)
                         raise DiffLimitExceeded(
                             'File Limit %s Exceeded', self.file_limit)
 
                     self._check_large_diff()
 
                     raw_diff, chunks, _stats = self._new_parse_lines(diff)
                     stats['binary'] = False
                     stats['added'] = _stats[0]
                     stats['deleted'] = _stats[1]
                     # explicit mark that it's a modified file
                     if op == OPS.MOD:
                         stats['ops'][MOD_FILENODE] = 'modified file'
 
                 except DiffLimitExceeded:
                     diff_container = lambda _diff: \
                         LimitedDiffContainer(
                             self.diff_limit, self.cur_diff_size, _diff)
 
                     limited_diff = True
                     chunks = []
 
             else:  # GIT format binary patch, or possibly empty diff
                 if head['bin_patch']:
                     # we have operation already extracted, but we mark simply
                     # it's a diff we wont show for binary files
                     stats['ops'][BIN_FILENODE] = 'binary diff hidden'
                 chunks = []
 
             # Hide content of deleted node by setting empty chunks
             if chunks and not self.show_full_diff and op == OPS.DEL:
                 # if not full diff mode show deleted file contents
                 # TODO: anderson: if the view is not too big, there is no way
                 # to see the content of the file
                 chunks = []
 
             chunks.insert(
                 0, [{'old_lineno': '',
                      'new_lineno': '',
                      'action': Action.CONTEXT,
                      'line': msg,
                      } for _op, msg in stats['ops'].items()
                     if _op not in [MOD_FILENODE]])
 
             original_filename = safe_unicode(head['a_path'])
             _files.append({
                 'original_filename': original_filename,
                 'filename': safe_unicode(head['b_path']),
                 'old_revision': head['a_blob_id'],
                 'new_revision': head['b_blob_id'],
                 'chunks': chunks,
                 'raw_diff': safe_unicode(raw_diff),
                 'operation': op,
                 'stats': stats,
                 'exceeds_limit': exceeds_limit,
                 'is_limited_diff': limited_diff,
             })
 
         sorter = lambda info: {OPS.ADD: 0, OPS.MOD: 1,
                                OPS.DEL: 2}.get(info['operation'])
 
         return diff_container(sorted(_files, key=sorter))
 
     # FIXME: NEWDIFFS: dan: this gets replaced by _new_parse_lines
     def _parse_lines(self, diff_iter):
         """
         Parse the diff an return data for the template.
         """
 
         stats = [0, 0]
         chunks = []
         raw_diff = []
 
         try:
             line = next(diff_iter)
 
             while line:
                 raw_diff.append(line)
                 lines = []
                 chunks.append(lines)
 
                 match = self._chunk_re.match(line)
 
                 if not match:
                     break
 
                 gr = match.groups()
                 (old_line, old_end,
                  new_line, new_end) = [int(x or 1) for x in gr[:-1]]
                 old_line -= 1
                 new_line -= 1
 
                 context = len(gr) == 5
                 old_end += old_line
                 new_end += new_line
 
                 if context:
                     # skip context only if it's first line
                     if int(gr[0]) > 1:
                         lines.append({
                             'old_lineno': '...',
                             'new_lineno': '...',
                             'action':     Action.CONTEXT,
                             'line':       line,
                         })
 
                 line = next(diff_iter)
 
                 while old_line < old_end or new_line < new_end:
                     command = ' '
                     if line:
                         command = line[0]
 
                     affects_old = affects_new = False
 
                     # ignore those if we don't expect them
                     if command in '#@':
                         continue
                     elif command == '+':
                         affects_new = True
                         action = Action.ADD
                         stats[0] += 1
                     elif command == '-':
                         affects_old = True
                         action = Action.DELETE
                         stats[1] += 1
                     else:
                         affects_old = affects_new = True
                         action = Action.UNMODIFIED
 
                     if not self._newline_marker.match(line):
                         old_line += affects_old
                         new_line += affects_new
                         lines.append({
                             'old_lineno':   affects_old and old_line or '',
                             'new_lineno':   affects_new and new_line or '',
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
                         raw_diff.append(line)
 
                     line = next(diff_iter)
 
                     if self._newline_marker.match(line):
                         # we need to append to lines, since this is not
                         # counted in the line specs of diff
                         lines.append({
                             'old_lineno':   '...',
                             'new_lineno':   '...',
                             'action':       Action.CONTEXT,
                             'line':         self._clean_line(line, command)
                         })
 
         except StopIteration:
             pass
         return ''.join(raw_diff), chunks, stats
 
     # FIXME: NEWDIFFS: dan: this replaces _parse_lines
     def _new_parse_lines(self, diff_iter):
         """
         Parse the diff an return data for the template.
         """
 
         stats = [0, 0]
         chunks = []
         raw_diff = []
 
         try:
             line = next(diff_iter)
 
             while line:
                 raw_diff.append(line)
                 # match header e.g @@ -0,0 +1 @@\n'
                 match = self._chunk_re.match(line)
 
                 if not match:
                     break
 
                 gr = match.groups()
                 (old_line, old_end,
                  new_line, new_end) = [int(x or 1) for x in gr[:-1]]
 
                 lines = []
                 hunk = {
                     'section_header': gr[-1],
                     'source_start': old_line,
                     'source_length': old_end,
                     'target_start': new_line,
                     'target_length': new_end,
                     'lines': lines,
                 }
                 chunks.append(hunk)
 
                 old_line -= 1
                 new_line -= 1
 
                 context = len(gr) == 5
                 old_end += old_line
                 new_end += new_line
 
                 line = next(diff_iter)
 
                 while old_line < old_end or new_line < new_end:
                     command = ' '
                     if line:
                         command = line[0]
 
                     affects_old = affects_new = False
 
                     # ignore those if we don't expect them
                     if command in '#@':
                         continue
                     elif command == '+':
                         affects_new = True
                         action = Action.ADD
                         stats[0] += 1
                     elif command == '-':
                         affects_old = True
                         action = Action.DELETE
                         stats[1] += 1
                     else:
                         affects_old = affects_new = True
                         action = Action.UNMODIFIED
 
                     if not self._newline_marker.match(line):
                         old_line += affects_old
                         new_line += affects_new
                         lines.append({
                             'old_lineno':   affects_old and old_line or '',
                             'new_lineno':   affects_new and new_line or '',
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
                     raw_diff.append(line)
 
                     line = next(diff_iter)
 
                     if self._newline_marker.match(line):
                         # we need to append to lines, since this is not
                         # counted in the line specs of diff
                         if affects_old:
                             action = Action.OLD_NO_NL
                         elif affects_new:
                             action = Action.NEW_NO_NL
                         else:
                             raise Exception('invalid context for no newline')
 
                         lines.append({
                             'old_lineno':   None,
                             'new_lineno':   None,
                             'action':       action,
                             'line':         self._clean_line(line, command)
                         })
 
         except StopIteration:
             pass
 
         return ''.join(raw_diff), chunks, stats
 
     def _safe_id(self, idstring):
         """Make a string safe for including in an id attribute.
 
         The HTML spec says that id attributes 'must begin with
         a letter ([A-Za-z]) and may be followed by any number
         of letters, digits ([0-9]), hyphens ("-"), underscores
         ("_"), colons (":"), and periods (".")'. These regexps
         are slightly over-zealous, in that they remove colons
         and periods unnecessarily.
 
         Whitespace is transformed into underscores, and then
         anything which is not a hyphen or a character that
         matches \w (alphanumerics and underscore) is removed.
 
         """
         # Transform all whitespace to underscore
         idstring = re.sub(r'\s', "_", '%s' % idstring)
         # Remove everything that is not a hyphen or a member of \w
         idstring = re.sub(r'(?!-)\W', "", idstring).lower()
         return idstring
 
     @classmethod
     def diff_splitter(cls, string):
         """
         Diff split that emulates .splitlines() but works only on \n
         """
         if not string:
             return
         elif string == '\n':
             yield '\n'
         else:
 
             has_newline = string.endswith('\n')
             elements = string.split('\n')
             if has_newline:
                 # skip last element as it's empty string from newlines
                 elements = elements[:-1]
 
             len_elements = len(elements)
 
             for cnt, line in enumerate(elements, start=1):
                 last_line = cnt == len_elements
                 if last_line and not has_newline:
                     yield safe_unicode(line)
                 else:
                     yield safe_unicode(line) + '\n'
 
     def prepare(self, inline_diff=True):
         """
         Prepare the passed udiff for HTML rendering.
 
         :return: A list of dicts with diff information.
         """
         parsed = self._parser(inline_diff=inline_diff)
         self.parsed = True
         self.parsed_diff = parsed
         return parsed
 
     def as_raw(self, diff_lines=None):
         """
         Returns raw diff as a byte string
         """
         return self._diff.raw
 
     def as_html(self, table_class='code-difftable', line_class='line',
                 old_lineno_class='lineno old', new_lineno_class='lineno new',
                 code_class='code', enable_comments=False, parsed_lines=None):
         """
         Return given diff as html table with customized css classes
         """
         # TODO(marcink): not sure how to pass in translator
         # here in an efficient way, leave the _ for proper gettext extraction
         _ = lambda s: s
 
         def _link_to_if(condition, label, url):
             """
             Generates a link if condition is meet or just the label if not.
             """
 
             if condition:
                 return '''<a href="%(url)s" class="tooltip"
                 title="%(title)s">%(label)s</a>''' % {
                     'title': _('Click to select line'),
                     'url': url,
                     'label': label
                 }
             else:
                 return label
         if not self.parsed:
             self.prepare()
 
         diff_lines = self.parsed_diff
         if parsed_lines:
             diff_lines = parsed_lines
 
         _html_empty = True
         _html = []
         _html.append('''<table class="%(table_class)s">\n''' % {
             'table_class': table_class
         })
 
         for diff in diff_lines:
             for line in diff['chunks']:
                 _html_empty = False
                 for change in line:
                     _html.append('''<tr class="%(lc)s %(action)s">\n''' % {
                         'lc': line_class,
                         'action': change['action']
                     })
                     anchor_old_id = ''
                     anchor_new_id = ''
                     anchor_old = "%(filename)s_o%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['old_lineno']
                     }
                     anchor_new = "%(filename)s_n%(oldline_no)s" % {
                         'filename': self._safe_id(diff['filename']),
                         'oldline_no': change['new_lineno']
                     }
                     cond_old = (change['old_lineno'] != '...' and
                                 change['old_lineno'])
                     cond_new = (change['new_lineno'] != '...' and
                                 change['new_lineno'])
                     if cond_old:
                         anchor_old_id = 'id="%s"' % anchor_old
                     if cond_new:
                         anchor_new_id = 'id="%s"' % anchor_new
 
                     if change['action'] != Action.CONTEXT:
                         anchor_link = True
                     else:
                         anchor_link = False
 
                     ###########################################################
                     # COMMENT ICONS
                     ###########################################################
                     _html.append('''\t<td class="add-comment-line"><span class="add-comment-content">''')
 
                     if enable_comments and change['action'] != Action.CONTEXT:
                         _html.append('''<a href="#"><span class="icon-comment-add"></span></a>''')
 
                     _html.append('''</span></td><td class="comment-toggle tooltip" title="Toggle Comment Thread"><i class="icon-comment"></i></td>\n''')
 
                     ###########################################################
                     # OLD LINE NUMBER
                     ###########################################################
                     _html.append('''\t<td %(a_id)s class="%(olc)s">''' % {
                         'a_id': anchor_old_id,
                         'olc': old_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(anchor_link, change['old_lineno'],
                                             '#%s' % anchor_old)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # NEW LINE NUMBER
                     ###########################################################
 
                     _html.append('''\t<td %(a_id)s class="%(nlc)s">''' % {
                         'a_id': anchor_new_id,
                         'nlc': new_lineno_class
                     })
 
                     _html.append('''%(link)s''' % {
                         'link': _link_to_if(anchor_link, change['new_lineno'],
                                             '#%s' % anchor_new)
                     })
                     _html.append('''</td>\n''')
                     ###########################################################
                     # CODE
                     ###########################################################
                     code_classes = [code_class]
                     if (not enable_comments or
                             change['action'] == Action.CONTEXT):
                         code_classes.append('no-comment')
                     _html.append('\t<td class="%s">' % ' '.join(code_classes))
                     _html.append('''\n\t\t<pre>%(code)s</pre>\n''' % {
                         'code': change['line']
                     })
 
                     _html.append('''\t</td>''')
                     _html.append('''\n</tr>\n''')
         _html.append('''</table>''')
         if _html_empty:
             return None
         return ''.join(_html)
 
     def stat(self):
         """
         Returns tuple of added, and removed lines for this instance
         """
         return self.adds, self.removes
 
     def get_context_of_line(
             self, path, diff_line=None, context_before=3, context_after=3):
         """
         Returns the context lines for the specified diff line.
 
         :type diff_line: :class:`DiffLineNumber`
         """
         assert self.parsed, "DiffProcessor is not initialized."
 
         if None not in diff_line:
             raise ValueError(
                 "Cannot specify both line numbers: {}".format(diff_line))
 
         file_diff = self._get_file_diff(path)
         chunk, idx = self._find_chunk_line_index(file_diff, diff_line)
 
         first_line_to_include = max(idx - context_before, 0)
         first_line_after_context = idx + context_after + 1
         context_lines = chunk[first_line_to_include:first_line_after_context]
 
         line_contents = [
             _context_line(line) for line in context_lines
             if _is_diff_content(line)]
         # TODO: johbo: Interim fixup, the diff chunks drop the final newline.
         # Once they are fixed, we can drop this line here.
         if line_contents:
             line_contents[-1] = (
                 line_contents[-1][0], line_contents[-1][1].rstrip('\n') + '\n')
         return line_contents
 
     def find_context(self, path, context, offset=0):
         """
         Finds the given `context` inside of the diff.
 
         Use the parameter `offset` to specify which offset the target line has
         inside of the given `context`. This way the correct diff line will be
         returned.
 
         :param offset: Shall be used to specify the offset of the main line
             within the given `context`.
         """
         if offset < 0 or offset >= len(context):
             raise ValueError(
                 "Only positive values up to the length of the context "
                 "minus one are allowed.")
 
         matches = []
         file_diff = self._get_file_diff(path)
 
         for chunk in file_diff['chunks']:
             context_iter = iter(context)
             for line_idx, line in enumerate(chunk):
                 try:
                     if _context_line(line) == next(context_iter):
                         continue
                 except StopIteration:
                     matches.append((line_idx, chunk))
                 context_iter = iter(context)
 
         # Increment position and triger StopIteration
         # if we had a match at the end
         line_idx += 1
         try:
             next(context_iter)
         except StopIteration:
             matches.append((line_idx, chunk))
 
         effective_offset = len(context) - offset
         found_at_diff_lines = [
             _line_to_diff_line_number(chunk[idx - effective_offset])
             for idx, chunk in matches]
 
         return found_at_diff_lines
 
     def _get_file_diff(self, path):
         for file_diff in self.parsed_diff:
             if file_diff['filename'] == path:
                 break
         else:
             raise FileNotInDiffException("File {} not in diff".format(path))
         return file_diff
 
     def _find_chunk_line_index(self, file_diff, diff_line):
         for chunk in file_diff['chunks']:
             for idx, line in enumerate(chunk):
                 if line['old_lineno'] == diff_line.old:
                     return chunk, idx
                 if line['new_lineno'] == diff_line.new:
                     return chunk, idx
         raise LineNotInDiffException(
             "The line {} is not part of the diff.".format(diff_line))
 
 
 def _is_diff_content(line):
     return line['action'] in (
         Action.UNMODIFIED, Action.ADD, Action.DELETE)
 
 
 def _context_line(line):
     return (line['action'], line['line'])
 
 
 DiffLineNumber = collections.namedtuple('DiffLineNumber', ['old', 'new'])
 
 
 def _line_to_diff_line_number(line):
     new_line_no = line['new_lineno'] or None
     old_line_no = line['old_lineno'] or None
     return DiffLineNumber(old=old_line_no, new=new_line_no)
 
 
 class FileNotInDiffException(Exception):
     """
     Raised when the context for a missing file is requested.
 
     If you request the context for a line in a file which is not part of the
     given diff, then this exception is raised.
     """
 
 
 class LineNotInDiffException(Exception):
     """
     Raised when the context for a missing line is requested.
 
     If you request the context for a line in a file and this line is not
     part of the given diff, then this exception is raised.
     """
 
 
 class DiffLimitExceeded(Exception):
     pass
 
 
 # NOTE(marcink): if diffs.mako change, probably this
 # needs a bump to next version
 CURRENT_DIFF_VERSION = 'v5'
 
 
 def _cleanup_cache_file(cached_diff_file):
     # cleanup file to not store it "damaged"
     try:
         os.remove(cached_diff_file)
     except Exception:
         log.exception('Failed to cleanup path %s', cached_diff_file)
 
 
 def _get_compression_mode(cached_diff_file):
     mode = 'bz2'
     if 'mode:plain' in cached_diff_file:
         mode = 'plain'
     elif 'mode:gzip' in cached_diff_file:
         mode = 'gzip'
     return mode
 
 
 def cache_diff(cached_diff_file, diff, commits):
     compression_mode = _get_compression_mode(cached_diff_file)
 
     struct = {
         'version': CURRENT_DIFF_VERSION,
         'diff': diff,
         'commits': commits
     }
 
     start = time.time()
     try:
         if compression_mode == 'plain':
             with open(cached_diff_file, 'wb') as f:
                 pickle.dump(struct, f)
         elif compression_mode == 'gzip':
             with gzip.GzipFile(cached_diff_file, 'wb') as f:
                 pickle.dump(struct, f)
         else:
             with bz2.BZ2File(cached_diff_file, 'wb') as f:
                 pickle.dump(struct, f)
     except Exception:
         log.warn('Failed to save cache', exc_info=True)
         _cleanup_cache_file(cached_diff_file)
 
     log.debug('Saved diff cache under %s in %.4fs', cached_diff_file, time.time() - start)
 
 
 def load_cached_diff(cached_diff_file):
     compression_mode = _get_compression_mode(cached_diff_file)
 
     default_struct = {
         'version': CURRENT_DIFF_VERSION,
         'diff': None,
         'commits': None
     }
 
     has_cache = os.path.isfile(cached_diff_file)
     if not has_cache:
         log.debug('Reading diff cache file failed %s', cached_diff_file)
         return default_struct
 
     data = None
 
     start = time.time()
     try:
         if compression_mode == 'plain':
             with open(cached_diff_file, 'rb') as f:
                 data = pickle.load(f)
         elif compression_mode == 'gzip':
             with gzip.GzipFile(cached_diff_file, 'rb') as f:
                 data = pickle.load(f)
         else:
             with bz2.BZ2File(cached_diff_file, 'rb') as f:
                 data = pickle.load(f)
     except Exception:
         log.warn('Failed to read diff cache file', exc_info=True)
 
     if not data:
         data = default_struct
 
     if not isinstance(data, dict):
         # old version of data ?
         data = default_struct
 
     # check version
     if data.get('version') != CURRENT_DIFF_VERSION:
         # purge cache
         _cleanup_cache_file(cached_diff_file)
         return default_struct
 
     log.debug('Loaded diff cache from %s in %.4fs', cached_diff_file, time.time() - start)
 
     return data
 
 
 def generate_diff_cache_key(*args):
     """
     Helper to generate a cache key using arguments
     """
     def arg_mapper(input_param):
         input_param = safe_str(input_param)
         # we cannot allow '/' in arguments since it would allow
         # subdirectory usage
         input_param.replace('/', '_')
         return input_param or None  # prevent empty string arguments
 
     return '_'.join([
         '{}' for i in range(len(args))]).format(*map(arg_mapper, args))
 
 
 def diff_cache_exist(cache_storage, *args):
     """
     Based on all generated arguments check and return a cache path
     """
     args = list(args) + ['mode:gzip']
     cache_key = generate_diff_cache_key(*args)
     cache_file_path = os.path.join(cache_storage, cache_key)
     # prevent path traversal attacks using some param that have e.g '../../'
     if not os.path.abspath(cache_file_path).startswith(cache_storage):
         raise ValueError('Final path must be within {}'.format(cache_storage))
 
     return cache_file_path

rhodecode/lib/enc_utils.py

@@ -1,48 +1,49 b''
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.lib.encrypt import encrypt_data, validate_and_decrypt_data
 from rhodecode.lib.encrypt2 import Encryptor
 
 ALLOWED_ALGOS = ['aes', 'fernet']
 
 
 def get_default_algo():
     import rhodecode
     return rhodecode.CONFIG.get('rhodecode.encrypted_values.algorithm') or 'aes'
 
 
 def encrypt_value(value: bytes, enc_key: bytes, algo: str = ''):
     if not algo:
         # not explicit algo, just use what's set by config
         algo = get_default_algo()
+
     if algo not in ALLOWED_ALGOS:
         ValueError(f'Bad encryption algorithm, should be {ALLOWED_ALGOS}, got: {algo}')
 
     enc_key = safe_bytes(enc_key)
     value = safe_bytes(value)
 
     if algo == 'aes':
         return encrypt_data(value, enc_key=enc_key)
     if algo == 'fernet':
         return Encryptor(enc_key).encrypt(value)
 
     return value
 
 
 def decrypt_value(value: bytes, enc_key: bytes, algo: str = '', strict_mode: bool = False):
 
     if not algo:
         # not explicit algo, just use what's set by config
         algo = get_default_algo()
     if algo not in ALLOWED_ALGOS:
         ValueError(f'Bad encryption algorithm, should be {ALLOWED_ALGOS}, got: {algo}')
 
     enc_key = safe_bytes(enc_key)
     value = safe_bytes(value)
     safe = not strict_mode
 
     if algo == 'aes':
         return validate_and_decrypt_data(value, enc_key, safe=safe)
     if algo == 'fernet':
         return Encryptor(enc_key).decrypt(value, safe=safe)
 
     return value

rhodecode/lib/encrypt.py

@@ -1,152 +1,152 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Generic encryption library for RhodeCode
 """
 
 import base64
 import logging
 
 from Crypto.Cipher import AES
 from Crypto import Random
 from Crypto.Hash import HMAC, SHA256
 
 from rhodecode.lib.str_utils import safe_bytes, safe_str
 from rhodecode.lib.exceptions import signature_verification_error
 
 
 class InvalidDecryptedValue(str):
 
     def __new__(cls, content):
         """
         This will generate something like this::
             <InvalidDecryptedValue(QkWusFgLJXR6m42v...)>
         And represent a safe indicator that encryption key is broken
         """
         content = '<{}({}...)>'.format(cls.__name__, content[:16])
         return str.__new__(cls, content)
 
 KEY_FORMAT = b'enc$aes_hmac${1}'
 
 
 class AESCipher(object):
 
     def __init__(self, key: bytes, hmac=False, strict_verification=True):
 
         if not key:
             raise ValueError('passed key variable is empty')
         self.strict_verification = strict_verification
         self.block_size = 32
         self.hmac_size = 32
         self.hmac = hmac
 
         self.key = SHA256.new(safe_bytes(key)).digest()
         self.hmac_key = SHA256.new(self.key).digest()
 
     def verify_hmac_signature(self, raw_data):
         org_hmac_signature = raw_data[-self.hmac_size:]
         data_without_sig = raw_data[:-self.hmac_size]
         recomputed_hmac = HMAC.new(
             self.hmac_key, data_without_sig, digestmod=SHA256).digest()
         return org_hmac_signature == recomputed_hmac
 
     def encrypt(self, raw: bytes):
         raw = self._pad(raw)
         iv = Random.new().read(AES.block_size)
         cipher = AES.new(self.key, AES.MODE_CBC, iv)
         enc_value = cipher.encrypt(raw)
 
         hmac_signature = b''
         if self.hmac:
             # compute hmac+sha256 on iv + enc text, we use
             # encrypt then mac method to create the signature
             hmac_signature = HMAC.new(
                 self.hmac_key, iv + enc_value, digestmod=SHA256).digest()
 
         return base64.b64encode(iv + enc_value + hmac_signature)
 
     def decrypt(self, enc, safe=True) -> bytes | InvalidDecryptedValue:
         enc_org = enc
         try:
             enc = base64.b64decode(enc)
         except Exception:
             logging.exception('Failed Base64 decode')
             raise signature_verification_error('Failed Base64 decode')
 
         if self.hmac and len(enc) > self.hmac_size:
             if self.verify_hmac_signature(enc):
                 # cut off the HMAC verification digest
                 enc = enc[:-self.hmac_size]
             else:
 
                 decrypt_fail = InvalidDecryptedValue(safe_str(enc_org))
                 if safe:
                     return decrypt_fail
                 raise signature_verification_error(decrypt_fail)
 
         iv = enc[:AES.block_size]
         cipher = AES.new(self.key, AES.MODE_CBC, iv)
         return self._unpad(cipher.decrypt(enc[AES.block_size:]))
 
     def _pad(self, s):
         block_pad = (self.block_size - len(s) % self.block_size)
         return s + block_pad * safe_bytes(chr(block_pad))
 
     @staticmethod
     def _unpad(s):
         return s[:-ord(s[len(s)-1:])]
 
 
 def validate_and_decrypt_data(enc_data, enc_key, enc_strict_mode=False, safe=True):
     enc_data = safe_str(enc_data)
 
     parts = enc_data.split('$', 3)
     if len(parts) != 3:
         raise ValueError(f'Encrypted Data has invalid format, expected {KEY_FORMAT}, got {parts}')
 
     enc_type = parts[1]
     enc_data_part = parts[2]
 
     if parts[0] != 'enc':
         # parts ok but without our header ?
         return enc_data
 
     # at that stage we know it's our encryption
     if enc_type == 'aes':
         decrypted_data = AESCipher(enc_key).decrypt(enc_data_part, safe=safe)
     elif enc_type == 'aes_hmac':
         decrypted_data = AESCipher(
             enc_key, hmac=True,
             strict_verification=enc_strict_mode).decrypt(enc_data_part, safe=safe)
 
     else:
         raise ValueError(
             f'Encryption type part is wrong, must be `aes` '
             f'or `aes_hmac`, got `{enc_type}` instead')
 
     return decrypted_data
 
 
 def encrypt_data(data, enc_key: bytes):
     enc_key = safe_bytes(enc_key)
     enc_value = AESCipher(enc_key, hmac=True).encrypt(safe_bytes(data))
     return KEY_FORMAT.replace(b'{1}', enc_value)

rhodecode/lib/exc_tracking.py

@@ -1,232 +1,231 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import time
 import sys
 import datetime
 import msgpack
 import logging
 import traceback
 import tempfile
 import glob
 
 log = logging.getLogger(__name__)
 
 # NOTE: Any changes should be synced with exc_tracking at vcsserver.lib.exc_tracking
 global_prefix = 'rhodecode'
 exc_store_dir_name = 'rc_exception_store_v1'
 
 
 def exc_serialize(exc_id, tb, exc_type, extra_data=None):
 
     data = {
         'version': 'v1',
         'exc_id': exc_id,
         'exc_utc_date': datetime.datetime.utcnow().isoformat(),
         'exc_timestamp': repr(time.time()),
         'exc_message': tb,
         'exc_type': exc_type,
     }
     if extra_data:
         data.update(extra_data)
     return msgpack.packb(data), data
 
 
 def exc_unserialize(tb):
     return msgpack.unpackb(tb)
 
 _exc_store = None
 
 
 def get_exc_store():
     """
     Get and create exception store if it's not existing
     """
     global _exc_store
     import rhodecode as app
 
     if _exc_store is not None:
         # quick global cache
         return _exc_store
 
     exc_store_dir = app.CONFIG.get('exception_tracker.store_path', '') or tempfile.gettempdir()
     _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
 
     _exc_store_path = os.path.abspath(_exc_store_path)
     if not os.path.isdir(_exc_store_path):
         os.makedirs(_exc_store_path)
         log.debug('Initializing exceptions store at %s', _exc_store_path)
         _exc_store = _exc_store_path
 
     return _exc_store_path
 
 
 def _store_exception(exc_id, exc_type_name, exc_traceback, prefix, send_email=None):
     """
     Low level function to store exception in the exception tracker
     """
     from pyramid.threadlocal import get_current_request
     import rhodecode as app
     request = get_current_request()
     extra_data = {}
     # NOTE(marcink): store request information into exc_data
     if request:
         extra_data['client_address'] = getattr(request, 'client_addr', '')
         extra_data['user_agent'] = getattr(request, 'user_agent', '')
         extra_data['method'] = getattr(request, 'method', '')
         extra_data['url'] = getattr(request, 'url', '')
 
     exc_store_path = get_exc_store()
     exc_data, org_data = exc_serialize(exc_id, exc_traceback, exc_type_name, extra_data=extra_data)
 
     exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
     if not os.path.isdir(exc_store_path):
         os.makedirs(exc_store_path)
     stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
     with open(stored_exc_path, 'wb') as f:
         f.write(exc_data)
     log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
 
     if send_email is None:
         # NOTE(marcink): read app config unless we specify explicitly
         send_email = app.CONFIG.get('exception_tracker.send_email', False)
 
     mail_server = app.CONFIG.get('smtp_server') or None
     send_email = send_email and mail_server
     if send_email and request:
         try:
             send_exc_email(request, exc_id, exc_type_name)
         except Exception:
             log.exception('Failed to send exception email')
             exc_info = sys.exc_info()
             store_exception(id(exc_info), exc_info, send_email=False)
 
 
 def send_exc_email(request, exc_id, exc_type_name):
     import rhodecode as app
     from rhodecode.apps._base import TemplateArgs
     from rhodecode.lib.utils2 import aslist
     from rhodecode.lib.celerylib import run_task, tasks
     from rhodecode.lib.base import attach_context_attributes
     from rhodecode.model.notification import EmailNotificationModel
 
     recipients = aslist(app.CONFIG.get('exception_tracker.send_email_recipients', ''))
     log.debug('Sending Email exception to: `%s`', recipients or 'all super admins')
 
     # NOTE(marcink): needed for email template rendering
     user_id = None
     if hasattr(request, 'user'):
         user_id = request.user.user_id
     attach_context_attributes(TemplateArgs(), request, user_id=user_id, is_api=True)
 
     email_kwargs = {
         'email_prefix': app.CONFIG.get('exception_tracker.email_prefix', '') or '[RHODECODE ERROR]',
         'exc_url': request.route_url('admin_settings_exception_tracker_show', exception_id=exc_id),
         'exc_id': exc_id,
         'exc_type_name': exc_type_name,
         'exc_traceback': read_exception(exc_id, prefix=None),
     }
 
     (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
         EmailNotificationModel.TYPE_EMAIL_EXCEPTION, **email_kwargs)
 
     run_task(tasks.send_email, recipients, subject,
              email_body_plaintext, email_body)
 
 
 def _prepare_exception(exc_info):
     exc_type, exc_value, exc_traceback = exc_info
     exc_type_name = exc_type.__name__
 
     tb = ''.join(traceback.format_exception(
         exc_type, exc_value, exc_traceback, None))
 
     return exc_type_name, tb
 
 
 def store_exception(exc_id, exc_info, prefix=global_prefix, send_email=None):
     """
     Example usage::
 
         exc_info = sys.exc_info()
         store_exception(id(exc_info), exc_info)
     """
 
     try:
         exc_type_name, exc_traceback = _prepare_exception(exc_info)
         _store_exception(exc_id=exc_id, exc_type_name=exc_type_name,
                          exc_traceback=exc_traceback, prefix=prefix, send_email=send_email)
         return exc_id, exc_type_name
     except Exception:
         log.exception('Failed to store exception `%s` information', exc_id)
         # there's no way this can fail, it will crash server badly if it does.
         pass
 
 
 def _find_exc_file(exc_id, prefix=global_prefix):
     exc_store_path = get_exc_store()
     if prefix:
         exc_id = '{}_{}'.format(exc_id, prefix)
     else:
         # search without a prefix
         exc_id = '{}'.format(exc_id)
 
     found_exc_id = None
     matches = glob.glob(os.path.join(exc_store_path, exc_id) + '*')
     if matches:
         found_exc_id = matches[0]
 
     return found_exc_id
 
 
 def _read_exception(exc_id, prefix):
     exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
     if exc_id_file_path:
         with open(exc_id_file_path, 'rb') as f:
             return exc_unserialize(f.read())
     else:
         log.debug('Exception File `%s` not found', exc_id_file_path)
     return None
 
 
 def read_exception(exc_id, prefix=global_prefix):
     try:
         return _read_exception(exc_id=exc_id, prefix=prefix)
     except Exception:
         log.exception('Failed to read exception `%s` information', exc_id)
         # there's no way this can fail, it will crash server badly if it does.
     return None
 
 
 def delete_exception(exc_id, prefix=global_prefix):
     try:
         exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
         if exc_id_file_path:
             os.remove(exc_id_file_path)
 
     except Exception:
         log.exception('Failed to remove exception `%s` information', exc_id)
         # there's no way this can fail, it will crash server badly if it does.
         pass
 
 
 def generate_id():
     return id(object())

rhodecode/lib/exceptions.py

@@ -1,183 +1,182 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Set of custom exceptions used in RhodeCode
 """
 
 from webob.exc import HTTPClientError
 from pyramid.httpexceptions import HTTPBadGateway
 
 
 class LdapUsernameError(Exception):
     pass
 
 
 class LdapPasswordError(Exception):
     pass
 
 
 class LdapConnectionError(Exception):
     pass
 
 
 class LdapImportError(Exception):
     pass
 
 
 class DefaultUserException(Exception):
     pass
 
 
 class UserOwnsReposException(Exception):
     pass
 
 
 class UserOwnsRepoGroupsException(Exception):
     pass
 
 
 class UserOwnsUserGroupsException(Exception):
     pass
 
 
 class UserOwnsPullRequestsException(Exception):
     pass
 
 
 class UserOwnsArtifactsException(Exception):
     pass
 
 
 class UserGroupAssignedException(Exception):
     pass
 
 
 class StatusChangeOnClosedPullRequestError(Exception):
     pass
 
 
 class AttachedForksError(Exception):
     pass
 
 
 class AttachedPullRequestsError(Exception):
     pass
 
 
 class RepoGroupAssignmentError(Exception):
     pass
 
 
 class NonRelativePathError(Exception):
     pass
 
 
 class HTTPRequirementError(HTTPClientError):
     title = explanation = 'Repository Requirement Missing'
     reason = None
 
     def __init__(self, message, *args, **kwargs):
         self.title = self.explanation = message
         super(HTTPRequirementError, self).__init__(*args, **kwargs)
         self.args = (message, )
 
 
 class HTTPLockedRC(HTTPClientError):
     """
     Special Exception For locked Repos in RhodeCode, the return code can
     be overwritten by _code keyword argument passed into constructors
     """
     code = 423
     title = explanation = 'Repository Locked'
     reason = None
 
     def __init__(self, message, *args, **kwargs):
         from rhodecode import CONFIG
         from rhodecode.lib.utils2 import safe_int
         _code = CONFIG.get('lock_ret_code')
         self.code = safe_int(_code, self.code)
         self.title = self.explanation = message
         super(HTTPLockedRC, self).__init__(*args, **kwargs)
         self.args = (message, )
 
 
 class HTTPBranchProtected(HTTPClientError):
     """
     Special Exception For Indicating that branch is protected in RhodeCode, the
     return code can be overwritten by _code keyword argument passed into constructors
     """
     code = 403
     title = explanation = 'Branch Protected'
     reason = None
 
     def __init__(self, message, *args, **kwargs):
         self.title = self.explanation = message
         super(HTTPBranchProtected, self).__init__(*args, **kwargs)
         self.args = (message, )
 
 
 class IMCCommitError(Exception):
     pass
 
 
 class UserCreationError(Exception):
     pass
 
 
 class NotAllowedToCreateUserError(Exception):
     pass
 
 
 class RepositoryCreationError(Exception):
     pass
 
 
 class VCSServerUnavailable(HTTPBadGateway):
     """ HTTP Exception class for VCS Server errors """
     code = 502
     title = 'VCS Server Error'
     causes = [
         'VCS Server is not running',
         'Incorrect vcs.server=host:port',
         'Incorrect vcs.server.protocol',
     ]
 
     def __init__(self, message=''):
         self.explanation = 'Could not connect to VCS Server'
         if message:
             self.explanation += ': ' + message
         super(VCSServerUnavailable, self).__init__()
 
 
 class ArtifactMetadataDuplicate(ValueError):
 
     def __init__(self, *args, **kwargs):
         self.err_section = kwargs.pop('err_section', None)
         self.err_key = kwargs.pop('err_key', None)
         super(ArtifactMetadataDuplicate, self).__init__(*args, **kwargs)
 
 
 class ArtifactMetadataBadValueType(ValueError):
     pass
 
 
 class CommentVersionMismatch(ValueError):
     pass

rhodecode/lib/ext_json_renderer.py

@@ -1,41 +1,40 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from rhodecode.lib import ext_json
 
 
 def pyramid_ext_json(info):
     """
     Custom json renderer for pyramid to use our ext_json lib
     """
     def _render(value, system):
         request = system.get('request')
         indent = None
         if request is not None:
             response = request.response
             ct = response.content_type
             if ct == response.default_content_type:
                 response.content_type = 'application/json'
             indent = getattr(request, 'ext_json_indent', None)
         if indent:
             return ext_json.formatted_json(value)
         return ext_json.json.dumps(value)
 
     return _render

rhodecode/lib/feedgenerator/__init__.py

@@ -1,21 +1,21 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 from .feedgenerator import Rss201rev2Feed, Atom1Feed
  No newline at end of file

rhodecode/lib/graphmod.py

@@ -1,142 +1,141 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Modified mercurial DAG graph functions that re-uses VCS structure
 
 It allows to have a shared codebase for DAG generation for hg, git, svn repos
 """
 
 nullrev = -1
 
 
 def grandparent(parent_idx_func, lowest_idx, roots, head):
     """
     Return all ancestors of head in roots which commit is
     greater or equal to lowest_idx.
     """
     pending = set([head])
     seen = set()
     kept = set()
     llowestrev = max(nullrev, lowest_idx)
     while pending:
         r = pending.pop()
         if r >= llowestrev and r not in seen:
             if r in roots:
                 kept.add(r)
             else:
                 pending.update(parent_idx_func(r))
             seen.add(r)
     return sorted(kept)
 
 
 def _dagwalker(repo, commits):
     if not commits:
         return
 
     def get_parent_indexes(idx):
         return [commit.idx for commit in repo[idx].parents]
 
     indexes = [commit['idx'] for commit in commits]
     lowest_idx = min(indexes)
     known_indexes = set(indexes)
 
     grandparnet_cache = {}
     for commit in commits:
         parents = sorted(set([p['idx'] for p in commit['parents']
                               if p['idx'] in known_indexes]))
         mpars = [p['idx'] for p in commit['parents'] if
                  p['idx'] != nullrev and p['idx'] not in parents]
         for mpar in mpars:
             gp = grandparnet_cache.get(mpar)
             if gp is None:
                 gp = grandparnet_cache[mpar] = grandparent(
                     get_parent_indexes, lowest_idx, indexes, mpar)
             if not gp:
                 parents.append(mpar)
             else:
                 parents.extend(g for g in gp if g not in parents)
 
         yield (commit['raw_id'], commit['idx'], parents, commit['branch'])
 
 
 def _colored(dag):
     """annotates a DAG with colored edge information
 
     For each DAG node this function emits tuples::
 
       ((col, color), [(col, nextcol, color)])
 
     with the following new elements:
 
       - Tuple (col, color) with column and color index for the current node
       - A list of tuples indicating the edges between the current node and its
         parents.
     """
     seen = []
     colors = {}
     newcolor = 1
 
     for commit_id, commit_idx, parents, branch in dag:
 
         # Compute seen and next_
         if commit_idx not in seen:
             seen.append(commit_idx)  # new head
             colors[commit_idx] = newcolor
             newcolor += 1
 
         col = seen.index(commit_idx)
         color = colors.pop(commit_idx)
         next_ = seen[:]
 
         # Add parents to next_
         addparents = [p for p in parents if p not in next_]
         next_[col:col + 1] = addparents
 
         # Set colors for the parents
         for i, p in enumerate(addparents):
             if i == 0:
                 colors[p] = color
             else:
                 colors[p] = newcolor
                 newcolor += 1
 
         # Add edges to the graph
         edges = []
         for ecol, eid in enumerate(seen):
             if eid in next_:
                 edges.append((ecol, next_.index(eid), colors[eid]))
             elif eid == commit_idx:
                 total_parents = len(parents)
                 edges.extend([
                     (ecol, next_.index(p),
                      _get_edge_color(p, total_parents, color, colors))
                     for p in parents])
 
         # Yield and move on
         yield (commit_id, (col, color), edges, branch)
         seen = next_
 
 
 def _get_edge_color(parent, total_parents, color, colors):
     if total_parents <= 1:
         return color
 
     return colors.get(parent, color)

rhodecode/lib/hash_utils.py

@@ -1,56 +1,56 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2011-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import hashlib
 from rhodecode.lib.str_utils import safe_bytes, safe_str
 
 
 def md5(s):
     return hashlib.md5(s).hexdigest()
 
 
 def md5_safe(s, return_type=''):
 
     val = md5(safe_bytes(s))
     if return_type == 'str':
         val = safe_str(val)
     return val
 
 
 def sha1(s):
     return hashlib.sha1(s).hexdigest()
 
 
 def sha1_safe(s, return_type=''):
     val = sha1(safe_bytes(s))
     if return_type == 'str':
         val = safe_str(val)
     return val
 
 
 def sha256(s):
     return hashlib.sha256(s).hexdigest()
 
 
 def sha256_safe(s, return_type=''):
     val = sha256(safe_bytes(s))
     if return_type == 'str':
         val = safe_str(val)
     return val

rhodecode/lib/helpers.py

@@ -1,2165 +1,2164 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Helper functions
 
 Consists of functions to typically be used within templates, but also
 available to Controllers. This module is available to both as 'h'.
 """
 import base64
 import collections
 
 import os
 import random
 import hashlib
 from io import StringIO
 import textwrap
 import urllib.request, urllib.parse, urllib.error
 import math
 import logging
 import re
 import time
 import string
 import hashlib
 import regex
 from collections import OrderedDict
 
 import pygments
 import itertools
 import fnmatch
 import bleach
 
 from datetime import datetime
 from functools import partial
 from pygments.formatters.html import HtmlFormatter
 from pygments.lexers import (
     get_lexer_by_name, get_lexer_for_filename, get_lexer_for_mimetype)
 
 from pyramid.threadlocal import get_current_request
 from tempita import looper
 from webhelpers2.html import literal, HTML, escape
 from webhelpers2.html._autolink import _auto_link_urls
 from webhelpers2.html.tools import (
     button_to, highlight, js_obfuscate, strip_links, strip_tags)
 
 from webhelpers2.text import (
     chop_at, collapse, convert_accented_entities,
     convert_misc_entities, lchop, plural, rchop, remove_formatting,
     replace_whitespace, urlify, truncate, wrap_paragraphs)
 from webhelpers2.date import time_ago_in_words
 
 from webhelpers2.html.tags import (
     _input, NotGiven, _make_safe_id_component as safeid,
     form as insecure_form,
     auto_discovery_link, checkbox, end_form, file,
     hidden, image, javascript_link, link_to, link_to_if, link_to_unless, ol,
     select as raw_select, stylesheet_link, submit, text, password, textarea,
     ul, radio, Options)
 
 from webhelpers2.number import format_byte_size
 
 from rhodecode.lib.action_parser import action_parser
 from rhodecode.lib.pagination import Page, RepoPage, SqlPage
 from rhodecode.lib import ext_json
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.lib.utils import repo_name_slug, get_custom_lexer
 from rhodecode.lib.utils2 import (
     str2bool, safe_unicode, safe_str,
     get_commit_safe, datetime_to_time, time_to_datetime, time_to_utcdatetime,
     AttributeDict, safe_int, md5, md5_safe, get_host_info)
 from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
 from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError
 from rhodecode.lib.vcs.backends.base import BaseChangeset, EmptyCommit
 from rhodecode.lib.vcs.conf.settings import ARCHIVE_SPECS
 from rhodecode.lib.index.search_utils import get_matching_line_offsets
 from rhodecode.config.conf import DATE_FORMAT, DATETIME_FORMAT
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.db import Permission, User, Repository, UserApiKeys, FileStore
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import IssueTrackerSettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 DEFAULT_USER = User.DEFAULT_USER
 DEFAULT_USER_EMAIL = User.DEFAULT_USER_EMAIL
 
 
 def asset(path, ver=None, **kwargs):
     """
     Helper to generate a static asset file path for rhodecode assets
 
     eg. h.asset('images/image.png', ver='3923')
 
     :param path: path of asset
     :param ver: optional version query param to append as ?ver=
     """
     request = get_current_request()
     query = {}
     query.update(kwargs)
     if ver:
         query = {'ver': ver}
     return request.static_path(
         'rhodecode:public/{}'.format(path), _query=query)
 
 
 default_html_escape_table = {
     ord('&'): u'&amp;',
     ord('<'): u'&lt;',
     ord('>'): u'&gt;',
     ord('"'): u'&quot;',
     ord("'"): u'&#39;',
 }
 
 
 def html_escape(text, html_escape_table=default_html_escape_table):
     """Produce entities within text."""
     return text.translate(html_escape_table)
 
 
 def str_json(*args, **kwargs):
     return ext_json.str_json(*args, **kwargs)
 
 
 def formatted_str_json(*args, **kwargs):
     return ext_json.formatted_str_json(*args, **kwargs)
 
 
 def chop_at_smart(s, sub, inclusive=False, suffix_if_chopped=None):
     """
     Truncate string ``s`` at the first occurrence of ``sub``.
 
     If ``inclusive`` is true, truncate just after ``sub`` rather than at it.
     """
     suffix_if_chopped = suffix_if_chopped or ''
     pos = s.find(sub)
     if pos == -1:
         return s
 
     if inclusive:
         pos += len(sub)
 
     chopped = s[:pos]
     left = s[pos:].strip()
 
     if left and suffix_if_chopped:
         chopped += suffix_if_chopped
 
     return chopped
 
 
 def shorter(text, size=20, prefix=False):
     postfix = '...'
     if len(text) > size:
         if prefix:
             # shorten in front
             return postfix + text[-(size - len(postfix)):]
         else:
             return text[:size - len(postfix)] + postfix
     return text
 
 
 def reset(name, value=None, id=NotGiven, type="reset", **attrs):
     """
     Reset button
     """
     return _input(type, name, value, id, attrs)
 
 
 def select(name, selected_values, options, id=NotGiven, **attrs):
 
     if isinstance(options, (list, tuple)):
         options_iter = options
         # Handle old value,label lists ... where value also can be value,label lists
         options = Options()
         for opt in options_iter:
             if isinstance(opt, tuple) and len(opt) == 2:
                 value, label = opt
             elif isinstance(opt, str):
                 value = label = opt
             else:
                 raise ValueError('invalid select option type %r' % type(opt))
 
             if isinstance(value, (list, tuple)):
                 option_group = options.add_optgroup(label)
                 for opt2 in value:
                     if isinstance(opt2, tuple) and len(opt2) == 2:
                         group_value, group_label = opt2
                     elif isinstance(opt2, str):
                         group_value = group_label = opt2
                     else:
                         raise ValueError('invalid select option type %r' % type(opt2))
 
                     option_group.add_option(group_label, group_value)
             else:
                 options.add_option(label, value)
 
     return raw_select(name, selected_values, options, id=id, **attrs)
 
 
 def branding(name, length=40):
     return truncate(name, length, indicator="")
 
 
 def FID(raw_id, path):
     """
     Creates a unique ID for filenode based on it's hash of path and commit
     it's safe to use in urls
 
     :param raw_id:
     :param path:
     """
 
     return 'c-%s-%s' % (short_id(raw_id), md5_safe(path)[:12])
 
 
 class _GetError(object):
     """Get error from form_errors, and represent it as span wrapped error
     message
 
     :param field_name: field to fetch errors for
     :param form_errors: form errors dict
     """
 
     def __call__(self, field_name, form_errors):
         tmpl = """<span class="error_msg">%s</span>"""
         if form_errors and field_name in form_errors:
             return literal(tmpl % form_errors.get(field_name))
 
 
 get_error = _GetError()
 
 
 class _ToolTip(object):
 
     def __call__(self, tooltip_title, trim_at=50):
         """
         Special function just to wrap our text into nice formatted
         autowrapped text
 
         :param tooltip_title:
         """
         tooltip_title = escape(tooltip_title)
         tooltip_title = tooltip_title.replace('<', '&lt;').replace('>', '&gt;')
         return tooltip_title
 
 
 tooltip = _ToolTip()
 
 files_icon = u'<i class="file-breadcrumb-copy tooltip icon-clipboard clipboard-action" data-clipboard-text="{}" title="Copy file path"></i>'
 
 
 def files_breadcrumbs(repo_name, repo_type, commit_id, file_path, landing_ref_name=None, at_ref=None,
                       limit_items=False, linkify_last_item=False, hide_last_item=False,
                       copy_path_icon=True):
     if isinstance(file_path, str):
         file_path = safe_unicode(file_path)
 
     if at_ref:
         route_qry = {'at': at_ref}
         default_landing_ref = at_ref or landing_ref_name or commit_id
     else:
         route_qry = None
         default_landing_ref = commit_id
 
     # first segment is a `HOME` link to repo files root location
     root_name = literal(u'<i class="icon-home"></i>')
 
     url_segments = [
         link_to(
             root_name,
             repo_files_by_ref_url(
                 repo_name,
                 repo_type,
                 f_path=None,  # None here is a special case for SVN repos,
                               # that won't prefix with a ref
                 ref_name=default_landing_ref,
                 commit_id=commit_id,
                 query=route_qry
             )
         )]
 
     path_segments = file_path.split('/')
     last_cnt = len(path_segments) - 1
     for cnt, segment in enumerate(path_segments):
         if not segment:
             continue
         segment_html = escape(segment)
 
         last_item = cnt == last_cnt
 
         if last_item and hide_last_item:
             # iterate over and hide last element
             continue
 
         if last_item and linkify_last_item is False:
             # plain version
             url_segments.append(segment_html)
         else:
             url_segments.append(
                 link_to(
                     segment_html,
                     repo_files_by_ref_url(
                         repo_name,
                         repo_type,
                         f_path='/'.join(path_segments[:cnt + 1]),
                         ref_name=default_landing_ref,
                         commit_id=commit_id,
                         query=route_qry
                     ),
                 ))
 
     limited_url_segments = url_segments[:1] + ['...'] + url_segments[-5:]
     if limit_items and len(limited_url_segments) < len(url_segments):
         url_segments = limited_url_segments
 
     full_path = file_path
     if copy_path_icon:
         icon = files_icon.format(escape(full_path))
     else:
         icon = ''
 
     if file_path == '':
         return root_name
     else:
         return literal(' / '.join(url_segments) + icon)
 
 
 def files_url_data(request):
     import urllib.request, urllib.parse, urllib.error
     matchdict = request.matchdict
 
     if 'f_path' not in matchdict:
         matchdict['f_path'] = ''
     else:
         matchdict['f_path'] = urllib.parse.quote(safe_str(matchdict['f_path']))
     if 'commit_id' not in matchdict:
         matchdict['commit_id'] = 'tip'
 
     return ext_json.str_json(matchdict)
 
 
 def repo_files_by_ref_url(db_repo_name, db_repo_type, f_path, ref_name, commit_id, query=None, ):
     _is_svn = is_svn(db_repo_type)
     final_f_path = f_path
 
     if _is_svn:
         """
         For SVN the ref_name cannot be used as a commit_id, it needs to be prefixed with
         actually commit_id followed by the ref_name. This should be done only in case
         This is a initial landing url, without additional paths.
 
         like: /1000/tags/1.0.0/?at=tags/1.0.0
         """
 
         if ref_name and ref_name != 'tip':
             # NOTE(marcink): for svn the ref_name is actually the stored path, so we prefix it
             # for SVN we only do this magic prefix if it's root, .eg landing revision
             # of files link. If we are in the tree we don't need this since we traverse the url
             # that has everything stored
             if f_path in ['', '/']:
                 final_f_path = '/'.join([ref_name, f_path])
 
         # SVN always needs a commit_id explicitly, without a named REF
         default_commit_id = commit_id
     else:
         """
         For git and mercurial we construct a new URL using the names instead of commit_id
         like: /master/some_path?at=master
         """
         # We currently do not support branches with slashes
         if '/' in ref_name:
             default_commit_id = commit_id
         else:
             default_commit_id = ref_name
 
     # sometimes we pass f_path as None, to indicate explicit no prefix,
     # we translate it to string to not have None
     final_f_path = final_f_path or ''
 
     files_url = route_path(
         'repo_files',
         repo_name=db_repo_name,
         commit_id=default_commit_id,
         f_path=final_f_path,
         _query=query
     )
     return files_url
 
 
 def code_highlight(code, lexer, formatter, use_hl_filter=False):
     """
     Lex ``code`` with ``lexer`` and format it with the formatter ``formatter``.
 
     If ``outfile`` is given and a valid file object (an object
     with a ``write`` method), the result will be written to it, otherwise
     it is returned as a string.
     """
     if use_hl_filter:
         # add HL filter
         from rhodecode.lib.index import search_utils
         lexer.add_filter(search_utils.ElasticSearchHLFilter())
     return pygments.format(pygments.lex(code, lexer), formatter)
 
 
 class CodeHtmlFormatter(HtmlFormatter):
     """
     My code Html Formatter for source codes
     """
 
     def wrap(self, source, outfile):
         return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<div id="L%s">%s</div>' % (cnt + 1, t)
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         dummyoutfile = StringIO.StringIO()
         lncount = 0
         for t, line in inner:
             if t:
                 lncount += 1
             dummyoutfile.write(line)
 
         fl = self.linenostart
         mw = len(str(lncount + fl - 1))
         sp = self.linenospecial
         st = self.linenostep
         la = self.lineanchors
         aln = self.anchorlinenos
         nocls = self.noclasses
         if sp:
             lines = []
 
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if i % sp == 0:
                         if aln:
                             lines.append('<a href="#%s%d" class="special">%*d</a>' %
                                          (la, i, mw, i))
                         else:
                             lines.append('<span class="special">%*d</span>' % (mw, i))
                     else:
                         if aln:
                             lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                         else:
                             lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
         else:
             lines = []
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if aln:
                         lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                     else:
                         lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
 
         # in case you wonder about the seemingly redundant <div> here: since the
         # content in the other cell also is wrapped in a div, some browsers in
         # some configurations seem to mess up the formatting...
         if nocls:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td><div class="linenodiv" '
                       'style="background-color: #f0f0f0; padding-right: 10px">'
                       '<pre style="line-height: 125%">' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         else:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td class="linenos"><div class="linenodiv"><pre>' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         yield 0, dummyoutfile.getvalue()
         yield 0, '</td></tr></table>'
 
 
 class SearchContentCodeHtmlFormatter(CodeHtmlFormatter):
     def __init__(self, **kw):
         # only show these line numbers if set
         self.only_lines = kw.pop('only_line_numbers', [])
         self.query_terms = kw.pop('query_terms', [])
         self.max_lines = kw.pop('max_lines', 5)
         self.line_context = kw.pop('line_context', 3)
         self.url = kw.pop('url', None)
 
         super(CodeHtmlFormatter, self).__init__(**kw)
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<pre>%s</pre>' % t
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         yield 0, '<table class="code-highlight %stable">' % self.cssclass
 
         last_shown_line_number = 0
         current_line_number = 1
 
         for t, line in inner:
             if not t:
                 yield t, line
                 continue
 
             if current_line_number in self.only_lines:
                 if last_shown_line_number + 1 != current_line_number:
                     yield 0, '<tr>'
                     yield 0, '<td class="line">...</td>'
                     yield 0, '<td id="hlcode" class="code"></td>'
                     yield 0, '</tr>'
 
                 yield 0, '<tr>'
                 if self.url:
                     yield 0, '<td class="line"><a href="%s#L%i">%i</a></td>' % (
                         self.url, current_line_number, current_line_number)
                 else:
                     yield 0, '<td class="line"><a href="">%i</a></td>' % (
                         current_line_number)
                 yield 0, '<td id="hlcode" class="code">' + line + '</td>'
                 yield 0, '</tr>'
 
                 last_shown_line_number = current_line_number
 
             current_line_number += 1
 
         yield 0, '</table>'
 
 
 def hsv_to_rgb(h, s, v):
     """ Convert hsv color values to rgb """
 
     if s == 0.0:
         return v, v, v
     i = int(h * 6.0)  # XXX assume int() truncates!
     f = (h * 6.0) - i
     p = v * (1.0 - s)
     q = v * (1.0 - s * f)
     t = v * (1.0 - s * (1.0 - f))
     i = i % 6
     if i == 0:
         return v, t, p
     if i == 1:
         return q, v, p
     if i == 2:
         return p, v, t
     if i == 3:
         return p, q, v
     if i == 4:
         return t, p, v
     if i == 5:
         return v, p, q
 
 
 def unique_color_generator(n=10000, saturation=0.10, lightness=0.95):
     """
     Generator for getting n of evenly distributed colors using
     hsv color and golden ratio. It always return same order of colors
 
     :param n: number of colors to generate
     :param saturation: saturation of returned colors
     :param lightness: lightness of returned colors
     :returns: RGB tuple
     """
 
     golden_ratio = 0.618033988749895
     h = 0.22717784590367374
 
     for _ in range(n):
         h += golden_ratio
         h %= 1
         HSV_tuple = [h, saturation, lightness]
         RGB_tuple = hsv_to_rgb(*HSV_tuple)
         yield map(lambda x: str(int(x * 256)), RGB_tuple)
 
 
 def color_hasher(n=10000, saturation=0.10, lightness=0.95):
     """
     Returns a function which when called with an argument returns a unique
     color for that argument, eg.
 
     :param n: number of colors to generate
     :param saturation: saturation of returned colors
     :param lightness: lightness of returned colors
     :returns: css RGB string
 
     >>> color_hash = color_hasher()
     >>> color_hash('hello')
     'rgb(34, 12, 59)'
     >>> color_hash('hello')
     'rgb(34, 12, 59)'
     >>> color_hash('other')
     'rgb(90, 224, 159)'
     """
 
     color_dict = {}
     cgenerator = unique_color_generator(
         saturation=saturation, lightness=lightness)
 
     def get_color_string(thing):
         if thing in color_dict:
             col = color_dict[thing]
         else:
             col = color_dict[thing] = next(cgenerator)
         return "rgb(%s)" % (', '.join(col))
 
     return get_color_string
 
 
 def get_lexer_safe(mimetype=None, filepath=None):
     """
     Tries to return a relevant pygments lexer using mimetype/filepath name,
     defaulting to plain text if none could be found
     """
     lexer = None
     try:
         if mimetype:
             lexer = get_lexer_for_mimetype(mimetype)
         if not lexer:
             lexer = get_lexer_for_filename(filepath)
     except pygments.util.ClassNotFound:
         pass
 
     if not lexer:
         lexer = get_lexer_by_name('text')
 
     return lexer
 
 
 def get_lexer_for_filenode(filenode):
     lexer = get_custom_lexer(filenode.extension) or filenode.lexer
     return lexer
 
 
 def pygmentize(filenode, **kwargs):
     """
     pygmentize function using pygments
 
     :param filenode:
     """
     lexer = get_lexer_for_filenode(filenode)
     return literal(code_highlight(filenode.content, lexer,
                                   CodeHtmlFormatter(**kwargs)))
 
 
 def is_following_repo(repo_name, user_id):
     from rhodecode.model.scm import ScmModel
     return ScmModel().is_following_repo(repo_name, user_id)
 
 
 class _Message(object):
     """A message returned by ``Flash.pop_messages()``.
 
     Converting the message to a string returns the message text. Instances
     also have the following attributes:
 
     * ``message``: the message text.
     * ``category``: the category specified when the message was created.
     """
 
     def __init__(self, category, message, sub_data=None):
         self.category = category
         self.message = message
         self.sub_data = sub_data or {}
 
     def __str__(self):
         return self.message
 
     __unicode__ = __str__
 
     def __html__(self):
         return escape(safe_unicode(self.message))
 
 
 class Flash(object):
     # List of allowed categories.  If None, allow any category.
     categories = ["warning", "notice", "error", "success"]
 
     # Default category if none is specified.
     default_category = "notice"
 
     def __init__(self, session_key="flash", categories=None,
                  default_category=None):
         """
         Instantiate a ``Flash`` object.
 
         ``session_key`` is the key to save the messages under in the user's
         session.
 
         ``categories`` is an optional list which overrides the default list
         of categories.
 
         ``default_category`` overrides the default category used for messages
         when none is specified.
         """
         self.session_key = session_key
         if categories is not None:
             self.categories = categories
         if default_category is not None:
             self.default_category = default_category
         if self.categories and self.default_category not in self.categories:
             raise ValueError(
                 "unrecognized default category %r" % (self.default_category,))
 
     def pop_messages(self, session=None, request=None):
         """
         Return all accumulated messages and delete them from the session.
 
         The return value is a list of ``Message`` objects.
         """
         messages = []
 
         if not session:
             if not request:
                 request = get_current_request()
             session = request.session
 
         # Pop the 'old' pylons flash messages. They are tuples of the form
         # (category, message)
         for cat, msg in session.pop(self.session_key, []):
             messages.append(_Message(cat, msg))
 
         # Pop the 'new' pyramid flash messages for each category as list
         # of strings.
         for cat in self.categories:
             for msg in session.pop_flash(queue=cat):
                 sub_data = {}
                 if hasattr(msg, 'rsplit'):
                     flash_data = msg.rsplit('|DELIM|', 1)
                     org_message = flash_data[0]
                     if len(flash_data) > 1:
                         sub_data = json.loads(flash_data[1])
                 else:
                     org_message = msg
 
                 messages.append(_Message(cat, org_message, sub_data=sub_data))
 
         # Map messages from the default queue to the 'notice' category.
         for msg in session.pop_flash():
             messages.append(_Message('notice', msg))
 
         session.save()
         return messages
 
     def json_alerts(self, session=None, request=None):
         payloads = []
         messages = flash.pop_messages(session=session, request=request) or []
         for message in messages:
             payloads.append({
                 'message': {
                     'message': u'{}'.format(message.message),
                     'level': message.category,
                     'force': True,
                     'subdata': message.sub_data
                 }
             })
         return safe_str(json.dumps(payloads))
 
     def __call__(self, message, category=None, ignore_duplicate=True,
                  session=None, request=None):
 
         if not session:
             if not request:
                 request = get_current_request()
             session = request.session
 
         session.flash(
             message, queue=category, allow_duplicate=not ignore_duplicate)
 
 
 flash = Flash()
 
 #==============================================================================
 # SCM FILTERS available via h.
 #==============================================================================
 from rhodecode.lib.vcs.utils import author_name, author_email
 from rhodecode.lib.utils2 import age, age_from_seconds
 from rhodecode.model.db import User, ChangesetStatus
 
 
 email = author_email
 
 
 def capitalize(raw_text):
     return raw_text.capitalize()
 
 
 def short_id(long_id):
     return long_id[:12]
 
 
 def hide_credentials(url):
     from rhodecode.lib.utils2 import credentials_filter
     return credentials_filter(url)
 
 
 import pytz
 import tzlocal
 local_timezone = tzlocal.get_localzone()
 
 
 def get_timezone(datetime_iso, time_is_local=False):
     tzinfo = '+00:00'
 
     # detect if we have a timezone info, otherwise, add it
     if time_is_local and isinstance(datetime_iso, datetime) and not datetime_iso.tzinfo:
         force_timezone = os.environ.get('RC_TIMEZONE', '')
         if force_timezone:
             force_timezone = pytz.timezone(force_timezone)
         timezone = force_timezone or local_timezone
         offset = timezone.localize(datetime_iso).strftime('%z')
         tzinfo = '{}:{}'.format(offset[:-2], offset[-2:])
     return tzinfo
 
 
 def age_component(datetime_iso, value=None, time_is_local=False, tooltip=True):
     title = value or format_date(datetime_iso)
     tzinfo = get_timezone(datetime_iso, time_is_local=time_is_local)
 
     return literal(
         '<time class="timeago {cls}" title="{tt_title}" datetime="{dt}{tzinfo}">{title}</time>'.format(
             cls='tooltip' if tooltip else '',
             tt_title=('{title}{tzinfo}'.format(title=title, tzinfo=tzinfo)) if tooltip else '',
             title=title, dt=datetime_iso, tzinfo=tzinfo
         ))
 
 
 def _shorten_commit_id(commit_id, commit_len=None):
     if commit_len is None:
         request = get_current_request()
         commit_len = request.call_context.visual.show_sha_length
     return commit_id[:commit_len]
 
 
 def show_id(commit, show_idx=None, commit_len=None):
     """
     Configurable function that shows ID
     by default it's r123:fffeeefffeee
 
     :param commit: commit instance
     """
     if show_idx is None:
         request = get_current_request()
         show_idx = request.call_context.visual.show_revision_number
 
     raw_id = _shorten_commit_id(commit.raw_id, commit_len=commit_len)
     if show_idx:
         return 'r%s:%s' % (commit.idx, raw_id)
     else:
         return '%s' % (raw_id, )
 
 
 def format_date(date):
     """
     use a standardized formatting for dates used in RhodeCode
 
     :param date: date/datetime object
     :return: formatted date
     """
 
     if date:
         _fmt = "%a, %d %b %Y %H:%M:%S"
         return safe_unicode(date.strftime(_fmt))
 
     return u""
 
 
 class _RepoChecker(object):
 
     def __init__(self, backend_alias):
         self._backend_alias = backend_alias
 
     def __call__(self, repository):
         if hasattr(repository, 'alias'):
             _type = repository.alias
         elif hasattr(repository, 'repo_type'):
             _type = repository.repo_type
         else:
             _type = repository
         return _type == self._backend_alias
 
 
 is_git = _RepoChecker('git')
 is_hg = _RepoChecker('hg')
 is_svn = _RepoChecker('svn')
 
 
 def get_repo_type_by_name(repo_name):
     repo = Repository.get_by_repo_name(repo_name)
     if repo:
         return repo.repo_type
 
 
 def is_svn_without_proxy(repository):
     if is_svn(repository):
         from rhodecode.model.settings import VcsSettingsModel
         conf = VcsSettingsModel().get_ui_settings_as_config_obj()
         return not str2bool(conf.get('vcs_svn_proxy', 'http_requests_enabled'))
     return False
 
 
 def discover_user(author):
     """
     Tries to discover RhodeCode User based on the author string. Author string
     is typically `FirstName LastName <[e-mail]>`
     """
 
     # if author is already an instance use it for extraction
     if isinstance(author, User):
         return author
 
     # Valid email in the attribute passed, see if they're in the system
     _email = author_email(author)
     if _email != '':
         user = User.get_by_email(_email, case_insensitive=True, cache=True)
         if user is not None:
             return user
 
     # Maybe it's a username, we try to extract it and fetch by username ?
     _author = author_name(author)
     user = User.get_by_username(_author, case_insensitive=True, cache=True)
     if user is not None:
         return user
 
     return None
 
 
 def email_or_none(author):
     # extract email from the commit string
     _email = author_email(author)
 
     # If we have an email, use it, otherwise
     # see if it contains a username we can get an email from
     if _email != '':
         return _email
     else:
         user = User.get_by_username(
             author_name(author), case_insensitive=True, cache=True)
 
     if user is not None:
             return user.email
 
     # No valid email, not a valid user in the system, none!
     return None
 
 
 def link_to_user(author, length=0, **kwargs):
     user = discover_user(author)
     # user can be None, but if we have it already it means we can re-use it
     # in the person() function, so we save 1 intensive-query
     if user:
         author = user
 
     display_person = person(author, 'username_or_name_or_email')
     if length:
         display_person = shorter(display_person, length)
 
     if user and user.username != user.DEFAULT_USER:
         return link_to(
             escape(display_person),
             route_path('user_profile', username=user.username),
             **kwargs)
     else:
         return escape(display_person)
 
 
 def link_to_group(users_group_name, **kwargs):
     return link_to(
         escape(users_group_name),
         route_path('user_group_profile', user_group_name=users_group_name),
         **kwargs)
 
 
 def person(author, show_attr="username_and_name"):
     user = discover_user(author)
     if user:
         return getattr(user, show_attr)
     else:
         _author = author_name(author)
         _email = email(author)
         return _author or _email
 
 
 def author_string(email):
     if email:
         user = User.get_by_email(email, case_insensitive=True, cache=True)
         if user:
             if user.first_name or user.last_name:
                 return '%s %s &lt;%s&gt;' % (
                     user.first_name, user.last_name, email)
             else:
                 return email
         else:
             return email
     else:
         return None
 
 
 def person_by_id(id_, show_attr="username_and_name"):
     # attr to return from fetched user
     person_getter = lambda usr: getattr(usr, show_attr)
 
     #maybe it's an ID ?
     if str(id_).isdigit() or isinstance(id_, int):
         id_ = int(id_)
         user = User.get(id_)
         if user is not None:
             return person_getter(user)
     return id_
 
 
 def gravatar_with_user(request, author, show_disabled=False, tooltip=False):
     _render = request.get_partial_renderer('rhodecode:templates/base/base.mako')
     return _render('gravatar_with_user', author, show_disabled=show_disabled, tooltip=tooltip)
 
 
 tags_paterns = OrderedDict((
     ('lang', (re.compile(r'\[(lang|language)\ \=\&gt;\ *([a-zA-Z\-\/\#\+\.]*)\]'),
              '<div class="metatag" tag="lang">\\2</div>')),
 
     ('see', (re.compile(r'\[see\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]'),
             '<div class="metatag" tag="see">see: \\1 </div>')),
 
     ('url', (re.compile(r'\[url\ \=\&gt;\ \[([a-zA-Z0-9\ \.\-\_]+)\]\((http://|https://|/)(.*?)\)\]'),
             '<div class="metatag" tag="url"> <a href="\\2\\3">\\1</a> </div>')),
 
     ('license', (re.compile(r'\[license\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]'),
                 '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/\\1">\\1</a></div>')),
 
     ('ref', (re.compile(r'\[(requires|recommends|conflicts|base)\ \=\&gt;\ *([a-zA-Z0-9\-\/]*)\]'),
             '<div class="metatag" tag="ref \\1">\\1: <a href="/\\2">\\2</a></div>')),
 
     ('state', (re.compile(r'\[(stable|featured|stale|dead|dev|deprecated)\]'),
               '<div class="metatag" tag="state \\1">\\1</div>')),
 
     # label in grey
     ('label', (re.compile(r'\[([a-z]+)\]'),
               '<div class="metatag" tag="label">\\1</div>')),
 
     # generic catch all in grey
     ('generic', (re.compile(r'\[([a-zA-Z0-9\.\-\_]+)\]'),
                 '<div class="metatag" tag="generic">\\1</div>')),
 ))
 
 
 def extract_metatags(value):
     """
     Extract supported meta-tags from given text value
     """
     tags = []
     if not value:
         return tags, ''
 
     for key, val in tags_paterns.items():
         pat, replace_html = val
         tags.extend([(key, x.group()) for x in pat.finditer(value)])
         value = pat.sub('', value)
 
     return tags, value
 
 
 def style_metatag(tag_type, value):
     """
     converts tags from value into html equivalent
     """
     if not value:
         return ''
 
     html_value = value
     tag_data = tags_paterns.get(tag_type)
     if tag_data:
         pat, replace_html = tag_data
         # convert to plain `unicode` instead of a markup tag to be used in
         # regex expressions. safe_unicode doesn't work here
         html_value = pat.sub(replace_html, value)
 
     return html_value
 
 
 def bool2icon(value, show_at_false=True):
     """
     Returns boolean value of a given value, represented as html element with
     classes that will represent icons
 
     :param value: given value to convert to html node
     """
 
     if value:  # does bool conversion
         return HTML.tag('i', class_="icon-true", title='True')
     else:  # not true as bool
         if show_at_false:
             return HTML.tag('i', class_="icon-false", title='False')
         return HTML.tag('i')
 
 
 def b64(inp):
     return base64.b64encode(inp)
 
 #==============================================================================
 # PERMS
 #==============================================================================
 from rhodecode.lib.auth import (
     HasPermissionAny, HasPermissionAll,
     HasRepoPermissionAny, HasRepoPermissionAll, HasRepoGroupPermissionAll,
     HasRepoGroupPermissionAny, HasRepoPermissionAnyApi, get_csrf_token,
     csrf_token_key, AuthUser)
 
 
 #==============================================================================
 # GRAVATAR URL
 #==============================================================================
 class InitialsGravatar(object):
     def __init__(self, email_address, first_name, last_name, size=30,
                  background=None, text_color='#fff'):
         self.size = size
         self.first_name = first_name
         self.last_name = last_name
         self.email_address = email_address
         self.background = background or self.str2color(email_address)
         self.text_color = text_color
 
     def get_color_bank(self):
         """
         returns a predefined list of colors that gravatars can use.
         Those are randomized distinct colors that guarantee readability and
         uniqueness.
 
         generated with: http://phrogz.net/css/distinct-colors.html
         """
         return [
             '#bf3030', '#a67f53', '#00ff00', '#5989b3', '#392040', '#d90000',
             '#402910', '#204020', '#79baf2', '#a700b3', '#bf6060', '#7f5320',
             '#008000', '#003059', '#ee00ff', '#ff0000', '#8c4b00', '#007300',
             '#005fb3', '#de73e6', '#ff4040', '#ffaa00', '#3df255', '#203140',
             '#47004d', '#591616', '#664400', '#59b365', '#0d2133', '#83008c',
             '#592d2d', '#bf9f60', '#73e682', '#1d3f73', '#73006b', '#402020',
             '#b2862d', '#397341', '#597db3', '#e600d6', '#a60000', '#736039',
             '#00b318', '#79aaf2', '#330d30', '#ff8080', '#403010', '#16591f',
             '#002459', '#8c4688', '#e50000', '#ffbf40', '#00732e', '#102340',
             '#bf60ac', '#8c4646', '#cc8800', '#00a642', '#1d3473', '#b32d98',
             '#660e00', '#ffd580', '#80ffb2', '#7391e6', '#733967', '#d97b6c',
             '#8c5e00', '#59b389', '#3967e6', '#590047', '#73281d', '#665200',
             '#00e67a', '#2d50b3', '#8c2377', '#734139', '#b2982d', '#16593a',
             '#001859', '#ff00aa', '#a65e53', '#ffcc00', '#0d3321', '#2d3959',
             '#731d56', '#401610', '#4c3d00', '#468c6c', '#002ca6', '#d936a3',
             '#d94c36', '#403920', '#36d9a3', '#0d1733', '#592d4a', '#993626',
             '#cca300', '#00734d', '#46598c', '#8c005e', '#7f1100', '#8c7000',
             '#00a66f', '#7382e6', '#b32d74', '#d9896c', '#ffe680', '#1d7362',
             '#364cd9', '#73003d', '#d93a00', '#998a4d', '#59b3a1', '#5965b3',
             '#e5007a', '#73341d', '#665f00', '#00b38f', '#0018b3', '#59163a',
             '#b2502d', '#bfb960', '#00ffcc', '#23318c', '#a6537f', '#734939',
             '#b2a700', '#104036', '#3d3df2', '#402031', '#e56739', '#736f39',
             '#79f2ea', '#000059', '#401029', '#4c1400', '#ffee00', '#005953',
             '#101040', '#990052', '#402820', '#403d10', '#00ffee', '#0000d9',
             '#ff80c4', '#a66953', '#eeff00', '#00ccbe', '#8080ff', '#e673a1',
             '#a62c00', '#474d00', '#1a3331', '#46468c', '#733950', '#662900',
             '#858c23', '#238c85', '#0f0073', '#b20047', '#d9986c', '#becc00',
             '#396f73', '#281d73', '#ff0066', '#ff6600', '#dee673', '#59adb3',
             '#6559b3', '#590024', '#b2622d', '#98b32d', '#36ced9', '#332d59',
             '#40001a', '#733f1d', '#526600', '#005359', '#242040', '#bf6079',
             '#735039', '#cef23d', '#007780', '#5630bf', '#66001b', '#b24700',
             '#acbf60', '#1d6273', '#25008c', '#731d34', '#a67453', '#50592d',
             '#00ccff', '#6600ff', '#ff0044', '#4c1f00', '#8a994d', '#79daf2',
             '#a173e6', '#d93662', '#402310', '#aaff00', '#2d98b3', '#8c40ff',
             '#592d39', '#ff8c40', '#354020', '#103640', '#1a0040', '#331a20',
             '#331400', '#334d00', '#1d5673', '#583973', '#7f0022', '#4c3626',
             '#88cc00', '#36a3d9', '#3d0073', '#d9364c', '#33241a', '#698c23',
             '#5995b3', '#300059', '#e57382', '#7f3300', '#366600', '#00aaff',
             '#3a1659', '#733941', '#663600', '#74b32d', '#003c59', '#7f53a6',
             '#73000f', '#ff8800', '#baf279', '#79caf2', '#291040', '#a6293a',
             '#b2742d', '#587339', '#0077b3', '#632699', '#400009', '#d9a66c',
             '#294010', '#2d4a59', '#aa00ff', '#4c131b', '#b25f00', '#5ce600',
             '#267399', '#a336d9', '#990014', '#664e33', '#86bf60', '#0088ff',
             '#7700b3', '#593a16', '#073300', '#1d4b73', '#ac60bf', '#e59539',
             '#4f8c46', '#368dd9', '#5c0073'
         ]
 
     def rgb_to_hex_color(self, rgb_tuple):
         """
         Converts an rgb_tuple passed to an hex color.
 
         :param rgb_tuple: tuple with 3 ints represents rgb color space
         """
         return '#' + ("".join(map(chr, rgb_tuple)).encode('hex'))
 
     def email_to_int_list(self, email_str):
         """
         Get every byte of the hex digest value of email and turn it to integer.
         It's going to be always between 0-255
         """
         digest = md5_safe(email_str.lower())
         return [int(digest[i * 2:i * 2 + 2], 16) for i in range(16)]
 
     def pick_color_bank_index(self, email_str, color_bank):
         return self.email_to_int_list(email_str)[0] % len(color_bank)
 
     def str2color(self, email_str):
         """
         Tries to map in a stable algorithm an email to color
 
         :param email_str:
         """
         color_bank = self.get_color_bank()
         # pick position (module it's length so we always find it in the
         # bank even if it's smaller than 256 values
         pos = self.pick_color_bank_index(email_str, color_bank)
         return color_bank[pos]
 
     def normalize_email(self, email_address):
         import unicodedata
         # default host used to fill in the fake/missing email
         default_host = 'localhost'
 
         if not email_address:
             email_address = '%s@%s' % (User.DEFAULT_USER, default_host)
 
         email_address = safe_unicode(email_address)
 
         if u'@' not in email_address:
             email_address = u'%s@%s' % (email_address, default_host)
 
         if email_address.endswith(u'@'):
             email_address = u'%s%s' % (email_address, default_host)
 
         email_address = unicodedata.normalize('NFKD', email_address)\
             .encode('ascii', 'ignore')
         return email_address
 
     def get_initials(self):
         """
         Returns 2 letter initials calculated based on the input.
         The algorithm picks first given email address, and takes first letter
         of part before @, and then the first letter of server name. In case
         the part before @ is in a format of `somestring.somestring2` it replaces
         the server letter with first letter of somestring2
 
         In case function was initialized with both first and lastname, this
         overrides the extraction from email by first letter of the first and
         last name. We add special logic to that functionality, In case Full name
         is compound, like Guido Von Rossum, we use last part of the last name
         (Von Rossum) picking `R`.
 
         Function also normalizes the non-ascii characters to they ascii
         representation, eg Ą => A
         """
         import unicodedata
         # replace non-ascii to ascii
         first_name = unicodedata.normalize(
             'NFKD', safe_unicode(self.first_name)).encode('ascii', 'ignore')
         last_name = unicodedata.normalize(
             'NFKD', safe_unicode(self.last_name)).encode('ascii', 'ignore')
 
         # do NFKD encoding, and also make sure email has proper format
         email_address = self.normalize_email(self.email_address)
 
         # first push the email initials
         prefix, server = email_address.split('@', 1)
 
         # check if prefix is maybe a 'first_name.last_name' syntax
         _dot_split = prefix.rsplit('.', 1)
         if len(_dot_split) == 2 and _dot_split[1]:
             initials = [_dot_split[0][0], _dot_split[1][0]]
         else:
             initials = [prefix[0], server[0]]
 
         # then try to replace either first_name or last_name
         fn_letter = (first_name or " ")[0].strip()
         ln_letter = (last_name.split(' ', 1)[-1] or " ")[0].strip()
 
         if fn_letter:
             initials[0] = fn_letter
 
         if ln_letter:
             initials[1] = ln_letter
 
         return ''.join(initials).upper()
 
     def get_img_data_by_type(self, font_family, img_type):
         default_user = """
         <svg xmlns="http://www.w3.org/2000/svg"
         version="1.1" x="0px" y="0px" width="{size}" height="{size}"
         viewBox="-15 -10 439.165 429.164"
 
         xml:space="preserve"
         style="background:{background};" >
 
         <path d="M204.583,216.671c50.664,0,91.74-48.075,
                  91.74-107.378c0-82.237-41.074-107.377-91.74-107.377
                  c-50.668,0-91.74,25.14-91.74,107.377C112.844,
                  168.596,153.916,216.671,
                  204.583,216.671z" fill="{text_color}"/>
         <path d="M407.164,374.717L360.88,
                  270.454c-2.117-4.771-5.836-8.728-10.465-11.138l-71.83-37.392
                  c-1.584-0.823-3.502-0.663-4.926,0.415c-20.316,
                  15.366-44.203,23.488-69.076,23.488c-24.877,
                  0-48.762-8.122-69.078-23.488
                  c-1.428-1.078-3.346-1.238-4.93-0.415L58.75,
                  259.316c-4.631,2.41-8.346,6.365-10.465,11.138L2.001,374.717
                  c-3.191,7.188-2.537,15.412,1.75,22.005c4.285,
                  6.592,11.537,10.526,19.4,10.526h362.861c7.863,0,15.117-3.936,
                  19.402-10.527 C409.699,390.129,
                  410.355,381.902,407.164,374.717z" fill="{text_color}"/>
         </svg>""".format(
             size=self.size,
             background='#979797',  # @grey4
             text_color=self.text_color,
             font_family=font_family)
 
         return {
             "default_user": default_user
         }[img_type]
 
     def get_img_data(self, svg_type=None):
         """
         generates the svg metadata for image
         """
         fonts = [
             '-apple-system',
              'BlinkMacSystemFont',
              'Segoe UI',
              'Roboto',
              'Oxygen-Sans',
              'Ubuntu',
              'Cantarell',
              'Helvetica Neue',
              'sans-serif'
         ]
         font_family = ','.join(fonts)
         if svg_type:
             return self.get_img_data_by_type(font_family, svg_type)
 
         initials = self.get_initials()
         img_data = """
         <svg xmlns="http://www.w3.org/2000/svg" pointer-events="none"
              width="{size}" height="{size}"
              style="width: 100%; height: 100%; background-color: {background}"
              viewBox="0 0 {size} {size}">
             <text text-anchor="middle" y="50%" x="50%" dy="0.35em"
                   pointer-events="auto" fill="{text_color}"
                   font-family="{font_family}"
                   style="font-weight: 400; font-size: {f_size}px;">{text}
             </text>
         </svg>""".format(
             size=self.size,
             f_size=self.size/2.05,  # scale the text inside the box nicely
             background=self.background,
             text_color=self.text_color,
             text=initials.upper(),
             font_family=font_family)
 
         return img_data
 
     def generate_svg(self, svg_type=None):
         img_data = self.get_img_data(svg_type)
         return "data:image/svg+xml;base64,%s" % base64.b64encode(img_data)
 
 
 def initials_gravatar(request, email_address, first_name, last_name, size=30, store_on_disk=False):
 
     svg_type = None
     if email_address == User.DEFAULT_USER_EMAIL:
         svg_type = 'default_user'
 
     klass = InitialsGravatar(email_address, first_name, last_name, size)
 
     if store_on_disk:
         from rhodecode.apps.file_store import utils as store_utils
         from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
             FileOverSizeException
         from rhodecode.model.db import Session
 
         image_key = md5_safe(email_address.lower()
                              + first_name.lower() + last_name.lower())
 
         storage = store_utils.get_file_storage(request.registry.settings)
         filename = '{}.svg'.format(image_key)
         subdir = 'gravatars'
         # since final name has a counter, we apply the 0
         uid = storage.apply_counter(0, store_utils.uid_filename(filename, randomized=False))
         store_uid = os.path.join(subdir, uid)
 
         db_entry = FileStore.get_by_store_uid(store_uid)
         if db_entry:
             return request.route_path('download_file', fid=store_uid)
 
         img_data = klass.get_img_data(svg_type=svg_type)
         img_file = store_utils.bytes_to_file_obj(img_data)
 
         try:
             store_uid, metadata = storage.save_file(
                 img_file, filename, directory=subdir,
                 extensions=['.svg'], randomized_name=False)
         except (FileNotAllowedException, FileOverSizeException):
             raise
 
         try:
             entry = FileStore.create(
                 file_uid=store_uid, filename=metadata["filename"],
                 file_hash=metadata["sha256"], file_size=metadata["size"],
                 file_display_name=filename,
                 file_description=u'user gravatar `{}`'.format(safe_unicode(filename)),
                 hidden=True, check_acl=False, user_id=1
             )
             Session().add(entry)
             Session().commit()
             log.debug('Stored upload in DB as %s', entry)
         except Exception:
             raise
 
         return request.route_path('download_file', fid=store_uid)
 
     else:
         return klass.generate_svg(svg_type=svg_type)
 
 
 def gravatar_external(request, gravatar_url_tmpl, email_address, size=30):
     return safe_str(gravatar_url_tmpl)\
         .replace('{email}', email_address) \
         .replace('{md5email}', md5_safe(email_address.lower())) \
         .replace('{netloc}', request.host) \
         .replace('{scheme}', request.scheme) \
         .replace('{size}', safe_str(size))
 
 
 def gravatar_url(email_address, size=30, request=None):
     request = request or get_current_request()
     _use_gravatar = request.call_context.visual.use_gravatar
 
     email_address = email_address or User.DEFAULT_USER_EMAIL
     if isinstance(email_address, str):
         # hashlib crashes on unicode items
         email_address = safe_str(email_address)
 
     # empty email or default user
     if not email_address or email_address == User.DEFAULT_USER_EMAIL:
         return initials_gravatar(request, User.DEFAULT_USER_EMAIL, '', '', size=size)
 
     if _use_gravatar:
         gravatar_url_tmpl = request.call_context.visual.gravatar_url \
                             or User.DEFAULT_GRAVATAR_URL
         return gravatar_external(request, gravatar_url_tmpl, email_address, size=size)
 
     else:
         return initials_gravatar(request, email_address, '', '', size=size)
 
 
 def breadcrumb_repo_link(repo):
     """
     Makes a breadcrumbs path link to repo
 
     ex::
         group >> subgroup >> repo
 
     :param repo: a Repository instance
     """
 
     path = [
         link_to(group.name, route_path('repo_group_home', repo_group_name=group.group_name),
                 title='last change:{}'.format(format_date(group.last_commit_change)))
         for group in repo.groups_with_parents
     ] + [
         link_to(repo.just_name, route_path('repo_summary', repo_name=repo.repo_name),
                 title='last change:{}'.format(format_date(repo.last_commit_change)))
     ]
 
     return literal(' &raquo; '.join(path))
 
 
 def breadcrumb_repo_group_link(repo_group):
     """
     Makes a breadcrumbs path link to repo
 
     ex::
         group >> subgroup
 
     :param repo_group: a Repository Group instance
     """
 
     path = [
         link_to(group.name,
                 route_path('repo_group_home', repo_group_name=group.group_name),
                 title='last change:{}'.format(format_date(group.last_commit_change)))
         for group in repo_group.parents
     ] + [
         link_to(repo_group.name,
                 route_path('repo_group_home', repo_group_name=repo_group.group_name),
                 title='last change:{}'.format(format_date(repo_group.last_commit_change)))
     ]
 
     return literal(' &raquo; '.join(path))
 
 
 def format_byte_size_binary(file_size):
     """
     Formats file/folder sizes to standard.
     """
     if file_size is None:
         file_size = 0
 
     formatted_size = format_byte_size(file_size, binary=True)
     return formatted_size
 
 
 def urlify_text(text_, safe=True, **href_attrs):
     """
     Extract urls from text and make html links out of them
     """
 
     url_pat = re.compile(r'''(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@#.&+]'''
                          '''|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)''')
 
     def url_func(match_obj):
         url_full = match_obj.groups()[0]
         a_options = dict(href_attrs)
         a_options['href'] = url_full
         a_text = url_full
         return HTML.tag("a", a_text, **a_options)
 
     _new_text = url_pat.sub(url_func, text_)
 
     if safe:
         return literal(_new_text)
     return _new_text
 
 
 def urlify_commits(text_, repo_name):
     """
     Extract commit ids from text and make link from them
 
     :param text_:
     :param repo_name: repo name to build the URL with
     """
 
     url_pat = re.compile(r'(^|\s)([0-9a-fA-F]{12,40})($|\s)')
 
     def url_func(match_obj):
         commit_id = match_obj.groups()[1]
         pref = match_obj.groups()[0]
         suf = match_obj.groups()[2]
 
         tmpl = (
             '%(pref)s<a class="tooltip-hovercard %(cls)s" href="%(url)s" data-hovercard-alt="%(hovercard_alt)s" data-hovercard-url="%(hovercard_url)s">'
             '%(commit_id)s</a>%(suf)s'
         )
         return tmpl % {
             'pref': pref,
             'cls': 'revision-link',
             'url': route_url(
                 'repo_commit', repo_name=repo_name, commit_id=commit_id),
             'commit_id': commit_id,
             'suf': suf,
             'hovercard_alt': 'Commit: {}'.format(commit_id),
             'hovercard_url': route_url(
                 'hovercard_repo_commit', repo_name=repo_name, commit_id=commit_id)
         }
 
     new_text = url_pat.sub(url_func, text_)
 
     return new_text
 
 
 def _process_url_func(match_obj, repo_name, uid, entry,
                       return_raw_data=False, link_format='html'):
     pref = ''
     if match_obj.group().startswith(' '):
         pref = ' '
 
     issue_id = ''.join(match_obj.groups())
 
     if link_format == 'html':
         tmpl = (
             '%(pref)s<a class="tooltip %(cls)s" href="%(url)s" title="%(title)s">'
             '%(issue-prefix)s%(id-repr)s'
             '</a>')
     elif link_format == 'html+hovercard':
         tmpl = (
             '%(pref)s<a class="tooltip-hovercard %(cls)s" href="%(url)s" data-hovercard-url="%(hovercard_url)s">'
             '%(issue-prefix)s%(id-repr)s'
             '</a>')
     elif link_format in ['rst', 'rst+hovercard']:
         tmpl = '`%(issue-prefix)s%(id-repr)s <%(url)s>`_'
     elif link_format in ['markdown', 'markdown+hovercard']:
         tmpl = '[%(pref)s%(issue-prefix)s%(id-repr)s](%(url)s)'
     else:
         raise ValueError('Bad link_format:{}'.format(link_format))
 
     (repo_name_cleaned,
      parent_group_name) = RepoGroupModel()._get_group_name_and_parent(repo_name)
 
     # variables replacement
     named_vars = {
         'id': issue_id,
         'repo': repo_name,
         'repo_name': repo_name_cleaned,
         'group_name': parent_group_name,
         # set dummy keys so we always have them
         'hostname': '',
         'netloc': '',
         'scheme': ''
     }
 
     request = get_current_request()
     if request:
         # exposes, hostname, netloc, scheme
         host_data = get_host_info(request)
         named_vars.update(host_data)
 
     # named regex variables
     named_vars.update(match_obj.groupdict())
     _url = string.Template(entry['url']).safe_substitute(**named_vars)
     desc = string.Template(escape(entry['desc'])).safe_substitute(**named_vars)
     hovercard_url = string.Template(entry.get('hovercard_url', '')).safe_substitute(**named_vars)
 
     def quote_cleaner(input_str):
         """Remove quotes as it's HTML"""
         return input_str.replace('"', '')
 
     data = {
         'pref': pref,
         'cls': quote_cleaner('issue-tracker-link'),
         'url': quote_cleaner(_url),
         'id-repr': issue_id,
         'issue-prefix': entry['pref'],
         'serv': entry['url'],
         'title': bleach.clean(desc, strip=True),
         'hovercard_url': hovercard_url
     }
 
     if return_raw_data:
         return {
             'id': issue_id,
             'url': _url
         }
     return tmpl % data
 
 
 def get_active_pattern_entries(repo_name):
     repo = None
     if repo_name:
         # Retrieving repo_name to avoid invalid repo_name to explode on
         # IssueTrackerSettingsModel but still passing invalid name further down
         repo = Repository.get_by_repo_name(repo_name, cache=True)
 
     settings_model = IssueTrackerSettingsModel(repo=repo)
     active_entries = settings_model.get_settings(cache=True)
     return active_entries
 
 
 pr_pattern_re = regex.compile(r'(?:(?:^!)|(?: !))(\d+)')
 
 allowed_link_formats = [
     'html', 'rst', 'markdown', 'html+hovercard', 'rst+hovercard', 'markdown+hovercard']
 
 compile_cache = {
 
 }
 
 
 def process_patterns(text_string, repo_name, link_format='html', active_entries=None):
 
     if link_format not in allowed_link_formats:
         raise ValueError('Link format can be only one of:{} got {}'.format(
                          allowed_link_formats, link_format))
     issues_data = []
     errors = []
     new_text = text_string
 
     if active_entries is None:
         log.debug('Fetch active issue tracker patterns for repo: %s', repo_name)
         active_entries = get_active_pattern_entries(repo_name)
 
     log.debug('Got %s pattern entries to process', len(active_entries))
 
     for uid, entry in active_entries.items():
 
         if not (entry['pat'] and entry['url']):
             log.debug('skipping due to missing data')
             continue
 
         log.debug('issue tracker entry: uid: `%s` PAT:%s URL:%s PREFIX:%s',
                   uid, entry['pat'], entry['url'], entry['pref'])
 
         if entry.get('pat_compiled'):
             pattern = entry['pat_compiled']
         elif entry['pat'] in compile_cache:
             pattern = compile_cache[entry['pat']]
         else:
             try:
                 pattern = regex.compile(r'%s' % entry['pat'])
             except regex.error as e:
                 regex_err = ValueError('{}:{}'.format(entry['pat'], e))
                 log.exception('issue tracker pattern: `%s` failed to compile', regex_err)
                 errors.append(regex_err)
                 continue
             compile_cache[entry['pat']] = pattern
 
         data_func = partial(
             _process_url_func, repo_name=repo_name, entry=entry, uid=uid,
             return_raw_data=True)
 
         for match_obj in pattern.finditer(text_string):
             issues_data.append(data_func(match_obj))
 
         url_func = partial(
             _process_url_func, repo_name=repo_name, entry=entry, uid=uid,
             link_format=link_format)
 
         new_text = pattern.sub(url_func, new_text)
         log.debug('processed prefix:uid `%s`', uid)
 
     # finally use global replace, eg !123 -> pr-link, those will not catch
     # if already similar pattern exists
     server_url = '${scheme}://${netloc}'
     pr_entry = {
         'pref': '!',
         'url': server_url + '/_admin/pull-requests/${id}',
         'desc': 'Pull Request !${id}',
         'hovercard_url': server_url + '/_hovercard/pull_request/${id}'
     }
     pr_url_func = partial(
         _process_url_func, repo_name=repo_name, entry=pr_entry, uid=None,
         link_format=link_format+'+hovercard')
     new_text = pr_pattern_re.sub(pr_url_func, new_text)
     log.debug('processed !pr pattern')
 
     return new_text, issues_data, errors
 
 
 def urlify_commit_message(commit_text, repository=None, active_pattern_entries=None,
                           issues_container_callback=None, error_container=None):
     """
     Parses given text message and makes proper links.
     issues are linked to given issue-server, and rest is a commit link
     """
 
     def escaper(_text):
         return _text.replace('<', '&lt;').replace('>', '&gt;')
 
     new_text = escaper(commit_text)
 
     # extract http/https links and make them real urls
     new_text = urlify_text(new_text, safe=False)
 
     # urlify commits - extract commit ids and make link out of them, if we have
     # the scope of repository present.
     if repository:
         new_text = urlify_commits(new_text, repository)
 
     # process issue tracker patterns
     new_text, issues, errors = process_patterns(
         new_text, repository or '', active_entries=active_pattern_entries)
 
     if issues_container_callback is not None:
         for issue in issues:
             issues_container_callback(issue)
 
     if error_container is not None:
         error_container.extend(errors)
 
     return literal(new_text)
 
 
 def render_binary(repo_name, file_obj):
     """
     Choose how to render a binary file
     """
 
     # unicode
     filename = file_obj.name
 
     # images
     for ext in ['*.png', '*.jpeg', '*.jpg', '*.ico', '*.gif']:
         if fnmatch.fnmatch(filename, pat=ext):
             src = route_path(
                 'repo_file_raw', repo_name=repo_name,
                 commit_id=file_obj.commit.raw_id,
                 f_path=file_obj.path)
 
             return literal(
                 '<img class="rendered-binary" alt="rendered-image" src="{}">'.format(src))
 
 
 def renderer_from_filename(filename, exclude=None):
     """
     choose a renderer based on filename, this works only for text based files
     """
 
     # ipython
     for ext in ['*.ipynb']:
         if fnmatch.fnmatch(filename, pat=ext):
             return 'jupyter'
 
     is_markup = MarkupRenderer.renderer_from_filename(filename, exclude=exclude)
     if is_markup:
         return is_markup
     return None
 
 
 def render(source, renderer='rst', mentions=False, relative_urls=None,
            repo_name=None, active_pattern_entries=None, issues_container_callback=None):
 
     def maybe_convert_relative_links(html_source):
         if relative_urls:
             return relative_links(html_source, relative_urls)
         return html_source
 
     if renderer == 'plain':
         return literal(
             MarkupRenderer.plain(source, leading_newline=False))
 
     elif renderer == 'rst':
         if repo_name:
             # process patterns on comments if we pass in repo name
             source, issues, errors = process_patterns(
                 source, repo_name, link_format='rst',
                 active_entries=active_pattern_entries)
             if issues_container_callback is not None:
                 for issue in issues:
                     issues_container_callback(issue)
 
         return literal(
             '<div class="rst-block">%s</div>' %
             maybe_convert_relative_links(
                 MarkupRenderer.rst(source, mentions=mentions)))
 
     elif renderer == 'markdown':
         if repo_name:
             # process patterns on comments if we pass in repo name
             source, issues, errors = process_patterns(
                 source, repo_name, link_format='markdown',
                 active_entries=active_pattern_entries)
             if issues_container_callback is not None:
                 for issue in issues:
                     issues_container_callback(issue)
 
 
         return literal(
             '<div class="markdown-block">%s</div>' %
             maybe_convert_relative_links(
                 MarkupRenderer.markdown(source, flavored=True,
                                         mentions=mentions)))
 
     elif renderer == 'jupyter':
         return literal(
             '<div class="ipynb">%s</div>' %
             maybe_convert_relative_links(
                 MarkupRenderer.jupyter(source)))
 
     # None means just show the file-source
     return None
 
 
 def commit_status(repo, commit_id):
     return ChangesetStatusModel().get_status(repo, commit_id)
 
 
 def commit_status_lbl(commit_status):
     return dict(ChangesetStatus.STATUSES).get(commit_status)
 
 
 def commit_time(repo_name, commit_id):
     repo = Repository.get_by_repo_name(repo_name)
     commit = repo.get_commit(commit_id=commit_id)
     return commit.date
 
 
 def get_permission_name(key):
     return dict(Permission.PERMS).get(key)
 
 
 def journal_filter_help(request):
     _ = request.translate
     from rhodecode.lib.audit_logger import ACTIONS
     actions = '\n'.join(textwrap.wrap(', '.join(sorted(ACTIONS.keys())), 80))
 
     return _(
         'Example filter terms:\n' +
         '     repository:vcs\n' +
         '     username:marcin\n' +
         '     username:(NOT marcin)\n' +
         '     action:*push*\n' +
         '     ip:127.0.0.1\n' +
         '     date:20120101\n' +
         '     date:[20120101100000 TO 20120102]\n' +
         '\n' +
         'Actions: {actions}\n' +
         '\n' +
         'Generate wildcards using \'*\' character:\n' +
         '     "repository:vcs*" - search everything starting with \'vcs\'\n' +
         '     "repository:*vcs*" - search for repository containing \'vcs\'\n' +
         '\n' +
         'Optional AND / OR operators in queries\n' +
         '     "repository:vcs OR repository:test"\n' +
         '     "username:test AND repository:test*"\n'
     ).format(actions=actions)
 
 
 def not_mapped_error(repo_name):
     from rhodecode.translation import _
     flash(_('%s repository is not mapped to db perhaps'
             ' it was created or renamed from the filesystem'
             ' please run the application again'
             ' in order to rescan repositories') % repo_name, category='error')
 
 
 def ip_range(ip_addr):
     from rhodecode.model.db import UserIpMap
     s, e = UserIpMap._get_ip_range(ip_addr)
     return '%s - %s' % (s, e)
 
 
 def form(url, method='post', needs_csrf_token=True, **attrs):
     """Wrapper around webhelpers.tags.form to prevent CSRF attacks."""
     if method.lower() != 'get' and needs_csrf_token:
         raise Exception(
             'Forms to POST/PUT/DELETE endpoints should have (in general) a ' +
             'CSRF token. If the endpoint does not require such token you can ' +
             'explicitly set the parameter needs_csrf_token to false.')
 
     return insecure_form(url, method=method, **attrs)
 
 
 def secure_form(form_url, method="POST", multipart=False, **attrs):
     """Start a form tag that points the action to an url. This
     form tag will also include the hidden field containing
     the auth token.
 
     The url options should be given either as a string, or as a
     ``url()`` function. The method for the form defaults to POST.
 
     Options:
 
     ``multipart``
         If set to True, the enctype is set to "multipart/form-data".
     ``method``
         The method to use when submitting the form, usually either
         "GET" or "POST". If "PUT", "DELETE", or another verb is used, a
         hidden input with name _method is added to simulate the verb
         over POST.
 
     """
 
     if 'request' in attrs:
         session = attrs['request'].session
         del attrs['request']
     else:
         raise ValueError(
             'Calling this form requires request= to be passed as argument')
 
     _form = insecure_form(form_url, method, multipart, **attrs)
     token = literal(
         '<input type="hidden" name="{}" value="{}">'.format(
             csrf_token_key, get_csrf_token(session)))
 
     return literal("%s\n%s" % (_form, token))
 
 
 def dropdownmenu(name, selected, options, enable_filter=False, **attrs):
     select_html = select(name, selected, options, **attrs)
 
     select2 = """
         <script>
             $(document).ready(function() {
                   $('#%s').select2({
                       containerCssClass: 'drop-menu %s',
                       dropdownCssClass: 'drop-menu-dropdown',
                       dropdownAutoWidth: true%s
                   });
             });
         </script>
     """
 
     filter_option = """,
                         minimumResultsForSearch: -1
     """
     input_id = attrs.get('id') or name
     extra_classes = ' '.join(attrs.pop('extra_classes', []))
     filter_enabled = "" if enable_filter else filter_option
     select_script = literal(select2 % (input_id, extra_classes, filter_enabled))
 
     return literal(select_html+select_script)
 
 
 def get_visual_attr(tmpl_context_var, attr_name):
     """
     A safe way to get a variable from visual variable of template context
 
     :param tmpl_context_var: instance of tmpl_context, usually present as `c`
     :param attr_name: name of the attribute we fetch from the c.visual
     """
     visual = getattr(tmpl_context_var, 'visual', None)
     if not visual:
         return
     else:
         return getattr(visual, attr_name, None)
 
 
 def get_last_path_part(file_node):
     if not file_node.path:
         return u'/'
 
     path = safe_unicode(file_node.path.split('/')[-1])
     return u'../' + path
 
 
 def route_url(*args, **kwargs):
     """
     Wrapper around pyramids `route_url` (fully qualified url) function.
     """
     req = get_current_request()
     return req.route_url(*args, **kwargs)
 
 
 def route_path(*args, **kwargs):
     """
     Wrapper around pyramids `route_path` function.
     """
     req = get_current_request()
     return req.route_path(*args, **kwargs)
 
 
 def route_path_or_none(*args, **kwargs):
     try:
         return route_path(*args, **kwargs)
     except KeyError:
         return None
 
 
 def current_route_path(request, **kw):
     new_args = request.GET.mixed()
     new_args.update(kw)
     return request.current_route_path(_query=new_args)
 
 
 def curl_api_example(method, args):
     args_json = json.dumps(OrderedDict([
         ('id', 1),
         ('auth_token', 'SECRET'),
         ('method', method),
         ('args', args)
     ]))
 
     return "curl {api_url} -X POST -H 'content-type:text/plain' --data-binary '{args_json}'".format(
         api_url=route_url('apiv2'),
         args_json=args_json
     )
 
 
 def api_call_example(method, args):
     """
     Generates an API call example via CURL
     """
     curl_call = curl_api_example(method, args)
 
     return literal(
         curl_call +
         "<br/><br/>SECRET can be found in <a href=\"{token_url}\">auth-tokens</a> page, "
         "and needs to be of `api calls` role."
         .format(token_url=route_url('my_account_auth_tokens')))
 
 
 def notification_description(notification, request):
     """
     Generate notification human readable description based on notification type
     """
     from rhodecode.model.notification import NotificationModel
     return NotificationModel().make_description(
         notification, translate=request.translate)
 
 
 def go_import_header(request, db_repo=None):
     """
     Creates a header for go-import functionality in Go Lang
     """
 
     if not db_repo:
         return
     if 'go-get' not in request.GET:
         return
 
     clone_url = db_repo.clone_url()
     prefix = re.split(r'^https?:\/\/', clone_url)[-1]
     # we have a repo and go-get flag,
     return literal('<meta name="go-import" content="{} {} {}">'.format(
         prefix, db_repo.repo_type, clone_url))
 
 
 def reviewer_as_json(*args, **kwargs):
     from rhodecode.apps.repository.utils import reviewer_as_json as _reviewer_as_json
     return _reviewer_as_json(*args, **kwargs)
 
 
 def get_repo_view_type(request):
     route_name = request.matched_route.name
     route_to_view_type = {
         'repo_changelog': 'commits',
         'repo_commits': 'commits',
         'repo_files': 'files',
         'repo_summary': 'summary',
         'repo_commit': 'commit'
     }
 
     return route_to_view_type.get(route_name)
 
 
 def is_active(menu_entry, selected):
     """
     Returns active class for selecting menus in templates
     <li class=${h.is_active('settings', current_active)}></li>
     """
     if not isinstance(menu_entry, list):
         menu_entry = [menu_entry]
 
     if selected in menu_entry:
         return "active"
 
 
 class IssuesRegistry(object):
     """
     issue_registry = IssuesRegistry()
     some_func(issues_callback=issues_registry(...))
     """
 
     def __init__(self):
         self.issues = []
         self.unique_issues = collections.defaultdict(lambda: [])
 
     def __call__(self, commit_dict=None):
         def callback(issue):
             if commit_dict and issue:
                 issue['commit'] = commit_dict
             self.issues.append(issue)
             self.unique_issues[issue['id']].append(issue)
         return callback
 
     def get_issues(self):
         return self.issues
 
     @property
     def issues_unique_count(self):
         return len(set(i['id'] for i in self.issues))

rhodecode/lib/hooks_base.py

@@ -1,538 +1,538 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2013-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Set of hooks run by RhodeCode Enterprise
 """
 
 import os
 import logging
 
 import rhodecode
 from rhodecode import events
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.utils2 import safe_str, user_agent_normalizer
 from rhodecode.lib.exceptions import (
     HTTPLockedRC, HTTPBranchProtected, UserCreationError)
 from rhodecode.model.db import Repository, User
 from rhodecode.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
 
 
 class HookResponse(object):
     def __init__(self, status, output):
         self.status = status
         self.output = output
 
     def __add__(self, other):
         other_status = getattr(other, 'status', 0)
         new_status = max(self.status, other_status)
         other_output = getattr(other, 'output', '')
         new_output = self.output + other_output
 
         return HookResponse(new_status, new_output)
 
     def __bool__(self):
         return self.status == 0
 
 
 def is_shadow_repo(extras):
     """
     Returns ``True`` if this is an action executed against a shadow repository.
     """
     return extras['is_shadow_repo']
 
 
 def _get_scm_size(alias, root_path):
 
     if not alias.startswith('.'):
         alias += '.'
 
     size_scm, size_root = 0, 0
     for path, unused_dirs, files in os.walk(safe_str(root_path)):
         if path.find(alias) != -1:
             for f in files:
                 try:
                     size_scm += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
         else:
             for f in files:
                 try:
                     size_root += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
 
     size_scm_f = h.format_byte_size_binary(size_scm)
     size_root_f = h.format_byte_size_binary(size_root)
     size_total_f = h.format_byte_size_binary(size_root + size_scm)
 
     return size_scm_f, size_root_f, size_total_f
 
 
 # actual hooks called by Mercurial internally, and GIT by our Python Hooks
 def repo_size(extras):
     """Present size of repository after push."""
     repo = Repository.get_by_repo_name(extras.repository)
     vcs_part = safe_str('.%s' % repo.repo_type)
     size_vcs, size_root, size_total = _get_scm_size(vcs_part,
                                                     repo.repo_full_path)
     msg = ('Repository `%s` size summary %s:%s repo:%s total:%s\n'
            % (repo.repo_name, vcs_part, size_vcs, size_root, size_total))
     return HookResponse(0, msg)
 
 
 def pre_push(extras):
     """
     Hook executed before pushing code.
 
     It bans pushing when the repository is locked.
     """
 
     user = User.get_by_username(extras.username)
     output = ''
     if extras.locked_by[0] and user.user_id != int(extras.locked_by[0]):
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output = _http_ret.title
         else:
             raise _http_ret
 
     hook_response = ''
     if not is_shadow_repo(extras):
         if extras.commit_ids and extras.check_branch_perms:
 
             auth_user = user.AuthUser()
             repo = Repository.get_by_repo_name(extras.repository)
             affected_branches = []
             if repo.repo_type == 'hg':
                 for entry in extras.commit_ids:
                     if entry['type'] == 'branch':
                         is_forced = bool(entry['multiple_heads'])
                         affected_branches.append([entry['name'], is_forced])
             elif repo.repo_type == 'git':
                 for entry in extras.commit_ids:
                     if entry['type'] == 'heads':
                         is_forced = bool(entry['pruned_sha'])
                         affected_branches.append([entry['name'], is_forced])
 
             for branch_name, is_forced in affected_branches:
 
                 rule, branch_perm = auth_user.get_rule_and_branch_permission(
                     extras.repository, branch_name)
                 if not branch_perm:
                     # no branch permission found for this branch, just keep checking
                     continue
 
                 if branch_perm == 'branch.push_force':
                     continue
                 elif branch_perm == 'branch.push' and is_forced is False:
                     continue
                 elif branch_perm == 'branch.push' and is_forced is True:
                     halt_message = 'Branch `{}` changes rejected by rule {}. ' \
                                    'FORCE PUSH FORBIDDEN.'.format(branch_name, rule)
                 else:
                     halt_message = 'Branch `{}` changes rejected by rule {}.'.format(
                         branch_name, rule)
 
                 if halt_message:
                     _http_ret = HTTPBranchProtected(halt_message)
                     raise _http_ret
 
         # Propagate to external components. This is done after checking the
         # lock, for consistent behavior.
         hook_response = pre_push_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPrePushEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def pre_pull(extras):
     """
     Hook executed before pulling the code.
 
     It bans pulling when the repository is locked.
     """
 
     output = ''
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output = _http_ret.title
         else:
             raise _http_ret
 
     # Propagate to external components. This is done after checking the
     # lock, for consistent behavior.
     hook_response = ''
     if not is_shadow_repo(extras):
         extras.hook_type = extras.hook_type or 'pre_pull'
         hook_response = pre_pull_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPrePullEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def post_pull(extras):
     """Hook executed after client pulls the code."""
 
     audit_user = audit_logger.UserWrap(
         username=extras.username,
         ip_addr=extras.ip)
     repo = audit_logger.RepoWrap(repo_name=extras.repository)
     audit_logger.store(
         'user.pull', action_data={'user_agent': extras.user_agent},
         user=audit_user, repo=repo, commit=True)
 
     statsd = StatsdClient.statsd
     if statsd:
         statsd.incr('rhodecode_pull_total', tags=[
             'user-agent:{}'.format(user_agent_normalizer(extras.user_agent)),
         ])
     output = ''
     # make lock is a tri state False, True, None. We only make lock on True
     if extras.make_lock is True and not is_shadow_repo(extras):
         user = User.get_by_username(extras.username)
         Repository.lock(Repository.get_by_repo_name(extras.repository),
                         user.user_id,
                         lock_reason=Repository.LOCK_PULL)
         msg = 'Made lock on repo `%s`' % (extras.repository,)
         output += msg
 
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output += _http_ret.title
 
     # Propagate to external components.
     hook_response = ''
     if not is_shadow_repo(extras):
         extras.hook_type = extras.hook_type or 'post_pull'
         hook_response = post_pull_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPullEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def post_push(extras):
     """Hook executed after user pushes to the repository."""
     commit_ids = extras.commit_ids
 
     # log the push call
     audit_user = audit_logger.UserWrap(
         username=extras.username, ip_addr=extras.ip)
     repo = audit_logger.RepoWrap(repo_name=extras.repository)
     audit_logger.store(
         'user.push', action_data={
             'user_agent': extras.user_agent,
             'commit_ids': commit_ids[:400]},
         user=audit_user, repo=repo, commit=True)
 
     statsd = StatsdClient.statsd
     if statsd:
         statsd.incr('rhodecode_push_total', tags=[
             'user-agent:{}'.format(user_agent_normalizer(extras.user_agent)),
         ])
 
     # Propagate to external components.
     output = ''
     # make lock is a tri state False, True, None. We only release lock on False
     if extras.make_lock is False and not is_shadow_repo(extras):
         Repository.unlock(Repository.get_by_repo_name(extras.repository))
         msg = 'Released lock on repo `{}`\n'.format(safe_str(extras.repository))
         output += msg
 
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         # TODO: johbo: if not?
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output += _http_ret.title
 
     if extras.new_refs:
         tmpl = '{}/{}/pull-request/new?{{ref_type}}={{ref_name}}'.format(
             safe_str(extras.server_url), safe_str(extras.repository))
 
         for branch_name in extras.new_refs['branches']:
             output += 'RhodeCode: open pull request link: {}\n'.format(
                 tmpl.format(ref_type='branch', ref_name=safe_str(branch_name)))
 
         for book_name in extras.new_refs['bookmarks']:
             output += 'RhodeCode: open pull request link: {}\n'.format(
                 tmpl.format(ref_type='bookmark', ref_name=safe_str(book_name)))
 
     hook_response = ''
     if not is_shadow_repo(extras):
         hook_response = post_push_extension(
             repo_store_path=Repository.base_path(),
             **extras)
         events.trigger(events.RepoPushEvent(
             repo_name=extras.repository, pushed_commit_ids=commit_ids, extras=extras))
 
     output += 'RhodeCode: push completed\n'
     return HookResponse(0, output) + hook_response
 
 
 def _locked_by_explanation(repo_name, user_name, reason):
     message = (
         'Repository `%s` locked by user `%s`. Reason:`%s`'
         % (repo_name, user_name, reason))
     return message
 
 
 def check_allowed_create_user(user_dict, created_by, **kwargs):
     # pre create hooks
     if pre_create_user.is_active():
         hook_result = pre_create_user(created_by=created_by, **user_dict)
         allowed = hook_result.status == 0
         if not allowed:
             reason = hook_result.output
             raise UserCreationError(reason)
 
 
 class ExtensionCallback(object):
     """
     Forwards a given call to rcextensions, sanitizes keyword arguments.
 
     Does check if there is an extension active for that hook. If it is
     there, it will forward all `kwargs_keys` keyword arguments to the
     extension callback.
     """
 
     def __init__(self, hook_name, kwargs_keys):
         self._hook_name = hook_name
         self._kwargs_keys = set(kwargs_keys)
 
     def __call__(self, *args, **kwargs):
         log.debug('Calling extension callback for `%s`', self._hook_name)
         callback = self._get_callback()
         if not callback:
             log.debug('extension callback `%s` not found, skipping...', self._hook_name)
             return
 
         kwargs_to_pass = {}
         for key in self._kwargs_keys:
             try:
                 kwargs_to_pass[key] = kwargs[key]
             except KeyError:
                 log.error('Failed to fetch %s key from given kwargs. '
                           'Expected keys: %s', key, self._kwargs_keys)
                 raise
 
         # backward compat for removed api_key for old hooks. This was it works
         # with older rcextensions that require api_key present
         if self._hook_name in ['CREATE_USER_HOOK', 'DELETE_USER_HOOK']:
             kwargs_to_pass['api_key'] = '_DEPRECATED_'
         return callback(**kwargs_to_pass)
 
     def is_active(self):
         return hasattr(rhodecode.EXTENSIONS, self._hook_name)
 
     def _get_callback(self):
         return getattr(rhodecode.EXTENSIONS, self._hook_name, None)
 
 
 pre_pull_extension = ExtensionCallback(
     hook_name='PRE_PULL_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'hook_type', 'user_agent', 'repo_store_path',))
 
 
 post_pull_extension = ExtensionCallback(
     hook_name='PULL_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'hook_type', 'user_agent', 'repo_store_path',))
 
 
 pre_push_extension = ExtensionCallback(
     hook_name='PRE_PUSH_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'repo_store_path', 'commit_ids', 'hook_type', 'user_agent',))
 
 
 post_push_extension = ExtensionCallback(
     hook_name='PUSH_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'repo_store_path', 'commit_ids', 'hook_type', 'user_agent',))
 
 
 pre_create_user = ExtensionCallback(
     hook_name='PRE_CREATE_USER_HOOK',
     kwargs_keys=(
         'username', 'password', 'email', 'firstname', 'lastname', 'active',
         'admin', 'created_by'))
 
 
 create_pull_request = ExtensionCallback(
     hook_name='CREATE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 merge_pull_request = ExtensionCallback(
     hook_name='MERGE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 close_pull_request = ExtensionCallback(
     hook_name='CLOSE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 review_pull_request = ExtensionCallback(
     hook_name='REVIEW_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 comment_pull_request = ExtensionCallback(
     hook_name='COMMENT_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'comment', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 comment_edit_pull_request = ExtensionCallback(
     hook_name='COMMENT_EDIT_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'comment', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 update_pull_request = ExtensionCallback(
     hook_name='UPDATE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 create_user = ExtensionCallback(
     hook_name='CREATE_USER_HOOK',
     kwargs_keys=(
         'username', 'full_name_or_username', 'full_contact', 'user_id',
         'name', 'firstname', 'short_contact', 'admin', 'lastname',
         'ip_addresses', 'extern_type', 'extern_name',
         'email', 'api_keys', 'last_login',
         'full_name', 'active', 'password', 'emails',
         'inherit_default_permissions', 'created_by', 'created_on'))
 
 
 delete_user = ExtensionCallback(
     hook_name='DELETE_USER_HOOK',
     kwargs_keys=(
         'username', 'full_name_or_username', 'full_contact', 'user_id',
         'name', 'firstname', 'short_contact', 'admin', 'lastname',
         'ip_addresses',
         'email', 'last_login',
         'full_name', 'active', 'password', 'emails',
         'inherit_default_permissions', 'deleted_by'))
 
 
 create_repository = ExtensionCallback(
     hook_name='CREATE_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id', 'created_by'))
 
 
 delete_repository = ExtensionCallback(
     hook_name='DELETE_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id', 'deleted_by', 'deleted_on'))
 
 
 comment_commit_repository = ExtensionCallback(
     hook_name='COMMENT_COMMIT_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id',
         'repository', 'created_by', 'comment', 'commit'))
 
 comment_edit_commit_repository = ExtensionCallback(
     hook_name='COMMENT_EDIT_COMMIT_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id',
         'repository', 'created_by', 'comment', 'commit'))
 
 
 create_repository_group = ExtensionCallback(
     hook_name='CREATE_REPO_GROUP_HOOK',
     kwargs_keys=(
         'group_name', 'group_parent_id', 'group_description',
         'group_id', 'user_id', 'created_by', 'created_on',
         'enable_locking'))

rhodecode/lib/hooks_daemon.py

@@ -1,365 +1,364 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import time
 import logging
 import tempfile
 import traceback
 import threading
 import socket
 import msgpack
 
 from http.server import BaseHTTPRequestHandler
 from socketserver import TCPServer
 
 import rhodecode
 from rhodecode.lib.exceptions import HTTPLockedRC, HTTPBranchProtected
 from rhodecode.model import meta
 from rhodecode.lib.base import bootstrap_request, bootstrap_config
 from rhodecode.lib import hooks_base
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.lib.ext_json import json
 from rhodecode.lib import rc_cache
 
 log = logging.getLogger(__name__)
 
 
 class HooksHttpHandler(BaseHTTPRequestHandler):
 
     def do_POST(self):
         hooks_proto, method, extras = self._read_request()
         log.debug('Handling HooksHttpHandler %s with %s proto', method, hooks_proto)
 
         txn_id = getattr(self.server, 'txn_id', None)
         if txn_id:
             log.debug('Computing TXN_ID based on `%s`:`%s`',
                       extras['repository'], extras['txn_id'])
             computed_txn_id = rc_cache.utils.compute_key_from_params(
                 extras['repository'], extras['txn_id'])
             if txn_id != computed_txn_id:
                 raise Exception(
                     'TXN ID fail: expected {} got {} instead'.format(
                         txn_id, computed_txn_id))
 
         request = getattr(self.server, 'request', None)
         try:
             hooks = Hooks(request=request, log_prefix='HOOKS: {} '.format(self.server.server_address))
             result = self._call_hook_method(hooks, method, extras)
         except Exception as e:
             exc_tb = traceback.format_exc()
             result = {
                 'exception': e.__class__.__name__,
                 'exception_traceback': exc_tb,
                 'exception_args': e.args
             }
         self._write_response(hooks_proto, result)
 
     def _read_request(self):
         length = int(self.headers['Content-Length'])
         hooks_proto = self.headers.get('rc-hooks-protocol') or 'json.v1'
         if hooks_proto == 'msgpack.v1':
             # support for new vcsserver msgpack based protocol hooks
             data = msgpack.unpackb(self.rfile.read(length), raw=False)
         else:
             body = self.rfile.read(length)
             data = json.loads(body)
 
         return hooks_proto, data['method'], data['extras']
 
     def _write_response(self, hooks_proto, result):
         self.send_response(200)
         if hooks_proto == 'msgpack.v1':
             self.send_header("Content-type", "application/msgpack")
             self.end_headers()
             self.wfile.write(msgpack.packb(result))
         else:
             self.send_header("Content-type", "text/json")
             self.end_headers()
             self.wfile.write(json.dumps(result))
 
     def _call_hook_method(self, hooks, method, extras):
         try:
             result = getattr(hooks, method)(extras)
         finally:
             meta.Session.remove()
         return result
 
     def log_message(self, format, *args):
         """
         This is an overridden method of BaseHTTPRequestHandler which logs using
         logging library instead of writing directly to stderr.
         """
 
         message = format % args
 
         log.debug(
             "HOOKS: %s - - [%s] %s", self.client_address,
             self.log_date_time_string(), message)
 
 
 class DummyHooksCallbackDaemon(object):
     hooks_uri = ''
 
     def __init__(self):
         self.hooks_module = Hooks.__module__
 
     def __enter__(self):
         log.debug('Running `%s` callback daemon', self.__class__.__name__)
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
 
 
 class ThreadedHookCallbackDaemon(object):
 
     _callback_thread = None
     _daemon = None
     _done = False
 
     def __init__(self, txn_id=None, host=None, port=None):
         self._prepare(txn_id=txn_id, host=host, port=port)
 
     def __enter__(self):
         log.debug('Running `%s` callback daemon', self.__class__.__name__)
         self._run()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         log.debug('Exiting `%s` callback daemon', self.__class__.__name__)
         self._stop()
 
     def _prepare(self, txn_id=None, host=None, port=None):
         raise NotImplementedError()
 
     def _run(self):
         raise NotImplementedError()
 
     def _stop(self):
         raise NotImplementedError()
 
 
 class HttpHooksCallbackDaemon(ThreadedHookCallbackDaemon):
     """
     Context manager which will run a callback daemon in a background thread.
     """
 
     hooks_uri = None
 
     # From Python docs: Polling reduces our responsiveness to a shutdown
     # request and wastes cpu at all other times.
     POLL_INTERVAL = 0.01
 
     @property
     def _hook_prefix(self):
         return 'HOOKS: {} '.format(self.hooks_uri)
 
     def get_hostname(self):
         return socket.gethostname() or '127.0.0.1'
 
     def get_available_port(self, min_port=20000, max_port=65535):
         from rhodecode.lib.utils2 import get_available_port as _get_port
         return _get_port(min_port, max_port)
 
     def _prepare(self, txn_id=None, host=None, port=None):
         from pyramid.threadlocal import get_current_request
 
         if not host or host == "*":
             host = self.get_hostname()
         if not port:
             port = self.get_available_port()
 
         server_address = (host, port)
         self.hooks_uri = '{}:{}'.format(host, port)
         self.txn_id = txn_id
         self._done = False
 
         log.debug(
             "%s Preparing HTTP callback daemon registering hook object: %s",
             self._hook_prefix, HooksHttpHandler)
 
         self._daemon = TCPServer(server_address, HooksHttpHandler)
         # inject transaction_id for later verification
         self._daemon.txn_id = self.txn_id
 
         # pass the WEB app request into daemon
         self._daemon.request = get_current_request()
 
     def _run(self):
         log.debug("Running event loop of callback daemon in background thread")
         callback_thread = threading.Thread(
             target=self._daemon.serve_forever,
             kwargs={'poll_interval': self.POLL_INTERVAL})
         callback_thread.daemon = True
         callback_thread.start()
         self._callback_thread = callback_thread
 
     def _stop(self):
         log.debug("Waiting for background thread to finish.")
         self._daemon.shutdown()
         self._callback_thread.join()
         self._daemon = None
         self._callback_thread = None
         if self.txn_id:
             txn_id_file = get_txn_id_data_path(self.txn_id)
             log.debug('Cleaning up TXN ID %s', txn_id_file)
             if os.path.isfile(txn_id_file):
                 os.remove(txn_id_file)
 
         log.debug("Background thread done.")
 
 
 def get_txn_id_data_path(txn_id):
     import rhodecode
 
     root = rhodecode.CONFIG.get('cache_dir') or tempfile.gettempdir()
     final_dir = os.path.join(root, 'svn_txn_id')
 
     if not os.path.isdir(final_dir):
         os.makedirs(final_dir)
     return os.path.join(final_dir, 'rc_txn_id_{}'.format(txn_id))
 
 
 def store_txn_id_data(txn_id, data_dict):
     if not txn_id:
         log.warning('Cannot store txn_id because it is empty')
         return
 
     path = get_txn_id_data_path(txn_id)
     try:
         with open(path, 'wb') as f:
             f.write(json.dumps(data_dict))
     except Exception:
         log.exception('Failed to write txn_id metadata')
 
 
 def get_txn_id_from_store(txn_id):
     """
     Reads txn_id from store and if present returns the data for callback manager
     """
     path = get_txn_id_data_path(txn_id)
     try:
         with open(path, 'rb') as f:
             return json.loads(f.read())
     except Exception:
         return {}
 
 
 def prepare_callback_daemon(extras, protocol, host, use_direct_calls, txn_id=None):
     txn_details = get_txn_id_from_store(txn_id)
     port = txn_details.get('port', 0)
     if use_direct_calls:
         callback_daemon = DummyHooksCallbackDaemon()
         extras['hooks_module'] = callback_daemon.hooks_module
     else:
         if protocol == 'http':
             callback_daemon = HttpHooksCallbackDaemon(
                 txn_id=txn_id, host=host, port=port)
         else:
             log.error('Unsupported callback daemon protocol "%s"', protocol)
             raise Exception('Unsupported callback daemon protocol.')
 
     extras['hooks_uri'] = callback_daemon.hooks_uri
     extras['hooks_protocol'] = protocol
     extras['time'] = time.time()
 
     # register txn_id
     extras['txn_id'] = txn_id
     log.debug('Prepared a callback daemon: %s at url `%s`',
               callback_daemon.__class__.__name__, callback_daemon.hooks_uri)
     return callback_daemon, extras
 
 
 class Hooks(object):
     """
     Exposes the hooks for remote call backs
     """
     def __init__(self, request=None, log_prefix=''):
         self.log_prefix = log_prefix
         self.request = request
 
     def repo_size(self, extras):
         log.debug("%sCalled repo_size of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.repo_size, extras)
 
     def pre_pull(self, extras):
         log.debug("%sCalled pre_pull of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.pre_pull, extras)
 
     def post_pull(self, extras):
         log.debug("%sCalled post_pull of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.post_pull, extras)
 
     def pre_push(self, extras):
         log.debug("%sCalled pre_push of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.pre_push, extras)
 
     def post_push(self, extras):
         log.debug("%sCalled post_push of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.post_push, extras)
 
     def _call_hook(self, hook, extras):
         extras = AttributeDict(extras)
         server_url = extras['server_url']
 
         extras.request = self.request
 
         try:
             result = hook(extras)
             if result is None:
                 raise Exception(
                     'Failed to obtain hook result from func: {}'.format(hook))
         except HTTPBranchProtected as handled_error:
             # Those special cases doesn't need error reporting. It's a case of
             # locked repo or protected branch
             result = AttributeDict({
                 'status': handled_error.code,
                 'output': handled_error.explanation
             })
         except (HTTPLockedRC, Exception) as error:
             # locked needs different handling since we need to also
             # handle PULL operations
             exc_tb = ''
             if not isinstance(error, HTTPLockedRC):
                 exc_tb = traceback.format_exc()
                 log.exception('%sException when handling hook %s', self.log_prefix, hook)
             error_args = error.args
             return {
                 'status': 128,
                 'output': '',
                 'exception': type(error).__name__,
                 'exception_traceback': exc_tb,
                 'exception_args': error_args,
             }
         finally:
             meta.Session.remove()
 
         log.debug('%sGot hook call response %s', self.log_prefix, result)
         return {
             'status': result.status,
             'output': result.output,
         }
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         pass

rhodecode/lib/hooks_utils.py

@@ -1,264 +1,263 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import webob
 from pyramid.threadlocal import get_current_request
 
 from rhodecode import events
 from rhodecode.lib import hooks_base
 from rhodecode.lib import utils2
 
 
 def _supports_repo_type(repo_type):
     if repo_type in ('hg', 'git'):
         return True
     return False
 
 
 def _get_vcs_operation_context(username, repo_name, repo_type, action):
     # NOTE(dan): import loop
     from rhodecode.lib.base import vcs_operation_context
 
     check_locking = action in ('pull', 'push')
 
     request = get_current_request()
 
     try:
         environ = request.environ
     except TypeError:
         # we might use this outside of request context
         environ = {}
 
     if not environ:
         environ = webob.Request.blank('').environ
 
     extras = vcs_operation_context(environ, repo_name, username, action, repo_type, check_locking)
     return utils2.AttributeDict(extras)
 
 
 def trigger_post_push_hook(username, action, hook_type, repo_name, repo_type, commit_ids):
     """
     Triggers push action hooks
 
     :param username: username who pushes
     :param action: push/push_local/push_remote
     :param hook_type: type of hook executed
     :param repo_name: name of repo
     :param repo_type: the type of SCM repo
     :param commit_ids: list of commit ids that we pushed
     """
     extras = _get_vcs_operation_context(username, repo_name, repo_type, action)
     extras.commit_ids = commit_ids
     extras.hook_type = hook_type
     hooks_base.post_push(extras)
 
 
 def trigger_comment_commit_hooks(username, repo_name, repo_type, repo, data=None):
     """
     Triggers when a comment is made on a commit
 
     :param username: username who creates the comment
     :param repo_name: name of target repo
     :param repo_type: the type of SCM target repo
     :param repo: the repo object we trigger the event for
     :param data: extra data for specific events e.g {'comment': comment_obj, 'commit': commit_obj}
     """
     if not _supports_repo_type(repo_type):
         return
 
     extras = _get_vcs_operation_context(username, repo_name, repo_type, 'comment_commit')
 
     comment = data['comment']
     commit = data['commit']
 
     events.trigger(events.RepoCommitCommentEvent(repo, commit, comment))
     extras.update(repo.get_dict())
 
     extras.commit = commit.serialize()
     extras.comment = comment.get_api_data()
     extras.created_by = username
     hooks_base.comment_commit_repository(**extras)
 
 
 def trigger_comment_commit_edit_hooks(username, repo_name, repo_type, repo, data=None):
     """
     Triggers when a comment is edited on a commit
 
     :param username: username who edits the comment
     :param repo_name: name of target repo
     :param repo_type: the type of SCM target repo
     :param repo: the repo object we trigger the event for
     :param data: extra data for specific events e.g {'comment': comment_obj, 'commit': commit_obj}
     """
     if not _supports_repo_type(repo_type):
         return
 
     extras = _get_vcs_operation_context(username, repo_name, repo_type, 'comment_commit')
 
     comment = data['comment']
     commit = data['commit']
 
     events.trigger(events.RepoCommitCommentEditEvent(repo, commit, comment))
     extras.update(repo.get_dict())
 
     extras.commit = commit.serialize()
     extras.comment = comment.get_api_data()
     extras.created_by = username
     hooks_base.comment_edit_commit_repository(**extras)
 
 
 def trigger_create_pull_request_hook(username, repo_name, repo_type, pull_request, data=None):
     """
     Triggers create pull request action hooks
 
     :param username: username who creates the pull request
     :param repo_name: name of target repo
     :param repo_type: the type of SCM target repo
     :param pull_request: the pull request that was created
     :param data: extra data for specific events e.g {'comment': comment_obj}
     """
     if not _supports_repo_type(repo_type):
         return
 
     extras = _get_vcs_operation_context(username, repo_name, repo_type, 'create_pull_request')
     events.trigger(events.PullRequestCreateEvent(pull_request))
     extras.update(pull_request.get_api_data(with_merge_state=False))
     hooks_base.create_pull_request(**extras)
 
 
 def trigger_merge_pull_request_hook(username, repo_name, repo_type, pull_request, data=None):
     """
     Triggers merge pull request action hooks
 
     :param username: username who creates the pull request
     :param repo_name: name of target repo
     :param repo_type: the type of SCM target repo
     :param pull_request: the pull request that was merged
     :param data: extra data for specific events e.g {'comment': comment_obj}
     """
     if not _supports_repo_type(repo_type):
         return
 
     extras = _get_vcs_operation_context(username, repo_name, repo_type, 'merge_pull_request')
     events.trigger(events.PullRequestMergeEvent(pull_request))
     extras.update(pull_request.get_api_data())
     hooks_base.merge_pull_request(**extras)
 
 
 def trigger_close_pull_request_hook(username, repo_name, repo_type, pull_request, data=None):
     """
     Triggers close pull request action hooks
 
     :param username: username who creates the pull request
     :param repo_name: name of target repo
     :param repo_type: the type of SCM target repo
     :param pull_request: the pull request that was closed
     :param data: extra data for specific events e.g {'comment': comment_obj}
     """
     if not _supports_repo_type(repo_type):
         return
 
     extras = _get_vcs_operation_context(username, repo_name, repo_type, 'close_pull_request')
     events.trigger(events.PullRequestCloseEvent(pull_request))
     extras.update(pull_request.get_api_data())
     hooks_base.close_pull_request(**extras)
 
 
 def trigger_review_pull_request_hook(username, repo_name, repo_type, pull_request, data=None):
     """
     Triggers review status change pull request action hooks
 
     :param username: username who creates the pull request
     :param repo_name: name of target repo
     :param repo_type: the type of SCM target repo
     :param pull_request: the pull request that review status changed
     :param data: extra data for specific events e.g {'comment': comment_obj}
     """
     if not _supports_repo_type(repo_type):
         return
 
     extras = _get_vcs_operation_context(username, repo_name, repo_type, 'review_pull_request')
     status = data.get('status')
     events.trigger(events.PullRequestReviewEvent(pull_request, status))
     extras.update(pull_request.get_api_data())
     hooks_base.review_pull_request(**extras)
 
 
 def trigger_comment_pull_request_hook(username, repo_name, repo_type, pull_request, data=None):
     """
     Triggers when a comment is made on a pull request
 
     :param username: username who creates the pull request
     :param repo_name: name of target repo
     :param repo_type: the type of SCM target repo
     :param pull_request: the pull request that comment was made on
     :param data: extra data for specific events e.g {'comment': comment_obj}
     """
     if not _supports_repo_type(repo_type):
         return
 
     extras = _get_vcs_operation_context(username, repo_name, repo_type, 'comment_pull_request')
 
     comment = data['comment']
     events.trigger(events.PullRequestCommentEvent(pull_request, comment))
     extras.update(pull_request.get_api_data())
     extras.comment = comment.get_api_data()
     hooks_base.comment_pull_request(**extras)
 
 
 def trigger_comment_pull_request_edit_hook(username, repo_name, repo_type, pull_request, data=None):
     """
     Triggers when a comment was edited on a pull request
 
     :param username: username who made the edit
     :param repo_name: name of target repo
     :param repo_type: the type of SCM target repo
     :param pull_request: the pull request that comment was made on
     :param data: extra data for specific events e.g {'comment': comment_obj}
     """
     if not _supports_repo_type(repo_type):
         return
 
     extras = _get_vcs_operation_context(username, repo_name, repo_type, 'comment_pull_request')
 
     comment = data['comment']
     events.trigger(events.PullRequestCommentEditEvent(pull_request, comment))
     extras.update(pull_request.get_api_data())
     extras.comment = comment.get_api_data()
     hooks_base.comment_edit_pull_request(**extras)
 
 
 def trigger_update_pull_request_hook(username, repo_name, repo_type, pull_request, data=None):
     """
     Triggers update pull request action hooks
 
     :param username: username who creates the pull request
     :param repo_name: name of target repo
     :param repo_type: the type of SCM target repo
     :param pull_request: the pull request that was updated
     :param data: extra data for specific events e.g {'comment': comment_obj}
     """
     if not _supports_repo_type(repo_type):
         return
 
     extras = _get_vcs_operation_context(username, repo_name, repo_type, 'update_pull_request')
     events.trigger(events.PullRequestUpdateEvent(pull_request))
     extras.update(pull_request.get_api_data())
     hooks_base.update_pull_request(**extras)

rhodecode/lib/html_filters.py

@@ -1,25 +1,25 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2020-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 # base64 filter e.g ${ example | base64,n }
 def base64(text):
     import base64
     from rhodecode.lib.helpers import safe_str, safe_bytes
     return safe_str(base64.encodebytes(safe_bytes(text)))

rhodecode/lib/index/__init__.py

@@ -1,155 +1,155 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Index schema for RhodeCode
 """
 
 import importlib
 import logging
 
 from rhodecode.lib.index.search_utils import normalize_text_for_matching
 
 log = logging.getLogger(__name__)
 
 # leave defaults for backward compat
 default_searcher = 'rhodecode.lib.index.whoosh'
 default_location = '%(here)s/data/index'
 
 ES_VERSION_2 = '2'
 ES_VERSION_6 = '6'
 # for legacy reasons we keep 2 compat as default
 DEFAULT_ES_VERSION = ES_VERSION_2
 
 try:
     from rhodecode_tools.lib.fts_index.elasticsearch_engine_6 import ES_CONFIG  # pragma: no cover
 except ImportError:
     log.warning('rhodecode_tools not available, use of full text search is limited')
     pass
 
 
 class BaseSearcher(object):
     query_lang_doc = ''
     es_version = None
     name = None
     DIRECTION_ASC = 'asc'
     DIRECTION_DESC = 'desc'
 
     def __init__(self):
         pass
 
     def cleanup(self):
         pass
 
     def search(self, query, document_type, search_user,
                repo_name=None, repo_group_name=None,
                raise_on_exc=True):
         raise Exception('NotImplemented')
 
     @staticmethod
     def query_to_mark(query, default_field=None):
         """
         Formats the query to mark token for jquery.mark.js highlighting. ES could
         have a different format optionally.
 
         :param default_field:
         :param query:
         """
         return ' '.join(normalize_text_for_matching(query).split())
 
     @property
     def is_es_6(self):
         return self.es_version == ES_VERSION_6
 
     def get_handlers(self):
         return {}
 
     @staticmethod
     def extract_search_tags(query):
         return []
 
     @staticmethod
     def escape_specials(val):
         """
         Handle and escape reserved chars for search
         """
         return val
 
     def sort_def(self, search_type, direction, sort_field):
         """
         Defines sorting for search. This function should decide if for given
         search_type, sorting can be done with sort_field.
 
         It also should translate common sort fields into backend specific. e.g elasticsearch
         """
         raise NotImplementedError()
 
     @staticmethod
     def get_sort(search_type, search_val):
         """
         Method used to parse the GET search sort value to a field and direction.
         e.g asc:lines == asc, lines
 
         There's also a legacy support for newfirst/oldfirst which defines commit
         sorting only
         """
 
         direction = BaseSearcher.DIRECTION_ASC
         sort_field = None
 
         if not search_val:
             return direction, sort_field
 
         if search_val.startswith('asc:'):
             sort_field = search_val[4:]
             direction = BaseSearcher.DIRECTION_ASC
         elif search_val.startswith('desc:'):
             sort_field = search_val[5:]
             direction = BaseSearcher.DIRECTION_DESC
         elif search_val == 'newfirst' and search_type == 'commit':
             sort_field = 'date'
             direction = BaseSearcher.DIRECTION_DESC
         elif search_val == 'oldfirst' and search_type == 'commit':
             sort_field = 'date'
             direction = BaseSearcher.DIRECTION_ASC
 
         return direction, sort_field
 
 
 def search_config(config, prefix='search.'):
     _config = {}
     for key in config.keys():
         if key.startswith(prefix):
             _config[key[len(prefix):]] = config[key]
     return _config
 
 
 def searcher_from_config(config, prefix='search.'):
     _config = search_config(config, prefix)
 
     if 'location' not in _config:
         _config['location'] = default_location
     if 'es_version' not in _config:
         # use old legacy ES version set to 2
         _config['es_version'] = '2'
 
     imported = importlib.import_module(_config.get('module', default_searcher))
     searcher = imported.Searcher(config=_config)
     return searcher

rhodecode/lib/index/search_utils.py

@@ -1,197 +1,197 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import re
 
 import pygments.filter
 import pygments.filters
 from pygments.token import Comment
 
 HL_BEG_MARKER = '__RCSearchHLMarkBEG__'
 HL_END_MARKER = '__RCSearchHLMarkEND__'
 HL_MARKER_RE = '{}(.*?){}'.format(HL_BEG_MARKER, HL_END_MARKER)
 
 
 class ElasticSearchHLFilter(pygments.filters.Filter):
     _names = [HL_BEG_MARKER, HL_END_MARKER]
 
     def __init__(self, **options):
         pygments.filters.Filter.__init__(self, **options)
 
     def filter(self, lexer, stream):
         def tokenize(_value):
             for token in re.split('({}|{})'.format(
                     self._names[0], self._names[1]), _value):
                 if token:
                     yield token
 
         hl = False
         for ttype, value in stream:
 
             if self._names[0] in value or self._names[1] in value:
                 for item in tokenize(value):
                     if item == self._names[0]:
                         # skip marker, but start HL
                         hl = True
                         continue
                     elif item == self._names[1]:
                         hl = False
                         continue
 
                     if hl:
                         yield Comment.ElasticMatch, item
                     else:
                         yield ttype, item
             else:
                 if hl:
                     yield Comment.ElasticMatch, value
                 else:
                     yield ttype, value
 
 
 def extract_phrases(text_query):
     """
     Extracts phrases from search term string making sure phrases
     contained in double quotes are kept together - and discarding empty values
     or fully whitespace values eg.
 
     'some   text "a phrase" more' => ['some', 'text', 'a phrase', 'more']
 
     """
 
     in_phrase = False
     buf = ''
     phrases = []
     for char in text_query:
         if in_phrase:
             if char == '"':  # end phrase
                 phrases.append(buf)
                 buf = ''
                 in_phrase = False
                 continue
             else:
                 buf += char
                 continue
         else:
             if char == '"':  # start phrase
                 in_phrase = True
                 phrases.append(buf)
                 buf = ''
                 continue
             elif char == ' ':
                 phrases.append(buf)
                 buf = ''
                 continue
             else:
                 buf += char
 
     phrases.append(buf)
     phrases = [phrase.strip() for phrase in phrases if phrase.strip()]
     return phrases
 
 
 def get_matching_phrase_offsets(text, phrases):
     """
     Returns a list of string offsets in `text` that the list of `terms` match
 
     >>> get_matching_phrase_offsets('some text here', ['some', 'here'])
     [(0, 4), (10, 14)]
 
     """
     phrases = phrases or []
     offsets = []
 
     for phrase in phrases:
         for match in re.finditer(phrase, text):
             offsets.append((match.start(), match.end()))
 
     return offsets
 
 
 def get_matching_markers_offsets(text, markers=None):
     """
     Returns a list of string offsets in `text` that the are between matching markers
 
     >>> get_matching_markers_offsets('$1some$2 text $1here$2 marked', ['\$1(.*?)\$2'])
     [(0, 5), (16, 22)]
 
     """
     markers = markers or [HL_MARKER_RE]
     offsets = []
 
     if markers:
         for mark in markers:
             for match in re.finditer(mark, text):
                 offsets.append((match.start(), match.end()))
 
     return offsets
 
 
 def normalize_text_for_matching(x):
     """
     Replaces all non alfanum characters to spaces and lower cases the string,
     useful for comparing two text strings without punctuation
     """
     return re.sub(r'[^\w]', ' ', x.lower())
 
 
 def get_matching_line_offsets(lines, terms=None, markers=None):
     """ Return a set of `lines` indices (starting from 1) matching a
     text search query, along with `context` lines above/below matching lines
 
     :param lines: list of strings representing lines
     :param terms: search term string to match in lines eg. 'some text'
     :param markers: instead of terms, use highlight markers instead that
         mark beginning and end for matched item. eg. ['START(.*?)END']
 
      eg.
 
     text = '''
     words words words
     words words words
     some text some
     words words words
     words words words
     text here what
     '''
     get_matching_line_offsets(text, 'text', context=1)
     6, {3: [(5, 9)], 6: [(0, 4)]]
 
     """
     matching_lines = {}
     line_index = 0
 
     if terms:
         phrases = [normalize_text_for_matching(phrase)
                    for phrase in extract_phrases(terms)]
 
         for line_index, line in enumerate(lines.splitlines(), start=1):
             normalized_line = normalize_text_for_matching(line)
             match_offsets = get_matching_phrase_offsets(normalized_line, phrases)
             if match_offsets:
                 matching_lines[line_index] = match_offsets
 
     else:
         markers = markers or [HL_MARKER_RE]
         for line_index, line in enumerate(lines.splitlines(), start=1):
             match_offsets = get_matching_markers_offsets(line, markers=markers)
             if match_offsets:
                 matching_lines[line_index] = match_offsets
 
     return line_index, matching_lines

rhodecode/lib/index/whoosh.py

@@ -1,311 +1,311 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Index schema for RhodeCode
 """
 
 
 import os
 import re
 import logging
 
 from whoosh import query as query_lib
 from whoosh.highlight import HtmlFormatter, ContextFragmenter
 from whoosh.index import create_in, open_dir, exists_in, EmptyIndexError
 from whoosh.qparser import QueryParser, QueryParserError
 
 import rhodecode.lib.helpers as h
 from rhodecode.lib.index import BaseSearcher
 from rhodecode.lib.utils2 import safe_unicode
 
 log = logging.getLogger(__name__)
 
 
 try:
     # we first try to import from rhodecode tools, fallback to copies if
     # we're unable to
     from rhodecode_tools.lib.fts_index.whoosh_schema import (
         ANALYZER, FILE_INDEX_NAME, FILE_SCHEMA, COMMIT_INDEX_NAME,
         COMMIT_SCHEMA)
 except ImportError:
     log.warning('rhodecode_tools schema not available, doing a fallback '
                 'import from `rhodecode.lib.index.whoosh_fallback_schema`')
     from rhodecode.lib.index.whoosh_fallback_schema import (
         ANALYZER, FILE_INDEX_NAME, FILE_SCHEMA, COMMIT_INDEX_NAME,
         COMMIT_SCHEMA)
 
 
 FORMATTER = HtmlFormatter('span', between='\n<span class="break">...</span>\n')
 FRAGMENTER = ContextFragmenter(200)
 
 log = logging.getLogger(__name__)
 
 
 class WhooshSearcher(BaseSearcher):
     # this also shows in UI
     query_lang_doc = 'http://whoosh.readthedocs.io/en/latest/querylang.html'
     name = 'whoosh'
 
     def __init__(self, config):
         super(Searcher, self).__init__()
         self.config = config
         if not os.path.isdir(self.config['location']):
             os.makedirs(self.config['location'])
 
         opener = create_in
         if exists_in(self.config['location'], indexname=FILE_INDEX_NAME):
             opener = open_dir
         file_index = opener(self.config['location'], schema=FILE_SCHEMA,
                             indexname=FILE_INDEX_NAME)
 
         opener = create_in
         if exists_in(self.config['location'], indexname=COMMIT_INDEX_NAME):
             opener = open_dir
         changeset_index = opener(self.config['location'], schema=COMMIT_SCHEMA,
                                  indexname=COMMIT_INDEX_NAME)
 
         self.commit_schema = COMMIT_SCHEMA
         self.commit_index = changeset_index
         self.file_schema = FILE_SCHEMA
         self.file_index = file_index
         self.searcher = None
 
     def cleanup(self):
         if self.searcher:
             self.searcher.close()
 
     def _extend_query(self, query):
         hashes = re.compile('([0-9a-f]{5,40})').findall(query)
         if hashes:
             hashes_or_query = ' OR '.join('commit_id:%s*' % h for h in hashes)
             query = u'(%s) OR %s' % (query, hashes_or_query)
         return query
 
     def sort_def(self, search_type, direction, sort_field):
 
         if search_type == 'commit':
             field_defs = {
                 'message': 'message',
                 'date': 'date',
                 'author_email': 'author',
             }
         elif search_type == 'path':
             field_defs = {
                 'file': 'path',
                 'size': 'size',
                 'lines': 'lines',
             }
         elif search_type == 'content':
             # NOTE(dan): content doesn't support any sorting
             field_defs = {}
         else:
             return ''
 
         if sort_field in field_defs:
             return field_defs[sort_field]
 
     def search(self, query, document_type, search_user,
                repo_name=None, repo_group_name=None,
                requested_page=1, page_limit=10, sort=None, raise_on_exc=True):
 
         original_query = query
         query = self._extend_query(query)
 
         log.debug(u'QUERY: %s on %s', query, document_type)
         result = {
             'results': [],
             'count': 0,
             'error': None,
             'runtime': 0
         }
         search_type, index_name, schema_defn = self._prepare_for_search(
             document_type)
         self._init_searcher(index_name)
         try:
             qp = QueryParser(search_type, schema=schema_defn)
             allowed_repos_filter = self._get_repo_filter(
                 search_user, repo_name)
             try:
                 query = qp.parse(safe_unicode(query))
                 log.debug('query: %s (%s)', query, repr(query))
 
                 reverse, sorted_by = False, None
                 direction, sort_field = self.get_sort(search_type, sort)
                 if sort_field:
                     sort_definition = self.sort_def(search_type, direction, sort_field)
                     if sort_definition:
                         sorted_by = sort_definition
                         if direction == Searcher.DIRECTION_DESC:
                             reverse = True
                         if direction == Searcher.DIRECTION_ASC:
                             reverse = False
 
                 whoosh_results = self.searcher.search(
                     query, filter=allowed_repos_filter, limit=None,
                     sortedby=sorted_by, reverse=reverse)
 
                 # fixes for 32k limit that whoosh uses for highlight
                 whoosh_results.fragmenter.charlimit = None
                 res_ln = whoosh_results.scored_length()
                 result['runtime'] = whoosh_results.runtime
                 result['count'] = res_ln
                 result['results'] = WhooshResultWrapper(
                     search_type, res_ln, whoosh_results)
 
             except QueryParserError:
                 result['error'] = 'Invalid search query. Try quoting it.'
         except (EmptyIndexError, IOError, OSError):
             msg = 'There is no index to search in. Please run whoosh indexer'
             log.exception(msg)
             result['error'] = msg
         except Exception:
             msg = 'An error occurred during this search operation'
             log.exception(msg)
             result['error'] = msg
 
         return result
 
     def statistics(self, translator):
         _ = translator
         stats = [
             {'key': _('Index Type'), 'value': 'Whoosh'},
             {'sep': True},
 
             {'key': _('File Index'), 'value': str(self.file_index)},
             {'key': _('Indexed documents'), 'value': self.file_index.doc_count()},
             {'key': _('Last update'), 'value': h.time_to_datetime(self.file_index.last_modified())},
 
             {'sep': True},
 
             {'key': _('Commit index'), 'value': str(self.commit_index)},
             {'key': _('Indexed documents'), 'value': str(self.commit_index.doc_count())},
             {'key': _('Last update'), 'value': h.time_to_datetime(self.commit_index.last_modified())}
         ]
         return stats
 
     def _get_repo_filter(self, auth_user, repo_name):
 
         allowed_to_search = [
             repo for repo, perm in
             auth_user.permissions['repositories'].items()
             if perm != 'repository.none']
 
         if repo_name:
             repo_filter = [query_lib.Term('repository', repo_name)]
 
         elif 'hg.admin' in auth_user.permissions.get('global', []):
             return None
 
         else:
             repo_filter = [query_lib.Term('repository', _rn)
                            for _rn in allowed_to_search]
             # in case we're not allowed to search anywhere, it's a trick
             # to tell whoosh we're filtering, on ALL results
             repo_filter = repo_filter or [query_lib.Term('repository', '')]
 
         return query_lib.Or(repo_filter)
 
     def _prepare_for_search(self, cur_type):
         search_type = {
             'content': 'content',
             'commit': 'message',
             'path': 'path',
             'repository': 'repository'
         }.get(cur_type, 'content')
 
         index_name = {
             'content': FILE_INDEX_NAME,
             'commit': COMMIT_INDEX_NAME,
             'path': FILE_INDEX_NAME
         }.get(cur_type, FILE_INDEX_NAME)
 
         schema_defn = {
             'content': self.file_schema,
             'commit': self.commit_schema,
             'path': self.file_schema
         }.get(cur_type, self.file_schema)
 
         log.debug('IDX: %s', index_name)
         log.debug('SCHEMA: %s', schema_defn)
         return search_type, index_name, schema_defn
 
     def _init_searcher(self, index_name):
         idx = open_dir(self.config['location'], indexname=index_name)
         self.searcher = idx.searcher()
         return self.searcher
 
 
 Searcher = WhooshSearcher
 
 
 class WhooshResultWrapper(object):
     def __init__(self, search_type, total_hits, results):
         self.search_type = search_type
         self.results = results
         self.total_hits = total_hits
 
     def __str__(self):
         return '<%s at %s>' % (self.__class__.__name__, len(self))
 
     def __repr__(self):
         return self.__str__()
 
     def __len__(self):
         return self.total_hits
 
     def __iter__(self):
         """
         Allows Iteration over results,and lazy generate content
 
         *Requires* implementation of ``__getitem__`` method.
         """
         for hit in self.results:
             yield self.get_full_content(hit)
 
     def __getitem__(self, key):
         """
         Slicing of resultWrapper
         """
         i, j = key.start, key.stop
         for hit in self.results[i:j]:
             yield self.get_full_content(hit)
 
     def get_full_content(self, hit):
         # TODO: marcink: this feels like an overkill, there's a lot of data
         # inside hit object, and we don't need all
         res = dict(hit)
         # elastic search uses that, we set it empty so it fallbacks to regular HL logic
         res['content_highlight'] = ''
 
         f_path = ''  # pragma: no cover
         if self.search_type in ['content', 'path']:
             f_path = res['path'][len(res['repository']):]
             f_path = f_path.lstrip(os.sep)
 
         if self.search_type == 'content':
             res.update({'content_short_hl': hit.highlights('content'),
                         'f_path': f_path})
         elif self.search_type == 'path':
             res.update({'f_path': f_path})
         elif self.search_type == 'message':
             res.update({'message_hl': hit.highlights('message')})
 
         return res

rhodecode/lib/index/whoosh_fallback_schema.py

@@ -1,75 +1,75 b''
-# -*- coding: utf-8 -*-
+
 
 # Copyright (C) 2012-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Whoosh fallback schema for RhodeCode in case rhodecode_tools defined one is
 not available
 """
 
 
 
 from whoosh.analysis import RegexTokenizer, LowercaseFilter
 from whoosh.formats import Characters
 from whoosh.fields import (
     TEXT, ID, STORED, NUMERIC, BOOLEAN, Schema, FieldType, DATETIME)
 
 # CUSTOM ANALYZER wordsplit + lowercase filter for case insensitive search
 ANALYZER = RegexTokenizer(expression=r"\w+") | LowercaseFilter()
 
 # FILE INDEX SCHEMA DEFINITION
 FILE_INDEX_NAME = 'FILE_INDEX'
 FILE_SCHEMA = Schema(
     fileid=ID(unique=True),  # Path
     repository=ID(stored=True),
     repository_id=NUMERIC(unique=True, stored=True),  # Numeric id of repo
     repo_name=TEXT(stored=True),
     owner=TEXT(),
     path=TEXT(stored=True),
     content=FieldType(format=Characters(), analyzer=ANALYZER,
                       scorable=True, stored=True),
     modtime=STORED(),
     md5=STORED(),
     extension=ID(stored=True),
     commit_id=TEXT(stored=True),
 
     size=NUMERIC(int, 64, signed=False, stored=True),
     mimetype=TEXT(stored=True),
     lines=NUMERIC(int, 64, signed=False, stored=True),
 )
 
 
 # COMMIT INDEX SCHEMA
 COMMIT_INDEX_NAME = 'COMMIT_INDEX'
 COMMIT_SCHEMA = Schema(
     commit_id=ID(unique=True, stored=True),
     repository=ID(unique=True, stored=True),
     repository_id=NUMERIC(unique=True, stored=True),
     commit_idx=NUMERIC(stored=True, sortable=True),
     commit_idx_sort=ID(),
     date=NUMERIC(int, 64, signed=False, stored=True, sortable=True),
     owner=TEXT(stored=True),
     author=TEXT(stored=True),
     message=FieldType(format=Characters(), analyzer=ANALYZER,
                       scorable=True, stored=True),
     parents=TEXT(stored=True),
     added=TEXT(stored=True),  # space separated names of added files
     removed=TEXT(stored=True),  # space separated names of removed files
     changed=TEXT(stored=True),  # space separated names of changed files
 )

rhodecode/lib/jsonalchemy.py

@@ -1,279 +1,278 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import collections
 
 import sqlalchemy
 from sqlalchemy import UnicodeText
 from sqlalchemy.ext.mutable import Mutable
 
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils2 import safe_unicode
 
 
 class JsonRaw(str):
     """
     Allows interacting with a JSON types field using a raw string.
 
     For example::
         db_instance = JsonTable()
         db_instance.enabled = True
         db_instance.json_data = JsonRaw('{"a": 4}')
 
     This will bypass serialization/checks, and allow storing
     raw values
     """
     pass
 
 
 # Set this to the standard dict if Order is not required
 DictClass = collections.OrderedDict
 
 
 class JSONEncodedObj(sqlalchemy.types.TypeDecorator):
     """
     Represents an immutable structure as a json-encoded string.
 
     If default is, for example, a dict, then a NULL value in the
     database will be exposed as an empty dict.
     """
 
     impl = UnicodeText
     safe = True
     enforce_unicode = True
 
     def __init__(self, *args, **kwargs):
         self.default = kwargs.pop('default', None)
         self.safe = kwargs.pop('safe_json', self.safe)
         self.enforce_unicode = kwargs.pop('enforce_unicode', self.enforce_unicode)
         self.dialect_map = kwargs.pop('dialect_map', {})
         super(JSONEncodedObj, self).__init__(*args, **kwargs)
 
     def load_dialect_impl(self, dialect):
         if dialect.name in self.dialect_map:
             return dialect.type_descriptor(self.dialect_map[dialect.name])
         return dialect.type_descriptor(self.impl)
 
     def process_bind_param(self, value, dialect):
         if isinstance(value, JsonRaw):
             value = value
         elif value is not None:
             value = json.dumps(value)
             if self.enforce_unicode:
                 value = safe_unicode(value)
         return value
 
     def process_result_value(self, value, dialect):
         if self.default is not None and (not value or value == '""'):
             return self.default()
 
         if value is not None:
             try:
                 value = json.loads(value, object_pairs_hook=DictClass)
             except Exception as e:
                 if self.safe and self.default is not None:
                     return self.default()
                 else:
                     raise
         return value
 
 
 class MutationObj(Mutable):
     @classmethod
     def coerce(cls, key, value):
         if isinstance(value, dict) and not isinstance(value, MutationDict):
             return MutationDict.coerce(key, value)
         if isinstance(value, list) and not isinstance(value, MutationList):
             return MutationList.coerce(key, value)
         return value
 
     def de_coerce(self):
         return self
 
     @classmethod
     def _listen_on_attribute(cls, attribute, coerce, parent_cls):
         key = attribute.key
         if parent_cls is not attribute.class_:
             return
 
         # rely on "propagate" here
         parent_cls = attribute.class_
 
         def load(state, *args):
             val = state.dict.get(key, None)
             if coerce:
                 val = cls.coerce(key, val)
                 state.dict[key] = val
             if isinstance(val, cls):
                 val._parents[state.obj()] = key
 
         def set(target, value, oldvalue, initiator):
             if not isinstance(value, cls):
                 value = cls.coerce(key, value)
             if isinstance(value, cls):
                 value._parents[target.obj()] = key
             if isinstance(oldvalue, cls):
                 oldvalue._parents.pop(target.obj(), None)
             return value
 
         def pickle(state, state_dict):
             val = state.dict.get(key, None)
             if isinstance(val, cls):
                 if 'ext.mutable.values' not in state_dict:
                     state_dict['ext.mutable.values'] = []
                 state_dict['ext.mutable.values'].append(val)
 
         def unpickle(state, state_dict):
             if 'ext.mutable.values' in state_dict:
                 for val in state_dict['ext.mutable.values']:
                     val._parents[state.obj()] = key
 
         sqlalchemy.event.listen(parent_cls, 'load', load, raw=True,
                                 propagate=True)
         sqlalchemy.event.listen(parent_cls, 'refresh', load, raw=True,
                                 propagate=True)
         sqlalchemy.event.listen(parent_cls, 'pickle', pickle, raw=True,
                                 propagate=True)
         sqlalchemy.event.listen(attribute, 'set', set, raw=True, retval=True,
                                 propagate=True)
         sqlalchemy.event.listen(parent_cls, 'unpickle', unpickle, raw=True,
                                 propagate=True)
 
 
 class MutationDict(MutationObj, DictClass):
     @classmethod
     def coerce(cls, key, value):
         """Convert plain dictionary to MutationDict"""
         self = MutationDict(
             (k, MutationObj.coerce(key, v)) for (k, v) in value.items())
         self._key = key
         return self
 
     def de_coerce(self):
         return dict(self)
 
     def __setitem__(self, key, value):
         # Due to the way OrderedDict works, this is called during __init__.
         # At this time we don't have a key set, but what is more, the value
         # being set has already been coerced. So special case this and skip.
         if hasattr(self, '_key'):
             value = MutationObj.coerce(self._key, value)
         DictClass.__setitem__(self, key, value)
         self.changed()
 
     def __delitem__(self, key):
         DictClass.__delitem__(self, key)
         self.changed()
 
     def __setstate__(self, state):
         self.__dict__ = state
 
     def __reduce_ex__(self, proto):
         # support pickling of MutationDicts
         d = dict(self)
         return (self.__class__, (d,))
 
 
 class MutationList(MutationObj, list):
     @classmethod
     def coerce(cls, key, value):
         """Convert plain list to MutationList"""
         self = MutationList((MutationObj.coerce(key, v) for v in value))
         self._key = key
         return self
 
     def de_coerce(self):
         return list(self)
 
     def __setitem__(self, idx, value):
         list.__setitem__(self, idx, MutationObj.coerce(self._key, value))
         self.changed()
 
     def __setslice__(self, start, stop, values):
         list.__setslice__(self, start, stop,
                           (MutationObj.coerce(self._key, v) for v in values))
         self.changed()
 
     def __delitem__(self, idx):
         list.__delitem__(self, idx)
         self.changed()
 
     def __delslice__(self, start, stop):
         list.__delslice__(self, start, stop)
         self.changed()
 
     def append(self, value):
         list.append(self, MutationObj.coerce(self._key, value))
         self.changed()
 
     def insert(self, idx, value):
         list.insert(self, idx, MutationObj.coerce(self._key, value))
         self.changed()
 
     def extend(self, values):
         list.extend(self, (MutationObj.coerce(self._key, v) for v in values))
         self.changed()
 
     def pop(self, *args, **kw):
         value = list.pop(self, *args, **kw)
         self.changed()
         return value
 
     def remove(self, value):
         list.remove(self, value)
         self.changed()
 
 
 def JsonType(impl=None, **kwargs):
     """
     Helper for using a mutation obj, it allows to use .with_variant easily.
     example::
 
         settings = Column('settings_json',
             MutationObj.as_mutable(
             JsonType(dialect_map=dict(mysql=UnicodeText(16384))))
     """
 
     if impl == 'list':
         return JSONEncodedObj(default=list, **kwargs)
     elif impl == 'dict':
         return JSONEncodedObj(default=DictClass, **kwargs)
     else:
         return JSONEncodedObj(**kwargs)
 
 
 JSON = MutationObj.as_mutable(JsonType())
 """
 A type to encode/decode JSON on the fly
 
 sqltype is the string type for the underlying DB column::
 
     Column(JSON) (defaults to UnicodeText)
 """
 
 JSONDict = MutationObj.as_mutable(JsonType('dict'))
 """
 A type to encode/decode JSON dictionaries on the fly
 """
 
 JSONList = MutationObj.as_mutable(JsonType('list'))
 """
 A type to encode/decode JSON lists` on the fly
 """

rhodecode/lib/logging_formatter.py

@@ -1,187 +1,186 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import sys
 import logging
 
 
 BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = list(range(30, 38))
 
 # Sequences
 RESET_SEQ = "\033[0m"
 COLOR_SEQ = "\033[0;%dm"
 BOLD_SEQ = "\033[1m"
 
 COLORS = {
     'CRITICAL': MAGENTA,
     'ERROR': RED,
     'WARNING': CYAN,
     'INFO': GREEN,
     'DEBUG': BLUE,
     'SQL': YELLOW
 }
 
 
 def _inject_req_id(record, with_prefix=True):
     from pyramid.threadlocal import get_current_request
     dummy = '00000000-0000-0000-0000-000000000000'
     req_id = None
 
     req = get_current_request()
     if req:
         req_id = getattr(req, 'req_id', None)
     if with_prefix:
         req_id = 'req_id:%-36s' % (req_id or dummy)
     else:
         req_id = (req_id or dummy)
     record.req_id = req_id
 
 
 def _add_log_to_debug_bucket(formatted_record):
     from pyramid.threadlocal import get_current_request
     req = get_current_request()
     if req:
         req.req_id_bucket.append(formatted_record)
 
 
 def one_space_trim(s):
     if s.find("  ") == -1:
         return s
     else:
         s = s.replace('  ', ' ')
         return one_space_trim(s)
 
 
 def format_sql(sql):
     sql = sql.replace('\n', '')
     sql = one_space_trim(sql)
     sql = sql\
         .replace(',', ',\n\t')\
         .replace('SELECT', '\n\tSELECT \n\t')\
         .replace('UPDATE', '\n\tUPDATE \n\t')\
         .replace('DELETE', '\n\tDELETE \n\t')\
         .replace('FROM', '\n\tFROM')\
         .replace('ORDER BY', '\n\tORDER BY')\
         .replace('LIMIT', '\n\tLIMIT')\
         .replace('WHERE', '\n\tWHERE')\
         .replace('AND', '\n\tAND')\
         .replace('LEFT', '\n\tLEFT')\
         .replace('INNER', '\n\tINNER')\
         .replace('INSERT', '\n\tINSERT')\
         .replace('DELETE', '\n\tDELETE')
     return sql
 
 
 class ExceptionAwareFormatter(logging.Formatter):
     """
     Extended logging formatter which prints out remote tracebacks.
     """
 
     def formatException(self, ei):
         ex_type, ex_value, ex_tb = ei
 
         local_tb = logging.Formatter.formatException(self, ei)
         if hasattr(ex_value, '_vcs_server_traceback'):
 
             def formatRemoteTraceback(remote_tb_lines):
                 result = ["\n +--- This exception occured remotely on VCSServer - Remote traceback:\n\n"]
                 result.append(remote_tb_lines)
                 result.append("\n +--- End of remote traceback\n")
                 return result
 
             try:
                 if ex_type is not None and ex_value is None and ex_tb is None:
                     # possible old (3.x) call syntax where caller is only
                     # providing exception object
                     if type(ex_type) is not type:
                         raise TypeError(
                             "invalid argument: ex_type should be an exception "
                             "type, or just supply no arguments at all")
                 if ex_type is None and ex_tb is None:
                     ex_type, ex_value, ex_tb = sys.exc_info()
 
                 remote_tb = getattr(ex_value, "_vcs_server_traceback", None)
 
                 if remote_tb:
                     remote_tb = formatRemoteTraceback(remote_tb)
                     return local_tb + ''.join(remote_tb)
             finally:
                 # clean up cycle to traceback, to allow proper GC
                 del ex_type, ex_value, ex_tb
 
         return local_tb
 
 
 class RequestTrackingFormatter(ExceptionAwareFormatter):
     def format(self, record):
         _inject_req_id(record)
         def_record = logging.Formatter.format(self, record)
         _add_log_to_debug_bucket(def_record)
         return def_record
 
 
 class ColorFormatter(ExceptionAwareFormatter):
 
     def format(self, record):
         """
         Changes record's levelname to use with COLORS enum
         """
         def_record = super(ColorFormatter, self).format(record)
 
         levelname = record.levelname
         start = COLOR_SEQ % (COLORS[levelname])
         end = RESET_SEQ
 
         colored_record = ''.join([start, def_record, end])
         return colored_record
 
 
 class ColorRequestTrackingFormatter(RequestTrackingFormatter):
 
     def format(self, record):
         """
         Changes record's levelname to use with COLORS enum
         """
         def_record = super(ColorRequestTrackingFormatter, self).format(record)
 
         levelname = record.levelname
         start = COLOR_SEQ % (COLORS[levelname])
         end = RESET_SEQ
 
         colored_record = ''.join([start, def_record, end])
         return colored_record
 
 
 class ColorFormatterSql(logging.Formatter):
 
     def format(self, record):
         """
         Changes record's levelname to use with COLORS enum
         """
 
         start = COLOR_SEQ % (COLORS['SQL'])
         def_record = format_sql(logging.Formatter.format(self, record))
         end = RESET_SEQ
 
         colored_record = ''.join([start, def_record, end])
         return colored_record
 
 # marcink: needs to stay with this name for backward .ini compatability
 Pyro4AwareFormatter = ExceptionAwareFormatter

rhodecode/lib/markdown_ext.py

@@ -1,176 +1,175 b''
-# -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import re
 import markdown
 import xml.etree.ElementTree as etree
 
 from markdown.extensions import Extension
 from markdown.extensions.fenced_code import FencedCodeExtension
 from markdown.extensions.tables import TableExtension
 from markdown.inlinepatterns import Pattern
 
 import gfm
 
 
 class InlineProcessor(Pattern):
     """
     Base class that inline patterns subclass.
     This is the newer style inline processor that uses a more
     efficient and flexible search approach.
     """
 
     def __init__(self, pattern, md=None):
         """
         Create an instant of an inline pattern.
         Keyword arguments:
         * pattern: A regular expression that matches a pattern
         """
         self.pattern = pattern
         self.compiled_re = re.compile(pattern, re.DOTALL | re.UNICODE)
 
         # Api for Markdown to pass safe_mode into instance
         self.safe_mode = False
         self.md = md
 
     def handleMatch(self, m, data):
         """Return a ElementTree element from the given match and the
         start and end index of the matched text.
         If `start` and/or `end` are returned as `None`, it will be
         assumed that the processor did not find a valid region of text.
         Subclasses should override this method.
         Keyword arguments:
         * m: A re match object containing a match of the pattern.
         * data: The buffer current under analysis
         Returns:
         * el: The ElementTree element, text or None.
         * start: The start of the region that has been matched or None.
         * end: The end of the region that has been matched or None.
         """
         pass  # pragma: no cover
 
 
 class SimpleTagInlineProcessor(InlineProcessor):
     """
     Return element of type `tag` with a text attribute of group(2)
     of a Pattern.
     """
     def __init__(self, pattern, tag):
         InlineProcessor.__init__(self, pattern)
         self.tag = tag
 
     def handleMatch(self, m, data):  # pragma: no cover
         el = etree.Element(self.tag)
         el.text = m.group(2)
         return el, m.start(0), m.end(0)
 
 
 class SubstituteTagInlineProcessor(SimpleTagInlineProcessor):
     """ Return an element of type `tag` with no children. """
     def handleMatch(self, m, data):
         return etree.Element(self.tag), m.start(0), m.end(0)
 
 
 class Nl2BrExtension(Extension):
     BR_RE = r'\n'
 
     def extendMarkdown(self, md, md_globals):
         br_tag = SubstituteTagInlineProcessor(self.BR_RE, 'br')
         md.inlinePatterns.add('nl', br_tag, '_end')
 
 
 class GithubFlavoredMarkdownExtension(Extension):
     """
     An extension that is as compatible as possible with GitHub-flavored
     Markdown (GFM).
 
     This extension aims to be compatible with the variant of GFM that GitHub
     uses for Markdown-formatted gists and files (including READMEs). This
     variant seems to have all the extensions described in the `GFM
     documentation`_, except:
 
     - Newlines in paragraphs are not transformed into ``br`` tags.
     - Intra-GitHub links to commits, repositories, and issues are not
       supported.
 
     If you need support for features specific to GitHub comments and issues,
     please use :class:`mdx_gfm.GithubFlavoredMarkdownExtension`.
 
     .. _GFM documentation: https://guides.github.com/features/mastering-markdown/
     """
 
     def extendMarkdown(self, md, md_globals):
         # Built-in extensions
         Nl2BrExtension().extendMarkdown(md, md_globals)
         FencedCodeExtension().extendMarkdown(md, md_globals)
         TableExtension().extendMarkdown(md, md_globals)
 
         # Custom extensions
         gfm.AutolinkExtension().extendMarkdown(md, md_globals)
         gfm.AutomailExtension().extendMarkdown(md, md_globals)
         gfm.HiddenHiliteExtension([
             ('guess_lang', 'False'),
             ('css_class', 'highlight')
         ]).extendMarkdown(md, md_globals)
         gfm.SemiSaneListExtension().extendMarkdown(md, md_globals)
         gfm.SpacedLinkExtension().extendMarkdown(md, md_globals)
         gfm.StrikethroughExtension().extendMarkdown(md, md_globals)
         gfm.TaskListExtension([
             ('list_attrs', {'class': 'checkbox'})
         ]).extendMarkdown(md, md_globals)
 
 
 # Global Vars
 URLIZE_RE = '(%s)' % '|'.join([
     r'<(?:f|ht)tps?://[^>]*>',
     r'\b(?:f|ht)tps?://[^)<>\s]+[^.,)<>\s]',
     r'\bwww\.[^)<>\s]+[^.,)<>\s]',
     r'[^(<\s]+\.(?:com|net|org)\b',
 ])
 
 
 class UrlizePattern(markdown.inlinepatterns.Pattern):
     """ Return a link Element given an autolink (`http://example/com`). """
     def handleMatch(self, m):
         url = m.group(2)
 
         if url.startswith('<'):
             url = url[1:-1]
 
         text = url
 
         if not url.split('://')[0] in ('http','https','ftp'):
             if '@' in url and not '/' in url:
                 url = 'mailto:' + url
             else:
                 url = 'http://' + url
 
         el = markdown.util.etree.Element("a")
         el.set('href', url)
         el.text = markdown.util.AtomicString(text)
         return el
 
 
 class UrlizeExtension(markdown.Extension):
     """ Urlize Extension for Python-Markdown. """
 
     def extendMarkdown(self, md, md_globals):
         """ Replace autolink with UrlizePattern """
         md.inlinePatterns['autolink'] = UrlizePattern(URLIZE_RE, md)