pull-requests: make the renderer stored and saved for each pull requests.

- gives backward compatability with previous versions which didn't set renderer
- allows versioning of renderer changes

rhodecode/lib/dbmigrate/versions/087_version_4_13_0.py

@@ -0,0 +1,40 b''
+import logging
+
+from sqlalchemy import *
+
+from rhodecode.model import meta
+from rhodecode.lib.dbmigrate.versions import _reset_base, notify
+
+log = logging.getLogger(__name__)
+
+
+def upgrade(migrate_engine):
+    """
+    Upgrade operations go here.
+    Don't create your own engine; bind migrate_engine to your metadata
+    """
+    _reset_base(migrate_engine)
+    from rhodecode.lib.dbmigrate.schema import db_4_11_0_0 as db
+
+    pull_request_table = db.PullRequest.__table__
+    pull_request_version_table = db.PullRequestVersion.__table__
+
+    renderer = Column('description_renderer', Unicode(64), nullable=True)
+    renderer.create(table=pull_request_table)
+
+    renderer_ver = Column('description_renderer', Unicode(64), nullable=True)
+    renderer_ver.create(table=pull_request_version_table)
+
+    # issue fixups
+    fixups(db, meta.Session)
+
+
+def downgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+
+def fixups(models, _SESSION):
+    pass
+
+

rhodecode/__init__.py

@@ -1,63 +1,63 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 
 RhodeCode, a web based repository management software
 versioning implementation: http://www.python.org/dev/peps/pep-0386/
 """
 
 import os
 import sys
 import platform
 
 VERSION = tuple(open(os.path.join(
     os.path.dirname(__file__), 'VERSION')).read().split('.'))
 
 BACKENDS = {
     'hg': 'Mercurial repository',
     'git': 'Git repository',
     'svn': 'Subversion repository',
 }
 
 CELERY_ENABLED = False
 CELERY_EAGER = False
 
 # link to config for pyramid
 CONFIG = {}
 
 # Populated with the settings dictionary from application init in
 # rhodecode.conf.environment.load_pyramid_environment
 PYRAMID_SETTINGS = {}
 
 # Linked module for extensions
 EXTENSIONS = {}
 
 __version__ = ('.'.join((str(each) for each in VERSION[:3])))
-__dbversion__ = 86  # defines current db version for migrations
+__dbversion__ = 87  # defines current db version for migrations
 __platform__ = platform.system()
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'
 __url__ = 'https://code.rhodecode.com'
 
 is_windows = __platform__ in ['Windows']
 is_unix = not is_windows
 is_test = False
 disable_error_handler = False

rhodecode/apps/repository/tests/test_repo_pullrequests.py

@@ -1,1203 +1,1206 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import mock
 import pytest
 
 import rhodecode
 from rhodecode.lib.vcs.backends.base import MergeResponse, MergeFailureReason
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib import helpers as h
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.db import (
     PullRequest, ChangesetStatus, UserLog, Notification, ChangesetComment)
 from rhodecode.model.meta import Session
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.user import UserModel
 from rhodecode.tests import (
     assert_session_flash, TEST_USER_ADMIN_LOGIN, TEST_USER_REGULAR_LOGIN)
 from rhodecode.tests.utils import AssertResponse
 
 
 def route_path(name, params=None, **kwargs):
     import urllib
 
     base_url = {
         'repo_changelog': '/{repo_name}/changelog',
         'repo_changelog_file': '/{repo_name}/changelog/{commit_id}/{f_path}',
         'pullrequest_show': '/{repo_name}/pull-request/{pull_request_id}',
         'pullrequest_show_all': '/{repo_name}/pull-request',
         'pullrequest_show_all_data': '/{repo_name}/pull-request-data',
         'pullrequest_repo_refs': '/{repo_name}/pull-request/refs/{target_repo_name:.*?[^/]}',
         'pullrequest_repo_destinations': '/{repo_name}/pull-request/repo-destinations',
         'pullrequest_new': '/{repo_name}/pull-request/new',
         'pullrequest_create': '/{repo_name}/pull-request/create',
         'pullrequest_update': '/{repo_name}/pull-request/{pull_request_id}/update',
         'pullrequest_merge': '/{repo_name}/pull-request/{pull_request_id}/merge',
         'pullrequest_delete': '/{repo_name}/pull-request/{pull_request_id}/delete',
         'pullrequest_comment_create': '/{repo_name}/pull-request/{pull_request_id}/comment',
         'pullrequest_comment_delete': '/{repo_name}/pull-request/{pull_request_id}/comment/{comment_id}/delete',
     }[name].format(**kwargs)
 
     if params:
         base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
     return base_url
 
 
 @pytest.mark.usefixtures('app', 'autologin_user')
 @pytest.mark.backends("git", "hg")
 class TestPullrequestsView(object):
 
     def test_index(self, backend):
         self.app.get(route_path(
             'pullrequest_new',
             repo_name=backend.repo_name))
 
     def test_option_menu_create_pull_request_exists(self, backend):
         repo_name = backend.repo_name
         response = self.app.get(h.route_path('repo_summary', repo_name=repo_name))
 
         create_pr_link = '<a href="%s">Create Pull Request</a>' % route_path(
             'pullrequest_new', repo_name=repo_name)
         response.mustcontain(create_pr_link)
 
     def test_create_pr_form_with_raw_commit_id(self, backend):
         repo = backend.repo
 
         self.app.get(
             route_path('pullrequest_new',
                 repo_name=repo.repo_name,
                 commit=repo.get_commit().raw_id),
             status=200)
 
     @pytest.mark.parametrize('pr_merge_enabled', [True, False])
     def test_show(self, pr_util, pr_merge_enabled):
         pull_request = pr_util.create_pull_request(
             mergeable=pr_merge_enabled, enable_notifications=False)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         for commit_id in pull_request.revisions:
             response.mustcontain(commit_id)
 
         assert pull_request.target_ref_parts.type in response
         assert pull_request.target_ref_parts.name in response
         target_clone_url = pull_request.target_repo.clone_url()
         assert target_clone_url in response
 
         assert 'class="pull-request-merge"' in response
         assert (
             'Server-side pull request merging is disabled.'
             in response) != pr_merge_enabled
 
     def test_close_status_visibility(self, pr_util, user_util, csrf_token):
         # Logout
         response = self.app.post(
             h.route_path('logout'),
             params={'csrf_token': csrf_token})
         # Login as regular user
         response = self.app.post(h.route_path('login'),
                                  {'username': TEST_USER_REGULAR_LOGIN,
                                   'password': 'test12'})
 
         pull_request = pr_util.create_pull_request(
             author=TEST_USER_REGULAR_LOGIN)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('Server-side pull request merging is disabled.')
 
         assert_response = response.assert_response()
         # for regular user without a merge permissions, we don't see it
         assert_response.no_element_exists('#close-pull-request-action')
 
         user_util.grant_user_permission_to_repo(
             pull_request.target_repo,
             UserModel().get_by_username(TEST_USER_REGULAR_LOGIN),
             'repository.write')
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('Server-side pull request merging is disabled.')
 
         assert_response = response.assert_response()
         # now regular user has a merge permissions, we have CLOSE button
         assert_response.one_element_exists('#close-pull-request-action')
 
     def test_show_invalid_commit_id(self, pr_util):
         # Simulating invalid revisions which will cause a lookup error
         pull_request = pr_util.create_pull_request()
         pull_request.revisions = ['invalid']
         Session().add(pull_request)
         Session().commit()
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         for commit_id in pull_request.revisions:
             response.mustcontain(commit_id)
 
     def test_show_invalid_source_reference(self, pr_util):
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'branch:b:invalid'
         Session().add(pull_request)
         Session().commit()
 
         self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
     def test_edit_title_description(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id
 
         response = self.app.post(
             route_path('pullrequest_update',
                 repo_name=pull_request.target_repo.repo_name,
                 pull_request_id=pull_request_id),
             params={
                 'edit_pull_request': 'true',
                 'title': 'New title',
                 'description': 'New description',
                 'csrf_token': csrf_token})
 
         assert_session_flash(
             response, u'Pull request title & description updated.',
             category='success')
 
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.title == 'New title'
         assert pull_request.description == 'New description'
 
     def test_edit_title_description_closed(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id
         repo_name = pull_request.target_repo.repo_name
         pr_util.close()
 
         response = self.app.post(
             route_path('pullrequest_update',
                        repo_name=repo_name, pull_request_id=pull_request_id),
             params={
                 'edit_pull_request': 'true',
                 'title': 'New title',
                 'description': 'New description',
                 'csrf_token': csrf_token}, status=200)
         assert_session_flash(
             response, u'Cannot update closed pull requests.',
             category='error')
 
     def test_update_invalid_source_reference(self, pr_util, csrf_token):
         from rhodecode.lib.vcs.backends.base import UpdateFailureReason
 
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'branch:invalid-branch:invalid-commit-id'
         Session().add(pull_request)
         Session().commit()
 
         pull_request_id = pull_request.pull_request_id
 
         response = self.app.post(
             route_path('pullrequest_update',
                 repo_name=pull_request.target_repo.repo_name,
                 pull_request_id=pull_request_id),
             params={'update_commits': 'true',
                     'csrf_token': csrf_token})
 
         expected_msg = str(PullRequestModel.UPDATE_STATUS_MESSAGES[
             UpdateFailureReason.MISSING_SOURCE_REF])
         assert_session_flash(response, expected_msg, category='error')
 
     def test_missing_target_reference(self, pr_util, csrf_token):
         from rhodecode.lib.vcs.backends.base import MergeFailureReason
         pull_request = pr_util.create_pull_request(
             approved=True, mergeable=True)
         pull_request.target_ref = 'branch:invalid-branch:invalid-commit-id'
         Session().add(pull_request)
         Session().commit()
 
         pull_request_id = pull_request.pull_request_id
         pull_request_url = route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.repo_name,
             pull_request_id=pull_request_id)
 
         response = self.app.get(pull_request_url)
 
         assertr = AssertResponse(response)
         expected_msg = PullRequestModel.MERGE_STATUS_MESSAGES[
             MergeFailureReason.MISSING_TARGET_REF]
         assertr.element_contains(
             'span[data-role="merge-message"]', str(expected_msg))
 
     def test_comment_and_close_pull_request_custom_message_approved(
             self, pr_util, csrf_token, xhr_header):
 
         pull_request = pr_util.create_pull_request(approved=True)
         pull_request_id = pull_request.pull_request_id
         author = pull_request.user_id
         repo = pull_request.target_repo.repo_id
 
         self.app.post(
             route_path('pullrequest_comment_create',
                 repo_name=pull_request.target_repo.scm_instance().name,
                 pull_request_id=pull_request_id),
             params={
                 'close_pull_request': '1',
                 'text': 'Closing a PR',
                 'csrf_token': csrf_token},
             extra_environ=xhr_header,)
 
         journal = UserLog.query()\
             .filter(UserLog.user_id == author)\
             .filter(UserLog.repository_id == repo) \
             .order_by('user_log_id') \
             .all()
         assert journal[-1].action == 'repo.pull_request.close'
 
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.is_closed()
 
         status = ChangesetStatusModel().get_status(
             pull_request.source_repo, pull_request=pull_request)
         assert status == ChangesetStatus.STATUS_APPROVED
         comments = ChangesetComment().query() \
             .filter(ChangesetComment.pull_request == pull_request) \
             .order_by(ChangesetComment.comment_id.asc())\
             .all()
         assert comments[-1].text == 'Closing a PR'
 
     def test_comment_force_close_pull_request_rejected(
             self, pr_util, csrf_token, xhr_header):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id
         PullRequestModel().update_reviewers(
             pull_request_id, [(1, ['reason'], False, []), (2, ['reason2'], False, [])],
             pull_request.author)
         author = pull_request.user_id
         repo = pull_request.target_repo.repo_id
 
         self.app.post(
             route_path('pullrequest_comment_create',
                 repo_name=pull_request.target_repo.scm_instance().name,
                 pull_request_id=pull_request_id),
             params={
                 'close_pull_request': '1',
                 'csrf_token': csrf_token},
             extra_environ=xhr_header)
 
         pull_request = PullRequest.get(pull_request_id)
 
         journal = UserLog.query()\
             .filter(UserLog.user_id == author, UserLog.repository_id == repo) \
             .order_by('user_log_id') \
             .all()
         assert journal[-1].action == 'repo.pull_request.close'
 
         # check only the latest status, not the review status
         status = ChangesetStatusModel().get_status(
             pull_request.source_repo, pull_request=pull_request)
         assert status == ChangesetStatus.STATUS_REJECTED
 
     def test_comment_and_close_pull_request(
             self, pr_util, csrf_token, xhr_header):
         pull_request = pr_util.create_pull_request()
         pull_request_id = pull_request.pull_request_id
 
         response = self.app.post(
             route_path('pullrequest_comment_create',
                 repo_name=pull_request.target_repo.scm_instance().name,
                 pull_request_id=pull_request.pull_request_id),
             params={
                 'close_pull_request': 'true',
                 'csrf_token': csrf_token},
             extra_environ=xhr_header)
 
         assert response.json
 
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.is_closed()
 
         # check only the latest status, not the review status
         status = ChangesetStatusModel().get_status(
             pull_request.source_repo, pull_request=pull_request)
         assert status == ChangesetStatus.STATUS_REJECTED
 
     def test_create_pull_request(self, backend, csrf_token):
         commits = [
             {'message': 'ancestor'},
             {'message': 'change'},
             {'message': 'change2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor'])
         source = backend.create_repo(heads=['change2'])
 
         response = self.app.post(
             route_path('pullrequest_create', repo_name=source.repo_name),
             [
                 ('source_repo', source.repo_name),
                 ('source_ref', 'branch:default:' + commit_ids['change2']),
                 ('target_repo', target.repo_name),
                 ('target_ref',  'branch:default:' + commit_ids['ancestor']),
                 ('common_ancestor', commit_ids['ancestor']),
+                ('pullrequest_title', 'Title'),
                 ('pullrequest_desc', 'Description'),
-                ('pullrequest_title', 'Title'),
+                ('description_renderer', 'markdown'),
                 ('__start__', 'review_members:sequence'),
                     ('__start__', 'reviewer:mapping'),
                         ('user_id', '1'),
                         ('__start__', 'reasons:sequence'),
                             ('reason', 'Some reason'),
                         ('__end__', 'reasons:sequence'),
                         ('__start__', 'rules:sequence'),
                         ('__end__', 'rules:sequence'),
                         ('mandatory', 'False'),
                     ('__end__', 'reviewer:mapping'),
                 ('__end__', 'review_members:sequence'),
                 ('__start__', 'revisions:sequence'),
                     ('revisions', commit_ids['change']),
                     ('revisions', commit_ids['change2']),
                 ('__end__', 'revisions:sequence'),
                 ('user', ''),
                 ('csrf_token', csrf_token),
             ],
             status=302)
 
         location = response.headers['Location']
         pull_request_id = location.rsplit('/', 1)[1]
         assert pull_request_id != 'new'
         pull_request = PullRequest.get(int(pull_request_id))
 
         # check that we have now both revisions
         assert pull_request.revisions == [commit_ids['change2'], commit_ids['change']]
         assert pull_request.source_ref == 'branch:default:' + commit_ids['change2']
         expected_target_ref = 'branch:default:' + commit_ids['ancestor']
         assert pull_request.target_ref == expected_target_ref
 
     def test_reviewer_notifications(self, backend, csrf_token):
         # We have to use the app.post for this test so it will create the
         # notifications properly with the new PR
         commits = [
             {'message': 'ancestor',
              'added': [FileNode('file_A', content='content_of_ancestor')]},
             {'message': 'change',
              'added': [FileNode('file_a', content='content_of_change')]},
             {'message': 'change-child'},
             {'message': 'ancestor-child', 'parents': ['ancestor'],
              'added': [
                 FileNode('file_B', content='content_of_ancestor_child')]},
             {'message': 'ancestor-child-2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor-child'])
         source = backend.create_repo(heads=['change'])
 
         response = self.app.post(
             route_path('pullrequest_create', repo_name=source.repo_name),
             [
                 ('source_repo', source.repo_name),
                 ('source_ref', 'branch:default:' + commit_ids['change']),
                 ('target_repo', target.repo_name),
                 ('target_ref',  'branch:default:' + commit_ids['ancestor-child']),
                 ('common_ancestor', commit_ids['ancestor']),
+                ('pullrequest_title', 'Title'),
                 ('pullrequest_desc', 'Description'),
-                ('pullrequest_title', 'Title'),
+                ('description_renderer', 'markdown'),
                 ('__start__', 'review_members:sequence'),
                     ('__start__', 'reviewer:mapping'),
                         ('user_id', '2'),
                         ('__start__', 'reasons:sequence'),
                             ('reason', 'Some reason'),
                         ('__end__', 'reasons:sequence'),
                         ('__start__', 'rules:sequence'),
                         ('__end__', 'rules:sequence'),
                         ('mandatory', 'False'),
                     ('__end__', 'reviewer:mapping'),
                 ('__end__', 'review_members:sequence'),
                 ('__start__', 'revisions:sequence'),
                     ('revisions', commit_ids['change']),
                 ('__end__', 'revisions:sequence'),
                 ('user', ''),
                 ('csrf_token', csrf_token),
             ],
             status=302)
 
         location = response.headers['Location']
 
         pull_request_id = location.rsplit('/', 1)[1]
         assert pull_request_id != 'new'
         pull_request = PullRequest.get(int(pull_request_id))
 
         # Check that a notification was made
         notifications = Notification.query()\
             .filter(Notification.created_by == pull_request.author.user_id,
                     Notification.type_ == Notification.TYPE_PULL_REQUEST,
                     Notification.subject.contains(
                         "wants you to review pull request #%s" % pull_request_id))
         assert len(notifications.all()) == 1
 
         # Change reviewers and check that a notification was made
         PullRequestModel().update_reviewers(
             pull_request.pull_request_id, [(1, [], False, [])],
             pull_request.author)
         assert len(notifications.all()) == 2
 
     def test_create_pull_request_stores_ancestor_commit_id(self, backend,
                                                            csrf_token):
         commits = [
             {'message': 'ancestor',
              'added': [FileNode('file_A', content='content_of_ancestor')]},
             {'message': 'change',
              'added': [FileNode('file_a', content='content_of_change')]},
             {'message': 'change-child'},
             {'message': 'ancestor-child', 'parents': ['ancestor'],
              'added': [
                 FileNode('file_B', content='content_of_ancestor_child')]},
             {'message': 'ancestor-child-2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor-child'])
         source = backend.create_repo(heads=['change'])
 
         response = self.app.post(
             route_path('pullrequest_create', repo_name=source.repo_name),
             [
                 ('source_repo', source.repo_name),
                 ('source_ref', 'branch:default:' + commit_ids['change']),
                 ('target_repo', target.repo_name),
                 ('target_ref',  'branch:default:' + commit_ids['ancestor-child']),
                 ('common_ancestor', commit_ids['ancestor']),
+                ('pullrequest_title', 'Title'),
                 ('pullrequest_desc', 'Description'),
-                ('pullrequest_title', 'Title'),
+                ('description_renderer', 'markdown'),
                 ('__start__', 'review_members:sequence'),
                     ('__start__', 'reviewer:mapping'),
                         ('user_id', '1'),
                         ('__start__', 'reasons:sequence'),
                             ('reason', 'Some reason'),
                         ('__end__', 'reasons:sequence'),
                         ('__start__', 'rules:sequence'),
                         ('__end__', 'rules:sequence'),
                         ('mandatory', 'False'),
                     ('__end__', 'reviewer:mapping'),
                 ('__end__', 'review_members:sequence'),
                 ('__start__', 'revisions:sequence'),
                     ('revisions', commit_ids['change']),
                 ('__end__', 'revisions:sequence'),
                 ('user', ''),
                 ('csrf_token', csrf_token),
             ],
             status=302)
 
         location = response.headers['Location']
 
         pull_request_id = location.rsplit('/', 1)[1]
         assert pull_request_id != 'new'
         pull_request = PullRequest.get(int(pull_request_id))
 
         # target_ref has to point to the ancestor's commit_id in order to
         # show the correct diff
         expected_target_ref = 'branch:default:' + commit_ids['ancestor']
         assert pull_request.target_ref == expected_target_ref
 
         # Check generated diff contents
         response = response.follow()
         assert 'content_of_ancestor' not in response.body
         assert 'content_of_ancestor-child' not in response.body
         assert 'content_of_change' in response.body
 
     def test_merge_pull_request_enabled(self, pr_util, csrf_token):
         # Clear any previous calls to rcextensions
         rhodecode.EXTENSIONS.calls.clear()
 
         pull_request = pr_util.create_pull_request(
             approved=True, mergeable=True)
         pull_request_id = pull_request.pull_request_id
         repo_name = pull_request.target_repo.scm_instance().name,
 
         response = self.app.post(
             route_path('pullrequest_merge',
                 repo_name=str(repo_name[0]),
                 pull_request_id=pull_request_id),
             params={'csrf_token': csrf_token}).follow()
 
         pull_request = PullRequest.get(pull_request_id)
 
         assert response.status_int == 200
         assert pull_request.is_closed()
         assert_pull_request_status(
             pull_request, ChangesetStatus.STATUS_APPROVED)
 
         # Check the relevant log entries were added
         user_logs = UserLog.query().order_by('-user_log_id').limit(3)
         actions = [log.action for log in user_logs]
         pr_commit_ids = PullRequestModel()._get_commit_ids(pull_request)
         expected_actions = [
              u'repo.pull_request.close',
              u'repo.pull_request.merge',
              u'repo.pull_request.comment.create'
         ]
         assert actions == expected_actions
 
         user_logs = UserLog.query().order_by('-user_log_id').limit(4)
         actions = [log for log in user_logs]
         assert actions[-1].action == 'user.push'
         assert actions[-1].action_data['commit_ids'] == pr_commit_ids
 
         # Check post_push rcextension was really executed
         push_calls = rhodecode.EXTENSIONS.calls['post_push']
         assert len(push_calls) == 1
         unused_last_call_args, last_call_kwargs = push_calls[0]
         assert last_call_kwargs['action'] == 'push'
         assert last_call_kwargs['pushed_revs'] == pr_commit_ids
 
     def test_merge_pull_request_disabled(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request(mergeable=False)
         pull_request_id = pull_request.pull_request_id
         pull_request = PullRequest.get(pull_request_id)
 
         response = self.app.post(
             route_path('pullrequest_merge',
                 repo_name=pull_request.target_repo.scm_instance().name,
                 pull_request_id=pull_request.pull_request_id),
             params={'csrf_token': csrf_token}).follow()
 
         assert response.status_int == 200
         response.mustcontain(
             'Merge is not currently possible because of below failed checks.')
         response.mustcontain('Server-side pull request merging is disabled.')
 
     @pytest.mark.skip_backends('svn')
     def test_merge_pull_request_not_approved(self, pr_util, csrf_token):
         pull_request = pr_util.create_pull_request(mergeable=True)
         pull_request_id = pull_request.pull_request_id
         repo_name = pull_request.target_repo.scm_instance().name
 
         response = self.app.post(
             route_path('pullrequest_merge',
                 repo_name=repo_name,
                 pull_request_id=pull_request_id),
             params={'csrf_token': csrf_token}).follow()
 
         assert response.status_int == 200
 
         response.mustcontain(
             'Merge is not currently possible because of below failed checks.')
         response.mustcontain('Pull request reviewer approval is pending.')
 
     def test_merge_pull_request_renders_failure_reason(
             self, user_regular, csrf_token, pr_util):
         pull_request = pr_util.create_pull_request(mergeable=True, approved=True)
         pull_request_id = pull_request.pull_request_id
         repo_name = pull_request.target_repo.scm_instance().name
 
         model_patcher = mock.patch.multiple(
             PullRequestModel,
             merge_repo=mock.Mock(return_value=MergeResponse(
                 True, False, 'STUB_COMMIT_ID', MergeFailureReason.PUSH_FAILED)),
             merge_status=mock.Mock(return_value=(True, 'WRONG_MESSAGE')))
 
         with model_patcher:
             response = self.app.post(
                 route_path('pullrequest_merge',
                            repo_name=repo_name,
                            pull_request_id=pull_request_id),
                 params={'csrf_token': csrf_token}, status=302)
 
         assert_session_flash(response, PullRequestModel.MERGE_STATUS_MESSAGES[
             MergeFailureReason.PUSH_FAILED])
 
     def test_update_source_revision(self, backend, csrf_token):
         commits = [
             {'message': 'ancestor'},
             {'message': 'change'},
             {'message': 'change-2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor'])
         source = backend.create_repo(heads=['change'])
 
         # create pr from a in source to A in target
         pull_request = PullRequest()
         pull_request.source_repo = source
         # TODO: johbo: Make sure that we write the source ref this way!
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['change'])
         pull_request.target_repo = target
 
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name,
             commit_id=commit_ids['ancestor'])
         pull_request.revisions = [commit_ids['change']]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(
             TEST_USER_ADMIN_LOGIN)
         Session().add(pull_request)
         Session().commit()
         pull_request_id = pull_request.pull_request_id
 
         # source has ancestor - change - change-2
         backend.pull_heads(source, heads=['change-2'])
 
         # update PR
         self.app.post(
             route_path('pullrequest_update',
                 repo_name=target.repo_name,
                 pull_request_id=pull_request_id),
             params={'update_commits': 'true',
                     'csrf_token': csrf_token})
 
         # check that we have now both revisions
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.revisions == [
             commit_ids['change-2'], commit_ids['change']]
 
         # TODO: johbo: this should be a test on its own
         response = self.app.get(route_path(
             'pullrequest_new',
             repo_name=target.repo_name))
         assert response.status_int == 200
         assert 'Pull request updated to' in response.body
         assert 'with 1 added, 0 removed commits.' in response.body
 
     def test_update_target_revision(self, backend, csrf_token):
         commits = [
             {'message': 'ancestor'},
             {'message': 'change'},
             {'message': 'ancestor-new', 'parents': ['ancestor']},
             {'message': 'change-rebased'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor'])
         source = backend.create_repo(heads=['change'])
 
         # create pr from a in source to A in target
         pull_request = PullRequest()
         pull_request.source_repo = source
         # TODO: johbo: Make sure that we write the source ref this way!
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name, commit_id=commit_ids['change'])
         pull_request.target_repo = target
         # TODO: johbo: Target ref should be branch based, since tip can jump
         # from branch to branch
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name,
             commit_id=commit_ids['ancestor'])
         pull_request.revisions = [commit_ids['change']]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(
             TEST_USER_ADMIN_LOGIN)
         Session().add(pull_request)
         Session().commit()
         pull_request_id = pull_request.pull_request_id
 
         # target has ancestor - ancestor-new
         # source has ancestor - ancestor-new - change-rebased
         backend.pull_heads(target, heads=['ancestor-new'])
         backend.pull_heads(source, heads=['change-rebased'])
 
         # update PR
         self.app.post(
             route_path('pullrequest_update',
                 repo_name=target.repo_name,
                 pull_request_id=pull_request_id),
             params={'update_commits': 'true',
                     'csrf_token': csrf_token},
             status=200)
 
         # check that we have now both revisions
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.revisions == [commit_ids['change-rebased']]
         assert pull_request.target_ref == 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name,
             commit_id=commit_ids['ancestor-new'])
 
         # TODO: johbo: This should be a test on its own
         response = self.app.get(route_path(
             'pullrequest_new',
             repo_name=target.repo_name))
         assert response.status_int == 200
         assert 'Pull request updated to' in response.body
         assert 'with 1 added, 1 removed commits.' in response.body
 
     def test_update_target_revision_with_removal_of_1_commit_git(self, backend_git, csrf_token):
         backend = backend_git
         commits = [
             {'message': 'master-commit-1'},
             {'message': 'master-commit-2-change-1'},
             {'message': 'master-commit-3-change-2'},
 
             {'message': 'feat-commit-1', 'parents': ['master-commit-1']},
             {'message': 'feat-commit-2'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['master-commit-3-change-2'])
         source = backend.create_repo(heads=['feat-commit-2'])
 
         # create pr from a in source to A in target
         pull_request = PullRequest()
         pull_request.source_repo = source
         # TODO: johbo: Make sure that we write the source ref this way!
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name,
             commit_id=commit_ids['master-commit-3-change-2'])
 
         pull_request.target_repo = target
         # TODO: johbo: Target ref should be branch based, since tip can jump
         # from branch to branch
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name,
             commit_id=commit_ids['feat-commit-2'])
 
         pull_request.revisions = [
             commit_ids['feat-commit-1'],
             commit_ids['feat-commit-2']
         ]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(
             TEST_USER_ADMIN_LOGIN)
         Session().add(pull_request)
         Session().commit()
         pull_request_id = pull_request.pull_request_id
 
         # PR is created, now we simulate a force-push into target,
         # that drops a 2 last commits
         vcsrepo = target.scm_instance()
         vcsrepo.config.clear_section('hooks')
         vcsrepo.run_git_command(['reset', '--soft', 'HEAD~2'])
 
         # update PR
         self.app.post(
             route_path('pullrequest_update',
                 repo_name=target.repo_name,
                 pull_request_id=pull_request_id),
             params={'update_commits': 'true',
                     'csrf_token': csrf_token},
             status=200)
 
         response = self.app.get(route_path(
             'pullrequest_new',
             repo_name=target.repo_name))
         assert response.status_int == 200
         response.mustcontain('Pull request updated to')
         response.mustcontain('with 0 added, 0 removed commits.')
 
     def test_update_of_ancestor_reference(self, backend, csrf_token):
         commits = [
             {'message': 'ancestor'},
             {'message': 'change'},
             {'message': 'change-2'},
             {'message': 'ancestor-new', 'parents': ['ancestor']},
             {'message': 'change-rebased'},
         ]
         commit_ids = backend.create_master_repo(commits)
         target = backend.create_repo(heads=['ancestor'])
         source = backend.create_repo(heads=['change'])
 
         # create pr from a in source to A in target
         pull_request = PullRequest()
         pull_request.source_repo = source
         # TODO: johbo: Make sure that we write the source ref this way!
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name,
             commit_id=commit_ids['change'])
         pull_request.target_repo = target
         # TODO: johbo: Target ref should be branch based, since tip can jump
         # from branch to branch
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name,
             commit_id=commit_ids['ancestor'])
         pull_request.revisions = [commit_ids['change']]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(
             TEST_USER_ADMIN_LOGIN)
         Session().add(pull_request)
         Session().commit()
         pull_request_id = pull_request.pull_request_id
 
         # target has ancestor - ancestor-new
         # source has ancestor - ancestor-new - change-rebased
         backend.pull_heads(target, heads=['ancestor-new'])
         backend.pull_heads(source, heads=['change-rebased'])
 
         # update PR
         self.app.post(
             route_path('pullrequest_update',
                 repo_name=target.repo_name,
                 pull_request_id=pull_request_id),
             params={'update_commits': 'true',
                     'csrf_token': csrf_token},
             status=200)
 
         # Expect the target reference to be updated correctly
         pull_request = PullRequest.get(pull_request_id)
         assert pull_request.revisions == [commit_ids['change-rebased']]
         expected_target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend.default_branch_name,
             commit_id=commit_ids['ancestor-new'])
         assert pull_request.target_ref == expected_target_ref
 
     def test_remove_pull_request_branch(self, backend_git, csrf_token):
         branch_name = 'development'
         commits = [
             {'message': 'initial-commit'},
             {'message': 'old-feature'},
             {'message': 'new-feature', 'branch': branch_name},
         ]
         repo = backend_git.create_repo(commits)
         commit_ids = backend_git.commit_ids
 
         pull_request = PullRequest()
         pull_request.source_repo = repo
         pull_request.target_repo = repo
         pull_request.source_ref = 'branch:{branch}:{commit_id}'.format(
             branch=branch_name, commit_id=commit_ids['new-feature'])
         pull_request.target_ref = 'branch:{branch}:{commit_id}'.format(
             branch=backend_git.default_branch_name,
             commit_id=commit_ids['old-feature'])
         pull_request.revisions = [commit_ids['new-feature']]
         pull_request.title = u"Test"
         pull_request.description = u"Description"
         pull_request.author = UserModel().get_by_username(
             TEST_USER_ADMIN_LOGIN)
         Session().add(pull_request)
         Session().commit()
 
         vcs = repo.scm_instance()
         vcs.remove_ref('refs/heads/{}'.format(branch_name))
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=repo.repo_name,
             pull_request_id=pull_request.pull_request_id))
 
         assert response.status_int == 200
         assert_response = AssertResponse(response)
         assert_response.element_contains(
             '#changeset_compare_view_content .alert strong',
             'Missing commits')
         assert_response.element_contains(
             '#changeset_compare_view_content .alert',
             'This pull request cannot be displayed, because one or more'
             ' commits no longer exist in the source repository.')
 
     def test_strip_commits_from_pull_request(
             self, backend, pr_util, csrf_token):
         commits = [
             {'message': 'initial-commit'},
             {'message': 'old-feature'},
             {'message': 'new-feature', 'parents': ['initial-commit']},
         ]
         pull_request = pr_util.create_pull_request(
             commits, target_head='initial-commit', source_head='new-feature',
             revisions=['new-feature'])
 
         vcs = pr_util.source_repository.scm_instance()
         if backend.alias == 'git':
             vcs.strip(pr_util.commit_ids['new-feature'], branch_name='master')
         else:
             vcs.strip(pr_util.commit_ids['new-feature'])
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pr_util.target_repository.repo_name,
             pull_request_id=pull_request.pull_request_id))
 
         assert response.status_int == 200
         assert_response = AssertResponse(response)
         assert_response.element_contains(
             '#changeset_compare_view_content .alert strong',
             'Missing commits')
         assert_response.element_contains(
             '#changeset_compare_view_content .alert',
             'This pull request cannot be displayed, because one or more'
             ' commits no longer exist in the source repository.')
         assert_response.element_contains(
             '#update_commits',
             'Update commits')
 
     def test_strip_commits_and_update(
             self, backend, pr_util, csrf_token):
         commits = [
             {'message': 'initial-commit'},
             {'message': 'old-feature'},
             {'message': 'new-feature', 'parents': ['old-feature']},
         ]
         pull_request = pr_util.create_pull_request(
             commits, target_head='old-feature', source_head='new-feature',
             revisions=['new-feature'], mergeable=True)
 
         vcs = pr_util.source_repository.scm_instance()
         if backend.alias == 'git':
             vcs.strip(pr_util.commit_ids['new-feature'], branch_name='master')
         else:
             vcs.strip(pr_util.commit_ids['new-feature'])
 
         response = self.app.post(
             route_path('pullrequest_update',
                 repo_name=pull_request.target_repo.repo_name,
                 pull_request_id=pull_request.pull_request_id),
             params={'update_commits': 'true',
                     'csrf_token': csrf_token})
 
         assert response.status_int == 200
         assert response.body == 'true'
 
         # Make sure that after update, it won't raise 500 errors
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pr_util.target_repository.repo_name,
             pull_request_id=pull_request.pull_request_id))
 
         assert response.status_int == 200
         assert_response = AssertResponse(response)
         assert_response.element_contains(
             '#changeset_compare_view_content .alert strong',
             'Missing commits')
 
     def test_branch_is_a_link(self, pr_util):
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'branch:origin:1234567890abcdef'
         pull_request.target_ref = 'branch:target:abcdef1234567890'
         Session().add(pull_request)
         Session().commit()
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
         assert response.status_int == 200
         assert_response = AssertResponse(response)
 
         origin = assert_response.get_element('.pr-origininfo .tag')
         origin_children = origin.getchildren()
         assert len(origin_children) == 1
         target = assert_response.get_element('.pr-targetinfo .tag')
         target_children = target.getchildren()
         assert len(target_children) == 1
 
         expected_origin_link = route_path(
             'repo_changelog',
             repo_name=pull_request.source_repo.scm_instance().name,
             params=dict(branch='origin'))
         expected_target_link = route_path(
             'repo_changelog',
             repo_name=pull_request.target_repo.scm_instance().name,
             params=dict(branch='target'))
         assert origin_children[0].attrib['href'] == expected_origin_link
         assert origin_children[0].text == 'branch: origin'
         assert target_children[0].attrib['href'] == expected_target_link
         assert target_children[0].text == 'branch: target'
 
     def test_bookmark_is_not_a_link(self, pr_util):
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'bookmark:origin:1234567890abcdef'
         pull_request.target_ref = 'bookmark:target:abcdef1234567890'
         Session().add(pull_request)
         Session().commit()
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
         assert response.status_int == 200
         assert_response = AssertResponse(response)
 
         origin = assert_response.get_element('.pr-origininfo .tag')
         assert origin.text.strip() == 'bookmark: origin'
         assert origin.getchildren() == []
 
         target = assert_response.get_element('.pr-targetinfo .tag')
         assert target.text.strip() == 'bookmark: target'
         assert target.getchildren() == []
 
     def test_tag_is_not_a_link(self, pr_util):
         pull_request = pr_util.create_pull_request()
         pull_request.source_ref = 'tag:origin:1234567890abcdef'
         pull_request.target_ref = 'tag:target:abcdef1234567890'
         Session().add(pull_request)
         Session().commit()
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
         assert response.status_int == 200
         assert_response = AssertResponse(response)
 
         origin = assert_response.get_element('.pr-origininfo .tag')
         assert origin.text.strip() == 'tag: origin'
         assert origin.getchildren() == []
 
         target = assert_response.get_element('.pr-targetinfo .tag')
         assert target.text.strip() == 'tag: target'
         assert target.getchildren() == []
 
     @pytest.mark.parametrize('mergeable', [True, False])
     def test_shadow_repository_link(
             self, mergeable, pr_util, http_host_only_stub):
         """
         Check that the pull request summary page displays a link to the shadow
         repository if the pull request is mergeable. If it is not mergeable
         the link should not be displayed.
         """
         pull_request = pr_util.create_pull_request(
             mergeable=mergeable, enable_notifications=False)
         target_repo = pull_request.target_repo.scm_instance()
         pr_id = pull_request.pull_request_id
         shadow_url = '{host}/{repo}/pull-request/{pr_id}/repository'.format(
             host=http_host_only_stub, repo=target_repo.name, pr_id=pr_id)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=target_repo.name,
             pull_request_id=pr_id))
 
         assertr = AssertResponse(response)
         if mergeable:
             assertr.element_value_contains('input.pr-mergeinfo', shadow_url)
             assertr.element_value_contains('input.pr-mergeinfo ', 'pr-merge')
         else:
             assertr.no_element_exists('.pr-mergeinfo')
 
 
 @pytest.mark.usefixtures('app')
 @pytest.mark.backends("git", "hg")
 class TestPullrequestsControllerDelete(object):
     def test_pull_request_delete_button_permissions_admin(
             self, autologin_user, user_admin, pr_util):
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('id="delete_pullrequest"')
         response.mustcontain('Confirm to delete this pull request')
 
     def test_pull_request_delete_button_permissions_owner(
             self, autologin_regular_user, user_regular, pr_util):
         pull_request = pr_util.create_pull_request(
             author=user_regular.username, enable_notifications=False)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('id="delete_pullrequest"')
         response.mustcontain('Confirm to delete this pull request')
 
     def test_pull_request_delete_button_permissions_forbidden(
             self, autologin_regular_user, user_regular, user_admin, pr_util):
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
         response.mustcontain(no=['id="delete_pullrequest"'])
         response.mustcontain(no=['Confirm to delete this pull request'])
 
     def test_pull_request_delete_button_permissions_can_update_cannot_delete(
             self, autologin_regular_user, user_regular, user_admin, pr_util,
             user_util):
 
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
 
         user_util.grant_user_permission_to_repo(
             pull_request.target_repo, user_regular,
             'repository.write')
 
         response = self.app.get(route_path(
             'pullrequest_show',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id))
 
         response.mustcontain('id="open_edit_pullrequest"')
         response.mustcontain('id="delete_pullrequest"')
         response.mustcontain(no=['Confirm to delete this pull request'])
 
     def test_delete_comment_returns_404_if_comment_does_not_exist(
             self, autologin_user, pr_util, user_admin, csrf_token, xhr_header):
 
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
 
         self.app.post(
             route_path(
             'pullrequest_comment_delete',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id,
             comment_id=1024404),
             extra_environ=xhr_header,
             params={'csrf_token': csrf_token},
             status=404
         )
 
     def test_delete_comment(
             self, autologin_user, pr_util, user_admin, csrf_token, xhr_header):
 
         pull_request = pr_util.create_pull_request(
             author=user_admin.username, enable_notifications=False)
         comment = pr_util.create_comment()
         comment_id = comment.comment_id
 
         response = self.app.post(
             route_path(
             'pullrequest_comment_delete',
             repo_name=pull_request.target_repo.scm_instance().name,
             pull_request_id=pull_request.pull_request_id,
             comment_id=comment_id),
             extra_environ=xhr_header,
             params={'csrf_token': csrf_token},
             status=200
         )
         assert response.body == 'true'
 
 
 def assert_pull_request_status(pull_request, expected_status):
     status = ChangesetStatusModel().calculated_review_status(
         pull_request=pull_request)
     assert status == expected_status
 
 
 @pytest.mark.parametrize('route', ['pullrequest_new', 'pullrequest_create'])
 @pytest.mark.usefixtures("autologin_user")
 def test_forbidde_to_repo_summary_for_svn_repositories(backend_svn, app, route):
     response = app.get(
         route_path(route, repo_name=backend_svn.repo_name), status=404)
 

rhodecode/apps/repository/views/repo_pull_requests.py

@@ -1,1316 +1,1324 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import collections
 
 import formencode
 import formencode.htmlfill
 import peppercorn
 from pyramid.httpexceptions import (
     HTTPFound, HTTPNotFound, HTTPForbidden, HTTPBadRequest)
 from pyramid.view import view_config
 from pyramid.renderers import render
 
 from rhodecode import events
 from rhodecode.apps._base import RepoAppView, DataGridAppView
 
 from rhodecode.lib import helpers as h, diffs, codeblocks, channelstream
 from rhodecode.lib.base import vcs_operation_context
 from rhodecode.lib.diffs import load_cached_diff, cache_diff, diff_cache_exist
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAny, HasRepoPermissionAnyDecorator,
     NotAnonymous, CSRFRequired)
 from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode
 from rhodecode.lib.vcs.backends.base import EmptyCommit, UpdateFailureReason
 from rhodecode.lib.vcs.exceptions import (CommitDoesNotExistError,
     RepositoryRequirementError, EmptyRepositoryError)
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (func, or_, PullRequest, PullRequestVersion,
     ChangesetComment, ChangesetStatus, Repository)
 from rhodecode.model.forms import PullRequestForm
 from rhodecode.model.meta import Session
 from rhodecode.model.pull_request import PullRequestModel, MergeCheck
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoPullRequestsView(RepoAppView, DataGridAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.REVIEW_STATUS_APPROVED = ChangesetStatus.STATUS_APPROVED
         c.REVIEW_STATUS_REJECTED = ChangesetStatus.STATUS_REJECTED
-
+        # backward compat., we use for OLD PRs a plain renderer
+        c.renderer = 'plain'
         return c
 
     def _get_pull_requests_list(
             self, repo_name, source, filter_type, opened_by, statuses):
 
         draw, start, limit = self._extract_chunk(self.request)
         search_q, order_by, order_dir = self._extract_ordering(self.request)
         _render = self.request.get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
 
         # pagination
 
         if filter_type == 'awaiting_review':
             pull_requests = PullRequestModel().get_awaiting_review(
                 repo_name, source=source, opened_by=opened_by,
                 statuses=statuses, offset=start, length=limit,
                 order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_awaiting_review(
                 repo_name, source=source, statuses=statuses,
                 opened_by=opened_by)
         elif filter_type == 'awaiting_my_review':
             pull_requests = PullRequestModel().get_awaiting_my_review(
                 repo_name, source=source, opened_by=opened_by,
                 user_id=self._rhodecode_user.user_id, statuses=statuses,
                 offset=start, length=limit, order_by=order_by,
                 order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_awaiting_my_review(
                 repo_name, source=source, user_id=self._rhodecode_user.user_id,
                 statuses=statuses, opened_by=opened_by)
         else:
             pull_requests = PullRequestModel().get_all(
                 repo_name, source=source, opened_by=opened_by,
                 statuses=statuses, offset=start, length=limit,
                 order_by=order_by, order_dir=order_dir)
             pull_requests_total_count = PullRequestModel().count_all(
                 repo_name, source=source, statuses=statuses,
                 opened_by=opened_by)
 
         data = []
         comments_model = CommentsModel()
         for pr in pull_requests:
             comments = comments_model.get_all_comments(
                 self.db_repo.repo_id, pull_request=pr)
 
             data.append({
                 'name': _render('pullrequest_name',
                                 pr.pull_request_id, pr.target_repo.repo_name),
                 'name_raw': pr.pull_request_id,
                 'status': _render('pullrequest_status',
                                   pr.calculated_review_status()),
                 'title': _render(
                     'pullrequest_title', pr.title, pr.description),
                 'description': h.escape(pr.description),
                 'updated_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.updated_on)),
                 'updated_on_raw': h.datetime_to_time(pr.updated_on),
                 'created_on': _render('pullrequest_updated_on',
                                       h.datetime_to_time(pr.created_on)),
                 'created_on_raw': h.datetime_to_time(pr.created_on),
                 'author': _render('pullrequest_author',
                                   pr.author.full_contact, ),
                 'author_raw': pr.author.full_name,
                 'comments': _render('pullrequest_comments', len(comments)),
                 'comments_raw': len(comments),
                 'closed': pr.is_closed(),
             })
 
         data = ({
             'draw': draw,
             'data': data,
             'recordsTotal': pull_requests_total_count,
             'recordsFiltered': pull_requests_total_count,
         })
         return data
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_show_all', request_method='GET',
         renderer='rhodecode:templates/pullrequests/pullrequests.mako')
     def pull_request_list(self):
         c = self.load_default_context()
 
         req_get = self.request.GET
         c.source = str2bool(req_get.get('source'))
         c.closed = str2bool(req_get.get('closed'))
         c.my = str2bool(req_get.get('my'))
         c.awaiting_review = str2bool(req_get.get('awaiting_review'))
         c.awaiting_my_review = str2bool(req_get.get('awaiting_my_review'))
 
         c.active = 'open'
         if c.my:
             c.active = 'my'
         if c.closed:
             c.active = 'closed'
         if c.awaiting_review and not c.source:
             c.active = 'awaiting'
         if c.source and not c.awaiting_review:
             c.active = 'source'
         if c.awaiting_my_review:
             c.active = 'awaiting_my'
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_show_all_data', request_method='GET',
         renderer='json_ext', xhr=True)
     def pull_request_list_data(self):
         self.load_default_context()
 
         # additional filters
         req_get = self.request.GET
         source = str2bool(req_get.get('source'))
         closed = str2bool(req_get.get('closed'))
         my = str2bool(req_get.get('my'))
         awaiting_review = str2bool(req_get.get('awaiting_review'))
         awaiting_my_review = str2bool(req_get.get('awaiting_my_review'))
 
         filter_type = 'awaiting_review' if awaiting_review \
             else 'awaiting_my_review' if awaiting_my_review \
             else None
 
         opened_by = None
         if my:
             opened_by = [self._rhodecode_user.user_id]
 
         statuses = [PullRequest.STATUS_NEW, PullRequest.STATUS_OPEN]
         if closed:
             statuses = [PullRequest.STATUS_CLOSED]
 
         data = self._get_pull_requests_list(
             repo_name=self.db_repo_name, source=source,
             filter_type=filter_type, opened_by=opened_by, statuses=statuses)
 
         return data
 
     def _is_diff_cache_enabled(self, target_repo):
         caching_enabled = self._get_general_setting(
             target_repo, 'rhodecode_diff_cache')
         log.debug('Diff caching enabled: %s', caching_enabled)
         return caching_enabled
 
     def _get_diffset(self, source_repo_name, source_repo,
                      source_ref_id, target_ref_id,
                      target_commit, source_commit, diff_limit, file_limit,
                      fulldiff):
 
         vcs_diff = PullRequestModel().get_diff(
             source_repo, source_ref_id, target_ref_id)
 
         diff_processor = diffs.DiffProcessor(
             vcs_diff, format='newdiff', diff_limit=diff_limit,
             file_limit=file_limit, show_full_diff=fulldiff)
 
         _parsed = diff_processor.prepare()
 
         diffset = codeblocks.DiffSet(
             repo_name=self.db_repo_name,
             source_repo_name=source_repo_name,
             source_node_getter=codeblocks.diffset_node_getter(target_commit),
             target_node_getter=codeblocks.diffset_node_getter(source_commit),
         )
         diffset = self.path_filter.render_patchset_filtered(
             diffset, _parsed, target_commit.raw_id, source_commit.raw_id)
 
         return diffset
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_show', request_method='GET',
         renderer='rhodecode:templates/pullrequests/pullrequest_show.mako')
     def pull_request_show(self):
         pull_request_id = self.request.matchdict['pull_request_id']
 
         c = self.load_default_context()
 
         version = self.request.GET.get('version')
         from_version = self.request.GET.get('from_version') or version
         merge_checks = self.request.GET.get('merge_checks')
         c.fulldiff = str2bool(self.request.GET.get('fulldiff'))
         force_refresh = str2bool(self.request.GET.get('force_refresh'))
 
         (pull_request_latest,
          pull_request_at_ver,
          pull_request_display_obj,
          at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=version)
         pr_closed = pull_request_latest.is_closed()
 
         if pr_closed and (version or from_version):
             # not allow to browse versions
             raise HTTPFound(h.route_path(
                 'pullrequest_show', repo_name=self.db_repo_name,
                 pull_request_id=pull_request_id))
 
         versions = pull_request_display_obj.versions()
 
         c.at_version = at_version
         c.at_version_num = (at_version
                             if at_version and at_version != 'latest'
                             else None)
         c.at_version_pos = ChangesetComment.get_index_from_version(
             c.at_version_num, versions)
 
         (prev_pull_request_latest,
          prev_pull_request_at_ver,
          prev_pull_request_display_obj,
          prev_at_version) = PullRequestModel().get_pr_version(
             pull_request_id, version=from_version)
 
         c.from_version = prev_at_version
         c.from_version_num = (prev_at_version
                               if prev_at_version and prev_at_version != 'latest'
                               else None)
         c.from_version_pos = ChangesetComment.get_index_from_version(
             c.from_version_num, versions)
 
         # define if we're in COMPARE mode or VIEW at version mode
         compare = at_version != prev_at_version
 
         # pull_requests repo_name we opened it against
         # ie. target_repo must match
         if self.db_repo_name != pull_request_at_ver.target_repo.repo_name:
             raise HTTPNotFound()
 
         c.shadow_clone_url = PullRequestModel().get_shadow_clone_url(
             pull_request_at_ver)
 
         c.pull_request = pull_request_display_obj
+        c.renderer = pull_request_at_ver.description_renderer or c.renderer
         c.pull_request_latest = pull_request_latest
 
         if compare or (at_version and not at_version == 'latest'):
             c.allowed_to_change_status = False
             c.allowed_to_update = False
             c.allowed_to_merge = False
             c.allowed_to_delete = False
             c.allowed_to_comment = False
             c.allowed_to_close = False
         else:
             can_change_status = PullRequestModel().check_user_change_status(
                 pull_request_at_ver, self._rhodecode_user)
             c.allowed_to_change_status = can_change_status and not pr_closed
 
             c.allowed_to_update = PullRequestModel().check_user_update(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_merge = PullRequestModel().check_user_merge(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_delete = PullRequestModel().check_user_delete(
                 pull_request_latest, self._rhodecode_user) and not pr_closed
             c.allowed_to_comment = not pr_closed
             c.allowed_to_close = c.allowed_to_merge and not pr_closed
 
         c.forbid_adding_reviewers = False
         c.forbid_author_to_review = False
         c.forbid_commit_author_to_review = False
 
         if pull_request_latest.reviewer_data and \
                         'rules' in pull_request_latest.reviewer_data:
             rules = pull_request_latest.reviewer_data['rules'] or {}
             try:
                 c.forbid_adding_reviewers = rules.get(
                     'forbid_adding_reviewers')
                 c.forbid_author_to_review = rules.get(
                     'forbid_author_to_review')
                 c.forbid_commit_author_to_review = rules.get(
                     'forbid_commit_author_to_review')
             except Exception:
                 pass
 
         # check merge capabilities
         _merge_check = MergeCheck.validate(
             pull_request_latest, user=self._rhodecode_user,
             translator=self.request.translate,
             force_shadow_repo_refresh=force_refresh)
         c.pr_merge_errors = _merge_check.error_details
         c.pr_merge_possible = not _merge_check.failed
         c.pr_merge_message = _merge_check.merge_msg
 
         c.pr_merge_info = MergeCheck.get_merge_conditions(
             pull_request_latest, translator=self.request.translate)
 
         c.pull_request_review_status = _merge_check.review_status
         if merge_checks:
             self.request.override_renderer = \
                 'rhodecode:templates/pullrequests/pullrequest_merge_checks.mako'
             return self._get_template_context(c)
 
         comments_model = CommentsModel()
 
         # reviewers and statuses
         c.pull_request_reviewers = pull_request_at_ver.reviewers_statuses()
         allowed_reviewers = [x[0].user_id for x in c.pull_request_reviewers]
 
         # GENERAL COMMENTS with versions #
         q = comments_model._all_general_comments_of_pull_request(pull_request_latest)
         q = q.order_by(ChangesetComment.comment_id.asc())
         general_comments = q
 
         # pick comments we want to render at current version
         c.comment_versions = comments_model.aggregate_comments(
             general_comments, versions, c.at_version_num)
         c.comments = c.comment_versions[c.at_version_num]['until']
 
         # INLINE COMMENTS with versions  #
         q = comments_model._all_inline_comments_of_pull_request(pull_request_latest)
         q = q.order_by(ChangesetComment.comment_id.asc())
         inline_comments = q
 
         c.inline_versions = comments_model.aggregate_comments(
             inline_comments, versions, c.at_version_num, inline=True)
 
         # inject latest version
         latest_ver = PullRequest.get_pr_display_object(
             pull_request_latest, pull_request_latest)
 
         c.versions = versions + [latest_ver]
 
         # if we use version, then do not show later comments
         # than current version
         display_inline_comments = collections.defaultdict(
             lambda: collections.defaultdict(list))
         for co in inline_comments:
             if c.at_version_num:
                 # pick comments that are at least UPTO given version, so we
                 # don't render comments for higher version
                 should_render = co.pull_request_version_id and \
                                 co.pull_request_version_id <= c.at_version_num
             else:
                 # showing all, for 'latest'
                 should_render = True
 
             if should_render:
                 display_inline_comments[co.f_path][co.line_no].append(co)
 
         # load diff data into template context, if we use compare mode then
         # diff is calculated based on changes between versions of PR
 
         source_repo = pull_request_at_ver.source_repo
         source_ref_id = pull_request_at_ver.source_ref_parts.commit_id
 
         target_repo = pull_request_at_ver.target_repo
         target_ref_id = pull_request_at_ver.target_ref_parts.commit_id
 
         if compare:
             # in compare switch the diff base to latest commit from prev version
             target_ref_id = prev_pull_request_display_obj.revisions[0]
 
         # despite opening commits for bookmarks/branches/tags, we always
         # convert this to rev to prevent changes after bookmark or branch change
         c.source_ref_type = 'rev'
         c.source_ref = source_ref_id
 
         c.target_ref_type = 'rev'
         c.target_ref = target_ref_id
 
         c.source_repo = source_repo
         c.target_repo = target_repo
 
         c.commit_ranges = []
         source_commit = EmptyCommit()
         target_commit = EmptyCommit()
         c.missing_requirements = False
 
         source_scm = source_repo.scm_instance()
         target_scm = target_repo.scm_instance()
 
         shadow_scm = None
         try:
             shadow_scm = pull_request_latest.get_shadow_repo()
         except Exception:
             log.debug('Failed to get shadow repo', exc_info=True)
         # try first the existing source_repo, and then shadow
         # repo if we can obtain one
         commits_source_repo = source_scm or shadow_scm
 
         c.commits_source_repo = commits_source_repo
         c.ancestor = None  # set it to None, to hide it from PR view
 
         # empty version means latest, so we keep this to prevent
         # double caching
         version_normalized = version or 'latest'
         from_version_normalized = from_version or 'latest'
 
         cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(
             target_repo)
         cache_file_path = diff_cache_exist(
             cache_path, 'pull_request', pull_request_id, version_normalized,
             from_version_normalized, source_ref_id, target_ref_id, c.fulldiff)
 
         caching_enabled = self._is_diff_cache_enabled(c.target_repo)
         force_recache = str2bool(self.request.GET.get('force_recache'))
 
         cached_diff = None
         if caching_enabled:
             cached_diff = load_cached_diff(cache_file_path)
 
         has_proper_commit_cache = (
                 cached_diff and cached_diff.get('commits')
                 and len(cached_diff.get('commits', [])) == 5
                 and cached_diff.get('commits')[0]
                 and cached_diff.get('commits')[3])
         if not force_recache and has_proper_commit_cache:
             diff_commit_cache = \
                 (ancestor_commit, commit_cache, missing_requirements,
                  source_commit, target_commit) = cached_diff['commits']
         else:
             diff_commit_cache = \
                 (ancestor_commit, commit_cache, missing_requirements,
                  source_commit, target_commit) = self.get_commits(
                     commits_source_repo,
                     pull_request_at_ver,
                     source_commit,
                     source_ref_id,
                     source_scm,
                     target_commit,
                     target_ref_id,
                     target_scm)
 
         # register our commit range
         for comm in commit_cache.values():
             c.commit_ranges.append(comm)
 
         c.missing_requirements = missing_requirements
         c.ancestor_commit = ancestor_commit
         c.statuses = source_repo.statuses(
             [x.raw_id for x in c.commit_ranges])
 
         # auto collapse if we have more than limit
         collapse_limit = diffs.DiffProcessor._collapse_commits_over
         c.collapse_all_commits = len(c.commit_ranges) > collapse_limit
         c.compare_mode = compare
 
         # diff_limit is the old behavior, will cut off the whole diff
         # if the limit is applied  otherwise will just hide the
         # big files from the front-end
         diff_limit = c.visual.cut_off_limit_diff
         file_limit = c.visual.cut_off_limit_file
 
         c.missing_commits = False
         if (c.missing_requirements
             or isinstance(source_commit, EmptyCommit)
             or source_commit == target_commit):
 
             c.missing_commits = True
         else:
             c.inline_comments = display_inline_comments
 
             has_proper_diff_cache = cached_diff and cached_diff.get('commits')
             if not force_recache and has_proper_diff_cache:
                 c.diffset = cached_diff['diff']
                 (ancestor_commit, commit_cache, missing_requirements,
                  source_commit, target_commit) = cached_diff['commits']
             else:
                 c.diffset = self._get_diffset(
                     c.source_repo.repo_name, commits_source_repo,
                     source_ref_id, target_ref_id,
                     target_commit, source_commit,
                     diff_limit, file_limit, c.fulldiff)
 
                 # save cached diff
                 if caching_enabled:
                     cache_diff(cache_file_path, c.diffset, diff_commit_cache)
 
             c.limited_diff = c.diffset.limited_diff
 
             # calculate removed files that are bound to comments
             comment_deleted_files = [
                 fname for fname in display_inline_comments
                 if fname not in c.diffset.file_stats]
 
             c.deleted_files_comments = collections.defaultdict(dict)
             for fname, per_line_comments in display_inline_comments.items():
                 if fname in comment_deleted_files:
                     c.deleted_files_comments[fname]['stats'] = 0
                     c.deleted_files_comments[fname]['comments'] = list()
                     for lno, comments in per_line_comments.items():
                         c.deleted_files_comments[fname]['comments'].extend(
                             comments)
 
         # this is a hack to properly display links, when creating PR, the
         # compare view and others uses different notation, and
         # compare_commits.mako renders links based on the target_repo.
         # We need to swap that here to generate it properly on the html side
         c.target_repo = c.source_repo
 
         c.commit_statuses = ChangesetStatus.STATUSES
 
         c.show_version_changes = not pr_closed
         if c.show_version_changes:
             cur_obj = pull_request_at_ver
             prev_obj = prev_pull_request_at_ver
 
             old_commit_ids = prev_obj.revisions
             new_commit_ids = cur_obj.revisions
             commit_changes = PullRequestModel()._calculate_commit_id_changes(
                 old_commit_ids, new_commit_ids)
             c.commit_changes_summary = commit_changes
 
             # calculate the diff for commits between versions
             c.commit_changes = []
             mark = lambda cs, fw: list(
                 h.itertools.izip_longest([], cs, fillvalue=fw))
             for c_type, raw_id in mark(commit_changes.added, 'a') \
                                 + mark(commit_changes.removed, 'r') \
                                 + mark(commit_changes.common, 'c'):
 
                 if raw_id in commit_cache:
                     commit = commit_cache[raw_id]
                 else:
                     try:
                         commit = commits_source_repo.get_commit(raw_id)
                     except CommitDoesNotExistError:
                         # in case we fail extracting still use "dummy" commit
                         # for display in commit diff
                         commit = h.AttributeDict(
                             {'raw_id': raw_id,
                              'message': 'EMPTY or MISSING COMMIT'})
                 c.commit_changes.append([c_type, commit])
 
             # current user review statuses for each version
             c.review_versions = {}
             if self._rhodecode_user.user_id in allowed_reviewers:
                 for co in general_comments:
                     if co.author.user_id == self._rhodecode_user.user_id:
                         status = co.status_change
                         if status:
                             _ver_pr = status[0].comment.pull_request_version_id
                             c.review_versions[_ver_pr] = status[0]
 
         return self._get_template_context(c)
 
     def get_commits(
             self, commits_source_repo, pull_request_at_ver, source_commit,
             source_ref_id, source_scm, target_commit, target_ref_id, target_scm):
         commit_cache = collections.OrderedDict()
         missing_requirements = False
         try:
             pre_load = ["author", "branch", "date", "message"]
             show_revs = pull_request_at_ver.revisions
             for rev in show_revs:
                 comm = commits_source_repo.get_commit(
                     commit_id=rev, pre_load=pre_load)
                 commit_cache[comm.raw_id] = comm
 
             # Order here matters, we first need to get target, and then
             # the source
             target_commit = commits_source_repo.get_commit(
                 commit_id=safe_str(target_ref_id))
 
             source_commit = commits_source_repo.get_commit(
                 commit_id=safe_str(source_ref_id))
         except CommitDoesNotExistError:
             log.warning(
                 'Failed to get commit from `{}` repo'.format(
                     commits_source_repo), exc_info=True)
         except RepositoryRequirementError:
             log.warning(
                 'Failed to get all required data from repo', exc_info=True)
             missing_requirements = True
         ancestor_commit = None
         try:
             ancestor_id = source_scm.get_common_ancestor(
                 source_commit.raw_id, target_commit.raw_id, target_scm)
             ancestor_commit = source_scm.get_commit(ancestor_id)
         except Exception:
             ancestor_commit = None
         return ancestor_commit, commit_cache, missing_requirements, source_commit, target_commit
 
     def assure_not_empty_repo(self):
         _ = self.request.translate
 
         try:
             self.db_repo.scm_instance().get_commit()
         except EmptyRepositoryError:
             h.flash(h.literal(_('There are no commits yet')),
                     category='warning')
             raise HTTPFound(
                 h.route_path('repo_summary', repo_name=self.db_repo.repo_name))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_new', request_method='GET',
         renderer='rhodecode:templates/pullrequests/pullrequest.mako')
     def pull_request_new(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         self.assure_not_empty_repo()
         source_repo = self.db_repo
 
         commit_id = self.request.GET.get('commit')
         branch_ref = self.request.GET.get('branch')
         bookmark_ref = self.request.GET.get('bookmark')
 
         try:
             source_repo_data = PullRequestModel().generate_repo_data(
                 source_repo, commit_id=commit_id,
                 branch=branch_ref, bookmark=bookmark_ref,
                 translator=self.request.translate)
         except CommitDoesNotExistError as e:
             log.exception(e)
             h.flash(_('Commit does not exist'), 'error')
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=source_repo.repo_name))
 
         default_target_repo = source_repo
 
         if source_repo.parent:
             parent_vcs_obj = source_repo.parent.scm_instance()
             if parent_vcs_obj and not parent_vcs_obj.is_empty():
                 # change default if we have a parent repo
                 default_target_repo = source_repo.parent
 
         target_repo_data = PullRequestModel().generate_repo_data(
             default_target_repo, translator=self.request.translate)
 
         selected_source_ref = source_repo_data['refs']['selected_ref']
         title_source_ref = ''
         if selected_source_ref:
             title_source_ref = selected_source_ref.split(':', 2)[1]
         c.default_title = PullRequestModel().generate_pullrequest_title(
             source=source_repo.repo_name,
             source_ref=title_source_ref,
             target=default_target_repo.repo_name
         )
 
         c.default_repo_data = {
             'source_repo_name': source_repo.repo_name,
             'source_refs_json': json.dumps(source_repo_data),
             'target_repo_name': default_target_repo.repo_name,
             'target_refs_json': json.dumps(target_repo_data),
         }
         c.default_source_ref = selected_source_ref
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_repo_refs', request_method='GET',
         renderer='json_ext', xhr=True)
     def pull_request_repo_refs(self):
         self.load_default_context()
         target_repo_name = self.request.matchdict['target_repo_name']
         repo = Repository.get_by_repo_name(target_repo_name)
         if not repo:
             raise HTTPNotFound()
 
         target_perm = HasRepoPermissionAny(
             'repository.read', 'repository.write', 'repository.admin')(
             target_repo_name)
         if not target_perm:
             raise HTTPNotFound()
 
         return PullRequestModel().generate_repo_data(
             repo, translator=self.request.translate)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @view_config(
         route_name='pullrequest_repo_destinations', request_method='GET',
         renderer='json_ext', xhr=True)
     def pull_request_repo_destinations(self):
         _ = self.request.translate
         filter_query = self.request.GET.get('query')
 
         query = Repository.query() \
             .order_by(func.length(Repository.repo_name)) \
             .filter(
                 or_(Repository.repo_name == self.db_repo.repo_name,
                     Repository.fork_id == self.db_repo.repo_id))
 
         if filter_query:
             ilike_expression = u'%{}%'.format(safe_unicode(filter_query))
             query = query.filter(
                 Repository.repo_name.ilike(ilike_expression))
 
         add_parent = False
         if self.db_repo.parent:
             if filter_query in self.db_repo.parent.repo_name:
                 parent_vcs_obj = self.db_repo.parent.scm_instance()
                 if parent_vcs_obj and not parent_vcs_obj.is_empty():
                     add_parent = True
 
         limit = 20 - 1 if add_parent else 20
         all_repos = query.limit(limit).all()
         if add_parent:
             all_repos += [self.db_repo.parent]
 
         repos = []
         for obj in ScmModel().get_repos(all_repos):
             repos.append({
                 'id': obj['name'],
                 'text': obj['name'],
                 'type': 'repo',
                 'repo_id': obj['dbrepo']['repo_id'],
                 'repo_type': obj['dbrepo']['repo_type'],
                 'private': obj['dbrepo']['private'],
 
             })
 
         data = {
             'more': False,
             'results': [{
                 'text': _('Repositories'),
                 'children': repos
             }] if repos else []
         }
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_create', request_method='POST',
         renderer=None)
     def pull_request_create(self):
         _ = self.request.translate
         self.assure_not_empty_repo()
         self.load_default_context()
 
         controls = peppercorn.parse(self.request.POST.items())
 
         try:
             form = PullRequestForm(
                 self.request.translate, self.db_repo.repo_id)()
             _form = form.to_python(controls)
         except formencode.Invalid as errors:
             if errors.error_dict.get('revisions'):
                 msg = 'Revisions: %s' % errors.error_dict['revisions']
             elif errors.error_dict.get('pullrequest_title'):
                 msg = errors.error_dict.get('pullrequest_title')
             else:
                 msg = _('Error creating pull request: {}').format(errors)
             log.exception(msg)
             h.flash(msg, 'error')
 
             # would rather just go back to form ...
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name))
 
         source_repo = _form['source_repo']
         source_ref = _form['source_ref']
         target_repo = _form['target_repo']
         target_ref = _form['target_ref']
         commit_ids = _form['revisions'][::-1]
 
         # find the ancestor for this pr
         source_db_repo = Repository.get_by_repo_name(_form['source_repo'])
         target_db_repo = Repository.get_by_repo_name(_form['target_repo'])
 
         # re-check permissions again here
         # source_repo we must have read permissions
 
         source_perm = HasRepoPermissionAny(
             'repository.read',
             'repository.write', 'repository.admin')(source_db_repo.repo_name)
         if not source_perm:
             msg = _('Not Enough permissions to source repo `{}`.'.format(
                 source_db_repo.repo_name))
             h.flash(msg, category='error')
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         # target repo we must have read permissions, and also later on
         # we want to check branch permissions here
         target_perm = HasRepoPermissionAny(
             'repository.read',
             'repository.write', 'repository.admin')(target_db_repo.repo_name)
         if not target_perm:
             msg = _('Not Enough permissions to target repo `{}`.'.format(
                 target_db_repo.repo_name))
             h.flash(msg, category='error')
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         source_scm = source_db_repo.scm_instance()
         target_scm = target_db_repo.scm_instance()
 
         source_commit = source_scm.get_commit(source_ref.split(':')[-1])
         target_commit = target_scm.get_commit(target_ref.split(':')[-1])
 
         ancestor = source_scm.get_common_ancestor(
             source_commit.raw_id, target_commit.raw_id, target_scm)
 
         # recalculate target ref based on ancestor
         target_ref_type, target_ref_name, __ = _form['target_ref'].split(':')
         target_ref = ':'.join((target_ref_type, target_ref_name, ancestor))
 
         get_default_reviewers_data, validate_default_reviewers = \
             PullRequestModel().get_reviewer_functions()
 
         # recalculate reviewers logic, to make sure we can validate this
         reviewer_rules = get_default_reviewers_data(
             self._rhodecode_db_user, source_db_repo,
             source_commit, target_db_repo, target_commit)
 
         given_reviewers = _form['review_members']
         reviewers = validate_default_reviewers(
             given_reviewers, reviewer_rules)
 
         pullrequest_title = _form['pullrequest_title']
         title_source_ref = source_ref.split(':', 2)[1]
         if not pullrequest_title:
             pullrequest_title = PullRequestModel().generate_pullrequest_title(
                 source=source_repo,
                 source_ref=title_source_ref,
                 target=target_repo
             )
 
         description = _form['pullrequest_desc']
+        description_renderer = _form['description_renderer']
 
         try:
             pull_request = PullRequestModel().create(
                 created_by=self._rhodecode_user.user_id,
                 source_repo=source_repo,
                 source_ref=source_ref,
                 target_repo=target_repo,
                 target_ref=target_ref,
                 revisions=commit_ids,
                 reviewers=reviewers,
                 title=pullrequest_title,
                 description=description,
+                description_renderer=description_renderer,
                 reviewer_data=reviewer_rules,
                 auth_user=self._rhodecode_user
             )
             Session().commit()
 
             h.flash(_('Successfully opened new pull request'),
                     category='success')
         except Exception:
             msg = _('Error occurred during creation of this pull request.')
             log.exception(msg)
             h.flash(msg, category='error')
 
             # copy the args back to redirect
             org_query = self.request.GET.mixed()
             raise HTTPFound(
                 h.route_path('pullrequest_new', repo_name=self.db_repo_name,
                              _query=org_query))
 
         raise HTTPFound(
             h.route_path('pullrequest_show', repo_name=target_repo,
                          pull_request_id=pull_request.pull_request_id))
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_update', request_method='POST',
         renderer='json_ext')
     def pull_request_update(self):
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         _ = self.request.translate
 
         self.load_default_context()
 
         if pull_request.is_closed():
             log.debug('update: forbidden because pull request is closed')
             msg = _(u'Cannot update closed pull requests.')
             h.flash(msg, category='error')
             return True
 
         # only owner or admin can update it
         allowed_to_update = PullRequestModel().check_user_update(
             pull_request, self._rhodecode_user)
         if allowed_to_update:
             controls = peppercorn.parse(self.request.POST.items())
 
             if 'review_members' in controls:
                 self._update_reviewers(
                     pull_request, controls['review_members'],
                     pull_request.reviewer_data)
             elif str2bool(self.request.POST.get('update_commits', 'false')):
                 self._update_commits(pull_request)
             elif str2bool(self.request.POST.get('edit_pull_request', 'false')):
                 self._edit_pull_request(pull_request)
             else:
                 raise HTTPBadRequest()
             return True
         raise HTTPForbidden()
 
     def _edit_pull_request(self, pull_request):
         _ = self.request.translate
+
         try:
             PullRequestModel().edit(
-                pull_request, self.request.POST.get('title'),
-                self.request.POST.get('description'), self._rhodecode_user)
+                pull_request,
+                self.request.POST.get('title'),
+                self.request.POST.get('description'),
+                self.request.POST.get('description_renderer'),
+                self._rhodecode_user)
         except ValueError:
             msg = _(u'Cannot update closed pull requests.')
             h.flash(msg, category='error')
             return
         else:
             Session().commit()
 
         msg = _(u'Pull request title & description updated.')
         h.flash(msg, category='success')
         return
 
     def _update_commits(self, pull_request):
         _ = self.request.translate
         resp = PullRequestModel().update_commits(pull_request)
 
         if resp.executed:
 
             if resp.target_changed and resp.source_changed:
                 changed = 'target and source repositories'
             elif resp.target_changed and not resp.source_changed:
                 changed = 'target repository'
             elif not resp.target_changed and resp.source_changed:
                 changed = 'source repository'
             else:
                 changed = 'nothing'
 
             msg = _(
                 u'Pull request updated to "{source_commit_id}" with '
                 u'{count_added} added, {count_removed} removed commits. '
                 u'Source of changes: {change_source}')
             msg = msg.format(
                 source_commit_id=pull_request.source_ref_parts.commit_id,
                 count_added=len(resp.changes.added),
                 count_removed=len(resp.changes.removed),
                 change_source=changed)
             h.flash(msg, category='success')
 
             channel = '/repo${}$/pr/{}'.format(
                 pull_request.target_repo.repo_name,
                 pull_request.pull_request_id)
             message = msg + (
                 ' - <a onclick="window.location.reload()">'
                 '<strong>{}</strong></a>'.format(_('Reload page')))
             channelstream.post_message(
                 channel, message, self._rhodecode_user.username,
                 registry=self.request.registry)
         else:
             msg = PullRequestModel.UPDATE_STATUS_MESSAGES[resp.reason]
             warning_reasons = [
                 UpdateFailureReason.NO_CHANGE,
                 UpdateFailureReason.WRONG_REF_TYPE,
             ]
             category = 'warning' if resp.reason in warning_reasons else 'error'
             h.flash(msg, category=category)
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_merge', request_method='POST',
         renderer='json_ext')
     def pull_request_merge(self):
         """
         Merge will perform a server-side merge of the specified
         pull request, if the pull request is approved and mergeable.
         After successful merging, the pull request is automatically
         closed, with a relevant comment.
         """
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
 
         self.load_default_context()
         check = MergeCheck.validate(pull_request, self._rhodecode_db_user,
                                     translator=self.request.translate)
         merge_possible = not check.failed
 
         for err_type, error_msg in check.errors:
             h.flash(error_msg, category=err_type)
 
         if merge_possible:
             log.debug("Pre-conditions checked, trying to merge.")
             extras = vcs_operation_context(
                 self.request.environ, repo_name=pull_request.target_repo.repo_name,
                 username=self._rhodecode_db_user.username, action='push',
                 scm=pull_request.target_repo.repo_type)
             self._merge_pull_request(
                 pull_request, self._rhodecode_db_user, extras)
         else:
             log.debug("Pre-conditions failed, NOT merging.")
 
         raise HTTPFound(
             h.route_path('pullrequest_show',
                          repo_name=pull_request.target_repo.repo_name,
                          pull_request_id=pull_request.pull_request_id))
 
     def _merge_pull_request(self, pull_request, user, extras):
         _ = self.request.translate
         merge_resp = PullRequestModel().merge_repo(pull_request, user, extras=extras)
 
         if merge_resp.executed:
             log.debug("The merge was successful, closing the pull request.")
             PullRequestModel().close_pull_request(
                 pull_request.pull_request_id, user)
             Session().commit()
             msg = _('Pull request was successfully merged and closed.')
             h.flash(msg, category='success')
         else:
             log.debug(
                 "The merge was not successful. Merge response: %s",
                 merge_resp)
             msg = PullRequestModel().merge_status_message(
                 merge_resp.failure_reason)
             h.flash(msg, category='error')
 
     def _update_reviewers(self, pull_request, review_members, reviewer_rules):
         _ = self.request.translate
         get_default_reviewers_data, validate_default_reviewers = \
             PullRequestModel().get_reviewer_functions()
 
         try:
             reviewers = validate_default_reviewers(review_members, reviewer_rules)
         except ValueError as e:
             log.error('Reviewers Validation: {}'.format(e))
             h.flash(e, category='error')
             return
 
         PullRequestModel().update_reviewers(
             pull_request, reviewers, self._rhodecode_user)
         h.flash(_('Pull request reviewers updated.'), category='success')
         Session().commit()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_delete', request_method='POST',
         renderer='json_ext')
     def pull_request_delete(self):
         _ = self.request.translate
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         self.load_default_context()
 
         pr_closed = pull_request.is_closed()
         allowed_to_delete = PullRequestModel().check_user_delete(
             pull_request, self._rhodecode_user) and not pr_closed
 
         # only owner can delete it !
         if allowed_to_delete:
             PullRequestModel().delete(pull_request, self._rhodecode_user)
             Session().commit()
             h.flash(_('Successfully deleted pull request'),
                     category='success')
             raise HTTPFound(h.route_path('pullrequest_show_all',
                                          repo_name=self.db_repo_name))
 
         log.warning('user %s tried to delete pull request without access',
                     self._rhodecode_user)
         raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_comment_create', request_method='POST',
         renderer='json_ext')
     def pull_request_comment_create(self):
         _ = self.request.translate
 
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
         pull_request_id = pull_request.pull_request_id
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         allowed_to_comment = PullRequestModel().check_user_comment(
             pull_request, self._rhodecode_user)
         if not allowed_to_comment:
             log.debug(
                 'comment: forbidden because pull request is from forbidden repo')
             raise HTTPForbidden()
 
         c = self.load_default_context()
 
         status = self.request.POST.get('changeset_status', None)
         text = self.request.POST.get('text')
         comment_type = self.request.POST.get('comment_type')
         resolves_comment_id = self.request.POST.get('resolves_comment_id', None)
         close_pull_request = self.request.POST.get('close_pull_request')
 
         # the logic here should work like following, if we submit close
         # pr comment, use `close_pull_request_with_comment` function
         # else handle regular comment logic
 
         if close_pull_request:
             # only owner or admin or person with write permissions
             allowed_to_close = PullRequestModel().check_user_update(
                 pull_request, self._rhodecode_user)
             if not allowed_to_close:
                 log.debug('comment: forbidden because not allowed to close '
                           'pull request %s', pull_request_id)
                 raise HTTPForbidden()
             comment, status = PullRequestModel().close_pull_request_with_comment(
                 pull_request, self._rhodecode_user, self.db_repo, message=text)
             Session().flush()
             events.trigger(
                 events.PullRequestCommentEvent(pull_request, comment))
 
         else:
             # regular comment case, could be inline, or one with status.
             # for that one we check also permissions
 
             allowed_to_change_status = PullRequestModel().check_user_change_status(
                 pull_request, self._rhodecode_user)
 
             if status and allowed_to_change_status:
                 message = (_('Status change %(transition_icon)s %(status)s')
                            % {'transition_icon': '>',
                               'status': ChangesetStatus.get_status_lbl(status)})
                 text = text or message
 
             comment = CommentsModel().create(
                 text=text,
                 repo=self.db_repo.repo_id,
                 user=self._rhodecode_user.user_id,
                 pull_request=pull_request,
                 f_path=self.request.POST.get('f_path'),
                 line_no=self.request.POST.get('line'),
                 status_change=(ChangesetStatus.get_status_lbl(status)
                                if status and allowed_to_change_status else None),
                 status_change_type=(status
                                     if status and allowed_to_change_status else None),
                 comment_type=comment_type,
                 resolves_comment_id=resolves_comment_id,
                 auth_user=self._rhodecode_user
             )
 
             if allowed_to_change_status:
                 # calculate old status before we change it
                 old_calculated_status = pull_request.calculated_review_status()
 
                 # get status if set !
                 if status:
                     ChangesetStatusModel().set_status(
                         self.db_repo.repo_id,
                         status,
                         self._rhodecode_user.user_id,
                         comment,
                         pull_request=pull_request
                     )
 
                 Session().flush()
                 # this is somehow required to get access to some relationship
                 # loaded on comment
                 Session().refresh(comment)
 
                 events.trigger(
                     events.PullRequestCommentEvent(pull_request, comment))
 
                 # we now calculate the status of pull request, and based on that
                 # calculation we set the commits status
                 calculated_status = pull_request.calculated_review_status()
                 if old_calculated_status != calculated_status:
                     PullRequestModel()._trigger_pull_request_hook(
                         pull_request, self._rhodecode_user, 'review_status_change')
 
         Session().commit()
 
         data = {
             'target_id': h.safeid(h.safe_unicode(
                 self.request.POST.get('f_path'))),
         }
         if comment:
             c.co = comment
             rendered_comment = render(
                 'rhodecode:templates/changeset/changeset_comment_block.mako',
                 self._get_template_context(c), self.request)
 
             data.update(comment.get_dict())
             data.update({'rendered_text': rendered_comment})
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     @view_config(
         route_name='pullrequest_comment_delete', request_method='POST',
         renderer='json_ext')
     def pull_request_comment_delete(self):
         pull_request = PullRequest.get_or_404(
             self.request.matchdict['pull_request_id'])
 
         comment = ChangesetComment.get_or_404(
             self.request.matchdict['comment_id'])
         comment_id = comment.comment_id
 
         if pull_request.is_closed():
             log.debug('comment: forbidden because pull request is closed')
             raise HTTPForbidden()
 
         if not comment:
             log.debug('Comment with id:%s not found, skipping', comment_id)
             # comment already deleted in another call probably
             return True
 
         if comment.pull_request.is_closed():
             # don't allow deleting comments on closed pull request
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = comment.author.user_id == self._rhodecode_user.user_id
         is_repo_comment = comment.repo.repo_name == self.db_repo_name
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if super_admin or comment_owner or comment_repo_admin:
             old_calculated_status = comment.pull_request.calculated_review_status()
             CommentsModel().delete(comment=comment, auth_user=self._rhodecode_user)
             Session().commit()
             calculated_status = comment.pull_request.calculated_review_status()
             if old_calculated_status != calculated_status:
                 PullRequestModel()._trigger_pull_request_hook(
                     comment.pull_request, self._rhodecode_user, 'review_status_change')
             return True
         else:
             log.warning('No permissions for user %s to delete comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()

rhodecode/lib/helpers.py

@@ -1,2095 +1,2101 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Helper functions
 
 Consists of functions to typically be used within templates, but also
 available to Controllers. This module is available to both as 'h'.
 """
 
 import os
 import random
 import hashlib
 import StringIO
 import textwrap
 import urllib
 import math
 import logging
 import re
 import urlparse
 import time
 import string
 import hashlib
 from collections import OrderedDict
 
 import pygments
 import itertools
 import fnmatch
 
 from datetime import datetime
 from functools import partial
 from pygments.formatters.html import HtmlFormatter
 from pygments import highlight as code_highlight
 from pygments.lexers import (
     get_lexer_by_name, get_lexer_for_filename, get_lexer_for_mimetype)
 
 from pyramid.threadlocal import get_current_request
 
 from webhelpers.html import literal, HTML, escape
 from webhelpers.html.tools import *
 from webhelpers.html.builder import make_tag
 from webhelpers.html.tags import auto_discovery_link, checkbox, css_classes, \
     end_form, file, form as wh_form, hidden, image, javascript_link, link_to, \
     link_to_if, link_to_unless, ol, required_legend, select, stylesheet_link, \
     submit, text, password, textarea, title, ul, xml_declaration, radio
 from webhelpers.html.tools import auto_link, button_to, highlight, \
     js_obfuscate, mail_to, strip_links, strip_tags, tag_re
 from webhelpers.text import chop_at, collapse, convert_accented_entities, \
     convert_misc_entities, lchop, plural, rchop, remove_formatting, \
     replace_whitespace, urlify, truncate, wrap_paragraphs
 from webhelpers.date import time_ago_in_words
 from webhelpers.paginate import Page as _Page
 from webhelpers.html.tags import _set_input_attrs, _set_id_attr, \
     convert_boolean_attrs, NotGiven, _make_safe_id_component
 from webhelpers2.number import format_byte_size
 
 from rhodecode.lib.action_parser import action_parser
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils import repo_name_slug, get_custom_lexer
 from rhodecode.lib.utils2 import str2bool, safe_unicode, safe_str, \
     get_commit_safe, datetime_to_time, time_to_datetime, time_to_utcdatetime, \
     AttributeDict, safe_int, md5, md5_safe
 from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
 from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError
 from rhodecode.lib.vcs.backends.base import BaseChangeset, EmptyCommit
 from rhodecode.config.conf import DATE_FORMAT, DATETIME_FORMAT
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.db import Permission, User, Repository
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import IssueTrackerSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 DEFAULT_USER = User.DEFAULT_USER
 DEFAULT_USER_EMAIL = User.DEFAULT_USER_EMAIL
 
 
 def asset(path, ver=None, **kwargs):
     """
     Helper to generate a static asset file path for rhodecode assets
 
     eg. h.asset('images/image.png', ver='3923')
 
     :param path: path of asset
     :param ver: optional version query param to append as ?ver=
     """
     request = get_current_request()
     query = {}
     query.update(kwargs)
     if ver:
         query = {'ver': ver}
     return request.static_path(
         'rhodecode:public/{}'.format(path), _query=query)
 
 
 default_html_escape_table = {
     ord('&'): u'&amp;',
     ord('<'): u'&lt;',
     ord('>'): u'&gt;',
     ord('"'): u'&quot;',
     ord("'"): u'&#39;',
 }
 
 
 def html_escape(text, html_escape_table=default_html_escape_table):
     """Produce entities within text."""
     return text.translate(html_escape_table)
 
 
 def chop_at_smart(s, sub, inclusive=False, suffix_if_chopped=None):
     """
     Truncate string ``s`` at the first occurrence of ``sub``.
 
     If ``inclusive`` is true, truncate just after ``sub`` rather than at it.
     """
     suffix_if_chopped = suffix_if_chopped or ''
     pos = s.find(sub)
     if pos == -1:
         return s
 
     if inclusive:
         pos += len(sub)
 
     chopped = s[:pos]
     left = s[pos:].strip()
 
     if left and suffix_if_chopped:
         chopped += suffix_if_chopped
 
     return chopped
 
 
 def shorter(text, size=20):
     postfix = '...'
     if len(text) > size:
         return text[:size - len(postfix)] + postfix
     return text
 
 
 def _reset(name, value=None, id=NotGiven, type="reset", **attrs):
     """
     Reset button
     """
     _set_input_attrs(attrs, type, name, value)
     _set_id_attr(attrs, id, name)
     convert_boolean_attrs(attrs, ["disabled"])
     return HTML.input(**attrs)
 
 reset = _reset
 safeid = _make_safe_id_component
 
 
 def branding(name, length=40):
     return truncate(name, length, indicator="")
 
 
 def FID(raw_id, path):
     """
     Creates a unique ID for filenode based on it's hash of path and commit
     it's safe to use in urls
 
     :param raw_id:
     :param path:
     """
 
     return 'c-%s-%s' % (short_id(raw_id), md5_safe(path)[:12])
 
 
 class _GetError(object):
     """Get error from form_errors, and represent it as span wrapped error
     message
 
     :param field_name: field to fetch errors for
     :param form_errors: form errors dict
     """
 
     def __call__(self, field_name, form_errors):
         tmpl = """<span class="error_msg">%s</span>"""
         if form_errors and field_name in form_errors:
             return literal(tmpl % form_errors.get(field_name))
 
 get_error = _GetError()
 
 
 class _ToolTip(object):
 
     def __call__(self, tooltip_title, trim_at=50):
         """
         Special function just to wrap our text into nice formatted
         autowrapped text
 
         :param tooltip_title:
         """
         tooltip_title = escape(tooltip_title)
         tooltip_title = tooltip_title.replace('<', '&lt;').replace('>', '&gt;')
         return tooltip_title
 tooltip = _ToolTip()
 
 
 def files_breadcrumbs(repo_name, commit_id, file_path):
     if isinstance(file_path, str):
         file_path = safe_unicode(file_path)
 
     # TODO: johbo: Is this always a url like path, or is this operating
     # system dependent?
     path_segments = file_path.split('/')
 
     repo_name_html = escape(repo_name)
     if len(path_segments) == 1 and path_segments[0] == '':
         url_segments = [repo_name_html]
     else:
         url_segments = [
             link_to(
                 repo_name_html,
                 route_path(
                     'repo_files',
                     repo_name=repo_name,
                     commit_id=commit_id,
                     f_path=''),
                 class_='pjax-link')]
 
     last_cnt = len(path_segments) - 1
     for cnt, segment in enumerate(path_segments):
         if not segment:
             continue
         segment_html = escape(segment)
 
         if cnt != last_cnt:
             url_segments.append(
                 link_to(
                     segment_html,
                     route_path(
                         'repo_files',
                         repo_name=repo_name,
                         commit_id=commit_id,
                         f_path='/'.join(path_segments[:cnt + 1])),
                     class_='pjax-link'))
         else:
             url_segments.append(segment_html)
 
     return literal('/'.join(url_segments))
 
 
 class CodeHtmlFormatter(HtmlFormatter):
     """
     My code Html Formatter for source codes
     """
 
     def wrap(self, source, outfile):
         return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<div id="L%s">%s</div>' % (cnt + 1, t)
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         dummyoutfile = StringIO.StringIO()
         lncount = 0
         for t, line in inner:
             if t:
                 lncount += 1
             dummyoutfile.write(line)
 
         fl = self.linenostart
         mw = len(str(lncount + fl - 1))
         sp = self.linenospecial
         st = self.linenostep
         la = self.lineanchors
         aln = self.anchorlinenos
         nocls = self.noclasses
         if sp:
             lines = []
 
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if i % sp == 0:
                         if aln:
                             lines.append('<a href="#%s%d" class="special">%*d</a>' %
                                          (la, i, mw, i))
                         else:
                             lines.append('<span class="special">%*d</span>' % (mw, i))
                     else:
                         if aln:
                             lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                         else:
                             lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
         else:
             lines = []
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if aln:
                         lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                     else:
                         lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
 
         # in case you wonder about the seemingly redundant <div> here: since the
         # content in the other cell also is wrapped in a div, some browsers in
         # some configurations seem to mess up the formatting...
         if nocls:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td><div class="linenodiv" '
                       'style="background-color: #f0f0f0; padding-right: 10px">'
                       '<pre style="line-height: 125%">' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         else:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td class="linenos"><div class="linenodiv"><pre>' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         yield 0, dummyoutfile.getvalue()
         yield 0, '</td></tr></table>'
 
 
 class SearchContentCodeHtmlFormatter(CodeHtmlFormatter):
     def __init__(self, **kw):
         # only show these line numbers if set
         self.only_lines = kw.pop('only_line_numbers', [])
         self.query_terms = kw.pop('query_terms', [])
         self.max_lines = kw.pop('max_lines', 5)
         self.line_context = kw.pop('line_context', 3)
         self.url = kw.pop('url', None)
 
         super(CodeHtmlFormatter, self).__init__(**kw)
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<pre>%s</pre>' % t
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         yield 0, '<table class="code-highlight %stable">' % self.cssclass
 
         last_shown_line_number = 0
         current_line_number = 1
 
         for t, line in inner:
             if not t:
                 yield t, line
                 continue
 
             if current_line_number in self.only_lines:
                 if last_shown_line_number + 1 != current_line_number:
                     yield 0, '<tr>'
                     yield 0, '<td class="line">...</td>'
                     yield 0, '<td id="hlcode" class="code"></td>'
                     yield 0, '</tr>'
 
                 yield 0, '<tr>'
                 if self.url:
                     yield 0, '<td class="line"><a href="%s#L%i">%i</a></td>' % (
                         self.url, current_line_number, current_line_number)
                 else:
                     yield 0, '<td class="line"><a href="">%i</a></td>' % (
                         current_line_number)
                 yield 0, '<td id="hlcode" class="code">' + line + '</td>'
                 yield 0, '</tr>'
 
                 last_shown_line_number = current_line_number
 
             current_line_number += 1
 
 
         yield 0, '</table>'
 
 
 def extract_phrases(text_query):
     """
     Extracts phrases from search term string making sure phrases
     contained in double quotes are kept together - and discarding empty values
     or fully whitespace values eg.
 
     'some   text "a phrase" more' => ['some', 'text', 'a phrase', 'more']
 
     """
 
     in_phrase = False
     buf = ''
     phrases = []
     for char in text_query:
         if in_phrase:
             if char == '"': # end phrase
                 phrases.append(buf)
                 buf = ''
                 in_phrase = False
                 continue
             else:
                 buf += char
                 continue
         else:
             if char == '"': # start phrase
                 in_phrase = True
                 phrases.append(buf)
                 buf = ''
                 continue
             elif char == ' ':
                 phrases.append(buf)
                 buf = ''
                 continue
             else:
                 buf += char
 
     phrases.append(buf)
     phrases = [phrase.strip() for phrase in phrases if phrase.strip()]
     return phrases
 
 
 def get_matching_offsets(text, phrases):
     """
     Returns a list of string offsets in `text` that the list of `terms` match
 
     >>> get_matching_offsets('some text here', ['some', 'here'])
     [(0, 4), (10, 14)]
 
     """
     offsets = []
     for phrase in phrases:
         for match in re.finditer(phrase, text):
             offsets.append((match.start(), match.end()))
 
     return offsets
 
 
 def normalize_text_for_matching(x):
     """
     Replaces all non alnum characters to spaces and lower cases the string,
     useful for comparing two text strings without punctuation
     """
     return re.sub(r'[^\w]', ' ', x.lower())
 
 
 def get_matching_line_offsets(lines, terms):
     """ Return a set of `lines` indices (starting from 1) matching a
     text search query, along with `context` lines above/below matching lines
 
     :param lines: list of strings representing lines
     :param terms: search term string to match in lines eg. 'some text'
     :param context: number of lines above/below a matching line to add to result
     :param max_lines: cut off for lines of interest
      eg.
 
     text = '''
     words words words
     words words words
     some text some
     words words words
     words words words
     text here what
     '''
     get_matching_line_offsets(text, 'text', context=1)
     {3: [(5, 9)], 6: [(0, 4)]]
 
     """
     matching_lines = {}
     phrases = [normalize_text_for_matching(phrase)
                for phrase in extract_phrases(terms)]
 
     for line_index, line in enumerate(lines, start=1):
         match_offsets = get_matching_offsets(
             normalize_text_for_matching(line), phrases)
         if match_offsets:
             matching_lines[line_index] = match_offsets
 
     return matching_lines
 
 
 def hsv_to_rgb(h, s, v):
     """ Convert hsv color values to rgb """
 
     if s == 0.0:
         return v, v, v
     i = int(h * 6.0)  # XXX assume int() truncates!
     f = (h * 6.0) - i
     p = v * (1.0 - s)
     q = v * (1.0 - s * f)
     t = v * (1.0 - s * (1.0 - f))
     i = i % 6
     if i == 0:
         return v, t, p
     if i == 1:
         return q, v, p
     if i == 2:
         return p, v, t
     if i == 3:
         return p, q, v
     if i == 4:
         return t, p, v
     if i == 5:
         return v, p, q
 
 
 def unique_color_generator(n=10000, saturation=0.10, lightness=0.95):
     """
     Generator for getting n of evenly distributed colors using
     hsv color and golden ratio. It always return same order of colors
 
     :param n: number of colors to generate
     :param saturation: saturation of returned colors
     :param lightness: lightness of returned colors
     :returns: RGB tuple
     """
 
     golden_ratio = 0.618033988749895
     h = 0.22717784590367374
 
     for _ in xrange(n):
         h += golden_ratio
         h %= 1
         HSV_tuple = [h, saturation, lightness]
         RGB_tuple = hsv_to_rgb(*HSV_tuple)
         yield map(lambda x: str(int(x * 256)), RGB_tuple)
 
 
 def color_hasher(n=10000, saturation=0.10, lightness=0.95):
     """
     Returns a function which when called with an argument returns a unique
     color for that argument, eg.
 
     :param n: number of colors to generate
     :param saturation: saturation of returned colors
     :param lightness: lightness of returned colors
     :returns: css RGB string
 
     >>> color_hash = color_hasher()
     >>> color_hash('hello')
     'rgb(34, 12, 59)'
     >>> color_hash('hello')
     'rgb(34, 12, 59)'
     >>> color_hash('other')
     'rgb(90, 224, 159)'
     """
 
     color_dict = {}
     cgenerator = unique_color_generator(
         saturation=saturation, lightness=lightness)
 
     def get_color_string(thing):
         if thing in color_dict:
             col = color_dict[thing]
         else:
             col = color_dict[thing] = cgenerator.next()
         return "rgb(%s)" % (', '.join(col))
 
     return get_color_string
 
 
 def get_lexer_safe(mimetype=None, filepath=None):
     """
     Tries to return a relevant pygments lexer using mimetype/filepath name,
     defaulting to plain text if none could be found
     """
     lexer = None
     try:
         if mimetype:
             lexer = get_lexer_for_mimetype(mimetype)
         if not lexer:
             lexer = get_lexer_for_filename(filepath)
     except pygments.util.ClassNotFound:
         pass
 
     if not lexer:
         lexer = get_lexer_by_name('text')
 
     return lexer
 
 
 def get_lexer_for_filenode(filenode):
     lexer = get_custom_lexer(filenode.extension) or filenode.lexer
     return lexer
 
 
 def pygmentize(filenode, **kwargs):
     """
     pygmentize function using pygments
 
     :param filenode:
     """
     lexer = get_lexer_for_filenode(filenode)
     return literal(code_highlight(filenode.content, lexer,
                                   CodeHtmlFormatter(**kwargs)))
 
 
 def is_following_repo(repo_name, user_id):
     from rhodecode.model.scm import ScmModel
     return ScmModel().is_following_repo(repo_name, user_id)
 
 
 class _Message(object):
     """A message returned by ``Flash.pop_messages()``.
 
     Converting the message to a string returns the message text. Instances
     also have the following attributes:
 
     * ``message``: the message text.
     * ``category``: the category specified when the message was created.
     """
 
     def __init__(self, category, message):
         self.category = category
         self.message = message
 
     def __str__(self):
         return self.message
 
     __unicode__ = __str__
 
     def __html__(self):
         return escape(safe_unicode(self.message))
 
 
 class Flash(object):
     # List of allowed categories.  If None, allow any category.
     categories = ["warning", "notice", "error", "success"]
 
     # Default category if none is specified.
     default_category = "notice"
 
     def __init__(self, session_key="flash", categories=None,
                  default_category=None):
         """
         Instantiate a ``Flash`` object.
 
         ``session_key`` is the key to save the messages under in the user's
         session.
 
         ``categories`` is an optional list which overrides the default list
         of categories.
 
         ``default_category`` overrides the default category used for messages
         when none is specified.
         """
         self.session_key = session_key
         if categories is not None:
             self.categories = categories
         if default_category is not None:
             self.default_category = default_category
         if self.categories and self.default_category not in self.categories:
             raise ValueError(
                 "unrecognized default category %r" % (self.default_category,))
 
     def pop_messages(self, session=None, request=None):
         """
         Return all accumulated messages and delete them from the session.
 
         The return value is a list of ``Message`` objects.
         """
         messages = []
 
         if not session:
             if not request:
                 request = get_current_request()
             session = request.session
 
         # Pop the 'old' pylons flash messages. They are tuples of the form
         # (category, message)
         for cat, msg in session.pop(self.session_key, []):
             messages.append(_Message(cat, msg))
 
         # Pop the 'new' pyramid flash messages for each category as list
         # of strings.
         for cat in self.categories:
             for msg in session.pop_flash(queue=cat):
                 messages.append(_Message(cat, msg))
         # Map messages from the default queue to the 'notice' category.
         for msg in session.pop_flash():
             messages.append(_Message('notice', msg))
 
         session.save()
         return messages
 
     def json_alerts(self, session=None, request=None):
         payloads = []
         messages = flash.pop_messages(session=session, request=request)
         if messages:
             for message in messages:
                 subdata = {}
                 if hasattr(message.message, 'rsplit'):
                     flash_data = message.message.rsplit('|DELIM|', 1)
                     org_message = flash_data[0]
                     if len(flash_data) > 1:
                         subdata = json.loads(flash_data[1])
                 else:
                     org_message = message.message
                 payloads.append({
                     'message': {
                         'message': u'{}'.format(org_message),
                         'level': message.category,
                         'force': True,
                         'subdata': subdata
                     }
                 })
         return json.dumps(payloads)
 
     def __call__(self, message, category=None, ignore_duplicate=False,
                  session=None, request=None):
 
         if not session:
             if not request:
                 request = get_current_request()
             session = request.session
 
         session.flash(
             message, queue=category, allow_duplicate=not ignore_duplicate)
 
 
 flash = Flash()
 
 #==============================================================================
 # SCM FILTERS available via h.
 #==============================================================================
 from rhodecode.lib.vcs.utils import author_name, author_email
 from rhodecode.lib.utils2 import credentials_filter, age as _age
 from rhodecode.model.db import User, ChangesetStatus
 
 age = _age
 capitalize = lambda x: x.capitalize()
 email = author_email
 short_id = lambda x: x[:12]
 hide_credentials = lambda x: ''.join(credentials_filter(x))
 
 
 import pytz
 import tzlocal
 local_timezone = tzlocal.get_localzone()
 
 
 def age_component(datetime_iso, value=None, time_is_local=False):
     title = value or format_date(datetime_iso)
     tzinfo = '+00:00'
 
     # detect if we have a timezone info, otherwise, add it
     if time_is_local and isinstance(datetime_iso, datetime) and not datetime_iso.tzinfo:
         force_timezone = os.environ.get('RC_TIMEZONE', '')
         if force_timezone:
             force_timezone = pytz.timezone(force_timezone)
         timezone = force_timezone or local_timezone
         offset = timezone.localize(datetime_iso).strftime('%z')
         tzinfo = '{}:{}'.format(offset[:-2], offset[-2:])
 
     return literal(
         '<time class="timeago tooltip" '
         'title="{1}{2}" datetime="{0}{2}">{1}</time>'.format(
             datetime_iso, title, tzinfo))
 
 
 def _shorten_commit_id(commit_id):
     from rhodecode import CONFIG
     def_len = safe_int(CONFIG.get('rhodecode_show_sha_length', 12))
     return commit_id[:def_len]
 
 
 def show_id(commit):
     """
     Configurable function that shows ID
     by default it's r123:fffeeefffeee
 
     :param commit: commit instance
     """
     from rhodecode import CONFIG
     show_idx = str2bool(CONFIG.get('rhodecode_show_revision_number', True))
 
     raw_id = _shorten_commit_id(commit.raw_id)
     if show_idx:
         return 'r%s:%s' % (commit.idx, raw_id)
     else:
         return '%s' % (raw_id, )
 
 
 def format_date(date):
     """
     use a standardized formatting for dates used in RhodeCode
 
     :param date: date/datetime object
     :return: formatted date
     """
 
     if date:
         _fmt = "%a, %d %b %Y %H:%M:%S"
         return safe_unicode(date.strftime(_fmt))
 
     return u""
 
 
 class _RepoChecker(object):
 
     def __init__(self, backend_alias):
         self._backend_alias = backend_alias
 
     def __call__(self, repository):
         if hasattr(repository, 'alias'):
             _type = repository.alias
         elif hasattr(repository, 'repo_type'):
             _type = repository.repo_type
         else:
             _type = repository
         return _type == self._backend_alias
 
 is_git = _RepoChecker('git')
 is_hg = _RepoChecker('hg')
 is_svn = _RepoChecker('svn')
 
 
 def get_repo_type_by_name(repo_name):
     repo = Repository.get_by_repo_name(repo_name)
     return repo.repo_type
 
 
 def is_svn_without_proxy(repository):
     if is_svn(repository):
         from rhodecode.model.settings import VcsSettingsModel
         conf = VcsSettingsModel().get_ui_settings_as_config_obj()
         return not str2bool(conf.get('vcs_svn_proxy', 'http_requests_enabled'))
     return False
 
 
 def discover_user(author):
     """
     Tries to discover RhodeCode User based on the autho string. Author string
     is typically `FirstName LastName <[e-mail]>`
     """
 
     # if author is already an instance use it for extraction
     if isinstance(author, User):
         return author
 
     # Valid email in the attribute passed, see if they're in the system
     _email = author_email(author)
     if _email != '':
         user = User.get_by_email(_email, case_insensitive=True, cache=True)
         if user is not None:
             return user
 
     # Maybe it's a username, we try to extract it and fetch by username ?
     _author = author_name(author)
     user = User.get_by_username(_author, case_insensitive=True, cache=True)
     if user is not None:
         return user
 
     return None
 
 
 def email_or_none(author):
     # extract email from the commit string
     _email = author_email(author)
 
     # If we have an email, use it, otherwise
     # see if it contains a username we can get an email from
     if _email != '':
         return _email
     else:
         user = User.get_by_username(
             author_name(author), case_insensitive=True, cache=True)
 
     if user is not None:
             return user.email
 
     # No valid email, not a valid user in the system, none!
     return None
 
 
 def link_to_user(author, length=0, **kwargs):
     user = discover_user(author)
     # user can be None, but if we have it already it means we can re-use it
     # in the person() function, so we save 1 intensive-query
     if user:
         author = user
 
     display_person = person(author, 'username_or_name_or_email')
     if length:
         display_person = shorter(display_person, length)
 
     if user:
         return link_to(
             escape(display_person),
             route_path('user_profile', username=user.username),
             **kwargs)
     else:
         return escape(display_person)
 
 
 def link_to_group(users_group_name, **kwargs):
     return link_to(
         escape(users_group_name),
         route_path('user_group_profile', user_group_name=users_group_name),
         **kwargs)
 
 
 def person(author, show_attr="username_and_name"):
     user = discover_user(author)
     if user:
         return getattr(user, show_attr)
     else:
         _author = author_name(author)
         _email = email(author)
         return _author or _email
 
 
 def author_string(email):
     if email:
         user = User.get_by_email(email, case_insensitive=True, cache=True)
         if user:
             if user.first_name or user.last_name:
                 return '%s %s &lt;%s&gt;' % (
                     user.first_name, user.last_name, email)
             else:
                 return email
         else:
             return email
     else:
         return None
 
 
 def person_by_id(id_, show_attr="username_and_name"):
     # attr to return from fetched user
     person_getter = lambda usr: getattr(usr, show_attr)
 
     #maybe it's an ID ?
     if str(id_).isdigit() or isinstance(id_, int):
         id_ = int(id_)
         user = User.get(id_)
         if user is not None:
             return person_getter(user)
     return id_
 
 
 def gravatar_with_user(request, author, show_disabled=False):
     _render = request.get_partial_renderer(
         'rhodecode:templates/base/base.mako')
     return _render('gravatar_with_user', author, show_disabled=show_disabled)
 
 
 tags_paterns = OrderedDict((
     ('lang', (re.compile(r'\[(lang|language)\ \=\&gt;\ *([a-zA-Z\-\/\#\+\.]*)\]'),
              '<div class="metatag" tag="lang">\\2</div>')),
 
     ('see', (re.compile(r'\[see\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]'),
             '<div class="metatag" tag="see">see: \\1 </div>')),
 
     ('url', (re.compile(r'\[url\ \=\&gt;\ \[([a-zA-Z0-9\ \.\-\_]+)\]\((http://|https://|/)(.*?)\)\]'),
             '<div class="metatag" tag="url"> <a href="\\2\\3">\\1</a> </div>')),
 
     ('license', (re.compile(r'\[license\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]'),
                 '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/\\1">\\1</a></div>')),
 
     ('ref', (re.compile(r'\[(requires|recommends|conflicts|base)\ \=\&gt;\ *([a-zA-Z0-9\-\/]*)\]'),
             '<div class="metatag" tag="ref \\1">\\1: <a href="/\\2">\\2</a></div>')),
 
     ('state', (re.compile(r'\[(stable|featured|stale|dead|dev|deprecated)\]'),
               '<div class="metatag" tag="state \\1">\\1</div>')),
 
     # label in grey
     ('label', (re.compile(r'\[([a-z]+)\]'),
               '<div class="metatag" tag="label">\\1</div>')),
 
     # generic catch all in grey
     ('generic', (re.compile(r'\[([a-zA-Z0-9\.\-\_]+)\]'),
                 '<div class="metatag" tag="generic">\\1</div>')),
 ))
 
 
 def extract_metatags(value):
     """
     Extract supported meta-tags from given text value
     """
     tags = []
     if not value:
         return tags, ''
 
     for key, val in tags_paterns.items():
         pat, replace_html = val
         tags.extend([(key, x.group()) for x in pat.finditer(value)])
         value = pat.sub('', value)
 
     return tags, value
 
 
 def style_metatag(tag_type, value):
     """
     converts tags from value into html equivalent
     """
     if not value:
         return ''
 
     html_value = value
     tag_data = tags_paterns.get(tag_type)
     if tag_data:
         pat, replace_html = tag_data
         # convert to plain `unicode` instead of a markup tag to be used in
         # regex expressions. safe_unicode doesn't work here
         html_value = pat.sub(replace_html, unicode(value))
 
     return html_value
 
 
 def bool2icon(value):
     """
     Returns boolean value of a given value, represented as html element with
     classes that will represent icons
 
     :param value: given value to convert to html node
     """
 
     if value:  # does bool conversion
         return HTML.tag('i', class_="icon-true")
     else:  # not true as bool
         return HTML.tag('i', class_="icon-false")
 
 
 #==============================================================================
 # PERMS
 #==============================================================================
 from rhodecode.lib.auth import HasPermissionAny, HasPermissionAll, \
 HasRepoPermissionAny, HasRepoPermissionAll, HasRepoGroupPermissionAll, \
 HasRepoGroupPermissionAny, HasRepoPermissionAnyApi, get_csrf_token, \
 csrf_token_key
 
 
 #==============================================================================
 # GRAVATAR URL
 #==============================================================================
 class InitialsGravatar(object):
     def __init__(self, email_address, first_name, last_name, size=30,
                  background=None, text_color='#fff'):
         self.size = size
         self.first_name = first_name
         self.last_name = last_name
         self.email_address = email_address
         self.background = background or self.str2color(email_address)
         self.text_color = text_color
 
     def get_color_bank(self):
         """
         returns a predefined list of colors that gravatars can use.
         Those are randomized distinct colors that guarantee readability and
         uniqueness.
 
         generated with: http://phrogz.net/css/distinct-colors.html
         """
         return [
             '#bf3030', '#a67f53', '#00ff00', '#5989b3', '#392040', '#d90000',
             '#402910', '#204020', '#79baf2', '#a700b3', '#bf6060', '#7f5320',
             '#008000', '#003059', '#ee00ff', '#ff0000', '#8c4b00', '#007300',
             '#005fb3', '#de73e6', '#ff4040', '#ffaa00', '#3df255', '#203140',
             '#47004d', '#591616', '#664400', '#59b365', '#0d2133', '#83008c',
             '#592d2d', '#bf9f60', '#73e682', '#1d3f73', '#73006b', '#402020',
             '#b2862d', '#397341', '#597db3', '#e600d6', '#a60000', '#736039',
             '#00b318', '#79aaf2', '#330d30', '#ff8080', '#403010', '#16591f',
             '#002459', '#8c4688', '#e50000', '#ffbf40', '#00732e', '#102340',
             '#bf60ac', '#8c4646', '#cc8800', '#00a642', '#1d3473', '#b32d98',
             '#660e00', '#ffd580', '#80ffb2', '#7391e6', '#733967', '#d97b6c',
             '#8c5e00', '#59b389', '#3967e6', '#590047', '#73281d', '#665200',
             '#00e67a', '#2d50b3', '#8c2377', '#734139', '#b2982d', '#16593a',
             '#001859', '#ff00aa', '#a65e53', '#ffcc00', '#0d3321', '#2d3959',
             '#731d56', '#401610', '#4c3d00', '#468c6c', '#002ca6', '#d936a3',
             '#d94c36', '#403920', '#36d9a3', '#0d1733', '#592d4a', '#993626',
             '#cca300', '#00734d', '#46598c', '#8c005e', '#7f1100', '#8c7000',
             '#00a66f', '#7382e6', '#b32d74', '#d9896c', '#ffe680', '#1d7362',
             '#364cd9', '#73003d', '#d93a00', '#998a4d', '#59b3a1', '#5965b3',
             '#e5007a', '#73341d', '#665f00', '#00b38f', '#0018b3', '#59163a',
             '#b2502d', '#bfb960', '#00ffcc', '#23318c', '#a6537f', '#734939',
             '#b2a700', '#104036', '#3d3df2', '#402031', '#e56739', '#736f39',
             '#79f2ea', '#000059', '#401029', '#4c1400', '#ffee00', '#005953',
             '#101040', '#990052', '#402820', '#403d10', '#00ffee', '#0000d9',
             '#ff80c4', '#a66953', '#eeff00', '#00ccbe', '#8080ff', '#e673a1',
             '#a62c00', '#474d00', '#1a3331', '#46468c', '#733950', '#662900',
             '#858c23', '#238c85', '#0f0073', '#b20047', '#d9986c', '#becc00',
             '#396f73', '#281d73', '#ff0066', '#ff6600', '#dee673', '#59adb3',
             '#6559b3', '#590024', '#b2622d', '#98b32d', '#36ced9', '#332d59',
             '#40001a', '#733f1d', '#526600', '#005359', '#242040', '#bf6079',
             '#735039', '#cef23d', '#007780', '#5630bf', '#66001b', '#b24700',
             '#acbf60', '#1d6273', '#25008c', '#731d34', '#a67453', '#50592d',
             '#00ccff', '#6600ff', '#ff0044', '#4c1f00', '#8a994d', '#79daf2',
             '#a173e6', '#d93662', '#402310', '#aaff00', '#2d98b3', '#8c40ff',
             '#592d39', '#ff8c40', '#354020', '#103640', '#1a0040', '#331a20',
             '#331400', '#334d00', '#1d5673', '#583973', '#7f0022', '#4c3626',
             '#88cc00', '#36a3d9', '#3d0073', '#d9364c', '#33241a', '#698c23',
             '#5995b3', '#300059', '#e57382', '#7f3300', '#366600', '#00aaff',
             '#3a1659', '#733941', '#663600', '#74b32d', '#003c59', '#7f53a6',
             '#73000f', '#ff8800', '#baf279', '#79caf2', '#291040', '#a6293a',
             '#b2742d', '#587339', '#0077b3', '#632699', '#400009', '#d9a66c',
             '#294010', '#2d4a59', '#aa00ff', '#4c131b', '#b25f00', '#5ce600',
             '#267399', '#a336d9', '#990014', '#664e33', '#86bf60', '#0088ff',
             '#7700b3', '#593a16', '#073300', '#1d4b73', '#ac60bf', '#e59539',
             '#4f8c46', '#368dd9', '#5c0073'
         ]
 
     def rgb_to_hex_color(self, rgb_tuple):
         """
         Converts an rgb_tuple passed to an hex color.
 
         :param rgb_tuple: tuple with 3 ints represents rgb color space
         """
         return '#' + ("".join(map(chr, rgb_tuple)).encode('hex'))
 
     def email_to_int_list(self, email_str):
         """
         Get every byte of the hex digest value of email and turn it to integer.
         It's going to be always between 0-255
         """
         digest = md5_safe(email_str.lower())
         return [int(digest[i * 2:i * 2 + 2], 16) for i in range(16)]
 
     def pick_color_bank_index(self, email_str, color_bank):
         return self.email_to_int_list(email_str)[0] % len(color_bank)
 
     def str2color(self, email_str):
         """
         Tries to map in a stable algorithm an email to color
 
         :param email_str:
         """
         color_bank = self.get_color_bank()
         # pick position (module it's length so we always find it in the
         # bank even if it's smaller than 256 values
         pos = self.pick_color_bank_index(email_str, color_bank)
         return color_bank[pos]
 
     def normalize_email(self, email_address):
         import unicodedata
         # default host used to fill in the fake/missing email
         default_host = u'localhost'
 
         if not email_address:
             email_address = u'%s@%s' % (User.DEFAULT_USER, default_host)
 
         email_address = safe_unicode(email_address)
 
         if u'@' not in email_address:
             email_address = u'%s@%s' % (email_address, default_host)
 
         if email_address.endswith(u'@'):
             email_address = u'%s%s' % (email_address, default_host)
 
         email_address = unicodedata.normalize('NFKD', email_address)\
             .encode('ascii', 'ignore')
         return email_address
 
     def get_initials(self):
         """
         Returns 2 letter initials calculated based on the input.
         The algorithm picks first given email address, and takes first letter
         of part before @, and then the first letter of server name. In case
         the part before @ is in a format of `somestring.somestring2` it replaces
         the server letter with first letter of somestring2
 
         In case function was initialized with both first and lastname, this
         overrides the extraction from email by first letter of the first and
         last name. We add special logic to that functionality, In case Full name
         is compound, like Guido Von Rossum, we use last part of the last name
         (Von Rossum) picking `R`.
 
         Function also normalizes the non-ascii characters to they ascii
         representation, eg Ą => A
         """
         import unicodedata
         # replace non-ascii to ascii
         first_name = unicodedata.normalize(
             'NFKD', safe_unicode(self.first_name)).encode('ascii', 'ignore')
         last_name = unicodedata.normalize(
             'NFKD', safe_unicode(self.last_name)).encode('ascii', 'ignore')
 
         # do NFKD encoding, and also make sure email has proper format
         email_address = self.normalize_email(self.email_address)
 
         # first push the email initials
         prefix, server = email_address.split('@', 1)
 
         # check if prefix is maybe a 'first_name.last_name' syntax
         _dot_split = prefix.rsplit('.', 1)
         if len(_dot_split) == 2 and _dot_split[1]:
             initials = [_dot_split[0][0], _dot_split[1][0]]
         else:
             initials = [prefix[0], server[0]]
 
         # then try to replace either first_name or last_name
         fn_letter = (first_name or " ")[0].strip()
         ln_letter = (last_name.split(' ', 1)[-1] or " ")[0].strip()
 
         if fn_letter:
             initials[0] = fn_letter
 
         if ln_letter:
             initials[1] = ln_letter
 
         return ''.join(initials).upper()
 
     def get_img_data_by_type(self, font_family, img_type):
         default_user = """
         <svg xmlns="http://www.w3.org/2000/svg"
         version="1.1" x="0px" y="0px" width="{size}" height="{size}"
         viewBox="-15 -10 439.165 429.164"
 
         xml:space="preserve"
         style="background:{background};" >
 
         <path d="M204.583,216.671c50.664,0,91.74-48.075,
                  91.74-107.378c0-82.237-41.074-107.377-91.74-107.377
                  c-50.668,0-91.74,25.14-91.74,107.377C112.844,
                  168.596,153.916,216.671,
                  204.583,216.671z" fill="{text_color}"/>
         <path d="M407.164,374.717L360.88,
                  270.454c-2.117-4.771-5.836-8.728-10.465-11.138l-71.83-37.392
                  c-1.584-0.823-3.502-0.663-4.926,0.415c-20.316,
                  15.366-44.203,23.488-69.076,23.488c-24.877,
                  0-48.762-8.122-69.078-23.488
                  c-1.428-1.078-3.346-1.238-4.93-0.415L58.75,
                  259.316c-4.631,2.41-8.346,6.365-10.465,11.138L2.001,374.717
                  c-3.191,7.188-2.537,15.412,1.75,22.005c4.285,
                  6.592,11.537,10.526,19.4,10.526h362.861c7.863,0,15.117-3.936,
                  19.402-10.527 C409.699,390.129,
                  410.355,381.902,407.164,374.717z" fill="{text_color}"/>
         </svg>""".format(
             size=self.size,
             background='#979797',  # @grey4
             text_color=self.text_color,
             font_family=font_family)
 
         return {
             "default_user": default_user
         }[img_type]
 
     def get_img_data(self, svg_type=None):
         """
         generates the svg metadata for image
         """
 
         font_family = ','.join([
             'proximanovaregular',
             'Proxima Nova Regular',
             'Proxima Nova',
             'Arial',
             'Lucida Grande',
             'sans-serif'
             ])
         if svg_type:
             return self.get_img_data_by_type(font_family, svg_type)
 
         initials = self.get_initials()
         img_data = """
         <svg xmlns="http://www.w3.org/2000/svg" pointer-events="none"
              width="{size}" height="{size}"
              style="width: 100%; height: 100%; background-color: {background}"
              viewBox="0 0 {size} {size}">
             <text text-anchor="middle" y="50%" x="50%" dy="0.35em"
                   pointer-events="auto" fill="{text_color}"
                   font-family="{font_family}"
                   style="font-weight: 400; font-size: {f_size}px;">{text}
             </text>
         </svg>""".format(
             size=self.size,
             f_size=self.size/1.85,  # scale the text inside the box nicely
             background=self.background,
             text_color=self.text_color,
             text=initials.upper(),
             font_family=font_family)
 
         return img_data
 
     def generate_svg(self, svg_type=None):
         img_data = self.get_img_data(svg_type)
         return "data:image/svg+xml;base64,%s" % img_data.encode('base64')
 
 
 def initials_gravatar(email_address, first_name, last_name, size=30):
     svg_type = None
     if email_address == User.DEFAULT_USER_EMAIL:
         svg_type = 'default_user'
     klass = InitialsGravatar(email_address, first_name, last_name, size)
     return klass.generate_svg(svg_type=svg_type)
 
 
 def gravatar_url(email_address, size=30, request=None):
     request = get_current_request()
     _use_gravatar = request.call_context.visual.use_gravatar
     _gravatar_url = request.call_context.visual.gravatar_url
 
     _gravatar_url = _gravatar_url or User.DEFAULT_GRAVATAR_URL
 
     email_address = email_address or User.DEFAULT_USER_EMAIL
     if isinstance(email_address, unicode):
         # hashlib crashes on unicode items
         email_address = safe_str(email_address)
 
     # empty email or default user
     if not email_address or email_address == User.DEFAULT_USER_EMAIL:
         return initials_gravatar(User.DEFAULT_USER_EMAIL, '', '', size=size)
 
     if _use_gravatar:
         # TODO: Disuse pyramid thread locals. Think about another solution to
         # get the host and schema here.
         request = get_current_request()
         tmpl = safe_str(_gravatar_url)
         tmpl = tmpl.replace('{email}', email_address)\
                    .replace('{md5email}', md5_safe(email_address.lower())) \
                    .replace('{netloc}', request.host)\
                    .replace('{scheme}', request.scheme)\
                    .replace('{size}', safe_str(size))
         return tmpl
     else:
         return initials_gravatar(email_address, '', '', size=size)
 
 
 class Page(_Page):
     """
     Custom pager to match rendering style with paginator
     """
 
     def _get_pos(self, cur_page, max_page, items):
         edge = (items / 2) + 1
         if (cur_page <= edge):
             radius = max(items / 2, items - cur_page)
         elif (max_page - cur_page) < edge:
             radius = (items - 1) - (max_page - cur_page)
         else:
             radius = items / 2
 
         left = max(1, (cur_page - (radius)))
         right = min(max_page, cur_page + (radius))
         return left, cur_page, right
 
     def _range(self, regexp_match):
         """
         Return range of linked pages (e.g. '1 2 [3] 4 5 6 7 8').
 
         Arguments:
 
         regexp_match
             A "re" (regular expressions) match object containing the
             radius of linked pages around the current page in
             regexp_match.group(1) as a string
 
         This function is supposed to be called as a callable in
         re.sub.
 
         """
         radius = int(regexp_match.group(1))
 
         # Compute the first and last page number within the radius
         # e.g. '1 .. 5 6 [7] 8 9 .. 12'
         # -> leftmost_page  = 5
         # -> rightmost_page = 9
         leftmost_page, _cur, rightmost_page = self._get_pos(self.page,
                                                             self.last_page,
                                                             (radius * 2) + 1)
         nav_items = []
 
         # Create a link to the first page (unless we are on the first page
         # or there would be no need to insert '..' spacers)
         if self.page != self.first_page and self.first_page < leftmost_page:
             nav_items.append(self._pagerlink(self.first_page, self.first_page))
 
         # Insert dots if there are pages between the first page
         # and the currently displayed page range
         if leftmost_page - self.first_page > 1:
             # Wrap in a SPAN tag if nolink_attr is set
             text = '..'
             if self.dotdot_attr:
                 text = HTML.span(c=text, **self.dotdot_attr)
             nav_items.append(text)
 
         for thispage in xrange(leftmost_page, rightmost_page + 1):
             # Hilight the current page number and do not use a link
             if thispage == self.page:
                 text = '%s' % (thispage,)
                 # Wrap in a SPAN tag if nolink_attr is set
                 if self.curpage_attr:
                     text = HTML.span(c=text, **self.curpage_attr)
                 nav_items.append(text)
             # Otherwise create just a link to that page
             else:
                 text = '%s' % (thispage,)
                 nav_items.append(self._pagerlink(thispage, text))
 
         # Insert dots if there are pages between the displayed
         # page numbers and the end of the page range
         if self.last_page - rightmost_page > 1:
             text = '..'
             # Wrap in a SPAN tag if nolink_attr is set
             if self.dotdot_attr:
                 text = HTML.span(c=text, **self.dotdot_attr)
             nav_items.append(text)
 
         # Create a link to the very last page (unless we are on the last
         # page or there would be no need to insert '..' spacers)
         if self.page != self.last_page and rightmost_page < self.last_page:
             nav_items.append(self._pagerlink(self.last_page, self.last_page))
 
         ## prerender links
         #_page_link = url.current()
         #nav_items.append(literal('<link rel="prerender" href="%s?page=%s">' % (_page_link, str(int(self.page)+1))))
         #nav_items.append(literal('<link rel="prefetch" href="%s?page=%s">' % (_page_link, str(int(self.page)+1))))
         return self.separator.join(nav_items)
 
     def pager(self, format='~2~', page_param='page', partial_param='partial',
         show_if_single_page=False, separator=' ', onclick=None,
         symbol_first='<<', symbol_last='>>',
         symbol_previous='<', symbol_next='>',
         link_attr={'class': 'pager_link', 'rel': 'prerender'},
         curpage_attr={'class': 'pager_curpage'},
         dotdot_attr={'class': 'pager_dotdot'}, **kwargs):
 
         self.curpage_attr = curpage_attr
         self.separator = separator
         self.pager_kwargs = kwargs
         self.page_param = page_param
         self.partial_param = partial_param
         self.onclick = onclick
         self.link_attr = link_attr
         self.dotdot_attr = dotdot_attr
 
         # Don't show navigator if there is no more than one page
         if self.page_count == 0 or (self.page_count == 1 and not show_if_single_page):
             return ''
 
         from string import Template
         # Replace ~...~ in token format by range of pages
         result = re.sub(r'~(\d+)~', self._range, format)
 
         # Interpolate '%' variables
         result = Template(result).safe_substitute({
             'first_page': self.first_page,
             'last_page': self.last_page,
             'page': self.page,
             'page_count': self.page_count,
             'items_per_page': self.items_per_page,
             'first_item': self.first_item,
             'last_item': self.last_item,
             'item_count': self.item_count,
             'link_first': self.page > self.first_page and \
                     self._pagerlink(self.first_page, symbol_first) or '',
             'link_last': self.page < self.last_page and \
                     self._pagerlink(self.last_page, symbol_last) or '',
             'link_previous': self.previous_page and \
                     self._pagerlink(self.previous_page, symbol_previous) \
                     or HTML.span(symbol_previous, class_="pg-previous disabled"),
             'link_next': self.next_page and \
                     self._pagerlink(self.next_page, symbol_next) \
                     or HTML.span(symbol_next, class_="pg-next disabled")
         })
 
         return literal(result)
 
 
 #==============================================================================
 # REPO PAGER, PAGER FOR REPOSITORY
 #==============================================================================
 class RepoPage(Page):
 
     def __init__(self, collection, page=1, items_per_page=20,
                  item_count=None, url=None, **kwargs):
 
         """Create a "RepoPage" instance. special pager for paging
         repository
         """
         self._url_generator = url
 
         # Safe the kwargs class-wide so they can be used in the pager() method
         self.kwargs = kwargs
 
         # Save a reference to the collection
         self.original_collection = collection
 
         self.collection = collection
 
         # The self.page is the number of the current page.
         # The first page has the number 1!
         try:
             self.page = int(page)  # make it int() if we get it as a string
         except (ValueError, TypeError):
             self.page = 1
 
         self.items_per_page = items_per_page
 
         # Unless the user tells us how many items the collections has
         # we calculate that ourselves.
         if item_count is not None:
             self.item_count = item_count
         else:
             self.item_count = len(self.collection)
 
         # Compute the number of the first and last available page
         if self.item_count > 0:
             self.first_page = 1
             self.page_count = int(math.ceil(float(self.item_count) /
                                             self.items_per_page))
             self.last_page = self.first_page + self.page_count - 1
 
             # Make sure that the requested page number is the range of
             # valid pages
             if self.page > self.last_page:
                 self.page = self.last_page
             elif self.page < self.first_page:
                 self.page = self.first_page
 
             # Note: the number of items on this page can be less than
             #       items_per_page if the last page is not full
             self.first_item = max(0, (self.item_count) - (self.page *
                                                           items_per_page))
             self.last_item = ((self.item_count - 1) - items_per_page *
                               (self.page - 1))
 
             self.items = list(self.collection[self.first_item:self.last_item + 1])
 
             # Links to previous and next page
             if self.page > self.first_page:
                 self.previous_page = self.page - 1
             else:
                 self.previous_page = None
 
             if self.page < self.last_page:
                 self.next_page = self.page + 1
             else:
                 self.next_page = None
 
         # No items available
         else:
             self.first_page = None
             self.page_count = 0
             self.last_page = None
             self.first_item = None
             self.last_item = None
             self.previous_page = None
             self.next_page = None
             self.items = []
 
         # This is a subclass of the 'list' type. Initialise the list now.
         list.__init__(self, reversed(self.items))
 
 
 def breadcrumb_repo_link(repo):
     """
     Makes a breadcrumbs path link to repo
 
     ex::
         group >> subgroup >> repo
 
     :param repo: a Repository instance
     """
 
     path = [
         link_to(group.name, route_path('repo_group_home', repo_group_name=group.group_name))
         for group in repo.groups_with_parents
     ] + [
         link_to(repo.just_name, route_path('repo_summary', repo_name=repo.repo_name))
     ]
 
     return literal(' &raquo; '.join(path))
 
 
 def format_byte_size_binary(file_size):
     """
     Formats file/folder sizes to standard.
     """
     if file_size is None:
         file_size = 0
 
     formatted_size = format_byte_size(file_size, binary=True)
     return formatted_size
 
 
 def urlify_text(text_, safe=True):
     """
     Extrac urls from text and make html links out of them
 
     :param text_:
     """
 
     url_pat = re.compile(r'''(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@#.&+]'''
                          '''|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)''')
 
     def url_func(match_obj):
         url_full = match_obj.groups()[0]
         return '<a href="%(url)s">%(url)s</a>' % ({'url': url_full})
     _newtext = url_pat.sub(url_func, text_)
     if safe:
         return literal(_newtext)
     return _newtext
 
 
 def urlify_commits(text_, repository):
     """
     Extract commit ids from text and make link from them
 
     :param text_:
     :param repository: repo name to build the URL with
     """
 
     URL_PAT = re.compile(r'(^|\s)([0-9a-fA-F]{12,40})($|\s)')
 
     def url_func(match_obj):
         commit_id = match_obj.groups()[1]
         pref = match_obj.groups()[0]
         suf = match_obj.groups()[2]
 
         tmpl = (
             '%(pref)s<a class="%(cls)s" href="%(url)s">'
             '%(commit_id)s</a>%(suf)s'
         )
         return tmpl % {
             'pref': pref,
             'cls': 'revision-link',
             'url': route_url('repo_commit', repo_name=repository,
                              commit_id=commit_id),
             'commit_id': commit_id,
             'suf': suf
         }
 
     newtext = URL_PAT.sub(url_func, text_)
 
     return newtext
 
 
 def _process_url_func(match_obj, repo_name, uid, entry,
                       return_raw_data=False, link_format='html'):
     pref = ''
     if match_obj.group().startswith(' '):
         pref = ' '
 
     issue_id = ''.join(match_obj.groups())
 
     if link_format == 'html':
         tmpl = (
             '%(pref)s<a class="%(cls)s" href="%(url)s">'
             '%(issue-prefix)s%(id-repr)s'
             '</a>')
     elif link_format == 'rst':
         tmpl = '`%(issue-prefix)s%(id-repr)s <%(url)s>`_'
     elif link_format == 'markdown':
         tmpl = '[%(issue-prefix)s%(id-repr)s](%(url)s)'
     else:
         raise ValueError('Bad link_format:{}'.format(link_format))
 
     (repo_name_cleaned,
      parent_group_name) = RepoGroupModel().\
         _get_group_name_and_parent(repo_name)
 
     # variables replacement
     named_vars = {
         'id': issue_id,
         'repo': repo_name,
         'repo_name': repo_name_cleaned,
         'group_name': parent_group_name
     }
     # named regex variables
     named_vars.update(match_obj.groupdict())
     _url = string.Template(entry['url']).safe_substitute(**named_vars)
 
     data = {
         'pref': pref,
         'cls': 'issue-tracker-link',
         'url': _url,
         'id-repr': issue_id,
         'issue-prefix': entry['pref'],
         'serv': entry['url'],
     }
     if return_raw_data:
         return {
             'id': issue_id,
             'url': _url
         }
     return tmpl % data
 
 
 def get_active_pattern_entries(repo_name):
     repo = None
     if repo_name:
         # Retrieving repo_name to avoid invalid repo_name to explode on
         # IssueTrackerSettingsModel but still passing invalid name further down
         repo = Repository.get_by_repo_name(repo_name, cache=True)
 
     settings_model = IssueTrackerSettingsModel(repo=repo)
     active_entries = settings_model.get_settings(cache=True)
     return active_entries
 
 
 def process_patterns(text_string, repo_name, link_format='html',
                      active_entries=None):
 
     allowed_formats = ['html', 'rst', 'markdown']
     if link_format not in allowed_formats:
         raise ValueError('Link format can be only one of:{} got {}'.format(
                          allowed_formats, link_format))
 
     active_entries = active_entries or get_active_pattern_entries(repo_name)
     issues_data = []
     newtext = text_string
 
     for uid, entry in active_entries.items():
         log.debug('found issue tracker entry with uid %s' % (uid,))
 
         if not (entry['pat'] and entry['url']):
             log.debug('skipping due to missing data')
             continue
 
         log.debug('issue tracker entry: uid: `%s` PAT:%s URL:%s PREFIX:%s'
                   % (uid, entry['pat'], entry['url'], entry['pref']))
 
         try:
             pattern = re.compile(r'%s' % entry['pat'])
         except re.error:
             log.exception(
                 'issue tracker pattern: `%s` failed to compile',
                 entry['pat'])
             continue
 
         data_func = partial(
             _process_url_func, repo_name=repo_name, entry=entry, uid=uid,
             return_raw_data=True)
 
         for match_obj in pattern.finditer(text_string):
             issues_data.append(data_func(match_obj))
 
         url_func = partial(
             _process_url_func, repo_name=repo_name, entry=entry, uid=uid,
             link_format=link_format)
 
         newtext = pattern.sub(url_func, newtext)
         log.debug('processed prefix:uid `%s`' % (uid,))
 
     return newtext, issues_data
 
 
 def urlify_commit_message(commit_text, repository=None,
                           active_pattern_entries=None):
     """
     Parses given text message and makes proper links.
     issues are linked to given issue-server, and rest is a commit link
 
     :param commit_text:
     :param repository:
     """
     def escaper(string):
         return string.replace('<', '&lt;').replace('>', '&gt;')
 
     newtext = escaper(commit_text)
 
     # extract http/https links and make them real urls
     newtext = urlify_text(newtext, safe=False)
 
     # urlify commits - extract commit ids and make link out of them, if we have
     # the scope of repository present.
     if repository:
         newtext = urlify_commits(newtext, repository)
 
     # process issue tracker patterns
     newtext, issues = process_patterns(newtext, repository or '',
                                        active_entries=active_pattern_entries)
 
     return literal(newtext)
 
 
 def render_binary(repo_name, file_obj):
     """
     Choose how to render a binary file
     """
     filename = file_obj.name
 
     # images
     for ext in ['*.png', '*.jpg', '*.ico', '*.gif']:
         if fnmatch.fnmatch(filename, pat=ext):
             alt = filename
             src = route_path(
                 'repo_file_raw', repo_name=repo_name,
                 commit_id=file_obj.commit.raw_id, f_path=file_obj.path)
             return literal('<img class="rendered-binary" alt="{}" src="{}">'.format(alt, src))
 
 
 def renderer_from_filename(filename, exclude=None):
     """
     choose a renderer based on filename, this works only for text based files
     """
 
     # ipython
     for ext in ['*.ipynb']:
         if fnmatch.fnmatch(filename, pat=ext):
             return 'jupyter'
 
     is_markup = MarkupRenderer.renderer_from_filename(filename, exclude=exclude)
     if is_markup:
         return is_markup
     return None
 
 
 def render(source, renderer='rst', mentions=False, relative_urls=None,
            repo_name=None):
 
     def maybe_convert_relative_links(html_source):
         if relative_urls:
             return relative_links(html_source, relative_urls)
         return html_source
 
-    if renderer == 'rst':
+    if renderer == 'plain':
+        return literal(
+            MarkupRenderer.plain(source, leading_newline=False))
+
+    elif renderer == 'rst':
         if repo_name:
             # process patterns on comments if we pass in repo name
             source, issues = process_patterns(
                 source, repo_name, link_format='rst')
 
         return literal(
             '<div class="rst-block">%s</div>' %
             maybe_convert_relative_links(
                 MarkupRenderer.rst(source, mentions=mentions)))
+
     elif renderer == 'markdown':
         if repo_name:
             # process patterns on comments if we pass in repo name
             source, issues = process_patterns(
                 source, repo_name, link_format='markdown')
 
         return literal(
             '<div class="markdown-block">%s</div>' %
             maybe_convert_relative_links(
                 MarkupRenderer.markdown(source, flavored=True,
                                         mentions=mentions)))
+
     elif renderer == 'jupyter':
         return literal(
             '<div class="ipynb">%s</div>' %
             maybe_convert_relative_links(
                 MarkupRenderer.jupyter(source)))
 
     # None means just show the file-source
     return None
 
 
 def commit_status(repo, commit_id):
     return ChangesetStatusModel().get_status(repo, commit_id)
 
 
 def commit_status_lbl(commit_status):
     return dict(ChangesetStatus.STATUSES).get(commit_status)
 
 
 def commit_time(repo_name, commit_id):
     repo = Repository.get_by_repo_name(repo_name)
     commit = repo.get_commit(commit_id=commit_id)
     return commit.date
 
 
 def get_permission_name(key):
     return dict(Permission.PERMS).get(key)
 
 
 def journal_filter_help(request):
     _ = request.translate
     from rhodecode.lib.audit_logger import ACTIONS
     actions = '\n'.join(textwrap.wrap(', '.join(sorted(ACTIONS.keys())), 80))
 
     return _(
         'Example filter terms:\n' +
         '     repository:vcs\n' +
         '     username:marcin\n' +
         '     username:(NOT marcin)\n' +
         '     action:*push*\n' +
         '     ip:127.0.0.1\n' +
         '     date:20120101\n' +
         '     date:[20120101100000 TO 20120102]\n' +
         '\n' +
         'Actions: {actions}\n' +
         '\n' +
         'Generate wildcards using \'*\' character:\n' +
         '     "repository:vcs*" - search everything starting with \'vcs\'\n' +
         '     "repository:*vcs*" - search for repository containing \'vcs\'\n' +
         '\n' +
         'Optional AND / OR operators in queries\n' +
         '     "repository:vcs OR repository:test"\n' +
         '     "username:test AND repository:test*"\n'
     ).format(actions=actions)
 
 
 def search_filter_help(searcher, request):
     _ = request.translate
 
     terms = ''
     return _(
         'Example filter terms for `{searcher}` search:\n' +
         '{terms}\n' +
         'Generate wildcards using \'*\' character:\n' +
         '     "repo_name:vcs*" - search everything starting with \'vcs\'\n' +
         '     "repo_name:*vcs*" - search for repository containing \'vcs\'\n' +
         '\n' +
         'Optional AND / OR operators in queries\n' +
         '     "repo_name:vcs OR repo_name:test"\n' +
         '     "owner:test AND repo_name:test*"\n' +
         'More: {search_doc}'
     ).format(searcher=searcher.name,
              terms=terms, search_doc=searcher.query_lang_doc)
 
 
 def not_mapped_error(repo_name):
     from rhodecode.translation import _
     flash(_('%s repository is not mapped to db perhaps'
             ' it was created or renamed from the filesystem'
             ' please run the application again'
             ' in order to rescan repositories') % repo_name, category='error')
 
 
 def ip_range(ip_addr):
     from rhodecode.model.db import UserIpMap
     s, e = UserIpMap._get_ip_range(ip_addr)
     return '%s - %s' % (s, e)
 
 
 def form(url, method='post', needs_csrf_token=True, **attrs):
     """Wrapper around webhelpers.tags.form to prevent CSRF attacks."""
     if method.lower() != 'get' and needs_csrf_token:
         raise Exception(
             'Forms to POST/PUT/DELETE endpoints should have (in general) a ' +
             'CSRF token. If the endpoint does not require such token you can ' +
             'explicitly set the parameter needs_csrf_token to false.')
 
     return wh_form(url, method=method, **attrs)
 
 
 def secure_form(form_url, method="POST", multipart=False, **attrs):
     """Start a form tag that points the action to an url. This
     form tag will also include the hidden field containing
     the auth token.
 
     The url options should be given either as a string, or as a
     ``url()`` function. The method for the form defaults to POST.
 
     Options:
 
     ``multipart``
         If set to True, the enctype is set to "multipart/form-data".
     ``method``
         The method to use when submitting the form, usually either
         "GET" or "POST". If "PUT", "DELETE", or another verb is used, a
         hidden input with name _method is added to simulate the verb
         over POST.
 
     """
     from webhelpers.pylonslib.secure_form import insecure_form
 
     if 'request' in attrs:
         session = attrs['request'].session
         del attrs['request']
     else:
         raise ValueError(
             'Calling this form requires request= to be passed as argument')
 
     form = insecure_form(form_url, method, multipart, **attrs)
     token = literal(
         '<input type="hidden" id="{}" name="{}" value="{}">'.format(
         csrf_token_key, csrf_token_key, get_csrf_token(session)))
 
     return literal("%s\n%s" % (form, token))
 
 
 def dropdownmenu(name, selected, options, enable_filter=False, **attrs):
     select_html = select(name, selected, options, **attrs)
     select2 = """
         <script>
             $(document).ready(function() {
                   $('#%s').select2({
                       containerCssClass: 'drop-menu',
                       dropdownCssClass: 'drop-menu-dropdown',
                       dropdownAutoWidth: true%s
                   });
             });
         </script>
     """
     filter_option = """,
                         minimumResultsForSearch: -1
     """
     input_id = attrs.get('id') or name
     filter_enabled = "" if enable_filter else filter_option
     select_script = literal(select2 % (input_id, filter_enabled))
 
     return literal(select_html+select_script)
 
 
 def get_visual_attr(tmpl_context_var, attr_name):
     """
     A safe way to get a variable from visual variable of template context
 
     :param tmpl_context_var: instance of tmpl_context, usually present as `c`
     :param attr_name: name of the attribute we fetch from the c.visual
     """
     visual = getattr(tmpl_context_var, 'visual', None)
     if not visual:
         return
     else:
         return getattr(visual, attr_name, None)
 
 
 def get_last_path_part(file_node):
     if not file_node.path:
         return u''
 
     path = safe_unicode(file_node.path.split('/')[-1])
     return u'../' + path
 
 
 def route_url(*args, **kwargs):
     """
     Wrapper around pyramids `route_url` (fully qualified url) function. 
     """
     req = get_current_request()
     return req.route_url(*args, **kwargs)
 
 
 def route_path(*args, **kwargs):
     """
     Wrapper around pyramids `route_path` function.
     """
     req = get_current_request()
     return req.route_path(*args, **kwargs)
 
 
 def route_path_or_none(*args, **kwargs):
     try:
         return route_path(*args, **kwargs)
     except KeyError:
         return None
 
 
 def current_route_path(request, **kw):
     new_args = request.GET.mixed()
     new_args.update(kw)
     return request.current_route_path(_query=new_args)
 
 
 def api_call_example(method, args):
     """
     Generates an API call example via CURL
     """
     args_json = json.dumps(OrderedDict([
         ('id', 1),
         ('auth_token', 'SECRET'),
         ('method', method),
         ('args', args)
     ]))
     return literal(
         "curl {api_url} -X POST -H 'content-type:text/plain' --data-binary '{data}'"
         "<br/><br/>SECRET can be found in <a href=\"{token_url}\">auth-tokens</a> page, "
         "and needs to be of `api calls` role."
         .format(
             api_url=route_url('apiv2'),
             token_url=route_url('my_account_auth_tokens'),
             data=args_json))
 
 
 def notification_description(notification, request):
     """
     Generate notification human readable description based on notification type
     """
     from rhodecode.model.notification import NotificationModel
     return NotificationModel().make_description(
         notification, translate=request.translate)
 
 
 def go_import_header(request, db_repo=None):
     """
     Creates a header for go-import functionality in Go Lang
     """
 
     if not db_repo:
         return
     if 'go-get' not in request.GET:
         return
 
     clone_url = db_repo.clone_url()
     prefix = re.split(r'^https?:\/\/', clone_url)[-1]
     # we have a repo and go-get flag,
     return literal('<meta name="go-import" content="{} {} {}">'.format(
         prefix, db_repo.repo_type, clone_url))
 
 
 def reviewer_as_json(*args, **kwargs):
     from rhodecode.apps.repository.utils import reviewer_as_json as _reviewer_as_json
     return _reviewer_as_json(*args, **kwargs)

rhodecode/lib/markup_renderer.py

@@ -1,515 +1,519 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2011-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Renderer for markup languages with ability to parse using rst or markdown
 """
 
 import re
 import os
 import lxml
 import logging
 import urlparse
 import bleach
 
 from mako.lookup import TemplateLookup
 from mako.template import Template as MakoTemplate
 
 from docutils.core import publish_parts
 from docutils.parsers.rst import directives
 from docutils import writers
 from docutils.writers import html4css1
 import markdown
 
 from rhodecode.lib.markdown_ext import GithubFlavoredMarkdownExtension
 from rhodecode.lib.utils2 import (
     safe_str, safe_unicode, md5_safe, MENTIONS_REGEX)
 
 log = logging.getLogger(__name__)
 
 # default renderer used to generate automated comments
 DEFAULT_COMMENTS_RENDERER = 'rst'
 
 
 class CustomHTMLTranslator(writers.html4css1.HTMLTranslator):
     """
     Custom HTML Translator used for sandboxing potential
     JS injections in ref links
     """
 
     def visit_reference(self, node):
         if 'refuri' in node.attributes:
             refuri = node['refuri']
             if ':' in refuri:
                 prefix, link = refuri.lstrip().split(':', 1)
                 if prefix == 'javascript':
                     # we don't allow javascript type of refs...
                     node['refuri'] = 'javascript:alert("SandBoxedJavascript")'
 
         # old style class requires this...
         return html4css1.HTMLTranslator.visit_reference(self, node)
 
 
 class RhodeCodeWriter(writers.html4css1.Writer):
     def __init__(self):
         writers.Writer.__init__(self)
         self.translator_class = CustomHTMLTranslator
 
 
 def relative_links(html_source, server_paths):
     if not html_source:
         return html_source
 
     try:
         from lxml.html import fromstring
         from lxml.html import tostring
     except ImportError:
         log.exception('Failed to import lxml')
         return html_source
 
     try:
         doc = lxml.html.fromstring(html_source)
     except Exception:
         return html_source
 
     for el in doc.cssselect('img, video'):
         src = el.attrib.get('src')
         if src:
             el.attrib['src'] = relative_path(src, server_paths['raw'])
 
     for el in doc.cssselect('a:not(.gfm)'):
         src = el.attrib.get('href')
         if src:
             raw_mode = el.attrib['href'].endswith('?raw=1')
             if raw_mode:
                 el.attrib['href'] = relative_path(src, server_paths['raw'])
             else:
                 el.attrib['href'] = relative_path(src, server_paths['standard'])
 
     return lxml.html.tostring(doc)
 
 
 def relative_path(path, request_path, is_repo_file=None):
     """
     relative link support, path is a rel path, and request_path is current
     server path (not absolute)
 
     e.g.
 
     path = '../logo.png'
     request_path= '/repo/files/path/file.md'
     produces: '/repo/files/logo.png'
     """
     # TODO(marcink): unicode/str support ?
     # maybe=> safe_unicode(urllib.quote(safe_str(final_path), '/:'))
 
     def dummy_check(p):
         return True  # assume default is a valid file path
 
     is_repo_file = is_repo_file or dummy_check
     if not path:
         return request_path
 
     path = safe_unicode(path)
     request_path = safe_unicode(request_path)
 
     if path.startswith((u'data:', u'javascript:', u'#', u':')):
         # skip data, anchor, invalid links
         return path
 
     is_absolute = bool(urlparse.urlparse(path).netloc)
     if is_absolute:
         return path
 
     if not request_path:
         return path
 
     if path.startswith(u'/'):
         path = path[1:]
 
     if path.startswith(u'./'):
         path = path[2:]
 
     parts = request_path.split('/')
     # compute how deep we need to traverse the request_path
     depth = 0
 
     if is_repo_file(request_path):
         # if request path is a VALID file, we use a relative path with
         # one level up
         depth += 1
 
     while path.startswith(u'../'):
         depth += 1
         path = path[3:]
 
     if depth > 0:
         parts = parts[:-depth]
 
     parts.append(path)
     final_path = u'/'.join(parts).lstrip(u'/')
 
     return u'/' + final_path
 
 
 class MarkupRenderer(object):
     RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES = ['include', 'meta', 'raw']
 
     MARKDOWN_PAT = re.compile(r'\.(md|mkdn?|mdown|markdown)$', re.IGNORECASE)
     RST_PAT = re.compile(r'\.re?st$', re.IGNORECASE)
     JUPYTER_PAT = re.compile(r'\.(ipynb)$', re.IGNORECASE)
     PLAIN_PAT = re.compile(r'^readme$', re.IGNORECASE)
 
     URL_PAT = re.compile(r'(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]'
                          r'|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)')
 
     extensions = ['codehilite', 'extra', 'def_list', 'sane_lists']
     output_format = 'html4'
     markdown_renderer = markdown.Markdown(
         extensions, enable_attributes=False, output_format=output_format)
 
     markdown_renderer_flavored = markdown.Markdown(
         extensions + [GithubFlavoredMarkdownExtension()],
         enable_attributes=False, output_format=output_format)
 
     # extension together with weights. Lower is first means we control how
     # extensions are attached to readme names with those.
     PLAIN_EXTS = [
         # prefer no extension
         ('', 0),  # special case that renders READMES names without extension
         ('.text', 2), ('.TEXT', 2),
         ('.txt', 3), ('.TXT', 3)
     ]
 
     RST_EXTS = [
         ('.rst', 1), ('.rest', 1),
         ('.RST', 2), ('.REST', 2)
     ]
 
     MARKDOWN_EXTS = [
         ('.md', 1), ('.MD', 1),
         ('.mkdn', 2), ('.MKDN', 2),
         ('.mdown', 3), ('.MDOWN', 3),
         ('.markdown', 4), ('.MARKDOWN', 4)
     ]
 
     def _detect_renderer(self, source, filename=None):
         """
         runs detection of what renderer should be used for generating html
         from a markup language
 
         filename can be also explicitly a renderer name
 
         :param source:
         :param filename:
         """
 
         if MarkupRenderer.MARKDOWN_PAT.findall(filename):
             detected_renderer = 'markdown'
         elif MarkupRenderer.RST_PAT.findall(filename):
             detected_renderer = 'rst'
         elif MarkupRenderer.JUPYTER_PAT.findall(filename):
             detected_renderer = 'jupyter'
         elif MarkupRenderer.PLAIN_PAT.findall(filename):
             detected_renderer = 'plain'
         else:
             detected_renderer = 'plain'
 
         return getattr(MarkupRenderer, detected_renderer)
 
     @classmethod
     def bleach_clean(cls, text):
         from .bleach_whitelist import markdown_attrs, markdown_tags
         allowed_tags = markdown_tags
         allowed_attrs = markdown_attrs
         return bleach.clean(text, tags=allowed_tags, attributes=allowed_attrs)
 
     @classmethod
     def renderer_from_filename(cls, filename, exclude):
         """
         Detect renderer markdown/rst from filename and optionally use exclude
         list to remove some options. This is mostly used in helpers.
         Returns None when no renderer can be detected.
         """
         def _filter(elements):
             if isinstance(exclude, (list, tuple)):
                 return [x for x in elements if x not in exclude]
             return elements
 
         if filename.endswith(
                 tuple(_filter([x[0] for x in cls.MARKDOWN_EXTS if x[0]]))):
             return 'markdown'
         if filename.endswith(tuple(_filter([x[0] for x in cls.RST_EXTS if x[0]]))):
             return 'rst'
 
         return None
 
     def render(self, source, filename=None):
         """
         Renders a given filename using detected renderer
         it detects renderers based on file extension or mimetype.
         At last it will just do a simple html replacing new lines with <br/>
 
         :param file_name:
         :param source:
         """
 
         renderer = self._detect_renderer(source, filename)
         readme_data = renderer(source)
         return readme_data
 
     @classmethod
     def _flavored_markdown(cls, text):
         """
         Github style flavored markdown
 
         :param text:
         """
 
         # Extract pre blocks.
         extractions = {}
 
         def pre_extraction_callback(matchobj):
             digest = md5_safe(matchobj.group(0))
             extractions[digest] = matchobj.group(0)
             return "{gfm-extraction-%s}" % digest
         pattern = re.compile(r'<pre>.*?</pre>', re.MULTILINE | re.DOTALL)
         text = re.sub(pattern, pre_extraction_callback, text)
 
         # Prevent foo_bar_baz from ending up with an italic word in the middle.
         def italic_callback(matchobj):
             s = matchobj.group(0)
             if list(s).count('_') >= 2:
                 return s.replace('_', r'\_')
             return s
         text = re.sub(r'^(?! {4}|\t)\w+_\w+_\w[\w_]*', italic_callback, text)
 
         # Insert pre block extractions.
         def pre_insert_callback(matchobj):
             return '\n\n' + extractions[matchobj.group(1)]
         text = re.sub(r'\{gfm-extraction-([0-9a-f]{32})\}',
                       pre_insert_callback, text)
 
         return text
 
     @classmethod
     def urlify_text(cls, text):
         def url_func(match_obj):
             url_full = match_obj.groups()[0]
             return '<a href="%(url)s">%(url)s</a>' % ({'url': url_full})
 
         return cls.URL_PAT.sub(url_func, text)
 
     @classmethod
-    def plain(cls, source, universal_newline=True):
+    def plain(cls, source, universal_newline=True, leading_newline=True):
         source = safe_unicode(source)
         if universal_newline:
             newline = '\n'
             source = newline.join(source.splitlines())
 
-        source = cls.urlify_text(source)
-        return '<br />' + source.replace("\n", '<br />')
+        rendered_source = cls.urlify_text(source)
+        source = ''
+        if leading_newline:
+            source += '<br />'
+        source += rendered_source.replace("\n", '<br />')
+        return source
 
     @classmethod
     def markdown(cls, source, safe=True, flavored=True, mentions=False,
                  clean_html=True):
         """
         returns markdown rendered code cleaned by the bleach library
         """
 
         if flavored:
             markdown_renderer = cls.markdown_renderer_flavored
         else:
             markdown_renderer = cls.markdown_renderer
 
         if mentions:
             mention_pat = re.compile(MENTIONS_REGEX)
 
             def wrapp(match_obj):
                 uname = match_obj.groups()[0]
                 return ' **@%(uname)s** ' % {'uname': uname}
             mention_hl = mention_pat.sub(wrapp, source).strip()
             # we extracted mentions render with this using Mentions false
             return cls.markdown(mention_hl, safe=safe, flavored=flavored,
                                 mentions=False)
 
         source = safe_unicode(source)
 
         try:
             if flavored:
                 source = cls._flavored_markdown(source)
             rendered = markdown_renderer.convert(source)
             if clean_html:
                 rendered = cls.bleach_clean(rendered)
             return rendered
         except Exception:
             log.exception('Error when rendering Markdown')
             if safe:
                 log.debug('Fallback to render in plain mode')
                 return cls.plain(source)
             else:
                 raise
 
     @classmethod
     def rst(cls, source, safe=True, mentions=False, clean_html=False):
         if mentions:
             mention_pat = re.compile(MENTIONS_REGEX)
 
             def wrapp(match_obj):
                 uname = match_obj.groups()[0]
                 return ' **@%(uname)s** ' % {'uname': uname}
             mention_hl = mention_pat.sub(wrapp, source).strip()
             # we extracted mentions render with this using Mentions false
             return cls.rst(mention_hl, safe=safe, mentions=False)
 
         source = safe_unicode(source)
         try:
             docutils_settings = dict(
                 [(alias, None) for alias in
                  cls.RESTRUCTUREDTEXT_DISALLOWED_DIRECTIVES])
 
             docutils_settings.update({
                 'input_encoding': 'unicode', 'report_level': 4})
 
             for k, v in docutils_settings.iteritems():
                 directives.register_directive(k, v)
 
             parts = publish_parts(source=source,
                                   writer=RhodeCodeWriter(),
                                   settings_overrides=docutils_settings)
             rendered = parts["fragment"]
             if clean_html:
                 rendered = cls.bleach_clean(rendered)
             return parts['html_title'] + rendered
         except Exception:
             log.exception('Error when rendering RST')
             if safe:
                 log.debug('Fallbacking to render in plain mode')
                 return cls.plain(source)
             else:
                 raise
 
     @classmethod
     def jupyter(cls, source, safe=True):
         from rhodecode.lib import helpers
 
         from traitlets.config import Config
         import nbformat
         from nbconvert import HTMLExporter
         from nbconvert.preprocessors import Preprocessor
 
         class CustomHTMLExporter(HTMLExporter):
             def _template_file_default(self):
                 return 'basic'
 
         class Sandbox(Preprocessor):
 
             def preprocess(self, nb, resources):
                 sandbox_text = 'SandBoxed(IPython.core.display.Javascript object)'
                 for cell in nb['cells']:
                     if safe and 'outputs' in cell:
                         for cell_output in cell['outputs']:
                             if 'data' in cell_output:
                                 if 'application/javascript' in cell_output['data']:
                                     cell_output['data']['text/plain'] = sandbox_text
                                     cell_output['data'].pop('application/javascript', None)
                 return nb, resources
 
         def _sanitize_resources(resources):
             """
             Skip/sanitize some of the CSS generated and included in jupyter
             so it doesn't messes up UI so much
             """
 
             # TODO(marcink): probably we should replace this with whole custom
             # CSS set that doesn't screw up, but jupyter generated html has some
             # special markers, so it requires Custom HTML exporter template with
             # _default_template_path_default, to achieve that
 
             # strip the reset CSS
             resources[0] = resources[0][resources[0].find('/*! Source'):]
             return resources
 
         def as_html(notebook):
             conf = Config()
             conf.CustomHTMLExporter.preprocessors = [Sandbox]
             html_exporter = CustomHTMLExporter(config=conf)
 
             (body, resources) = html_exporter.from_notebook_node(notebook)
             header = '<!-- ## IPYTHON NOTEBOOK RENDERING ## -->'
             js = MakoTemplate(r'''
             <!-- Load mathjax -->
                 <!-- MathJax configuration -->
                 <script type="text/x-mathjax-config">
                 MathJax.Hub.Config({
                     jax: ["input/TeX","output/HTML-CSS", "output/PreviewHTML"],
                     extensions: ["tex2jax.js","MathMenu.js","MathZoom.js", "fast-preview.js", "AssistiveMML.js", "[Contrib]/a11y/accessibility-menu.js"],
                     TeX: {
                         extensions: ["AMSmath.js","AMSsymbols.js","noErrors.js","noUndefined.js"]
                     },
                     tex2jax: {
                         inlineMath: [ ['$','$'], ["\\(","\\)"] ],
                         displayMath: [ ['$$','$$'], ["\\[","\\]"] ],
                         processEscapes: true,
                         processEnvironments: true
                     },
                     // Center justify equations in code and markdown cells. Elsewhere
                     // we use CSS to left justify single line equations in code cells.
                     displayAlign: 'center',
                     "HTML-CSS": {
                         styles: {'.MathJax_Display': {"margin": 0}},
                         linebreaks: { automatic: true },
                         availableFonts: ["STIX", "TeX"]
                     },
                     showMathMenu: false
                 });
                 </script>
                 <!-- End of mathjax configuration -->
                 <script src="${h.asset('js/src/math_jax/MathJax.js')}"></script>
             ''').render(h=helpers)
 
             css = '<style>{}</style>'.format(
                 ''.join(_sanitize_resources(resources['inlining']['css'])))
 
             body = '\n'.join([header, css, js, body])
             return body, resources
 
         notebook = nbformat.reads(source, as_version=4)
         (body, resources) = as_html(notebook)
         return body
 
 
 class RstTemplateRenderer(object):
 
     def __init__(self):
         base = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
         rst_template_dirs = [os.path.join(base, 'templates', 'rst_templates')]
         self.template_store = TemplateLookup(
             directories=rst_template_dirs,
             input_encoding='utf-8',
             imports=['from rhodecode.lib import helpers as h'])
 
     def _get_template(self, templatename):
         return self.template_store.get_template(templatename)
 
     def render(self, template_name, **kwargs):
         template = self._get_template(template_name)
         return template.render(**kwargs)

rhodecode/model/forms.py

@@ -1,620 +1,621 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 this is forms validation classes
 http://formencode.org/module-formencode.validators.html
 for list off all availible validators
 
 we can create our own validators
 
 The table below outlines the options which can be used in a schema in addition to the validators themselves
 pre_validators          []     These validators will be applied before the schema
 chained_validators      []     These validators will be applied after the schema
 allow_extra_fields      False     If True, then it is not an error when keys that aren't associated with a validator are present
 filter_extra_fields     False     If True, then keys that aren't associated with a validator are removed
 if_key_missing          NoDefault If this is given, then any keys that aren't available but are expected will be replaced with this value (and then validated). This does not override a present .if_missing attribute on validators. NoDefault is a special FormEncode class to mean that no default values has been specified and therefore missing keys shouldn't take a default value.
 ignore_key_missing      False     If True, then missing keys will be missing in the result, if the validator doesn't have .if_missing on it already
 
 
 <name> = formencode.validators.<name of validator>
 <name> must equal form name
 list=[1,2,3,4,5]
 for SELECT use formencode.All(OneOf(list), Int())
 
 """
 
 import deform
 import logging
 import formencode
 
 from pkg_resources import resource_filename
 from formencode import All, Pipe
 
 from pyramid.threadlocal import get_current_request
 
 from rhodecode import BACKENDS
 from rhodecode.lib import helpers
 from rhodecode.model import validators as v
 
 log = logging.getLogger(__name__)
 
 
 deform_templates = resource_filename('deform', 'templates')
 rhodecode_templates = resource_filename('rhodecode', 'templates/forms')
 search_path = (rhodecode_templates, deform_templates)
 
 
 class RhodecodeFormZPTRendererFactory(deform.ZPTRendererFactory):
     """ Subclass of ZPTRendererFactory to add rhodecode context variables """
     def __call__(self, template_name, **kw):
         kw['h'] = helpers
         kw['request'] = get_current_request()
         return self.load(template_name)(**kw)
 
 
 form_renderer = RhodecodeFormZPTRendererFactory(search_path)
 deform.Form.set_default_renderer(form_renderer)
 
 
 def LoginForm(localizer):
     _ = localizer
 
     class _LoginForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         username = v.UnicodeString(
             strip=True,
             min=1,
             not_empty=True,
             messages={
                 'empty': _(u'Please enter a login'),
                 'tooShort': _(u'Enter a value %(min)i characters long or more')
             }
         )
 
         password = v.UnicodeString(
             strip=False,
             min=3,
             max=72,
             not_empty=True,
             messages={
                 'empty': _(u'Please enter a password'),
                 'tooShort': _(u'Enter %(min)i characters or more')}
         )
 
         remember = v.StringBoolean(if_missing=False)
 
         chained_validators = [v.ValidAuth(localizer)]
     return _LoginForm
 
 
 def UserForm(localizer, edit=False, available_languages=None, old_data=None):
     old_data = old_data or {}
     available_languages = available_languages or []
     _ = localizer
 
     class _UserForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         username = All(v.UnicodeString(strip=True, min=1, not_empty=True),
                        v.ValidUsername(localizer, edit, old_data))
         if edit:
             new_password = All(
                 v.ValidPassword(localizer),
                 v.UnicodeString(strip=False, min=6, max=72, not_empty=False)
             )
             password_confirmation = All(
                 v.ValidPassword(localizer),
                 v.UnicodeString(strip=False, min=6, max=72, not_empty=False),
             )
             admin = v.StringBoolean(if_missing=False)
         else:
             password = All(
                 v.ValidPassword(localizer),
                 v.UnicodeString(strip=False, min=6, max=72, not_empty=True)
             )
             password_confirmation = All(
                 v.ValidPassword(localizer),
                 v.UnicodeString(strip=False, min=6, max=72, not_empty=False)
             )
 
         password_change = v.StringBoolean(if_missing=False)
         create_repo_group = v.StringBoolean(if_missing=False)
 
         active = v.StringBoolean(if_missing=False)
         firstname = v.UnicodeString(strip=True, min=1, not_empty=False)
         lastname = v.UnicodeString(strip=True, min=1, not_empty=False)
         email = All(v.UniqSystemEmail(localizer, old_data), v.Email(not_empty=True))
         extern_name = v.UnicodeString(strip=True)
         extern_type = v.UnicodeString(strip=True)
         language = v.OneOf(available_languages, hideList=False,
                            testValueList=True, if_missing=None)
         chained_validators = [v.ValidPasswordsMatch(localizer)]
     return _UserForm
 
 
 def UserGroupForm(localizer, edit=False, old_data=None, allow_disabled=False):
     old_data = old_data or {}
     _ = localizer
 
     class _UserGroupForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
 
         users_group_name = All(
             v.UnicodeString(strip=True, min=1, not_empty=True),
             v.ValidUserGroup(localizer, edit, old_data)
         )
         user_group_description = v.UnicodeString(strip=True, min=1,
                                                  not_empty=False)
 
         users_group_active = v.StringBoolean(if_missing=False)
 
         if edit:
             # this is user group owner
             user = All(
                 v.UnicodeString(not_empty=True),
                 v.ValidRepoUser(localizer, allow_disabled))
     return _UserGroupForm
 
 
 def RepoGroupForm(localizer, edit=False, old_data=None, available_groups=None,
                   can_create_in_root=False, allow_disabled=False):
     _ = localizer
     old_data = old_data or {}
     available_groups = available_groups or []
 
     class _RepoGroupForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
 
         group_name = All(v.UnicodeString(strip=True, min=1, not_empty=True),
                          v.SlugifyName(localizer),)
         group_description = v.UnicodeString(strip=True, min=1,
                                             not_empty=False)
         group_copy_permissions = v.StringBoolean(if_missing=False)
 
         group_parent_id = v.OneOf(available_groups, hideList=False,
                                   testValueList=True, not_empty=True)
         enable_locking = v.StringBoolean(if_missing=False)
         chained_validators = [
             v.ValidRepoGroup(localizer, edit, old_data, can_create_in_root)]
 
         if edit:
             # this is repo group owner
             user = All(
                 v.UnicodeString(not_empty=True),
                 v.ValidRepoUser(localizer, allow_disabled))
     return _RepoGroupForm
 
 
 def RegisterForm(localizer, edit=False, old_data=None):
     _ = localizer
     old_data = old_data or {}
 
     class _RegisterForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         username = All(
             v.ValidUsername(localizer, edit, old_data),
             v.UnicodeString(strip=True, min=1, not_empty=True)
         )
         password = All(
             v.ValidPassword(localizer),
             v.UnicodeString(strip=False, min=6, max=72, not_empty=True)
         )
         password_confirmation = All(
             v.ValidPassword(localizer),
             v.UnicodeString(strip=False, min=6, max=72, not_empty=True)
         )
         active = v.StringBoolean(if_missing=False)
         firstname = v.UnicodeString(strip=True, min=1, not_empty=False)
         lastname = v.UnicodeString(strip=True, min=1, not_empty=False)
         email = All(v.UniqSystemEmail(localizer, old_data), v.Email(not_empty=True))
 
         chained_validators = [v.ValidPasswordsMatch(localizer)]
     return _RegisterForm
 
 
 def PasswordResetForm(localizer):
     _ = localizer
 
     class _PasswordResetForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         email = All(v.ValidSystemEmail(localizer), v.Email(not_empty=True))
     return _PasswordResetForm
 
 
 def RepoForm(localizer, edit=False, old_data=None, repo_groups=None,
              landing_revs=None, allow_disabled=False):
     _ = localizer
     old_data = old_data or {}
     repo_groups = repo_groups or []
     landing_revs = landing_revs or []
     supported_backends = BACKENDS.keys()
 
     class _RepoForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         repo_name = All(v.UnicodeString(strip=True, min=1, not_empty=True),
                         v.SlugifyName(localizer), v.CannotHaveGitSuffix(localizer))
         repo_group = All(v.CanWriteGroup(localizer, old_data),
                          v.OneOf(repo_groups, hideList=True))
         repo_type = v.OneOf(supported_backends, required=False,
                             if_missing=old_data.get('repo_type'))
         repo_description = v.UnicodeString(strip=True, min=1, not_empty=False)
         repo_private = v.StringBoolean(if_missing=False)
         repo_landing_rev = v.OneOf(landing_revs, hideList=True)
         repo_copy_permissions = v.StringBoolean(if_missing=False)
         clone_uri = All(v.UnicodeString(strip=True, min=1, not_empty=False))
 
         repo_enable_statistics = v.StringBoolean(if_missing=False)
         repo_enable_downloads = v.StringBoolean(if_missing=False)
         repo_enable_locking = v.StringBoolean(if_missing=False)
 
         if edit:
             # this is repo owner
             user = All(
                 v.UnicodeString(not_empty=True),
                 v.ValidRepoUser(localizer, allow_disabled))
             clone_uri_change = v.UnicodeString(
                 not_empty=False, if_missing=v.Missing)
 
         chained_validators = [v.ValidCloneUri(localizer),
                               v.ValidRepoName(localizer, edit, old_data)]
     return _RepoForm
 
 
 def RepoPermsForm(localizer):
     _ = localizer
 
     class _RepoPermsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         chained_validators = [v.ValidPerms(localizer, type_='repo')]
     return _RepoPermsForm
 
 
 def RepoGroupPermsForm(localizer, valid_recursive_choices):
     _ = localizer
 
     class _RepoGroupPermsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         recursive = v.OneOf(valid_recursive_choices)
         chained_validators = [v.ValidPerms(localizer, type_='repo_group')]
     return _RepoGroupPermsForm
 
 
 def UserGroupPermsForm(localizer):
     _ = localizer
 
     class _UserPermsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         chained_validators = [v.ValidPerms(localizer, type_='user_group')]
     return _UserPermsForm
 
 
 def RepoFieldForm(localizer):
     _ = localizer
 
     class _RepoFieldForm(formencode.Schema):
         filter_extra_fields = True
         allow_extra_fields = True
 
         new_field_key = All(v.FieldKey(localizer),
                             v.UnicodeString(strip=True, min=3, not_empty=True))
         new_field_value = v.UnicodeString(not_empty=False, if_missing=u'')
         new_field_type = v.OneOf(['str', 'unicode', 'list', 'tuple'],
                                  if_missing='str')
         new_field_label = v.UnicodeString(not_empty=False)
         new_field_desc = v.UnicodeString(not_empty=False)
     return _RepoFieldForm
 
 
 def RepoForkForm(localizer, edit=False, old_data=None,
                  supported_backends=BACKENDS.keys(), repo_groups=None,
                  landing_revs=None):
     _ = localizer
     old_data = old_data or {}
     repo_groups = repo_groups or []
     landing_revs = landing_revs or []
 
     class _RepoForkForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         repo_name = All(v.UnicodeString(strip=True, min=1, not_empty=True),
                         v.SlugifyName(localizer))
         repo_group = All(v.CanWriteGroup(localizer, ),
                          v.OneOf(repo_groups, hideList=True))
         repo_type = All(v.ValidForkType(localizer, old_data), v.OneOf(supported_backends))
         description = v.UnicodeString(strip=True, min=1, not_empty=True)
         private = v.StringBoolean(if_missing=False)
         copy_permissions = v.StringBoolean(if_missing=False)
         fork_parent_id = v.UnicodeString()
         chained_validators = [v.ValidForkName(localizer, edit, old_data)]
         landing_rev = v.OneOf(landing_revs, hideList=True)
     return _RepoForkForm
 
 
 def ApplicationSettingsForm(localizer):
     _ = localizer
 
     class _ApplicationSettingsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         rhodecode_title = v.UnicodeString(strip=True, max=40, not_empty=False)
         rhodecode_realm = v.UnicodeString(strip=True, min=1, not_empty=True)
         rhodecode_pre_code = v.UnicodeString(strip=True, min=1, not_empty=False)
         rhodecode_post_code = v.UnicodeString(strip=True, min=1, not_empty=False)
         rhodecode_captcha_public_key = v.UnicodeString(strip=True, min=1, not_empty=False)
         rhodecode_captcha_private_key = v.UnicodeString(strip=True, min=1, not_empty=False)
         rhodecode_create_personal_repo_group = v.StringBoolean(if_missing=False)
         rhodecode_personal_repo_group_pattern = v.UnicodeString(strip=True, min=1, not_empty=False)
     return _ApplicationSettingsForm
 
 
 def ApplicationVisualisationForm(localizer):
     from rhodecode.model.db import Repository
     _ = localizer
 
     class _ApplicationVisualisationForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         rhodecode_show_public_icon = v.StringBoolean(if_missing=False)
         rhodecode_show_private_icon = v.StringBoolean(if_missing=False)
         rhodecode_stylify_metatags = v.StringBoolean(if_missing=False)
 
         rhodecode_repository_fields = v.StringBoolean(if_missing=False)
         rhodecode_lightweight_journal = v.StringBoolean(if_missing=False)
         rhodecode_dashboard_items = v.Int(min=5, not_empty=True)
         rhodecode_admin_grid_items = v.Int(min=5, not_empty=True)
         rhodecode_show_version = v.StringBoolean(if_missing=False)
         rhodecode_use_gravatar = v.StringBoolean(if_missing=False)
         rhodecode_markup_renderer = v.OneOf(['markdown', 'rst'])
         rhodecode_gravatar_url = v.UnicodeString(min=3)
         rhodecode_clone_uri_tmpl = v.UnicodeString(not_empty=False, if_empty=Repository.DEFAULT_CLONE_URI)
         rhodecode_clone_uri_ssh_tmpl = v.UnicodeString(not_empty=False, if_empty=Repository.DEFAULT_CLONE_URI_SSH)
         rhodecode_support_url = v.UnicodeString()
         rhodecode_show_revision_number = v.StringBoolean(if_missing=False)
         rhodecode_show_sha_length = v.Int(min=4, not_empty=True)
     return _ApplicationVisualisationForm
 
 
 class _BaseVcsSettingsForm(formencode.Schema):
 
     allow_extra_fields = True
     filter_extra_fields = False
     hooks_changegroup_repo_size = v.StringBoolean(if_missing=False)
     hooks_changegroup_push_logger = v.StringBoolean(if_missing=False)
     hooks_outgoing_pull_logger = v.StringBoolean(if_missing=False)
 
     # PR/Code-review
     rhodecode_pr_merge_enabled = v.StringBoolean(if_missing=False)
     rhodecode_use_outdated_comments = v.StringBoolean(if_missing=False)
 
     # hg
     extensions_largefiles = v.StringBoolean(if_missing=False)
     extensions_evolve = v.StringBoolean(if_missing=False)
     phases_publish = v.StringBoolean(if_missing=False)
 
     rhodecode_hg_use_rebase_for_merging = v.StringBoolean(if_missing=False)
     rhodecode_hg_close_branch_before_merging = v.StringBoolean(if_missing=False)
 
     # git
     vcs_git_lfs_enabled = v.StringBoolean(if_missing=False)
     rhodecode_git_use_rebase_for_merging = v.StringBoolean(if_missing=False)
     rhodecode_git_close_branch_before_merging = v.StringBoolean(if_missing=False)
 
     # svn
     vcs_svn_proxy_http_requests_enabled = v.StringBoolean(if_missing=False)
     vcs_svn_proxy_http_server_url = v.UnicodeString(strip=True, if_missing=None)
 
     # cache
     rhodecode_diff_cache = v.StringBoolean(if_missing=False)
 
 
 def ApplicationUiSettingsForm(localizer):
     _ = localizer
 
     class _ApplicationUiSettingsForm(_BaseVcsSettingsForm):
         web_push_ssl = v.StringBoolean(if_missing=False)
         paths_root_path = All(
             v.ValidPath(localizer),
             v.UnicodeString(strip=True, min=1, not_empty=True)
         )
         largefiles_usercache = All(
             v.ValidPath(localizer),
             v.UnicodeString(strip=True, min=2, not_empty=True))
         vcs_git_lfs_store_location = All(
             v.ValidPath(localizer),
             v.UnicodeString(strip=True, min=2, not_empty=True))
         extensions_hgsubversion = v.StringBoolean(if_missing=False)
         extensions_hggit = v.StringBoolean(if_missing=False)
         new_svn_branch = v.ValidSvnPattern(localizer, section='vcs_svn_branch')
         new_svn_tag = v.ValidSvnPattern(localizer, section='vcs_svn_tag')
     return _ApplicationUiSettingsForm
 
 
 def RepoVcsSettingsForm(localizer, repo_name):
     _ = localizer
 
     class _RepoVcsSettingsForm(_BaseVcsSettingsForm):
         inherit_global_settings = v.StringBoolean(if_missing=False)
         new_svn_branch = v.ValidSvnPattern(localizer,
             section='vcs_svn_branch', repo_name=repo_name)
         new_svn_tag = v.ValidSvnPattern(localizer,
             section='vcs_svn_tag', repo_name=repo_name)
     return _RepoVcsSettingsForm
 
 
 def LabsSettingsForm(localizer):
     _ = localizer
 
     class _LabSettingsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
     return _LabSettingsForm
 
 
 def ApplicationPermissionsForm(
         localizer, register_choices, password_reset_choices,
         extern_activate_choices):
     _ = localizer
 
     class _DefaultPermissionsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
 
         anonymous = v.StringBoolean(if_missing=False)
         default_register = v.OneOf(register_choices)
         default_register_message = v.UnicodeString()
         default_password_reset = v.OneOf(password_reset_choices)
         default_extern_activate = v.OneOf(extern_activate_choices)
     return _DefaultPermissionsForm
 
 
 def ObjectPermissionsForm(localizer, repo_perms_choices, group_perms_choices,
                           user_group_perms_choices):
     _ = localizer
 
     class _ObjectPermissionsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         overwrite_default_repo = v.StringBoolean(if_missing=False)
         overwrite_default_group = v.StringBoolean(if_missing=False)
         overwrite_default_user_group = v.StringBoolean(if_missing=False)
         default_repo_perm = v.OneOf(repo_perms_choices)
         default_group_perm = v.OneOf(group_perms_choices)
         default_user_group_perm = v.OneOf(user_group_perms_choices)
     return _ObjectPermissionsForm
 
 
 def UserPermissionsForm(localizer, create_choices, create_on_write_choices,
                         repo_group_create_choices, user_group_create_choices,
                         fork_choices, inherit_default_permissions_choices):
     _ = localizer
 
     class _DefaultPermissionsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
 
         anonymous = v.StringBoolean(if_missing=False)
 
         default_repo_create = v.OneOf(create_choices)
         default_repo_create_on_write = v.OneOf(create_on_write_choices)
         default_user_group_create = v.OneOf(user_group_create_choices)
         default_repo_group_create = v.OneOf(repo_group_create_choices)
         default_fork_create = v.OneOf(fork_choices)
         default_inherit_default_permissions = v.OneOf(inherit_default_permissions_choices)
     return _DefaultPermissionsForm
 
 
 def UserIndividualPermissionsForm(localizer):
     _ = localizer
 
     class _DefaultPermissionsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
 
         inherit_default_permissions = v.StringBoolean(if_missing=False)
     return _DefaultPermissionsForm
 
 
 def DefaultsForm(localizer, edit=False, old_data=None, supported_backends=BACKENDS.keys()):
     _ = localizer
     old_data = old_data or {}
 
     class _DefaultsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         default_repo_type = v.OneOf(supported_backends)
         default_repo_private = v.StringBoolean(if_missing=False)
         default_repo_enable_statistics = v.StringBoolean(if_missing=False)
         default_repo_enable_downloads = v.StringBoolean(if_missing=False)
         default_repo_enable_locking = v.StringBoolean(if_missing=False)
     return _DefaultsForm
 
 
 def AuthSettingsForm(localizer):
     _ = localizer
 
     class _AuthSettingsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         auth_plugins = All(v.ValidAuthPlugins(localizer),
                            v.UniqueListFromString(localizer)(not_empty=True))
     return _AuthSettingsForm
 
 
 def UserExtraEmailForm(localizer):
     _ = localizer
 
     class _UserExtraEmailForm(formencode.Schema):
         email = All(v.UniqSystemEmail(localizer), v.Email(not_empty=True))
     return _UserExtraEmailForm
 
 
 def UserExtraIpForm(localizer):
     _ = localizer
 
     class _UserExtraIpForm(formencode.Schema):
         ip = v.ValidIp(localizer)(not_empty=True)
     return _UserExtraIpForm
 
 
 def PullRequestForm(localizer, repo_id):
     _ = localizer
 
     class ReviewerForm(formencode.Schema):
         user_id = v.Int(not_empty=True)
         reasons = All()
         rules = All(v.UniqueList(localizer, convert=int)())
         mandatory = v.StringBoolean()
 
     class _PullRequestForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
 
         common_ancestor = v.UnicodeString(strip=True, required=True)
         source_repo = v.UnicodeString(strip=True, required=True)
         source_ref = v.UnicodeString(strip=True, required=True)
         target_repo = v.UnicodeString(strip=True, required=True)
         target_ref = v.UnicodeString(strip=True, required=True)
         revisions = All(#v.NotReviewedRevisions(localizer, repo_id)(),
                         v.UniqueList(localizer)(not_empty=True))
         review_members = formencode.ForEach(ReviewerForm())
         pullrequest_title = v.UnicodeString(strip=True, required=True, min=3, max=255)
         pullrequest_desc = v.UnicodeString(strip=True, required=False)
+        description_renderer = v.UnicodeString(strip=True, required=False)
 
     return _PullRequestForm
 
 
 def IssueTrackerPatternsForm(localizer):
     _ = localizer
 
     class _IssueTrackerPatternsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         chained_validators = [v.ValidPattern(localizer)]
     return _IssueTrackerPatternsForm

rhodecode/model/pull_request.py

@@ -1,1701 +1,1704 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 pull request model for RhodeCode
 """
 
 
 import json
 import logging
 import datetime
 import urllib
 import collections
 
 from pyramid.threadlocal import get_current_request
 
 from rhodecode import events
 from rhodecode.translation import lazy_ugettext#, _
 from rhodecode.lib import helpers as h, hooks_utils, diffs
 from rhodecode.lib import audit_logger
 from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.markup_renderer import (
     DEFAULT_COMMENTS_RENDERER, RstTemplateRenderer)
 from rhodecode.lib.utils2 import safe_unicode, safe_str, md5_safe
 from rhodecode.lib.vcs.backends.base import (
     Reference, MergeResponse, MergeFailureReason, UpdateFailureReason)
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.exceptions import (
     CommitDoesNotExistError, EmptyRepositoryError)
 from rhodecode.model import BaseModel
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
     or_, PullRequest, PullRequestReviewers, ChangesetStatus,
     PullRequestVersion, ChangesetComment, Repository, RepoReviewRule)
 from rhodecode.model.meta import Session
 from rhodecode.model.notification import NotificationModel, \
     EmailNotificationModel
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import VcsSettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 # Data structure to hold the response data when updating commits during a pull
 # request update.
 UpdateResponse = collections.namedtuple('UpdateResponse', [
     'executed', 'reason', 'new', 'old', 'changes',
     'source_changed', 'target_changed'])
 
 
 class PullRequestModel(BaseModel):
 
     cls = PullRequest
 
     DIFF_CONTEXT = 3
 
     MERGE_STATUS_MESSAGES = {
         MergeFailureReason.NONE: lazy_ugettext(
             'This pull request can be automatically merged.'),
         MergeFailureReason.UNKNOWN: lazy_ugettext(
             'This pull request cannot be merged because of an unhandled'
             ' exception.'),
         MergeFailureReason.MERGE_FAILED: lazy_ugettext(
             'This pull request cannot be merged because of merge conflicts.'),
         MergeFailureReason.PUSH_FAILED: lazy_ugettext(
             'This pull request could not be merged because push to target'
             ' failed.'),
         MergeFailureReason.TARGET_IS_NOT_HEAD: lazy_ugettext(
             'This pull request cannot be merged because the target is not a'
             ' head.'),
         MergeFailureReason.HG_SOURCE_HAS_MORE_BRANCHES: lazy_ugettext(
             'This pull request cannot be merged because the source contains'
             ' more branches than the target.'),
         MergeFailureReason.HG_TARGET_HAS_MULTIPLE_HEADS: lazy_ugettext(
             'This pull request cannot be merged because the target has'
             ' multiple heads.'),
         MergeFailureReason.TARGET_IS_LOCKED: lazy_ugettext(
             'This pull request cannot be merged because the target repository'
             ' is locked.'),
         MergeFailureReason._DEPRECATED_MISSING_COMMIT: lazy_ugettext(
             'This pull request cannot be merged because the target or the '
             'source reference is missing.'),
         MergeFailureReason.MISSING_TARGET_REF: lazy_ugettext(
             'This pull request cannot be merged because the target '
             'reference is missing.'),
         MergeFailureReason.MISSING_SOURCE_REF: lazy_ugettext(
             'This pull request cannot be merged because the source '
             'reference is missing.'),
         MergeFailureReason.SUBREPO_MERGE_FAILED: lazy_ugettext(
             'This pull request cannot be merged because of conflicts related '
             'to sub repositories.'),
     }
 
     UPDATE_STATUS_MESSAGES = {
         UpdateFailureReason.NONE: lazy_ugettext(
             'Pull request update successful.'),
         UpdateFailureReason.UNKNOWN: lazy_ugettext(
             'Pull request update failed because of an unknown error.'),
         UpdateFailureReason.NO_CHANGE: lazy_ugettext(
             'No update needed because the source and target have not changed.'),
         UpdateFailureReason.WRONG_REF_TYPE: lazy_ugettext(
             'Pull request cannot be updated because the reference type is '
             'not supported for an update. Only Branch, Tag or Bookmark is allowed.'),
         UpdateFailureReason.MISSING_TARGET_REF: lazy_ugettext(
             'This pull request cannot be updated because the target '
             'reference is missing.'),
         UpdateFailureReason.MISSING_SOURCE_REF: lazy_ugettext(
             'This pull request cannot be updated because the source '
             'reference is missing.'),
     }
 
     def __get_pull_request(self, pull_request):
         return self._get_instance((
             PullRequest, PullRequestVersion), pull_request)
 
     def _check_perms(self, perms, pull_request, user, api=False):
         if not api:
             return h.HasRepoPermissionAny(*perms)(
                 user=user, repo_name=pull_request.target_repo.repo_name)
         else:
             return h.HasRepoPermissionAnyApi(*perms)(
                 user=user, repo_name=pull_request.target_repo.repo_name)
 
     def check_user_read(self, pull_request, user, api=False):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         return self._check_perms(_perms, pull_request, user, api)
 
     def check_user_merge(self, pull_request, user, api=False):
         _perms = ('repository.admin', 'repository.write', 'hg.admin',)
         return self._check_perms(_perms, pull_request, user, api)
 
     def check_user_update(self, pull_request, user, api=False):
         owner = user.user_id == pull_request.user_id
         return self.check_user_merge(pull_request, user, api) or owner
 
     def check_user_delete(self, pull_request, user):
         owner = user.user_id == pull_request.user_id
         _perms = ('repository.admin',)
         return self._check_perms(_perms, pull_request, user) or owner
 
     def check_user_change_status(self, pull_request, user, api=False):
         reviewer = user.user_id in [x.user_id for x in
                                     pull_request.reviewers]
         return self.check_user_update(pull_request, user, api) or reviewer
 
     def check_user_comment(self, pull_request, user):
         owner = user.user_id == pull_request.user_id
         return self.check_user_read(pull_request, user) or owner
 
     def get(self, pull_request):
         return self.__get_pull_request(pull_request)
 
     def _prepare_get_all_query(self, repo_name, source=False, statuses=None,
                                opened_by=None, order_by=None,
                                order_dir='desc'):
         repo = None
         if repo_name:
             repo = self._get_repo(repo_name)
 
         q = PullRequest.query()
 
         # source or target
         if repo and source:
             q = q.filter(PullRequest.source_repo == repo)
         elif repo:
             q = q.filter(PullRequest.target_repo == repo)
 
         # closed,opened
         if statuses:
             q = q.filter(PullRequest.status.in_(statuses))
 
         # opened by filter
         if opened_by:
             q = q.filter(PullRequest.user_id.in_(opened_by))
 
         if order_by:
             order_map = {
                 'name_raw': PullRequest.pull_request_id,
                 'title': PullRequest.title,
                 'updated_on_raw': PullRequest.updated_on,
                 'target_repo': PullRequest.target_repo_id
             }
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_all(self, repo_name, source=False, statuses=None,
                   opened_by=None):
         """
         Count the number of pull requests for a specific repository.
 
         :param repo_name: target or source repo
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :returns: int number of pull requests
         """
         q = self._prepare_get_all_query(
             repo_name, source=source, statuses=statuses, opened_by=opened_by)
 
         return q.count()
 
     def get_all(self, repo_name, source=False, statuses=None, opened_by=None,
                 offset=0, length=None, order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository.
 
         :param repo_name: target or source repo
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
         q = self._prepare_get_all_query(
             repo_name, source=source, statuses=statuses, opened_by=opened_by,
             order_by=order_by, order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def count_awaiting_review(self, repo_name, source=False, statuses=None,
                               opened_by=None):
         """
         Count the number of pull requests for a specific repository that are
         awaiting review.
 
         :param repo_name: target or source repo
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :returns: int number of pull requests
         """
         pull_requests = self.get_awaiting_review(
             repo_name, source=source, statuses=statuses, opened_by=opened_by)
 
         return len(pull_requests)
 
     def get_awaiting_review(self, repo_name, source=False, statuses=None,
                             opened_by=None, offset=0, length=None,
                             order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository that are awaiting
         review.
 
         :param repo_name: target or source repo
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
         pull_requests = self.get_all(
             repo_name, source=source, statuses=statuses, opened_by=opened_by,
             order_by=order_by, order_dir=order_dir)
 
         _filtered_pull_requests = []
         for pr in pull_requests:
             status = pr.calculated_review_status()
             if status in [ChangesetStatus.STATUS_NOT_REVIEWED,
                           ChangesetStatus.STATUS_UNDER_REVIEW]:
                 _filtered_pull_requests.append(pr)
         if length:
             return _filtered_pull_requests[offset:offset+length]
         else:
             return _filtered_pull_requests
 
     def count_awaiting_my_review(self, repo_name, source=False, statuses=None,
                                  opened_by=None, user_id=None):
         """
         Count the number of pull requests for a specific repository that are
         awaiting review from a specific user.
 
         :param repo_name: target or source repo
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :param user_id: reviewer user of the pull request
         :returns: int number of pull requests
         """
         pull_requests = self.get_awaiting_my_review(
             repo_name, source=source, statuses=statuses, opened_by=opened_by,
             user_id=user_id)
 
         return len(pull_requests)
 
     def get_awaiting_my_review(self, repo_name, source=False, statuses=None,
                                opened_by=None, user_id=None, offset=0,
                                length=None, order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository that are awaiting
         review from a specific user.
 
         :param repo_name: target or source repo
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :param user_id: reviewer user of the pull request
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
         pull_requests = self.get_all(
             repo_name, source=source, statuses=statuses, opened_by=opened_by,
             order_by=order_by, order_dir=order_dir)
 
         _my = PullRequestModel().get_not_reviewed(user_id)
         my_participation = []
         for pr in pull_requests:
             if pr in _my:
                 my_participation.append(pr)
         _filtered_pull_requests = my_participation
         if length:
             return _filtered_pull_requests[offset:offset+length]
         else:
             return _filtered_pull_requests
 
     def get_not_reviewed(self, user_id):
         return [
             x.pull_request for x in PullRequestReviewers.query().filter(
                 PullRequestReviewers.user_id == user_id).all()
         ]
 
     def _prepare_participating_query(self, user_id=None, statuses=None,
                                      order_by=None, order_dir='desc'):
         q = PullRequest.query()
         if user_id:
             reviewers_subquery = Session().query(
                 PullRequestReviewers.pull_request_id).filter(
                 PullRequestReviewers.user_id == user_id).subquery()
             user_filter = or_(
                 PullRequest.user_id == user_id,
                 PullRequest.pull_request_id.in_(reviewers_subquery)
             )
             q = PullRequest.query().filter(user_filter)
 
         # closed,opened
         if statuses:
             q = q.filter(PullRequest.status.in_(statuses))
 
         if order_by:
             order_map = {
                 'name_raw': PullRequest.pull_request_id,
                 'title': PullRequest.title,
                 'updated_on_raw': PullRequest.updated_on,
                 'target_repo': PullRequest.target_repo_id
             }
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_im_participating_in(self, user_id=None, statuses=None):
         q = self._prepare_participating_query(user_id, statuses=statuses)
         return q.count()
 
     def get_im_participating_in(
             self, user_id=None, statuses=None, offset=0,
             length=None, order_by=None, order_dir='desc'):
         """
         Get all Pull requests that i'm participating in, or i have opened
         """
 
         q = self._prepare_participating_query(
             user_id, statuses=statuses, order_by=order_by,
             order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def get_versions(self, pull_request):
         """
         returns version of pull request sorted by ID descending
         """
         return PullRequestVersion.query()\
             .filter(PullRequestVersion.pull_request == pull_request)\
             .order_by(PullRequestVersion.pull_request_version_id.asc())\
             .all()
 
     def get_pr_version(self, pull_request_id, version=None):
         at_version = None
 
         if version and version == 'latest':
             pull_request_ver = PullRequest.get(pull_request_id)
             pull_request_obj = pull_request_ver
             _org_pull_request_obj = pull_request_obj
             at_version = 'latest'
         elif version:
             pull_request_ver = PullRequestVersion.get_or_404(version)
             pull_request_obj = pull_request_ver
             _org_pull_request_obj = pull_request_ver.pull_request
             at_version = pull_request_ver.pull_request_version_id
         else:
             _org_pull_request_obj = pull_request_obj = PullRequest.get_or_404(
                 pull_request_id)
 
         pull_request_display_obj = PullRequest.get_pr_display_object(
             pull_request_obj, _org_pull_request_obj)
 
         return _org_pull_request_obj, pull_request_obj, \
                pull_request_display_obj, at_version
 
     def create(self, created_by, source_repo, source_ref, target_repo,
                target_ref, revisions, reviewers, title, description=None,
+               description_renderer=None,
                reviewer_data=None, translator=None, auth_user=None):
         translator = translator or get_current_request().translate
 
         created_by_user = self._get_user(created_by)
         auth_user = auth_user or created_by_user
         source_repo = self._get_repo(source_repo)
         target_repo = self._get_repo(target_repo)
 
         pull_request = PullRequest()
         pull_request.source_repo = source_repo
         pull_request.source_ref = source_ref
         pull_request.target_repo = target_repo
         pull_request.target_ref = target_ref
         pull_request.revisions = revisions
         pull_request.title = title
         pull_request.description = description
+        pull_request.description_renderer = description_renderer
         pull_request.author = created_by_user
         pull_request.reviewer_data = reviewer_data
 
         Session().add(pull_request)
         Session().flush()
 
         reviewer_ids = set()
         # members / reviewers
         for reviewer_object in reviewers:
             user_id, reasons, mandatory, rules = reviewer_object
             user = self._get_user(user_id)
 
             # skip duplicates
             if user.user_id in reviewer_ids:
                 continue
 
             reviewer_ids.add(user.user_id)
 
             reviewer = PullRequestReviewers()
             reviewer.user = user
             reviewer.pull_request = pull_request
             reviewer.reasons = reasons
             reviewer.mandatory = mandatory
 
             # NOTE(marcink): pick only first rule for now
             rule_id = list(rules)[0] if rules else None
             rule = RepoReviewRule.get(rule_id) if rule_id else None
             if rule:
                 review_group = rule.user_group_vote_rule()
                 if review_group:
                     # NOTE(marcink):
                     # again, can be that user is member of more,
                     # but we pick the first same, as default reviewers algo
                     review_group = review_group[0]
 
                     rule_data = {
                         'rule_name':
                             rule.review_rule_name,
                         'rule_user_group_entry_id':
                             review_group.repo_review_rule_users_group_id,
                         'rule_user_group_name':
                             review_group.users_group.users_group_name,
                         'rule_user_group_members':
                             [x.user.username for x in review_group.users_group.members],
                     }
                     # e.g {'vote_rule': -1, 'mandatory': True}
                     rule_data.update(review_group.rule_data())
 
                     reviewer.rule_data = rule_data
 
             Session().add(reviewer)
             Session().flush()
 
         # Set approval status to "Under Review" for all commits which are
         # part of this pull request.
         ChangesetStatusModel().set_status(
             repo=target_repo,
             status=ChangesetStatus.STATUS_UNDER_REVIEW,
             user=created_by_user,
             pull_request=pull_request
         )
         # we commit early at this point. This has to do with a fact
         # that before queries do some row-locking. And because of that
         # we need to commit and finish transation before below validate call
         # that for large repos could be long resulting in long row locks
         Session().commit()
 
         # prepare workspace, and run initial merge simulation
         MergeCheck.validate(
             pull_request, user=created_by_user, translator=translator)
 
         self.notify_reviewers(pull_request, reviewer_ids)
         self._trigger_pull_request_hook(
             pull_request, created_by_user, 'create')
 
         creation_data = pull_request.get_api_data(with_merge_state=False)
         self._log_audit_action(
             'repo.pull_request.create', {'data': creation_data},
             auth_user, pull_request)
 
         return pull_request
 
     def _trigger_pull_request_hook(self, pull_request, user, action):
         pull_request = self.__get_pull_request(pull_request)
         target_scm = pull_request.target_repo.scm_instance()
         if action == 'create':
             trigger_hook = hooks_utils.trigger_log_create_pull_request_hook
         elif action == 'merge':
             trigger_hook = hooks_utils.trigger_log_merge_pull_request_hook
         elif action == 'close':
             trigger_hook = hooks_utils.trigger_log_close_pull_request_hook
         elif action == 'review_status_change':
             trigger_hook = hooks_utils.trigger_log_review_pull_request_hook
         elif action == 'update':
             trigger_hook = hooks_utils.trigger_log_update_pull_request_hook
         else:
             return
 
         trigger_hook(
             username=user.username,
             repo_name=pull_request.target_repo.repo_name,
             repo_alias=target_scm.alias,
             pull_request=pull_request)
 
     def _get_commit_ids(self, pull_request):
         """
         Return the commit ids of the merged pull request.
 
         This method is not dealing correctly yet with the lack of autoupdates
         nor with the implicit target updates.
         For example: if a commit in the source repo is already in the target it
         will be reported anyways.
         """
         merge_rev = pull_request.merge_rev
         if merge_rev is None:
             raise ValueError('This pull request was not merged yet')
 
         commit_ids = list(pull_request.revisions)
         if merge_rev not in commit_ids:
             commit_ids.append(merge_rev)
 
         return commit_ids
 
     def merge_repo(self, pull_request, user, extras):
         log.debug("Merging pull request %s", pull_request.pull_request_id)
         merge_state = self._merge_pull_request(pull_request, user, extras)
         if merge_state.executed:
             log.debug(
                 "Merge was successful, updating the pull request comments.")
             self._comment_and_close_pr(pull_request, user, merge_state)
 
             self._log_audit_action(
                 'repo.pull_request.merge',
                 {'merge_state': merge_state.__dict__},
                 user, pull_request)
 
         else:
             log.warn("Merge failed, not updating the pull request.")
         return merge_state
 
     def _merge_pull_request(self, pull_request, user, extras, merge_msg=None):
         target_vcs = pull_request.target_repo.scm_instance()
         source_vcs = pull_request.source_repo.scm_instance()
         target_ref = self._refresh_reference(
             pull_request.target_ref_parts, target_vcs)
 
         message = merge_msg or (
             'Merge pull request #%(pr_id)s from '
             '%(source_repo)s %(source_ref_name)s\n\n %(pr_title)s') % {
                 'pr_id': pull_request.pull_request_id,
                 'source_repo': source_vcs.name,
                 'source_ref_name': pull_request.source_ref_parts.name,
                 'pr_title': pull_request.title
             }
 
         workspace_id = self._workspace_id(pull_request)
         repo_id = pull_request.target_repo.repo_id
         use_rebase = self._use_rebase_for_merging(pull_request)
         close_branch = self._close_branch_before_merging(pull_request)
 
         callback_daemon, extras = prepare_callback_daemon(
             extras, protocol=vcs_settings.HOOKS_PROTOCOL,
             host=vcs_settings.HOOKS_HOST,
             use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
 
         with callback_daemon:
             # TODO: johbo: Implement a clean way to run a config_override
             # for a single call.
             target_vcs.config.set(
                 'rhodecode', 'RC_SCM_DATA', json.dumps(extras))
             merge_state = target_vcs.merge(
                 repo_id, workspace_id, target_ref, source_vcs,
                 pull_request.source_ref_parts,
                 user_name=user.username, user_email=user.email,
                 message=message, use_rebase=use_rebase,
                 close_branch=close_branch)
         return merge_state
 
     def _comment_and_close_pr(self, pull_request, user, merge_state, close_msg=None):
         pull_request.merge_rev = merge_state.merge_ref.commit_id
         pull_request.updated_on = datetime.datetime.now()
         close_msg = close_msg or 'Pull request merged and closed'
 
         CommentsModel().create(
             text=safe_unicode(close_msg),
             repo=pull_request.target_repo.repo_id,
             user=user.user_id,
             pull_request=pull_request.pull_request_id,
             f_path=None,
             line_no=None,
             closing_pr=True
         )
 
         Session().add(pull_request)
         Session().flush()
         # TODO: paris: replace invalidation with less radical solution
         ScmModel().mark_for_invalidation(
             pull_request.target_repo.repo_name)
         self._trigger_pull_request_hook(pull_request, user, 'merge')
 
     def has_valid_update_type(self, pull_request):
         source_ref_type = pull_request.source_ref_parts.type
         return source_ref_type in ['book', 'branch', 'tag']
 
     def update_commits(self, pull_request):
         """
         Get the updated list of commits for the pull request
         and return the new pull request version and the list
         of commits processed by this update action
         """
         pull_request = self.__get_pull_request(pull_request)
         source_ref_type = pull_request.source_ref_parts.type
         source_ref_name = pull_request.source_ref_parts.name
         source_ref_id = pull_request.source_ref_parts.commit_id
 
         target_ref_type = pull_request.target_ref_parts.type
         target_ref_name = pull_request.target_ref_parts.name
         target_ref_id = pull_request.target_ref_parts.commit_id
 
         if not self.has_valid_update_type(pull_request):
             log.debug(
                 "Skipping update of pull request %s due to ref type: %s",
                 pull_request, source_ref_type)
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.WRONG_REF_TYPE,
                 old=pull_request, new=None, changes=None,
                 source_changed=False, target_changed=False)
 
         # source repo
         source_repo = pull_request.source_repo.scm_instance()
         try:
             source_commit = source_repo.get_commit(commit_id=source_ref_name)
         except CommitDoesNotExistError:
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.MISSING_SOURCE_REF,
                 old=pull_request, new=None, changes=None,
                 source_changed=False, target_changed=False)
 
         source_changed = source_ref_id != source_commit.raw_id
 
         # target repo
         target_repo = pull_request.target_repo.scm_instance()
         try:
             target_commit = target_repo.get_commit(commit_id=target_ref_name)
         except CommitDoesNotExistError:
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.MISSING_TARGET_REF,
                 old=pull_request, new=None, changes=None,
                 source_changed=False, target_changed=False)
         target_changed = target_ref_id != target_commit.raw_id
 
         if not (source_changed or target_changed):
             log.debug("Nothing changed in pull request %s", pull_request)
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.NO_CHANGE,
                 old=pull_request, new=None, changes=None,
                 source_changed=target_changed, target_changed=source_changed)
 
         change_in_found = 'target repo' if target_changed else 'source repo'
         log.debug('Updating pull request because of change in %s detected',
                   change_in_found)
 
         # Finally there is a need for an update, in case of source change
         # we create a new version, else just an update
         if source_changed:
             pull_request_version = self._create_version_from_snapshot(pull_request)
             self._link_comments_to_version(pull_request_version)
         else:
             try:
                 ver = pull_request.versions[-1]
             except IndexError:
                 ver = None
 
             pull_request.pull_request_version_id = \
                 ver.pull_request_version_id if ver else None
             pull_request_version = pull_request
 
         try:
             if target_ref_type in ('tag', 'branch', 'book'):
                 target_commit = target_repo.get_commit(target_ref_name)
             else:
                 target_commit = target_repo.get_commit(target_ref_id)
         except CommitDoesNotExistError:
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.MISSING_TARGET_REF,
                 old=pull_request, new=None, changes=None,
                 source_changed=source_changed, target_changed=target_changed)
 
         # re-compute commit ids
         old_commit_ids = pull_request.revisions
         pre_load = ["author", "branch", "date", "message"]
         commit_ranges = target_repo.compare(
             target_commit.raw_id, source_commit.raw_id, source_repo, merge=True,
             pre_load=pre_load)
 
         ancestor = target_repo.get_common_ancestor(
             target_commit.raw_id, source_commit.raw_id, source_repo)
 
         pull_request.source_ref = '%s:%s:%s' % (
             source_ref_type, source_ref_name, source_commit.raw_id)
         pull_request.target_ref = '%s:%s:%s' % (
             target_ref_type, target_ref_name, ancestor)
 
         pull_request.revisions = [
             commit.raw_id for commit in reversed(commit_ranges)]
         pull_request.updated_on = datetime.datetime.now()
         Session().add(pull_request)
         new_commit_ids = pull_request.revisions
 
         old_diff_data, new_diff_data = self._generate_update_diffs(
             pull_request, pull_request_version)
 
         # calculate commit and file changes
         changes = self._calculate_commit_id_changes(
             old_commit_ids, new_commit_ids)
         file_changes = self._calculate_file_changes(
             old_diff_data, new_diff_data)
 
         # set comments as outdated if DIFFS changed
         CommentsModel().outdate_comments(
             pull_request, old_diff_data=old_diff_data,
             new_diff_data=new_diff_data)
 
         commit_changes = (changes.added or changes.removed)
         file_node_changes = (
             file_changes.added or file_changes.modified or file_changes.removed)
         pr_has_changes = commit_changes or file_node_changes
 
         # Add an automatic comment to the pull request, in case
         # anything has changed
         if pr_has_changes:
             update_comment = CommentsModel().create(
                 text=self._render_update_message(changes, file_changes),
                 repo=pull_request.target_repo,
                 user=pull_request.author,
                 pull_request=pull_request,
                 send_email=False, renderer=DEFAULT_COMMENTS_RENDERER)
 
             # Update status to "Under Review" for added commits
             for commit_id in changes.added:
                 ChangesetStatusModel().set_status(
                     repo=pull_request.source_repo,
                     status=ChangesetStatus.STATUS_UNDER_REVIEW,
                     comment=update_comment,
                     user=pull_request.author,
                     pull_request=pull_request,
                     revision=commit_id)
 
         log.debug(
             'Updated pull request %s, added_ids: %s, common_ids: %s, '
             'removed_ids: %s', pull_request.pull_request_id,
             changes.added, changes.common, changes.removed)
         log.debug(
             'Updated pull request with the following file changes: %s',
             file_changes)
 
         log.info(
             "Updated pull request %s from commit %s to commit %s, "
             "stored new version %s of this pull request.",
             pull_request.pull_request_id, source_ref_id,
             pull_request.source_ref_parts.commit_id,
             pull_request_version.pull_request_version_id)
         Session().commit()
         self._trigger_pull_request_hook(
             pull_request, pull_request.author, 'update')
 
         return UpdateResponse(
             executed=True, reason=UpdateFailureReason.NONE,
             old=pull_request, new=pull_request_version, changes=changes,
             source_changed=source_changed, target_changed=target_changed)
 
     def _create_version_from_snapshot(self, pull_request):
         version = PullRequestVersion()
         version.title = pull_request.title
         version.description = pull_request.description
         version.status = pull_request.status
         version.created_on = datetime.datetime.now()
         version.updated_on = pull_request.updated_on
         version.user_id = pull_request.user_id
         version.source_repo = pull_request.source_repo
         version.source_ref = pull_request.source_ref
         version.target_repo = pull_request.target_repo
         version.target_ref = pull_request.target_ref
 
         version._last_merge_source_rev = pull_request._last_merge_source_rev
         version._last_merge_target_rev = pull_request._last_merge_target_rev
         version.last_merge_status = pull_request.last_merge_status
         version.shadow_merge_ref = pull_request.shadow_merge_ref
         version.merge_rev = pull_request.merge_rev
         version.reviewer_data = pull_request.reviewer_data
 
         version.revisions = pull_request.revisions
         version.pull_request = pull_request
         Session().add(version)
         Session().flush()
 
         return version
 
     def _generate_update_diffs(self, pull_request, pull_request_version):
 
         diff_context = (
             self.DIFF_CONTEXT +
             CommentsModel.needed_extra_diff_context())
 
         source_repo = pull_request_version.source_repo
         source_ref_id = pull_request_version.source_ref_parts.commit_id
         target_ref_id = pull_request_version.target_ref_parts.commit_id
         old_diff = self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id, context=diff_context)
 
         source_repo = pull_request.source_repo
         source_ref_id = pull_request.source_ref_parts.commit_id
         target_ref_id = pull_request.target_ref_parts.commit_id
 
         new_diff = self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id, context=diff_context)
 
         old_diff_data = diffs.DiffProcessor(old_diff)
         old_diff_data.prepare()
         new_diff_data = diffs.DiffProcessor(new_diff)
         new_diff_data.prepare()
 
         return old_diff_data, new_diff_data
 
     def _link_comments_to_version(self, pull_request_version):
         """
         Link all unlinked comments of this pull request to the given version.
 
         :param pull_request_version: The `PullRequestVersion` to which
             the comments shall be linked.
 
         """
         pull_request = pull_request_version.pull_request
         comments = ChangesetComment.query()\
             .filter(
                 # TODO: johbo: Should we query for the repo at all here?
                 # Pending decision on how comments of PRs are to be related
                 # to either the source repo, the target repo or no repo at all.
                 ChangesetComment.repo_id == pull_request.target_repo.repo_id,
                 ChangesetComment.pull_request == pull_request,
                 ChangesetComment.pull_request_version == None)\
             .order_by(ChangesetComment.comment_id.asc())
 
         # TODO: johbo: Find out why this breaks if it is done in a bulk
         # operation.
         for comment in comments:
             comment.pull_request_version_id = (
                 pull_request_version.pull_request_version_id)
             Session().add(comment)
 
     def _calculate_commit_id_changes(self, old_ids, new_ids):
         added = [x for x in new_ids if x not in old_ids]
         common = [x for x in new_ids if x in old_ids]
         removed = [x for x in old_ids if x not in new_ids]
         total = new_ids
         return ChangeTuple(added, common, removed, total)
 
     def _calculate_file_changes(self, old_diff_data, new_diff_data):
 
         old_files = OrderedDict()
         for diff_data in old_diff_data.parsed_diff:
             old_files[diff_data['filename']] = md5_safe(diff_data['raw_diff'])
 
         added_files = []
         modified_files = []
         removed_files = []
         for diff_data in new_diff_data.parsed_diff:
             new_filename = diff_data['filename']
             new_hash = md5_safe(diff_data['raw_diff'])
 
             old_hash = old_files.get(new_filename)
             if not old_hash:
                 # file is not present in old diff, means it's added
                 added_files.append(new_filename)
             else:
                 if new_hash != old_hash:
                     modified_files.append(new_filename)
                 # now remove a file from old, since we have seen it already
                 del old_files[new_filename]
 
         # removed files is when there are present in old, but not in NEW,
         # since we remove old files that are present in new diff, left-overs
         # if any should be the removed files
         removed_files.extend(old_files.keys())
 
         return FileChangeTuple(added_files, modified_files, removed_files)
 
     def _render_update_message(self, changes, file_changes):
         """
         render the message using DEFAULT_COMMENTS_RENDERER (RST renderer),
         so it's always looking the same disregarding on which default
         renderer system is using.
 
         :param changes: changes named tuple
         :param file_changes: file changes named tuple
 
         """
         new_status = ChangesetStatus.get_status_lbl(
             ChangesetStatus.STATUS_UNDER_REVIEW)
 
         changed_files = (
             file_changes.added + file_changes.modified + file_changes.removed)
 
         params = {
             'under_review_label': new_status,
             'added_commits': changes.added,
             'removed_commits': changes.removed,
             'changed_files': changed_files,
             'added_files': file_changes.added,
             'modified_files': file_changes.modified,
             'removed_files': file_changes.removed,
         }
         renderer = RstTemplateRenderer()
         return renderer.render('pull_request_update.mako', **params)
 
-    def edit(self, pull_request, title, description, user):
+    def edit(self, pull_request, title, description, description_renderer, user):
         pull_request = self.__get_pull_request(pull_request)
         old_data = pull_request.get_api_data(with_merge_state=False)
         if pull_request.is_closed():
             raise ValueError('This pull request is closed')
         if title:
             pull_request.title = title
         pull_request.description = description
         pull_request.updated_on = datetime.datetime.now()
+        pull_request.description_renderer = description_renderer
         Session().add(pull_request)
         self._log_audit_action(
             'repo.pull_request.edit', {'old_data': old_data},
             user, pull_request)
 
     def update_reviewers(self, pull_request, reviewer_data, user):
         """
         Update the reviewers in the pull request
 
         :param pull_request: the pr to update
         :param reviewer_data: list of tuples
             [(user, ['reason1', 'reason2'], mandatory_flag, [rules])]
         """
         pull_request = self.__get_pull_request(pull_request)
         if pull_request.is_closed():
             raise ValueError('This pull request is closed')
 
         reviewers = {}
         for user_id, reasons, mandatory, rules in reviewer_data:
             if isinstance(user_id, (int, basestring)):
                 user_id = self._get_user(user_id).user_id
             reviewers[user_id] = {
                 'reasons': reasons, 'mandatory': mandatory}
 
         reviewers_ids = set(reviewers.keys())
         current_reviewers = PullRequestReviewers.query()\
             .filter(PullRequestReviewers.pull_request ==
                     pull_request).all()
         current_reviewers_ids = set([x.user.user_id for x in current_reviewers])
 
         ids_to_add = reviewers_ids.difference(current_reviewers_ids)
         ids_to_remove = current_reviewers_ids.difference(reviewers_ids)
 
         log.debug("Adding %s reviewers", ids_to_add)
         log.debug("Removing %s reviewers", ids_to_remove)
         changed = False
         for uid in ids_to_add:
             changed = True
             _usr = self._get_user(uid)
             reviewer = PullRequestReviewers()
             reviewer.user = _usr
             reviewer.pull_request = pull_request
             reviewer.reasons = reviewers[uid]['reasons']
             # NOTE(marcink): mandatory shouldn't be changed now
             # reviewer.mandatory = reviewers[uid]['reasons']
             Session().add(reviewer)
             self._log_audit_action(
                 'repo.pull_request.reviewer.add', {'data': reviewer.get_dict()},
                 user, pull_request)
 
         for uid in ids_to_remove:
             changed = True
             reviewers = PullRequestReviewers.query()\
                 .filter(PullRequestReviewers.user_id == uid,
                         PullRequestReviewers.pull_request == pull_request)\
                 .all()
             # use .all() in case we accidentally added the same person twice
             # this CAN happen due to the lack of DB checks
             for obj in reviewers:
                 old_data = obj.get_dict()
                 Session().delete(obj)
                 self._log_audit_action(
                     'repo.pull_request.reviewer.delete',
                     {'old_data': old_data}, user, pull_request)
 
         if changed:
             pull_request.updated_on = datetime.datetime.now()
             Session().add(pull_request)
 
         self.notify_reviewers(pull_request, ids_to_add)
         return ids_to_add, ids_to_remove
 
     def get_url(self, pull_request, request=None, permalink=False):
         if not request:
             request = get_current_request()
 
         if permalink:
             return request.route_url(
                 'pull_requests_global',
                 pull_request_id=pull_request.pull_request_id,)
         else:
             return request.route_url('pullrequest_show',
                 repo_name=safe_str(pull_request.target_repo.repo_name),
                 pull_request_id=pull_request.pull_request_id,)
 
     def get_shadow_clone_url(self, pull_request, request=None):
         """
         Returns qualified url pointing to the shadow repository. If this pull
         request is closed there is no shadow repository and ``None`` will be
         returned.
         """
         if pull_request.is_closed():
             return None
         else:
             pr_url = urllib.unquote(self.get_url(pull_request, request=request))
             return safe_unicode('{pr_url}/repository'.format(pr_url=pr_url))
 
     def notify_reviewers(self, pull_request, reviewers_ids):
         # notification to reviewers
         if not reviewers_ids:
             return
 
         pull_request_obj = pull_request
         # get the current participants of this pull request
         recipients = reviewers_ids
         notification_type = EmailNotificationModel.TYPE_PULL_REQUEST
 
         pr_source_repo = pull_request_obj.source_repo
         pr_target_repo = pull_request_obj.target_repo
 
         pr_url = h.route_url('pullrequest_show',
             repo_name=pr_target_repo.repo_name,
             pull_request_id=pull_request_obj.pull_request_id,)
 
         # set some variables for email notification
         pr_target_repo_url = h.route_url(
             'repo_summary', repo_name=pr_target_repo.repo_name)
 
         pr_source_repo_url = h.route_url(
             'repo_summary', repo_name=pr_source_repo.repo_name)
 
         # pull request specifics
         pull_request_commits = [
             (x.raw_id, x.message)
             for x in map(pr_source_repo.get_commit, pull_request.revisions)]
 
         kwargs = {
             'user': pull_request.author,
             'pull_request': pull_request_obj,
             'pull_request_commits': pull_request_commits,
 
             'pull_request_target_repo': pr_target_repo,
             'pull_request_target_repo_url': pr_target_repo_url,
 
             'pull_request_source_repo': pr_source_repo,
             'pull_request_source_repo_url': pr_source_repo_url,
 
             'pull_request_url': pr_url,
         }
 
         # pre-generate the subject for notification itself
         (subject,
          _h, _e,  # we don't care about those
          body_plaintext) = EmailNotificationModel().render_email(
             notification_type, **kwargs)
 
         # create notification objects, and emails
         NotificationModel().create(
             created_by=pull_request.author,
             notification_subject=subject,
             notification_body=body_plaintext,
             notification_type=notification_type,
             recipients=recipients,
             email_kwargs=kwargs,
         )
 
     def delete(self, pull_request, user):
         pull_request = self.__get_pull_request(pull_request)
         old_data = pull_request.get_api_data(with_merge_state=False)
         self._cleanup_merge_workspace(pull_request)
         self._log_audit_action(
             'repo.pull_request.delete', {'old_data': old_data},
             user, pull_request)
         Session().delete(pull_request)
 
     def close_pull_request(self, pull_request, user):
         pull_request = self.__get_pull_request(pull_request)
         self._cleanup_merge_workspace(pull_request)
         pull_request.status = PullRequest.STATUS_CLOSED
         pull_request.updated_on = datetime.datetime.now()
         Session().add(pull_request)
         self._trigger_pull_request_hook(
             pull_request, pull_request.author, 'close')
 
         pr_data = pull_request.get_api_data(with_merge_state=False)
         self._log_audit_action(
             'repo.pull_request.close', {'data': pr_data}, user, pull_request)
 
     def close_pull_request_with_comment(
             self, pull_request, user, repo, message=None):
 
         pull_request_review_status = pull_request.calculated_review_status()
 
         if pull_request_review_status == ChangesetStatus.STATUS_APPROVED:
             # approved only if we have voting consent
             status = ChangesetStatus.STATUS_APPROVED
         else:
             status = ChangesetStatus.STATUS_REJECTED
         status_lbl = ChangesetStatus.get_status_lbl(status)
 
         default_message = (
             'Closing with status change {transition_icon} {status}.'
         ).format(transition_icon='>', status=status_lbl)
         text = message or default_message
 
         # create a comment, and link it to new status
         comment = CommentsModel().create(
             text=text,
             repo=repo.repo_id,
             user=user.user_id,
             pull_request=pull_request.pull_request_id,
             status_change=status_lbl,
             status_change_type=status,
             closing_pr=True
         )
 
         # calculate old status before we change it
         old_calculated_status = pull_request.calculated_review_status()
         ChangesetStatusModel().set_status(
             repo.repo_id,
             status,
             user.user_id,
             comment=comment,
             pull_request=pull_request.pull_request_id
         )
 
         Session().flush()
         events.trigger(events.PullRequestCommentEvent(pull_request, comment))
         # we now calculate the status of pull request again, and based on that
         # calculation trigger status change. This might happen in cases
         # that non-reviewer admin closes a pr, which means his vote doesn't
         # change the status, while if he's a reviewer this might change it.
         calculated_status = pull_request.calculated_review_status()
         if old_calculated_status != calculated_status:
             self._trigger_pull_request_hook(
                 pull_request, user, 'review_status_change')
 
         # finally close the PR
         PullRequestModel().close_pull_request(
             pull_request.pull_request_id, user)
 
         return comment, status
 
     def merge_status(self, pull_request, translator=None,
                      force_shadow_repo_refresh=False):
         _ = translator or get_current_request().translate
 
         if not self._is_merge_enabled(pull_request):
             return False, _('Server-side pull request merging is disabled.')
         if pull_request.is_closed():
             return False, _('This pull request is closed.')
         merge_possible, msg = self._check_repo_requirements(
             target=pull_request.target_repo, source=pull_request.source_repo,
             translator=_)
         if not merge_possible:
             return merge_possible, msg
 
         try:
             resp = self._try_merge(
                 pull_request,
                 force_shadow_repo_refresh=force_shadow_repo_refresh)
             log.debug("Merge response: %s", resp)
             status = resp.possible, self.merge_status_message(
                 resp.failure_reason)
         except NotImplementedError:
             status = False, _('Pull request merging is not supported.')
 
         return status
 
     def _check_repo_requirements(self, target, source, translator):
         """
         Check if `target` and `source` have compatible requirements.
 
         Currently this is just checking for largefiles.
         """
         _ = translator
         target_has_largefiles = self._has_largefiles(target)
         source_has_largefiles = self._has_largefiles(source)
         merge_possible = True
         message = u''
 
         if target_has_largefiles != source_has_largefiles:
             merge_possible = False
             if source_has_largefiles:
                 message = _(
                     'Target repository large files support is disabled.')
             else:
                 message = _(
                     'Source repository large files support is disabled.')
 
         return merge_possible, message
 
     def _has_largefiles(self, repo):
         largefiles_ui = VcsSettingsModel(repo=repo).get_ui_settings(
             'extensions', 'largefiles')
         return largefiles_ui and largefiles_ui[0].active
 
     def _try_merge(self, pull_request, force_shadow_repo_refresh=False):
         """
         Try to merge the pull request and return the merge status.
         """
         log.debug(
             "Trying out if the pull request %s can be merged. Force_refresh=%s",
             pull_request.pull_request_id, force_shadow_repo_refresh)
         target_vcs = pull_request.target_repo.scm_instance()
 
         # Refresh the target reference.
         try:
             target_ref = self._refresh_reference(
                 pull_request.target_ref_parts, target_vcs)
         except CommitDoesNotExistError:
             merge_state = MergeResponse(
                 False, False, None, MergeFailureReason.MISSING_TARGET_REF)
             return merge_state
 
         target_locked = pull_request.target_repo.locked
         if target_locked and target_locked[0]:
             log.debug("The target repository is locked.")
             merge_state = MergeResponse(
                 False, False, None, MergeFailureReason.TARGET_IS_LOCKED)
         elif force_shadow_repo_refresh or self._needs_merge_state_refresh(
                 pull_request, target_ref):
             log.debug("Refreshing the merge status of the repository.")
             merge_state = self._refresh_merge_state(
                 pull_request, target_vcs, target_ref)
         else:
             possible = pull_request.\
                 last_merge_status == MergeFailureReason.NONE
             merge_state = MergeResponse(
                 possible, False, None, pull_request.last_merge_status)
 
         return merge_state
 
     def _refresh_reference(self, reference, vcs_repository):
         if reference.type in ('branch', 'book'):
             name_or_id = reference.name
         else:
             name_or_id = reference.commit_id
         refreshed_commit = vcs_repository.get_commit(name_or_id)
         refreshed_reference = Reference(
             reference.type, reference.name, refreshed_commit.raw_id)
         return refreshed_reference
 
     def _needs_merge_state_refresh(self, pull_request, target_reference):
         return not(
             pull_request.revisions and
             pull_request.revisions[0] == pull_request._last_merge_source_rev and
             target_reference.commit_id == pull_request._last_merge_target_rev)
 
     def _refresh_merge_state(self, pull_request, target_vcs, target_reference):
         workspace_id = self._workspace_id(pull_request)
         source_vcs = pull_request.source_repo.scm_instance()
         repo_id = pull_request.target_repo.repo_id
         use_rebase = self._use_rebase_for_merging(pull_request)
         close_branch = self._close_branch_before_merging(pull_request)
         merge_state = target_vcs.merge(
             repo_id, workspace_id,
             target_reference, source_vcs, pull_request.source_ref_parts,
             dry_run=True, use_rebase=use_rebase,
             close_branch=close_branch)
 
         # Do not store the response if there was an unknown error.
         if merge_state.failure_reason != MergeFailureReason.UNKNOWN:
             pull_request._last_merge_source_rev = \
                 pull_request.source_ref_parts.commit_id
             pull_request._last_merge_target_rev = target_reference.commit_id
             pull_request.last_merge_status = merge_state.failure_reason
             pull_request.shadow_merge_ref = merge_state.merge_ref
             Session().add(pull_request)
             Session().commit()
 
         return merge_state
 
     def _workspace_id(self, pull_request):
         workspace_id = 'pr-%s' % pull_request.pull_request_id
         return workspace_id
 
     def merge_status_message(self, status_code):
         """
         Return a human friendly error message for the given merge status code.
         """
         return self.MERGE_STATUS_MESSAGES[status_code]
 
     def generate_repo_data(self, repo, commit_id=None, branch=None,
                            bookmark=None, translator=None):
         from rhodecode.model.repo import RepoModel
 
         all_refs, selected_ref = \
             self._get_repo_pullrequest_sources(
                 repo.scm_instance(), commit_id=commit_id,
                 branch=branch, bookmark=bookmark, translator=translator)
 
         refs_select2 = []
         for element in all_refs:
             children = [{'id': x[0], 'text': x[1]} for x in element[0]]
             refs_select2.append({'text': element[1], 'children': children})
 
         return {
             'user': {
                 'user_id': repo.user.user_id,
                 'username': repo.user.username,
                 'firstname': repo.user.first_name,
                 'lastname': repo.user.last_name,
                 'gravatar_link': h.gravatar_url(repo.user.email, 14),
             },
             'name': repo.repo_name,
             'link': RepoModel().get_url(repo),
             'description': h.chop_at_smart(repo.description_safe, '\n'),
             'refs': {
                 'all_refs': all_refs,
                 'selected_ref': selected_ref,
                 'select2_refs': refs_select2
             }
         }
 
     def generate_pullrequest_title(self, source, source_ref, target):
         return u'{source}#{at_ref} to {target}'.format(
             source=source,
             at_ref=source_ref,
             target=target,
         )
 
     def _cleanup_merge_workspace(self, pull_request):
         # Merging related cleanup
         repo_id = pull_request.target_repo.repo_id
         target_scm = pull_request.target_repo.scm_instance()
         workspace_id = self._workspace_id(pull_request)
 
         try:
             target_scm.cleanup_merge_workspace(repo_id, workspace_id)
         except NotImplementedError:
             pass
 
     def _get_repo_pullrequest_sources(
             self, repo, commit_id=None, branch=None, bookmark=None,
             translator=None):
         """
         Return a structure with repo's interesting commits, suitable for
         the selectors in pullrequest controller
 
         :param commit_id: a commit that must be in the list somehow
             and selected by default
         :param branch: a branch that must be in the list and selected
             by default - even if closed
         :param bookmark: a bookmark that must be in the list and selected
         """
         _ = translator or get_current_request().translate
 
         commit_id = safe_str(commit_id) if commit_id else None
         branch = safe_str(branch) if branch else None
         bookmark = safe_str(bookmark) if bookmark else None
 
         selected = None
 
         # order matters: first source that has commit_id in it will be selected
         sources = []
         sources.append(('book', repo.bookmarks.items(), _('Bookmarks'), bookmark))
         sources.append(('branch', repo.branches.items(), _('Branches'), branch))
 
         if commit_id:
             ref_commit = (h.short_id(commit_id), commit_id)
             sources.append(('rev', [ref_commit], _('Commit IDs'), commit_id))
 
         sources.append(
             ('branch', repo.branches_closed.items(), _('Closed Branches'), branch),
         )
 
         groups = []
         for group_key, ref_list, group_name, match in sources:
             group_refs = []
             for ref_name, ref_id in ref_list:
                 ref_key = '%s:%s:%s' % (group_key, ref_name, ref_id)
                 group_refs.append((ref_key, ref_name))
 
                 if not selected:
                     if set([commit_id, match]) & set([ref_id, ref_name]):
                         selected = ref_key
 
             if group_refs:
                 groups.append((group_refs, group_name))
 
         if not selected:
             ref = commit_id or branch or bookmark
             if ref:
                 raise CommitDoesNotExistError(
                     'No commit refs could be found matching: %s' % ref)
             elif repo.DEFAULT_BRANCH_NAME in repo.branches:
                 selected = 'branch:%s:%s' % (
                     repo.DEFAULT_BRANCH_NAME,
                     repo.branches[repo.DEFAULT_BRANCH_NAME]
                 )
             elif repo.commit_ids:
                 # make the user select in this case
                 selected = None
             else:
                 raise EmptyRepositoryError()
         return groups, selected
 
     def get_diff(self, source_repo, source_ref_id, target_ref_id, context=DIFF_CONTEXT):
         return self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id, context=context)
 
     def _get_diff_from_pr_or_version(
             self, source_repo, source_ref_id, target_ref_id, context):
         target_commit = source_repo.get_commit(
             commit_id=safe_str(target_ref_id))
         source_commit = source_repo.get_commit(
             commit_id=safe_str(source_ref_id))
         if isinstance(source_repo, Repository):
             vcs_repo = source_repo.scm_instance()
         else:
             vcs_repo = source_repo
 
         # TODO: johbo: In the context of an update, we cannot reach
         # the old commit anymore with our normal mechanisms. It needs
         # some sort of special support in the vcs layer to avoid this
         # workaround.
         if (source_commit.raw_id == vcs_repo.EMPTY_COMMIT_ID and
                 vcs_repo.alias == 'git'):
             source_commit.raw_id = safe_str(source_ref_id)
 
         log.debug('calculating diff between '
                   'source_ref:%s and target_ref:%s for repo `%s`',
                   target_ref_id, source_ref_id,
                   safe_unicode(vcs_repo.path))
 
         vcs_diff = vcs_repo.get_diff(
             commit1=target_commit, commit2=source_commit, context=context)
         return vcs_diff
 
     def _is_merge_enabled(self, pull_request):
         return self._get_general_setting(
             pull_request, 'rhodecode_pr_merge_enabled')
 
     def _use_rebase_for_merging(self, pull_request):
         repo_type = pull_request.target_repo.repo_type
         if repo_type == 'hg':
             return self._get_general_setting(
                 pull_request, 'rhodecode_hg_use_rebase_for_merging')
         elif repo_type == 'git':
             return self._get_general_setting(
                 pull_request, 'rhodecode_git_use_rebase_for_merging')
 
         return False
 
     def _close_branch_before_merging(self, pull_request):
         repo_type = pull_request.target_repo.repo_type
         if repo_type == 'hg':
             return self._get_general_setting(
                 pull_request, 'rhodecode_hg_close_branch_before_merging')
         elif repo_type == 'git':
             return self._get_general_setting(
                 pull_request, 'rhodecode_git_close_branch_before_merging')
 
         return False
 
     def _get_general_setting(self, pull_request, settings_key, default=False):
         settings_model = VcsSettingsModel(repo=pull_request.target_repo)
         settings = settings_model.get_general_settings()
         return settings.get(settings_key, default)
 
     def _log_audit_action(self, action, action_data, user, pull_request):
         audit_logger.store(
             action=action,
             action_data=action_data,
             user=user,
             repo=pull_request.target_repo)
 
     def get_reviewer_functions(self):
         """
         Fetches functions for validation and fetching default reviewers.
         If available we use the EE package, else we fallback to CE
         package functions
         """
         try:
             from rc_reviewers.utils import get_default_reviewers_data
             from rc_reviewers.utils import validate_default_reviewers
         except ImportError:
             from rhodecode.apps.repository.utils import \
                 get_default_reviewers_data
             from rhodecode.apps.repository.utils import \
                 validate_default_reviewers
 
         return get_default_reviewers_data, validate_default_reviewers
 
 
 class MergeCheck(object):
     """
     Perform Merge Checks and returns a check object which stores information
     about merge errors, and merge conditions
     """
     TODO_CHECK = 'todo'
     PERM_CHECK = 'perm'
     REVIEW_CHECK = 'review'
     MERGE_CHECK = 'merge'
 
     def __init__(self):
         self.review_status = None
         self.merge_possible = None
         self.merge_msg = ''
         self.failed = None
         self.errors = []
         self.error_details = OrderedDict()
 
     def push_error(self, error_type, message, error_key, details):
         self.failed = True
         self.errors.append([error_type, message])
         self.error_details[error_key] = dict(
             details=details,
             error_type=error_type,
             message=message
         )
 
     @classmethod
     def validate(cls, pull_request, user, translator, fail_early=False,
                  force_shadow_repo_refresh=False):
         _ = translator
         merge_check = cls()
 
         # permissions to merge
         user_allowed_to_merge = PullRequestModel().check_user_merge(
             pull_request, user)
         if not user_allowed_to_merge:
             log.debug("MergeCheck: cannot merge, approval is pending.")
 
             msg = _('User `{}` not allowed to perform merge.').format(user.username)
             merge_check.push_error('error', msg, cls.PERM_CHECK, user.username)
             if fail_early:
                 return merge_check
 
         # review status, must be always present
         review_status = pull_request.calculated_review_status()
         merge_check.review_status = review_status
 
         status_approved = review_status == ChangesetStatus.STATUS_APPROVED
         if not status_approved:
             log.debug("MergeCheck: cannot merge, approval is pending.")
 
             msg = _('Pull request reviewer approval is pending.')
 
             merge_check.push_error(
                 'warning', msg, cls.REVIEW_CHECK, review_status)
 
             if fail_early:
                 return merge_check
 
         # left over TODOs
         todos = CommentsModel().get_unresolved_todos(pull_request)
         if todos:
             log.debug("MergeCheck: cannot merge, {} "
                       "unresolved todos left.".format(len(todos)))
 
             if len(todos) == 1:
                 msg = _('Cannot merge, {} TODO still not resolved.').format(
                     len(todos))
             else:
                 msg = _('Cannot merge, {} TODOs still not resolved.').format(
                     len(todos))
 
             merge_check.push_error('warning', msg, cls.TODO_CHECK, todos)
 
             if fail_early:
                 return merge_check
 
         # merge possible, here is the filesystem simulation + shadow repo
         merge_status, msg = PullRequestModel().merge_status(
             pull_request, translator=translator,
             force_shadow_repo_refresh=force_shadow_repo_refresh)
         merge_check.merge_possible = merge_status
         merge_check.merge_msg = msg
         if not merge_status:
             log.debug(
                 "MergeCheck: cannot merge, pull request merge not possible.")
             merge_check.push_error('warning', msg, cls.MERGE_CHECK, None)
 
             if fail_early:
                 return merge_check
 
         log.debug('MergeCheck: is failed: %s', merge_check.failed)
         return merge_check
 
     @classmethod
     def get_merge_conditions(cls, pull_request, translator):
         _ = translator
         merge_details = {}
 
         model = PullRequestModel()
         use_rebase = model._use_rebase_for_merging(pull_request)
 
         if use_rebase:
             merge_details['merge_strategy'] = dict(
                 details={},
                 message=_('Merge strategy: rebase')
             )
         else:
             merge_details['merge_strategy'] = dict(
                 details={},
                 message=_('Merge strategy: explicit merge commit')
             )
 
         close_branch = model._close_branch_before_merging(pull_request)
         if close_branch:
             repo_type = pull_request.target_repo.repo_type
             if repo_type == 'hg':
                 close_msg = _('Source branch will be closed after merge.')
             elif repo_type == 'git':
                 close_msg = _('Source branch will be deleted after merge.')
 
             merge_details['close_branch'] = dict(
                 details={},
                 message=close_msg
             )
 
         return merge_details
 
 ChangeTuple = collections.namedtuple(
     'ChangeTuple', ['added', 'common', 'removed', 'total'])
 
 FileChangeTuple = collections.namedtuple(
     'FileChangeTuple', ['added', 'modified', 'removed'])

rhodecode/public/js/src/rhodecode/pullrequests.js

@@ -1,550 +1,551 b''
 // # Copyright (C) 2010-2018 RhodeCode GmbH
 // #
 // # This program is free software: you can redistribute it and/or modify
 // # it under the terms of the GNU Affero General Public License, version 3
 // # (only), as published by the Free Software Foundation.
 // #
 // # This program is distributed in the hope that it will be useful,
 // # but WITHOUT ANY WARRANTY; without even the implied warranty of
 // # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 // # GNU General Public License for more details.
 // #
 // # You should have received a copy of the GNU Affero General Public License
 // # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 // #
 // # This program is dual-licensed. If you wish to learn more about the
 // # RhodeCode Enterprise Edition, including its added features, Support services,
 // # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 var prButtonLockChecks = {
   'compare': false,
   'reviewers': false
 };
 
 /**
  * lock button until all checks and loads are made. E.g reviewer calculation
  * should prevent from submitting a PR
  * @param lockEnabled
  * @param msg
  * @param scope
  */
 var prButtonLock = function(lockEnabled, msg, scope) {
       scope = scope || 'all';
       if (scope == 'all'){
           prButtonLockChecks['compare'] = !lockEnabled;
           prButtonLockChecks['reviewers'] = !lockEnabled;
       } else if (scope == 'compare') {
           prButtonLockChecks['compare'] = !lockEnabled;
       } else if (scope == 'reviewers'){
           prButtonLockChecks['reviewers'] = !lockEnabled;
       }
       var checksMeet = prButtonLockChecks.compare && prButtonLockChecks.reviewers;
       if (lockEnabled) {
           $('#pr_submit').attr('disabled', 'disabled');
       }
       else if (checksMeet) {
           $('#pr_submit').removeAttr('disabled');
       }
 
       if (msg) {
           $('#pr_open_message').html(msg);
       }
 };
 
 
 /**
 Generate Title and Description for a PullRequest.
 In case of 1 commits, the title and description is that one commit
 in case of multiple commits, we iterate on them with max N number of commits,
 and build description in a form
 - commitN
 - commitN+1
 ...
 
 Title is then constructed from branch names, or other references,
 replacing '-' and '_' into spaces
 
 * @param sourceRef
 * @param elements
 * @param limit
 * @returns {*[]}
 */
 var getTitleAndDescription = function(sourceRef, elements, limit) {
   var title = '';
   var desc = '';
 
   $.each($(elements).get().reverse().slice(0, limit), function(idx, value) {
       var rawMessage = $(value).find('td.td-description .message').data('messageRaw');
       desc += '- ' + rawMessage.split('\n')[0].replace(/\n+$/, "") + '\n';
   });
   // only 1 commit, use commit message as title
   if (elements.length === 1) {
       title = $(elements[0]).find('td.td-description .message').data('messageRaw').split('\n')[0];
   }
   else {
       // use reference name
       title = sourceRef.replace(/-/g, ' ').replace(/_/g, ' ').capitalizeFirstLetter();
   }
 
   return [title, desc]
 };
 
 
 
 ReviewersController = function () {
     var self = this;
     this.$reviewRulesContainer = $('#review_rules');
     this.$rulesList = this.$reviewRulesContainer.find('.pr-reviewer-rules');
     this.forbidReviewUsers = undefined;
     this.$reviewMembers = $('#review_members');
     this.currentRequest = null;
 
     this.defaultForbidReviewUsers = function() {
         return [
             {'username': 'default',
              'user_id': templateContext.default_user.user_id}
         ];
     };
 
     this.hideReviewRules = function() {
         self.$reviewRulesContainer.hide();
     };
 
     this.showReviewRules = function() {
         self.$reviewRulesContainer.show();
     };
 
     this.addRule = function(ruleText) {
         self.showReviewRules();
         return '<div>- {0}</div>'.format(ruleText)
     };
 
     this.loadReviewRules = function(data) {
         // reset forbidden Users
         this.forbidReviewUsers = self.defaultForbidReviewUsers();
 
         // reset state of review rules
         self.$rulesList.html('');
 
         if (!data || data.rules === undefined || $.isEmptyObject(data.rules)) {
             // default rule, case for older repo that don't have any rules stored
             self.$rulesList.append(
                 self.addRule(
                     _gettext('All reviewers must vote.'))
             );
             return self.forbidReviewUsers
         }
 
         if (data.rules.voting !== undefined) {
             if (data.rules.voting < 0) {
                 self.$rulesList.append(
                     self.addRule(
                          _gettext('All individual reviewers must vote.'))
                 )
             } else if (data.rules.voting === 1) {
                 self.$rulesList.append(
                     self.addRule(
                         _gettext('At least {0} reviewer must vote.').format(data.rules.voting))
                 )
 
             } else {
                 self.$rulesList.append(
                     self.addRule(
                         _gettext('At least {0} reviewers must vote.').format(data.rules.voting))
                 )
             }
         }
 
         if (data.rules.voting_groups !== undefined) {
             $.each(data.rules.voting_groups, function(index, rule_data) {
                 self.$rulesList.append(
                     self.addRule(rule_data.text)
                 )
             });
         }
 
         if (data.rules.use_code_authors_for_review) {
             self.$rulesList.append(
                 self.addRule(
                     _gettext('Reviewers picked from source code changes.'))
             )
         }
         if (data.rules.forbid_adding_reviewers) {
             $('#add_reviewer_input').remove();
             self.$rulesList.append(
                 self.addRule(
                     _gettext('Adding new reviewers is forbidden.'))
             )
         }
         if (data.rules.forbid_author_to_review) {
             self.forbidReviewUsers.push(data.rules_data.pr_author);
             self.$rulesList.append(
                 self.addRule(
                     _gettext('Author is not allowed to be a reviewer.'))
             )
         }
         if (data.rules.forbid_commit_author_to_review) {
 
             if (data.rules_data.forbidden_users) {
                 $.each(data.rules_data.forbidden_users, function(index, member_data) {
                     self.forbidReviewUsers.push(member_data)
                 });
 
             }
 
             self.$rulesList.append(
                 self.addRule(
                     _gettext('Commit Authors are not allowed to be a reviewer.'))
             )
         }
 
         return self.forbidReviewUsers
     };
 
     this.loadDefaultReviewers = function(sourceRepo, sourceRef, targetRepo, targetRef) {
 
         if (self.currentRequest) {
             // make sure we cleanup old running requests before triggering this
             // again
             self.currentRequest.abort();
         }
 
         $('.calculate-reviewers').show();
         // reset reviewer members
         self.$reviewMembers.empty();
 
         prButtonLock(true, null, 'reviewers');
         $('#user').hide(); // hide user autocomplete before load
 
         if (sourceRef.length !== 3 || targetRef.length !== 3) {
             // don't load defaults in case we're missing some refs...
             $('.calculate-reviewers').hide();
             return
         }
 
         var url = pyroutes.url('repo_default_reviewers_data',
                 {
                     'repo_name': templateContext.repo_name,
                     'source_repo': sourceRepo,
                     'source_ref': sourceRef[2],
                     'target_repo': targetRepo,
                     'target_ref': targetRef[2]
                 });
 
         self.currentRequest = $.get(url)
             .done(function(data) {
                 self.currentRequest = null;
 
                 // review rules
                 self.loadReviewRules(data);
 
                 for (var i = 0; i < data.reviewers.length; i++) {
                   var reviewer = data.reviewers[i];
                   self.addReviewMember(
                       reviewer, reviewer.reasons, reviewer.mandatory);
                 }
                 $('.calculate-reviewers').hide();
                 prButtonLock(false, null, 'reviewers');
                 $('#user').show(); // show user autocomplete after load
             });
     };
 
     // check those, refactor
     this.removeReviewMember = function(reviewer_id, mark_delete) {
         var reviewer = $('#reviewer_{0}'.format(reviewer_id));
 
         if(typeof(mark_delete) === undefined){
             mark_delete = false;
         }
 
         if(mark_delete === true){
             if (reviewer){
                 // now delete the input
                 $('#reviewer_{0} input'.format(reviewer_id)).remove();
                 // mark as to-delete
                 var obj = $('#reviewer_{0}_name'.format(reviewer_id));
                 obj.addClass('to-delete');
                 obj.css({"text-decoration":"line-through", "opacity": 0.5});
             }
         }
         else{
             $('#reviewer_{0}'.format(reviewer_id)).remove();
         }
     };
     this.reviewMemberEntry = function() {
 
     };
     this.addReviewMember = function(reviewer_obj, reasons, mandatory) {
         var members = self.$reviewMembers.get(0);
         var id = reviewer_obj.user_id;
         var username = reviewer_obj.username;
 
         var reasons = reasons || [];
         var mandatory = mandatory || false;
 
         // register IDS to check if we don't have this ID already in
         var currentIds = [];
         var _els = self.$reviewMembers.find('li').toArray();
         for (el in _els){
             currentIds.push(_els[el].id)
         }
 
         var userAllowedReview = function(userId) {
             var allowed = true;
             $.each(self.forbidReviewUsers, function(index, member_data) {
                 if (parseInt(userId) === member_data['user_id']) {
                     allowed = false;
                     return false // breaks the loop
                 }
             });
             return allowed
         };
 
         var userAllowed = userAllowedReview(id);
         if (!userAllowed){
            alert(_gettext('User `{0}` not allowed to be a reviewer').format(username));
         } else {
             // only add if it's not there
             var alreadyReviewer = currentIds.indexOf('reviewer_'+id) != -1;
 
             if (alreadyReviewer) {
                 alert(_gettext('User `{0}` already in reviewers').format(username));
             } else {
                 members.innerHTML += renderTemplate('reviewMemberEntry', {
                         'member': reviewer_obj,
                         'mandatory': mandatory,
                         'allowed_to_update': true,
                         'review_status': 'not_reviewed',
                         'review_status_label': _gettext('Not Reviewed'),
                         'reasons': reasons,
                         'create': true
                         });
             }
         }
 
     };
 
     this.updateReviewers = function(repo_name, pull_request_id){
         var postData = $('#reviewers input').serialize();
         _updatePullRequest(repo_name, pull_request_id, postData);
     };
 
 };
 
 
 var _updatePullRequest = function(repo_name, pull_request_id, postData) {
     var url = pyroutes.url(
         'pullrequest_update',
         {"repo_name": repo_name, "pull_request_id": pull_request_id});
     if (typeof postData === 'string' ) {
         postData += '&csrf_token=' + CSRF_TOKEN;
     } else {
         postData.csrf_token = CSRF_TOKEN;
     }
     var success = function(o) {
         window.location.reload();
     };
     ajaxPOST(url, postData, success);
 };
 
 /**
  * PULL REQUEST update commits
  */
 var updateCommits = function(repo_name, pull_request_id) {
     var postData = {
         'update_commits': true};
     _updatePullRequest(repo_name, pull_request_id, postData);
 };
 
 
 /**
  * PULL REQUEST edit info
  */
-var editPullRequest = function(repo_name, pull_request_id, title, description) {
+var editPullRequest = function(repo_name, pull_request_id, title, description, renderer) {
     var url = pyroutes.url(
         'pullrequest_update',
         {"repo_name": repo_name, "pull_request_id": pull_request_id});
 
     var postData = {
         'title': title,
         'description': description,
+        'description_renderer': renderer,
         'edit_pull_request': true,
         'csrf_token': CSRF_TOKEN
     };
     var success = function(o) {
         window.location.reload();
     };
     ajaxPOST(url, postData, success);
 };
 
 
 /**
  * Reviewer autocomplete
  */
 var ReviewerAutoComplete = function(inputId) {
   $(inputId).autocomplete({
     serviceUrl: pyroutes.url('user_autocomplete_data'),
     minChars:2,
     maxHeight:400,
     deferRequestBy: 300, //miliseconds
     showNoSuggestionNotice: true,
     tabDisabled: true,
     autoSelectFirst: true,
     params: { user_id: templateContext.rhodecode_user.user_id, user_groups:true, user_groups_expand:true, skip_default_user:true },
     formatResult: autocompleteFormatResult,
     lookupFilter: autocompleteFilterResult,
     onSelect: function(element, data) {
         var mandatory = false;
         var reasons = [_gettext('added manually by "{0}"').format(templateContext.rhodecode_user.username)];
 
         // add whole user groups
         if (data.value_type == 'user_group') {
             reasons.push(_gettext('member of "{0}"').format(data.value_display));
 
             $.each(data.members, function(index, member_data) {
                 var reviewer = member_data;
                 reviewer['user_id'] = member_data['id'];
                 reviewer['gravatar_link'] = member_data['icon_link'];
                 reviewer['user_link'] = member_data['profile_link'];
                 reviewer['rules'] = [];
                 reviewersController.addReviewMember(reviewer, reasons, mandatory);
             })
         }
         // add single user
         else {
             var reviewer = data;
             reviewer['user_id'] = data['id'];
             reviewer['gravatar_link'] = data['icon_link'];
             reviewer['user_link'] = data['profile_link'];
             reviewer['rules'] = [];
             reviewersController.addReviewMember(reviewer, reasons, mandatory);
         }
 
       $(inputId).val('');
     }
   });
 };
 
 
 VersionController = function () {
     var self = this;
     this.$verSource = $('input[name=ver_source]');
     this.$verTarget = $('input[name=ver_target]');
     this.$showVersionDiff = $('#show-version-diff');
 
     this.adjustRadioSelectors = function (curNode) {
         var getVal = function (item) {
             if (item == 'latest') {
                 return Number.MAX_SAFE_INTEGER
             }
             else {
                 return parseInt(item)
             }
         };
 
         var curVal = getVal($(curNode).val());
         var cleared = false;
 
         $.each(self.$verSource, function (index, value) {
             var elVal = getVal($(value).val());
 
             if (elVal > curVal) {
                 if ($(value).is(':checked')) {
                     cleared = true;
                 }
                 $(value).attr('disabled', 'disabled');
                 $(value).removeAttr('checked');
                 $(value).css({'opacity': 0.1});
             }
             else {
                 $(value).css({'opacity': 1});
                 $(value).removeAttr('disabled');
             }
         });
 
         if (cleared) {
             // if we unchecked an active, set the next one to same loc.
             $(this.$verSource).filter('[value={0}]'.format(
                 curVal)).attr('checked', 'checked');
         }
 
         self.setLockAction(false,
             $(curNode).data('verPos'),
             $(this.$verSource).filter(':checked').data('verPos')
         );
     };
 
 
     this.attachVersionListener = function () {
         self.$verTarget.change(function (e) {
             self.adjustRadioSelectors(this)
         });
         self.$verSource.change(function (e) {
             self.adjustRadioSelectors(self.$verTarget.filter(':checked'))
         });
     };
 
     this.init = function () {
 
         var curNode = self.$verTarget.filter(':checked');
         self.adjustRadioSelectors(curNode);
         self.setLockAction(true);
         self.attachVersionListener();
 
     };
 
     this.setLockAction = function (state, selectedVersion, otherVersion) {
         var $showVersionDiff = this.$showVersionDiff;
 
         if (state) {
             $showVersionDiff.attr('disabled', 'disabled');
             $showVersionDiff.addClass('disabled');
             $showVersionDiff.html($showVersionDiff.data('labelTextLocked'));
         }
         else {
             $showVersionDiff.removeAttr('disabled');
             $showVersionDiff.removeClass('disabled');
 
             if (selectedVersion == otherVersion) {
                 $showVersionDiff.html($showVersionDiff.data('labelTextShow'));
             } else {
                 $showVersionDiff.html($showVersionDiff.data('labelTextDiff'));
             }
         }
 
     };
 
     this.showVersionDiff = function () {
         var target = self.$verTarget.filter(':checked');
         var source = self.$verSource.filter(':checked');
 
         if (target.val() && source.val()) {
             var params = {
                 'pull_request_id': templateContext.pull_request_data.pull_request_id,
                 'repo_name': templateContext.repo_name,
                 'version': target.val(),
                 'from_version': source.val()
             };
             window.location = pyroutes.url('pullrequest_show', params)
         }
 
         return false;
     };
 
     this.toggleVersionView = function (elem) {
 
         if (this.$showVersionDiff.is(':visible')) {
             $('.version-pr').hide();
             this.$showVersionDiff.hide();
             $(elem).html($(elem).data('toggleOn'))
         } else {
             $('.version-pr').show();
             this.$showVersionDiff.show();
             $(elem).html($(elem).data('toggleOff'))
         }
 
         return false
     }
 
 };
  No newline at end of file