code: code cleanups, use is None instead of == None.

rhodecode/apps/admin/views/settings.py

@@ -1,781 +1,780 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import logging
 import collections
 
 import datetime
 import formencode
 import formencode.htmlfill
 
 import rhodecode
 from pyramid.view import view_config
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.apps._base import BaseAppView
 from rhodecode.apps.admin.navigation import navigation_list
 from rhodecode.apps.svn_support.config_keys import generate_config
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.lib.celerylib import tasks, run_task
 from rhodecode.lib.utils import repo2db_mapper
 from rhodecode.lib.utils2 import str2bool, safe_unicode, AttributeDict
 from rhodecode.lib.index import searcher_from_config
 
 from rhodecode.model.db import RhodeCodeUi, Repository
 from rhodecode.model.forms import (ApplicationSettingsForm,
     ApplicationUiSettingsForm, ApplicationVisualisationForm,
     LabsSettingsForm, IssueTrackerPatternsForm)
 from rhodecode.model.repo_group import RepoGroupModel
 
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.notification import EmailNotificationModel
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import (
     IssueTrackerSettingsModel, VcsSettingsModel, SettingNotFound,
     SettingsModel)
 
 
 log = logging.getLogger(__name__)
 
 
 class AdminSettingsView(BaseAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         c.labs_active = str2bool(
             rhodecode.CONFIG.get('labs_settings_active', 'true'))
         c.navlist = navigation_list(self.request)
 
         return c
 
     @classmethod
     def _get_ui_settings(cls):
         ret = RhodeCodeUi.query().all()
 
         if not ret:
             raise Exception('Could not get application ui settings !')
         settings = {}
         for each in ret:
             k = each.ui_key
             v = each.ui_value
             if k == '/':
                 k = 'root_path'
 
             if k in ['push_ssl', 'publish', 'enabled']:
                 v = str2bool(v)
 
             if k.find('.') != -1:
                 k = k.replace('.', '_')
 
             if each.ui_section in ['hooks', 'extensions']:
                 v = each.ui_active
 
             settings[each.ui_section + '_' + k] = v
         return settings
 
     @classmethod
     def _form_defaults(cls):
         defaults = SettingsModel().get_all_settings()
         defaults.update(cls._get_ui_settings())
 
         defaults.update({
             'new_svn_branch': '',
             'new_svn_tag': '',
         })
         return defaults
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='admin_settings_vcs', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_vcs(self):
         c = self.load_default_context()
         c.active = 'vcs'
         model = VcsSettingsModel()
         c.svn_branch_patterns = model.get_global_svn_branch_patterns()
         c.svn_tag_patterns = model.get_global_svn_tag_patterns()
 
         settings = self.request.registry.settings
         c.svn_proxy_generate_config = settings[generate_config]
 
         defaults = self._form_defaults()
 
         model.create_largeobjects_dirs_if_needed(defaults['paths_root_path'])
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='admin_settings_vcs_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_vcs_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'vcs'
 
         model = VcsSettingsModel()
         c.svn_branch_patterns = model.get_global_svn_branch_patterns()
         c.svn_tag_patterns = model.get_global_svn_tag_patterns()
 
         settings = self.request.registry.settings
         c.svn_proxy_generate_config = settings[generate_config]
 
         application_form = ApplicationUiSettingsForm(self.request.translate)()
 
         try:
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
             h.flash(
                 _("Some form inputs contain invalid data."),
                 category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         try:
             if c.visual.allow_repo_location_change:
-                model.update_global_path_setting(
-                    form_result['paths_root_path'])
+                model.update_global_path_setting(form_result['paths_root_path'])
 
             model.update_global_ssl_setting(form_result['web_push_ssl'])
             model.update_global_hook_settings(form_result)
 
             model.create_or_update_global_svn_settings(form_result)
             model.create_or_update_global_hg_settings(form_result)
             model.create_or_update_global_git_settings(form_result)
             model.create_or_update_global_pr_settings(form_result)
         except Exception:
             log.exception("Exception while updating settings")
             h.flash(_('Error occurred during updating '
                       'application settings'), category='error')
         else:
             Session().commit()
             h.flash(_('Updated VCS settings'), category='success')
             raise HTTPFound(h.route_path('admin_settings_vcs'))
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='admin_settings_vcs_svn_pattern_delete', request_method='POST',
         renderer='json_ext', xhr=True)
     def settings_vcs_delete_svn_pattern(self):
         delete_pattern_id = self.request.POST.get('delete_svn_pattern')
         model = VcsSettingsModel()
         try:
             model.delete_global_svn_pattern(delete_pattern_id)
         except SettingNotFound:
             log.exception(
                 'Failed to delete svn_pattern with id %s', delete_pattern_id)
             raise HTTPNotFound()
 
         Session().commit()
         return True
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='admin_settings_mapping', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_mapping(self):
         c = self.load_default_context()
         c.active = 'mapping'
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='admin_settings_mapping_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_mapping_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'mapping'
         rm_obsolete = self.request.POST.get('destroy', False)
         invalidate_cache = self.request.POST.get('invalidate', False)
         log.debug(
             'rescanning repo location with destroy obsolete=%s', rm_obsolete)
 
         if invalidate_cache:
             log.debug('invalidating all repositories cache')
             for repo in Repository.get_all():
                 ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
 
         filesystem_repos = ScmModel().repo_scan()
         added, removed = repo2db_mapper(filesystem_repos, rm_obsolete)
         _repr = lambda l: ', '.join(map(safe_unicode, l)) or '-'
         h.flash(_('Repositories successfully '
                   'rescanned added: %s ; removed: %s') %
                 (_repr(added), _repr(removed)),
                 category='success')
         raise HTTPFound(h.route_path('admin_settings_mapping'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='admin_settings', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     @view_config(
         route_name='admin_settings_global', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_global(self):
         c = self.load_default_context()
         c.active = 'global'
         c.personal_repo_group_default_pattern = RepoGroupModel()\
             .get_personal_group_name_pattern()
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='admin_settings_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     @view_config(
         route_name='admin_settings_global_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_global_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'global'
         c.personal_repo_group_default_pattern = RepoGroupModel()\
             .get_personal_group_name_pattern()
         application_form = ApplicationSettingsForm(self.request.translate)()
         try:
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
             h.flash(
                 _("Some form inputs contain invalid data."),
                 category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         settings = [
             ('title', 'rhodecode_title', 'unicode'),
             ('realm', 'rhodecode_realm', 'unicode'),
             ('pre_code', 'rhodecode_pre_code', 'unicode'),
             ('post_code', 'rhodecode_post_code', 'unicode'),
             ('captcha_public_key', 'rhodecode_captcha_public_key', 'unicode'),
             ('captcha_private_key', 'rhodecode_captcha_private_key', 'unicode'),
             ('create_personal_repo_group', 'rhodecode_create_personal_repo_group', 'bool'),
             ('personal_repo_group_pattern', 'rhodecode_personal_repo_group_pattern', 'unicode'),
         ]
         try:
             for setting, form_key, type_ in settings:
                 sett = SettingsModel().create_or_update_setting(
                     setting, form_result[form_key], type_)
                 Session().add(sett)
 
             Session().commit()
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated application settings'), category='success')
         except Exception:
             log.exception("Exception while updating application settings")
             h.flash(
                 _('Error occurred during updating application settings'),
                 category='error')
 
         raise HTTPFound(h.route_path('admin_settings_global'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='admin_settings_visual', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_visual(self):
         c = self.load_default_context()
         c.active = 'visual'
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='admin_settings_visual_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_visual_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'visual'
         application_form = ApplicationVisualisationForm(self.request.translate)()
         try:
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
             h.flash(
                 _("Some form inputs contain invalid data."),
                 category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         try:
             settings = [
                 ('show_public_icon', 'rhodecode_show_public_icon', 'bool'),
                 ('show_private_icon', 'rhodecode_show_private_icon', 'bool'),
                 ('stylify_metatags', 'rhodecode_stylify_metatags', 'bool'),
                 ('repository_fields', 'rhodecode_repository_fields', 'bool'),
                 ('dashboard_items', 'rhodecode_dashboard_items', 'int'),
                 ('admin_grid_items', 'rhodecode_admin_grid_items', 'int'),
                 ('show_version', 'rhodecode_show_version', 'bool'),
                 ('use_gravatar', 'rhodecode_use_gravatar', 'bool'),
                 ('markup_renderer', 'rhodecode_markup_renderer', 'unicode'),
                 ('gravatar_url', 'rhodecode_gravatar_url', 'unicode'),
                 ('clone_uri_tmpl', 'rhodecode_clone_uri_tmpl', 'unicode'),
                 ('clone_uri_ssh_tmpl', 'rhodecode_clone_uri_ssh_tmpl', 'unicode'),
                 ('support_url', 'rhodecode_support_url', 'unicode'),
                 ('show_revision_number', 'rhodecode_show_revision_number', 'bool'),
                 ('show_sha_length', 'rhodecode_show_sha_length', 'int'),
             ]
             for setting, form_key, type_ in settings:
                 sett = SettingsModel().create_or_update_setting(
                     setting, form_result[form_key], type_)
                 Session().add(sett)
 
             Session().commit()
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated visualisation settings'), category='success')
         except Exception:
             log.exception("Exception updating visualization settings")
             h.flash(_('Error occurred during updating '
                       'visualisation settings'),
                     category='error')
 
         raise HTTPFound(h.route_path('admin_settings_visual'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='admin_settings_issuetracker', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_issuetracker(self):
         c = self.load_default_context()
         c.active = 'issuetracker'
         defaults = SettingsModel().get_all_settings()
 
         entry_key = 'rhodecode_issuetracker_pat_'
 
         c.issuetracker_entries = {}
         for k, v in defaults.items():
             if k.startswith(entry_key):
                 uid = k[len(entry_key):]
                 c.issuetracker_entries[uid] = None
 
         for uid in c.issuetracker_entries:
             c.issuetracker_entries[uid] = AttributeDict({
                 'pat': defaults.get('rhodecode_issuetracker_pat_' + uid),
                 'url': defaults.get('rhodecode_issuetracker_url_' + uid),
                 'pref': defaults.get('rhodecode_issuetracker_pref_' + uid),
                 'desc': defaults.get('rhodecode_issuetracker_desc_' + uid),
             })
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='admin_settings_issuetracker_test', request_method='POST',
         renderer='string', xhr=True)
     def settings_issuetracker_test(self):
         return h.urlify_commit_message(
             self.request.POST.get('test_text', ''),
             'repo_group/test_repo1')
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='admin_settings_issuetracker_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_issuetracker_update(self):
         _ = self.request.translate
         self.load_default_context()
         settings_model = IssueTrackerSettingsModel()
 
         try:
             form = IssueTrackerPatternsForm(self.request.translate)()
             data = form.to_python(self.request.POST)
         except formencode.Invalid as errors:
             log.exception('Failed to add new pattern')
             error = errors
             h.flash(_('Invalid issue tracker pattern: {}'.format(error)),
                     category='error')
             raise HTTPFound(h.route_path('admin_settings_issuetracker'))
 
         if data:
             for uid in data.get('delete_patterns', []):
                 settings_model.delete_entries(uid)
 
             for pattern in data.get('patterns', []):
                 for setting, value, type_ in pattern:
                     sett = settings_model.create_or_update_setting(
                         setting, value, type_)
                     Session().add(sett)
 
                 Session().commit()
 
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated issue tracker entries'), category='success')
         raise HTTPFound(h.route_path('admin_settings_issuetracker'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='admin_settings_issuetracker_delete', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_issuetracker_delete(self):
         _ = self.request.translate
         self.load_default_context()
         uid = self.request.POST.get('uid')
         try:
             IssueTrackerSettingsModel().delete_entries(uid)
         except Exception:
             log.exception('Failed to delete issue tracker setting %s', uid)
             raise HTTPNotFound()
         h.flash(_('Removed issue tracker entry'), category='success')
         raise HTTPFound(h.route_path('admin_settings_issuetracker'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='admin_settings_email', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_email(self):
         c = self.load_default_context()
         c.active = 'email'
         c.rhodecode_ini = rhodecode.CONFIG
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='admin_settings_email_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_email_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'email'
 
         test_email = self.request.POST.get('test_email')
 
         if not test_email:
             h.flash(_('Please enter email address'), category='error')
             raise HTTPFound(h.route_path('admin_settings_email'))
 
         email_kwargs = {
             'date': datetime.datetime.now(),
             'user': c.rhodecode_user,
             'rhodecode_version': c.rhodecode_version
         }
 
         (subject, headers, email_body,
          email_body_plaintext) = EmailNotificationModel().render_email(
             EmailNotificationModel.TYPE_EMAIL_TEST, **email_kwargs)
 
         recipients = [test_email] if test_email else None
 
         run_task(tasks.send_email, recipients, subject,
                  email_body_plaintext, email_body)
 
         h.flash(_('Send email task created'), category='success')
         raise HTTPFound(h.route_path('admin_settings_email'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='admin_settings_hooks', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_hooks(self):
         c = self.load_default_context()
         c.active = 'hooks'
 
         model = SettingsModel()
         c.hooks = model.get_builtin_hooks()
         c.custom_hooks = model.get_custom_hooks()
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='admin_settings_hooks_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     @view_config(
         route_name='admin_settings_hooks_delete', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_hooks_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'hooks'
         if c.visual.allow_custom_hooks_settings:
             ui_key = self.request.POST.get('new_hook_ui_key')
             ui_value = self.request.POST.get('new_hook_ui_value')
 
             hook_id = self.request.POST.get('hook_id')
             new_hook = False
 
             model = SettingsModel()
             try:
                 if ui_value and ui_key:
                     model.create_or_update_hook(ui_key, ui_value)
                     h.flash(_('Added new hook'), category='success')
                     new_hook = True
                 elif hook_id:
                     RhodeCodeUi.delete(hook_id)
                     Session().commit()
 
                 # check for edits
                 update = False
                 _d = self.request.POST.dict_of_lists()
                 for k, v in zip(_d.get('hook_ui_key', []),
                                 _d.get('hook_ui_value_new', [])):
                     model.create_or_update_hook(k, v)
                     update = True
 
                 if update and not new_hook:
                     h.flash(_('Updated hooks'), category='success')
                 Session().commit()
             except Exception:
                 log.exception("Exception during hook creation")
                 h.flash(_('Error occurred during hook creation'),
                         category='error')
 
         raise HTTPFound(h.route_path('admin_settings_hooks'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='admin_settings_search', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_search(self):
         c = self.load_default_context()
         c.active = 'search'
 
         searcher = searcher_from_config(self.request.registry.settings)
         c.statistics = searcher.statistics(self.request.translate)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='admin_settings_automation', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_automation(self):
         c = self.load_default_context()
         c.active = 'automation'
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @view_config(
         route_name='admin_settings_labs', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_labs(self):
         c = self.load_default_context()
         if not c.labs_active:
             raise HTTPFound(h.route_path('admin_settings'))
 
         c.active = 'labs'
         c.lab_settings = _LAB_SETTINGS
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     @view_config(
         route_name='admin_settings_labs_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
     def settings_labs_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'labs'
 
         application_form = LabsSettingsForm(self.request.translate)()
         try:
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
             h.flash(
                 _("Some form inputs contain invalid data."),
                 category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.error_dict or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         try:
             session = Session()
             for setting in _LAB_SETTINGS:
                 setting_name = setting.key[len('rhodecode_'):]
                 sett = SettingsModel().create_or_update_setting(
                     setting_name, form_result[setting.key], setting.type)
                 session.add(sett)
 
         except Exception:
             log.exception('Exception while updating lab settings')
             h.flash(_('Error occurred during updating labs settings'),
                     category='error')
         else:
             Session().commit()
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated Labs settings'), category='success')
             raise HTTPFound(h.route_path('admin_settings_labs'))
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
 
 # :param key: name of the setting including the 'rhodecode_' prefix
 # :param type: the RhodeCodeSetting type to use.
 # :param group: the i18ned group in which we should dispaly this setting
 # :param label: the i18ned label we should display for this setting
 # :param help: the i18ned help we should dispaly for this setting
 LabSetting = collections.namedtuple(
     'LabSetting', ('key', 'type', 'group', 'label', 'help'))
 
 
 # This list has to be kept in sync with the form
 # rhodecode.model.forms.LabsSettingsForm.
 _LAB_SETTINGS = [
 
 ]

rhodecode/lib/diff_match_patch.py

@@ -1,1919 +1,1919 b''
 #!/usr/bin/python2.4
 
 from __future__ import division
 
 """Diff Match and Patch
 
 Copyright 2006 Google Inc.
 http://code.google.com/p/google-diff-match-patch/
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
   http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
 
 """Functions for diff, match and patch.
 
 Computes the difference between two texts to create a patch.
 Applies the patch onto another text, allowing for errors.
 """
 
 __author__ = '[e-mail] (Neil Fraser)'
 
 import math
 import re
 import sys
 import time
 import urllib
 
 class diff_match_patch:
   """Class containing the diff, match and patch methods.
 
   Also contains the behaviour settings.
   """
 
   def __init__(self):
     """Inits a diff_match_patch object with default settings.
     Redefine these in your program to override the defaults.
     """
 
     # Number of seconds to map a diff before giving up (0 for infinity).
     self.Diff_Timeout = 1.0
     # Cost of an empty edit operation in terms of edit characters.
     self.Diff_EditCost = 4
     # At what point is no match declared (0.0 = perfection, 1.0 = very loose).
     self.Match_Threshold = 0.5
     # How far to search for a match (0 = exact location, 1000+ = broad match).
     # A match this many characters away from the expected location will add
     # 1.0 to the score (0.0 is a perfect match).
     self.Match_Distance = 1000
     # When deleting a large block of text (over ~64 characters), how close do
     # the contents have to be to match the expected contents. (0.0 = perfection,
     # 1.0 = very loose).  Note that Match_Threshold controls how closely the
     # end points of a delete need to match.
     self.Patch_DeleteThreshold = 0.5
     # Chunk size for context length.
     self.Patch_Margin = 4
 
     # The number of bits in an int.
     # Python has no maximum, thus to disable patch splitting set to 0.
     # However to avoid long patches in certain pathological cases, use 32.
     # Multiple short patches (using native ints) are much faster than long ones.
     self.Match_MaxBits = 32
 
   #  DIFF FUNCTIONS
 
   # The data structure representing a diff is an array of tuples:
   # [(DIFF_DELETE, "Hello"), (DIFF_INSERT, "Goodbye"), (DIFF_EQUAL, " world.")]
   # which means: delete "Hello", add "Goodbye" and keep " world."
   DIFF_DELETE = -1
   DIFF_INSERT = 1
   DIFF_EQUAL = 0
 
   def diff_main(self, text1, text2, checklines=True, deadline=None):
     """Find the differences between two texts.  Simplifies the problem by
       stripping any common prefix or suffix off the texts before diffing.
 
     Args:
       text1: Old string to be diffed.
       text2: New string to be diffed.
       checklines: Optional speedup flag.  If present and false, then don't run
         a line-level diff first to identify the changed areas.
         Defaults to true, which does a faster, slightly less optimal diff.
       deadline: Optional time when the diff should be complete by.  Used
         internally for recursive calls.  Users should set DiffTimeout instead.
 
     Returns:
       Array of changes.
     """
     # Set a deadline by which time the diff must be complete.
-    if deadline == None:
+    if deadline is None:
       # Unlike in most languages, Python counts time in seconds.
       if self.Diff_Timeout <= 0:
         deadline = sys.maxint
       else:
         deadline = time.time() + self.Diff_Timeout
 
     # Check for null inputs.
-    if text1 == None or text2 == None:
+    if text1 is None or text2 is None:
       raise ValueError("Null inputs. (diff_main)")
 
     # Check for equality (speedup).
     if text1 == text2:
       if text1:
         return [(self.DIFF_EQUAL, text1)]
       return []
 
     # Trim off common prefix (speedup).
     commonlength = self.diff_commonPrefix(text1, text2)
     commonprefix = text1[:commonlength]
     text1 = text1[commonlength:]
     text2 = text2[commonlength:]
 
     # Trim off common suffix (speedup).
     commonlength = self.diff_commonSuffix(text1, text2)
     if commonlength == 0:
       commonsuffix = ''
     else:
       commonsuffix = text1[-commonlength:]
       text1 = text1[:-commonlength]
       text2 = text2[:-commonlength]
 
     # Compute the diff on the middle block.
     diffs = self.diff_compute(text1, text2, checklines, deadline)
 
     # Restore the prefix and suffix.
     if commonprefix:
       diffs[:0] = [(self.DIFF_EQUAL, commonprefix)]
     if commonsuffix:
       diffs.append((self.DIFF_EQUAL, commonsuffix))
     self.diff_cleanupMerge(diffs)
     return diffs
 
   def diff_compute(self, text1, text2, checklines, deadline):
     """Find the differences between two texts.  Assumes that the texts do not
       have any common prefix or suffix.
 
     Args:
       text1: Old string to be diffed.
       text2: New string to be diffed.
       checklines: Speedup flag.  If false, then don't run a line-level diff
         first to identify the changed areas.
         If true, then run a faster, slightly less optimal diff.
       deadline: Time when the diff should be complete by.
 
     Returns:
       Array of changes.
     """
     if not text1:
       # Just add some text (speedup).
       return [(self.DIFF_INSERT, text2)]
 
     if not text2:
       # Just delete some text (speedup).
       return [(self.DIFF_DELETE, text1)]
 
     if len(text1) > len(text2):
       (longtext, shorttext) = (text1, text2)
     else:
       (shorttext, longtext) = (text1, text2)
     i = longtext.find(shorttext)
     if i != -1:
       # Shorter text is inside the longer text (speedup).
       diffs = [(self.DIFF_INSERT, longtext[:i]), (self.DIFF_EQUAL, shorttext),
                (self.DIFF_INSERT, longtext[i + len(shorttext):])]
       # Swap insertions for deletions if diff is reversed.
       if len(text1) > len(text2):
         diffs[0] = (self.DIFF_DELETE, diffs[0][1])
         diffs[2] = (self.DIFF_DELETE, diffs[2][1])
       return diffs
 
     if len(shorttext) == 1:
       # Single character string.
       # After the previous speedup, the character can't be an equality.
       return [(self.DIFF_DELETE, text1), (self.DIFF_INSERT, text2)]
 
     # Check to see if the problem can be split in two.
     hm = self.diff_halfMatch(text1, text2)
     if hm:
       # A half-match was found, sort out the return data.
       (text1_a, text1_b, text2_a, text2_b, mid_common) = hm
       # Send both pairs off for separate processing.
       diffs_a = self.diff_main(text1_a, text2_a, checklines, deadline)
       diffs_b = self.diff_main(text1_b, text2_b, checklines, deadline)
       # Merge the results.
       return diffs_a + [(self.DIFF_EQUAL, mid_common)] + diffs_b
 
     if checklines and len(text1) > 100 and len(text2) > 100:
       return self.diff_lineMode(text1, text2, deadline)
 
     return self.diff_bisect(text1, text2, deadline)
 
   def diff_lineMode(self, text1, text2, deadline):
     """Do a quick line-level diff on both strings, then rediff the parts for
       greater accuracy.
       This speedup can produce non-minimal diffs.
 
     Args:
       text1: Old string to be diffed.
       text2: New string to be diffed.
       deadline: Time when the diff should be complete by.
 
     Returns:
       Array of changes.
     """
 
     # Scan the text on a line-by-line basis first.
     (text1, text2, linearray) = self.diff_linesToChars(text1, text2)
 
     diffs = self.diff_main(text1, text2, False, deadline)
 
     # Convert the diff back to original text.
     self.diff_charsToLines(diffs, linearray)
     # Eliminate freak matches (e.g. blank lines)
     self.diff_cleanupSemantic(diffs)
 
     # Rediff any replacement blocks, this time character-by-character.
     # Add a dummy entry at the end.
     diffs.append((self.DIFF_EQUAL, ''))
     pointer = 0
     count_delete = 0
     count_insert = 0
     text_delete = ''
     text_insert = ''
     while pointer < len(diffs):
       if diffs[pointer][0] == self.DIFF_INSERT:
         count_insert += 1
         text_insert += diffs[pointer][1]
       elif diffs[pointer][0] == self.DIFF_DELETE:
         count_delete += 1
         text_delete += diffs[pointer][1]
       elif diffs[pointer][0] == self.DIFF_EQUAL:
         # Upon reaching an equality, check for prior redundancies.
         if count_delete >= 1 and count_insert >= 1:
           # Delete the offending records and add the merged ones.
           a = self.diff_main(text_delete, text_insert, False, deadline)
           diffs[pointer - count_delete - count_insert : pointer] = a
           pointer = pointer - count_delete - count_insert + len(a)
         count_insert = 0
         count_delete = 0
         text_delete = ''
         text_insert = ''
 
       pointer += 1
 
     diffs.pop()  # Remove the dummy entry at the end.
 
     return diffs
 
   def diff_bisect(self, text1, text2, deadline):
     """Find the 'middle snake' of a diff, split the problem in two
       and return the recursively constructed diff.
       See Myers 1986 paper: An O(ND) Difference Algorithm and Its Variations.
 
     Args:
       text1: Old string to be diffed.
       text2: New string to be diffed.
       deadline: Time at which to bail if not yet complete.
 
     Returns:
       Array of diff tuples.
     """
 
     # Cache the text lengths to prevent multiple calls.
     text1_length = len(text1)
     text2_length = len(text2)
     max_d = (text1_length + text2_length + 1) // 2
     v_offset = max_d
     v_length = 2 * max_d
     v1 = [-1] * v_length
     v1[v_offset + 1] = 0
     v2 = v1[:]
     delta = text1_length - text2_length
     # If the total number of characters is odd, then the front path will
     # collide with the reverse path.
     front = (delta % 2 != 0)
     # Offsets for start and end of k loop.
     # Prevents mapping of space beyond the grid.
     k1start = 0
     k1end = 0
     k2start = 0
     k2end = 0
     for d in xrange(max_d):
       # Bail out if deadline is reached.
       if time.time() > deadline:
         break
 
       # Walk the front path one step.
       for k1 in xrange(-d + k1start, d + 1 - k1end, 2):
         k1_offset = v_offset + k1
         if k1 == -d or (k1 != d and
             v1[k1_offset - 1] < v1[k1_offset + 1]):
           x1 = v1[k1_offset + 1]
         else:
           x1 = v1[k1_offset - 1] + 1
         y1 = x1 - k1
         while (x1 < text1_length and y1 < text2_length and
                text1[x1] == text2[y1]):
           x1 += 1
           y1 += 1
         v1[k1_offset] = x1
         if x1 > text1_length:
           # Ran off the right of the graph.
           k1end += 2
         elif y1 > text2_length:
           # Ran off the bottom of the graph.
           k1start += 2
         elif front:
           k2_offset = v_offset + delta - k1
           if k2_offset >= 0 and k2_offset < v_length and v2[k2_offset] != -1:
             # Mirror x2 onto top-left coordinate system.
             x2 = text1_length - v2[k2_offset]
             if x1 >= x2:
               # Overlap detected.
               return self.diff_bisectSplit(text1, text2, x1, y1, deadline)
 
       # Walk the reverse path one step.
       for k2 in xrange(-d + k2start, d + 1 - k2end, 2):
         k2_offset = v_offset + k2
         if k2 == -d or (k2 != d and
             v2[k2_offset - 1] < v2[k2_offset + 1]):
           x2 = v2[k2_offset + 1]
         else:
           x2 = v2[k2_offset - 1] + 1
         y2 = x2 - k2
         while (x2 < text1_length and y2 < text2_length and
                text1[-x2 - 1] == text2[-y2 - 1]):
           x2 += 1
           y2 += 1
         v2[k2_offset] = x2
         if x2 > text1_length:
           # Ran off the left of the graph.
           k2end += 2
         elif y2 > text2_length:
           # Ran off the top of the graph.
           k2start += 2
         elif not front:
           k1_offset = v_offset + delta - k2
           if k1_offset >= 0 and k1_offset < v_length and v1[k1_offset] != -1:
             x1 = v1[k1_offset]
             y1 = v_offset + x1 - k1_offset
             # Mirror x2 onto top-left coordinate system.
             x2 = text1_length - x2
             if x1 >= x2:
               # Overlap detected.
               return self.diff_bisectSplit(text1, text2, x1, y1, deadline)
 
     # Diff took too long and hit the deadline or
     # number of diffs equals number of characters, no commonality at all.
     return [(self.DIFF_DELETE, text1), (self.DIFF_INSERT, text2)]
 
   def diff_bisectSplit(self, text1, text2, x, y, deadline):
     """Given the location of the 'middle snake', split the diff in two parts
     and recurse.
 
     Args:
       text1: Old string to be diffed.
       text2: New string to be diffed.
       x: Index of split point in text1.
       y: Index of split point in text2.
       deadline: Time at which to bail if not yet complete.
 
     Returns:
       Array of diff tuples.
     """
     text1a = text1[:x]
     text2a = text2[:y]
     text1b = text1[x:]
     text2b = text2[y:]
 
     # Compute both diffs serially.
     diffs = self.diff_main(text1a, text2a, False, deadline)
     diffsb = self.diff_main(text1b, text2b, False, deadline)
 
     return diffs + diffsb
 
   def diff_linesToChars(self, text1, text2):
     """Split two texts into an array of strings.  Reduce the texts to a string
     of hashes where each Unicode character represents one line.
 
     Args:
       text1: First string.
       text2: Second string.
 
     Returns:
       Three element tuple, containing the encoded text1, the encoded text2 and
       the array of unique strings.  The zeroth element of the array of unique
       strings is intentionally blank.
     """
     lineArray = []  # e.g. lineArray[4] == "Hello\n"
     lineHash = {}   # e.g. lineHash["Hello\n"] == 4
 
     # "\x00" is a valid character, but various debuggers don't like it.
     # So we'll insert a junk entry to avoid generating a null character.
     lineArray.append('')
 
     def diff_linesToCharsMunge(text):
       """Split a text into an array of strings.  Reduce the texts to a string
       of hashes where each Unicode character represents one line.
       Modifies linearray and linehash through being a closure.
 
       Args:
         text: String to encode.
 
       Returns:
         Encoded string.
       """
       chars = []
       # Walk the text, pulling out a substring for each line.
       # text.split('\n') would would temporarily double our memory footprint.
       # Modifying text would create many large strings to garbage collect.
       lineStart = 0
       lineEnd = -1
       while lineEnd < len(text) - 1:
         lineEnd = text.find('\n', lineStart)
         if lineEnd == -1:
           lineEnd = len(text) - 1
         line = text[lineStart:lineEnd + 1]
         lineStart = lineEnd + 1
 
         if line in lineHash:
           chars.append(unichr(lineHash[line]))
         else:
           lineArray.append(line)
           lineHash[line] = len(lineArray) - 1
           chars.append(unichr(len(lineArray) - 1))
       return "".join(chars)
 
     chars1 = diff_linesToCharsMunge(text1)
     chars2 = diff_linesToCharsMunge(text2)
     return (chars1, chars2, lineArray)
 
   def diff_charsToLines(self, diffs, lineArray):
     """Rehydrate the text in a diff from a string of line hashes to real lines
     of text.
 
     Args:
       diffs: Array of diff tuples.
       lineArray: Array of unique strings.
     """
     for x in xrange(len(diffs)):
       text = []
       for char in diffs[x][1]:
         text.append(lineArray[ord(char)])
       diffs[x] = (diffs[x][0], "".join(text))
 
   def diff_commonPrefix(self, text1, text2):
     """Determine the common prefix of two strings.
 
     Args:
       text1: First string.
       text2: Second string.
 
     Returns:
       The number of characters common to the start of each string.
     """
     # Quick check for common null cases.
     if not text1 or not text2 or text1[0] != text2[0]:
       return 0
     # Binary search.
     # Performance analysis: http://neil.fraser.name/news/2007/10/09/
     pointermin = 0
     pointermax = min(len(text1), len(text2))
     pointermid = pointermax
     pointerstart = 0
     while pointermin < pointermid:
       if text1[pointerstart:pointermid] == text2[pointerstart:pointermid]:
         pointermin = pointermid
         pointerstart = pointermin
       else:
         pointermax = pointermid
       pointermid = (pointermax - pointermin) // 2 + pointermin
     return pointermid
 
   def diff_commonSuffix(self, text1, text2):
     """Determine the common suffix of two strings.
 
     Args:
       text1: First string.
       text2: Second string.
 
     Returns:
       The number of characters common to the end of each string.
     """
     # Quick check for common null cases.
     if not text1 or not text2 or text1[-1] != text2[-1]:
       return 0
     # Binary search.
     # Performance analysis: http://neil.fraser.name/news/2007/10/09/
     pointermin = 0
     pointermax = min(len(text1), len(text2))
     pointermid = pointermax
     pointerend = 0
     while pointermin < pointermid:
       if (text1[-pointermid:len(text1) - pointerend] ==
           text2[-pointermid:len(text2) - pointerend]):
         pointermin = pointermid
         pointerend = pointermin
       else:
         pointermax = pointermid
       pointermid = (pointermax - pointermin) // 2 + pointermin
     return pointermid
 
   def diff_commonOverlap(self, text1, text2):
     """Determine if the suffix of one string is the prefix of another.
 
     Args:
       text1 First string.
       text2 Second string.
 
     Returns:
       The number of characters common to the end of the first
       string and the start of the second string.
     """
     # Cache the text lengths to prevent multiple calls.
     text1_length = len(text1)
     text2_length = len(text2)
     # Eliminate the null case.
     if text1_length == 0 or text2_length == 0:
       return 0
     # Truncate the longer string.
     if text1_length > text2_length:
       text1 = text1[-text2_length:]
     elif text1_length < text2_length:
       text2 = text2[:text1_length]
     text_length = min(text1_length, text2_length)
     # Quick check for the worst case.
     if text1 == text2:
       return text_length
 
     # Start by looking for a single character match
     # and increase length until no match is found.
     # Performance analysis: http://neil.fraser.name/news/2010/11/04/
     best = 0
     length = 1
     while True:
       pattern = text1[-length:]
       found = text2.find(pattern)
       if found == -1:
         return best
       length += found
       if found == 0 or text1[-length:] == text2[:length]:
         best = length
         length += 1
 
   def diff_halfMatch(self, text1, text2):
     """Do the two texts share a substring which is at least half the length of
     the longer text?
     This speedup can produce non-minimal diffs.
 
     Args:
       text1: First string.
       text2: Second string.
 
     Returns:
       Five element Array, containing the prefix of text1, the suffix of text1,
       the prefix of text2, the suffix of text2 and the common middle.  Or None
       if there was no match.
     """
     if self.Diff_Timeout <= 0:
       # Don't risk returning a non-optimal diff if we have unlimited time.
       return None
     if len(text1) > len(text2):
       (longtext, shorttext) = (text1, text2)
     else:
       (shorttext, longtext) = (text1, text2)
     if len(longtext) < 4 or len(shorttext) * 2 < len(longtext):
       return None  # Pointless.
 
     def diff_halfMatchI(longtext, shorttext, i):
       """Does a substring of shorttext exist within longtext such that the
       substring is at least half the length of longtext?
       Closure, but does not reference any external variables.
 
       Args:
         longtext: Longer string.
         shorttext: Shorter string.
         i: Start index of quarter length substring within longtext.
 
       Returns:
         Five element Array, containing the prefix of longtext, the suffix of
         longtext, the prefix of shorttext, the suffix of shorttext and the
         common middle.  Or None if there was no match.
       """
       seed = longtext[i:i + len(longtext) // 4]
       best_common = ''
       j = shorttext.find(seed)
       while j != -1:
         prefixLength = self.diff_commonPrefix(longtext[i:], shorttext[j:])
         suffixLength = self.diff_commonSuffix(longtext[:i], shorttext[:j])
         if len(best_common) < suffixLength + prefixLength:
           best_common = (shorttext[j - suffixLength:j] +
               shorttext[j:j + prefixLength])
           best_longtext_a = longtext[:i - suffixLength]
           best_longtext_b = longtext[i + prefixLength:]
           best_shorttext_a = shorttext[:j - suffixLength]
           best_shorttext_b = shorttext[j + prefixLength:]
         j = shorttext.find(seed, j + 1)
 
       if len(best_common) * 2 >= len(longtext):
         return (best_longtext_a, best_longtext_b,
                 best_shorttext_a, best_shorttext_b, best_common)
       else:
         return None
 
     # First check if the second quarter is the seed for a half-match.
     hm1 = diff_halfMatchI(longtext, shorttext, (len(longtext) + 3) // 4)
     # Check again based on the third quarter.
     hm2 = diff_halfMatchI(longtext, shorttext, (len(longtext) + 1) // 2)
     if not hm1 and not hm2:
       return None
     elif not hm2:
       hm = hm1
     elif not hm1:
       hm = hm2
     else:
       # Both matched.  Select the longest.
       if len(hm1[4]) > len(hm2[4]):
         hm = hm1
       else:
         hm = hm2
 
     # A half-match was found, sort out the return data.
     if len(text1) > len(text2):
       (text1_a, text1_b, text2_a, text2_b, mid_common) = hm
     else:
       (text2_a, text2_b, text1_a, text1_b, mid_common) = hm
     return (text1_a, text1_b, text2_a, text2_b, mid_common)
 
   def diff_cleanupSemantic(self, diffs):
     """Reduce the number of edits by eliminating semantically trivial
     equalities.
 
     Args:
       diffs: Array of diff tuples.
     """
     changes = False
     equalities = []  # Stack of indices where equalities are found.
     lastequality = None  # Always equal to diffs[equalities[-1]][1]
     pointer = 0  # Index of current position.
     # Number of chars that changed prior to the equality.
     length_insertions1, length_deletions1 = 0, 0
     # Number of chars that changed after the equality.
     length_insertions2, length_deletions2 = 0, 0
     while pointer < len(diffs):
       if diffs[pointer][0] == self.DIFF_EQUAL:  # Equality found.
         equalities.append(pointer)
         length_insertions1, length_insertions2 = length_insertions2, 0
         length_deletions1, length_deletions2 = length_deletions2, 0
         lastequality = diffs[pointer][1]
       else:  # An insertion or deletion.
         if diffs[pointer][0] == self.DIFF_INSERT:
           length_insertions2 += len(diffs[pointer][1])
         else:
           length_deletions2 += len(diffs[pointer][1])
         # Eliminate an equality that is smaller or equal to the edits on both
         # sides of it.
         if (lastequality and (len(lastequality) <=
             max(length_insertions1, length_deletions1)) and
             (len(lastequality) <= max(length_insertions2, length_deletions2))):
           # Duplicate record.
           diffs.insert(equalities[-1], (self.DIFF_DELETE, lastequality))
           # Change second copy to insert.
           diffs[equalities[-1] + 1] = (self.DIFF_INSERT,
               diffs[equalities[-1] + 1][1])
           # Throw away the equality we just deleted.
           equalities.pop()
           # Throw away the previous equality (it needs to be reevaluated).
           if len(equalities):
             equalities.pop()
           if len(equalities):
             pointer = equalities[-1]
           else:
             pointer = -1
           # Reset the counters.
           length_insertions1, length_deletions1 = 0, 0
           length_insertions2, length_deletions2 = 0, 0
           lastequality = None
           changes = True
       pointer += 1
 
     # Normalize the diff.
     if changes:
       self.diff_cleanupMerge(diffs)
     self.diff_cleanupSemanticLossless(diffs)
 
     # Find any overlaps between deletions and insertions.
     # e.g: <del>abcxxx</del><ins>xxxdef</ins>
     #   -> <del>abc</del>xxx<ins>def</ins>
     # e.g: <del>xxxabc</del><ins>defxxx</ins>
     #   -> <ins>def</ins>xxx<del>abc</del>
     # Only extract an overlap if it is as big as the edit ahead or behind it.
     pointer = 1
     while pointer < len(diffs):
       if (diffs[pointer - 1][0] == self.DIFF_DELETE and
           diffs[pointer][0] == self.DIFF_INSERT):
         deletion = diffs[pointer - 1][1]
         insertion = diffs[pointer][1]
         overlap_length1 = self.diff_commonOverlap(deletion, insertion)
         overlap_length2 = self.diff_commonOverlap(insertion, deletion)
         if overlap_length1 >= overlap_length2:
           if (overlap_length1 >= len(deletion) / 2.0 or
               overlap_length1 >= len(insertion) / 2.0):
             # Overlap found.  Insert an equality and trim the surrounding edits.
             diffs.insert(pointer, (self.DIFF_EQUAL,
                                    insertion[:overlap_length1]))
             diffs[pointer - 1] = (self.DIFF_DELETE,
                                   deletion[:len(deletion) - overlap_length1])
             diffs[pointer + 1] = (self.DIFF_INSERT,
                                   insertion[overlap_length1:])
             pointer += 1
         else:
           if (overlap_length2 >= len(deletion) / 2.0 or
               overlap_length2 >= len(insertion) / 2.0):
             # Reverse overlap found.
             # Insert an equality and swap and trim the surrounding edits.
             diffs.insert(pointer, (self.DIFF_EQUAL, deletion[:overlap_length2]))
             diffs[pointer - 1] = (self.DIFF_INSERT,
                                   insertion[:len(insertion) - overlap_length2])
             diffs[pointer + 1] = (self.DIFF_DELETE, deletion[overlap_length2:])
             pointer += 1
         pointer += 1
       pointer += 1
 
   def diff_cleanupSemanticLossless(self, diffs):
     """Look for single edits surrounded on both sides by equalities
     which can be shifted sideways to align the edit to a word boundary.
     e.g: The c<ins>at c</ins>ame. -> The <ins>cat </ins>came.
 
     Args:
       diffs: Array of diff tuples.
     """
 
     def diff_cleanupSemanticScore(one, two):
       """Given two strings, compute a score representing whether the
       internal boundary falls on logical boundaries.
       Scores range from 6 (best) to 0 (worst).
       Closure, but does not reference any external variables.
 
       Args:
         one: First string.
         two: Second string.
 
       Returns:
         The score.
       """
       if not one or not two:
         # Edges are the best.
         return 6
 
       # Each port of this function behaves slightly differently due to
       # subtle differences in each language's definition of things like
       # 'whitespace'.  Since this function's purpose is largely cosmetic,
       # the choice has been made to use each language's native features
       # rather than force total conformity.
       char1 = one[-1]
       char2 = two[0]
       nonAlphaNumeric1 = not char1.isalnum()
       nonAlphaNumeric2 = not char2.isalnum()
       whitespace1 = nonAlphaNumeric1 and char1.isspace()
       whitespace2 = nonAlphaNumeric2 and char2.isspace()
       lineBreak1 = whitespace1 and (char1 == "\r" or char1 == "\n")
       lineBreak2 = whitespace2 and (char2 == "\r" or char2 == "\n")
       blankLine1 = lineBreak1 and self.BLANKLINEEND.search(one)
       blankLine2 = lineBreak2 and self.BLANKLINESTART.match(two)
 
       if blankLine1 or blankLine2:
         # Five points for blank lines.
         return 5
       elif lineBreak1 or lineBreak2:
         # Four points for line breaks.
         return 4
       elif nonAlphaNumeric1 and not whitespace1 and whitespace2:
         # Three points for end of sentences.
         return 3
       elif whitespace1 or whitespace2:
         # Two points for whitespace.
         return 2
       elif nonAlphaNumeric1 or nonAlphaNumeric2:
         # One point for non-alphanumeric.
         return 1
       return 0
 
     pointer = 1
     # Intentionally ignore the first and last element (don't need checking).
     while pointer < len(diffs) - 1:
       if (diffs[pointer - 1][0] == self.DIFF_EQUAL and
           diffs[pointer + 1][0] == self.DIFF_EQUAL):
         # This is a single edit surrounded by equalities.
         equality1 = diffs[pointer - 1][1]
         edit = diffs[pointer][1]
         equality2 = diffs[pointer + 1][1]
 
         # First, shift the edit as far left as possible.
         commonOffset = self.diff_commonSuffix(equality1, edit)
         if commonOffset:
           commonString = edit[-commonOffset:]
           equality1 = equality1[:-commonOffset]
           edit = commonString + edit[:-commonOffset]
           equality2 = commonString + equality2
 
         # Second, step character by character right, looking for the best fit.
         bestEquality1 = equality1
         bestEdit = edit
         bestEquality2 = equality2
         bestScore = (diff_cleanupSemanticScore(equality1, edit) +
             diff_cleanupSemanticScore(edit, equality2))
         while edit and equality2 and edit[0] == equality2[0]:
           equality1 += edit[0]
           edit = edit[1:] + equality2[0]
           equality2 = equality2[1:]
           score = (diff_cleanupSemanticScore(equality1, edit) +
               diff_cleanupSemanticScore(edit, equality2))
           # The >= encourages trailing rather than leading whitespace on edits.
           if score >= bestScore:
             bestScore = score
             bestEquality1 = equality1
             bestEdit = edit
             bestEquality2 = equality2
 
         if diffs[pointer - 1][1] != bestEquality1:
           # We have an improvement, save it back to the diff.
           if bestEquality1:
             diffs[pointer - 1] = (diffs[pointer - 1][0], bestEquality1)
           else:
             del diffs[pointer - 1]
             pointer -= 1
           diffs[pointer] = (diffs[pointer][0], bestEdit)
           if bestEquality2:
             diffs[pointer + 1] = (diffs[pointer + 1][0], bestEquality2)
           else:
             del diffs[pointer + 1]
             pointer -= 1
       pointer += 1
 
   # Define some regex patterns for matching boundaries.
   BLANKLINEEND = re.compile(r"\n\r?\n$");
   BLANKLINESTART = re.compile(r"^\r?\n\r?\n");
 
   def diff_cleanupEfficiency(self, diffs):
     """Reduce the number of edits by eliminating operationally trivial
     equalities.
 
     Args:
       diffs: Array of diff tuples.
     """
     changes = False
     equalities = []  # Stack of indices where equalities are found.
     lastequality = None  # Always equal to diffs[equalities[-1]][1]
     pointer = 0  # Index of current position.
     pre_ins = False  # Is there an insertion operation before the last equality.
     pre_del = False  # Is there a deletion operation before the last equality.
     post_ins = False  # Is there an insertion operation after the last equality.
     post_del = False  # Is there a deletion operation after the last equality.
     while pointer < len(diffs):
       if diffs[pointer][0] == self.DIFF_EQUAL:  # Equality found.
         if (len(diffs[pointer][1]) < self.Diff_EditCost and
             (post_ins or post_del)):
           # Candidate found.
           equalities.append(pointer)
           pre_ins = post_ins
           pre_del = post_del
           lastequality = diffs[pointer][1]
         else:
           # Not a candidate, and can never become one.
           equalities = []
           lastequality = None
 
         post_ins = post_del = False
       else:  # An insertion or deletion.
         if diffs[pointer][0] == self.DIFF_DELETE:
           post_del = True
         else:
           post_ins = True
 
         # Five types to be split:
         # <ins>A</ins><del>B</del>XY<ins>C</ins><del>D</del>
         # <ins>A</ins>X<ins>C</ins><del>D</del>
         # <ins>A</ins><del>B</del>X<ins>C</ins>
         # <ins>A</del>X<ins>C</ins><del>D</del>
         # <ins>A</ins><del>B</del>X<del>C</del>
 
         if lastequality and ((pre_ins and pre_del and post_ins and post_del) or
                              ((len(lastequality) < self.Diff_EditCost / 2) and
                               (pre_ins + pre_del + post_ins + post_del) == 3)):
           # Duplicate record.
           diffs.insert(equalities[-1], (self.DIFF_DELETE, lastequality))
           # Change second copy to insert.
           diffs[equalities[-1] + 1] = (self.DIFF_INSERT,
               diffs[equalities[-1] + 1][1])
           equalities.pop()  # Throw away the equality we just deleted.
           lastequality = None
           if pre_ins and pre_del:
             # No changes made which could affect previous entry, keep going.
             post_ins = post_del = True
             equalities = []
           else:
             if len(equalities):
               equalities.pop()  # Throw away the previous equality.
             if len(equalities):
               pointer = equalities[-1]
             else:
               pointer = -1
             post_ins = post_del = False
           changes = True
       pointer += 1
 
     if changes:
       self.diff_cleanupMerge(diffs)
 
   def diff_cleanupMerge(self, diffs):
     """Reorder and merge like edit sections.  Merge equalities.
     Any edit section can move as long as it doesn't cross an equality.
 
     Args:
       diffs: Array of diff tuples.
     """
     diffs.append((self.DIFF_EQUAL, ''))  # Add a dummy entry at the end.
     pointer = 0
     count_delete = 0
     count_insert = 0
     text_delete = ''
     text_insert = ''
     while pointer < len(diffs):
       if diffs[pointer][0] == self.DIFF_INSERT:
         count_insert += 1
         text_insert += diffs[pointer][1]
         pointer += 1
       elif diffs[pointer][0] == self.DIFF_DELETE:
         count_delete += 1
         text_delete += diffs[pointer][1]
         pointer += 1
       elif diffs[pointer][0] == self.DIFF_EQUAL:
         # Upon reaching an equality, check for prior redundancies.
         if count_delete + count_insert > 1:
           if count_delete != 0 and count_insert != 0:
             # Factor out any common prefixies.
             commonlength = self.diff_commonPrefix(text_insert, text_delete)
             if commonlength != 0:
               x = pointer - count_delete - count_insert - 1
               if x >= 0 and diffs[x][0] == self.DIFF_EQUAL:
                 diffs[x] = (diffs[x][0], diffs[x][1] +
                             text_insert[:commonlength])
               else:
                 diffs.insert(0, (self.DIFF_EQUAL, text_insert[:commonlength]))
                 pointer += 1
               text_insert = text_insert[commonlength:]
               text_delete = text_delete[commonlength:]
             # Factor out any common suffixies.
             commonlength = self.diff_commonSuffix(text_insert, text_delete)
             if commonlength != 0:
               diffs[pointer] = (diffs[pointer][0], text_insert[-commonlength:] +
                   diffs[pointer][1])
               text_insert = text_insert[:-commonlength]
               text_delete = text_delete[:-commonlength]
           # Delete the offending records and add the merged ones.
           if count_delete == 0:
             diffs[pointer - count_insert : pointer] = [
                 (self.DIFF_INSERT, text_insert)]
           elif count_insert == 0:
             diffs[pointer - count_delete : pointer] = [
                 (self.DIFF_DELETE, text_delete)]
           else:
             diffs[pointer - count_delete - count_insert : pointer] = [
                 (self.DIFF_DELETE, text_delete),
                 (self.DIFF_INSERT, text_insert)]
           pointer = pointer - count_delete - count_insert + 1
           if count_delete != 0:
             pointer += 1
           if count_insert != 0:
             pointer += 1
         elif pointer != 0 and diffs[pointer - 1][0] == self.DIFF_EQUAL:
           # Merge this equality with the previous one.
           diffs[pointer - 1] = (diffs[pointer - 1][0],
                                 diffs[pointer - 1][1] + diffs[pointer][1])
           del diffs[pointer]
         else:
           pointer += 1
 
         count_insert = 0
         count_delete = 0
         text_delete = ''
         text_insert = ''
 
     if diffs[-1][1] == '':
       diffs.pop()  # Remove the dummy entry at the end.
 
     # Second pass: look for single edits surrounded on both sides by equalities
     # which can be shifted sideways to eliminate an equality.
     # e.g: A<ins>BA</ins>C -> <ins>AB</ins>AC
     changes = False
     pointer = 1
     # Intentionally ignore the first and last element (don't need checking).
     while pointer < len(diffs) - 1:
       if (diffs[pointer - 1][0] == self.DIFF_EQUAL and
           diffs[pointer + 1][0] == self.DIFF_EQUAL):
         # This is a single edit surrounded by equalities.
         if diffs[pointer][1].endswith(diffs[pointer - 1][1]):
           # Shift the edit over the previous equality.
           diffs[pointer] = (diffs[pointer][0],
               diffs[pointer - 1][1] +
               diffs[pointer][1][:-len(diffs[pointer - 1][1])])
           diffs[pointer + 1] = (diffs[pointer + 1][0],
                                 diffs[pointer - 1][1] + diffs[pointer + 1][1])
           del diffs[pointer - 1]
           changes = True
         elif diffs[pointer][1].startswith(diffs[pointer + 1][1]):
           # Shift the edit over the next equality.
           diffs[pointer - 1] = (diffs[pointer - 1][0],
                                 diffs[pointer - 1][1] + diffs[pointer + 1][1])
           diffs[pointer] = (diffs[pointer][0],
               diffs[pointer][1][len(diffs[pointer + 1][1]):] +
               diffs[pointer + 1][1])
           del diffs[pointer + 1]
           changes = True
       pointer += 1
 
     # If shifts were made, the diff needs reordering and another shift sweep.
     if changes:
       self.diff_cleanupMerge(diffs)
 
   def diff_xIndex(self, diffs, loc):
     """loc is a location in text1, compute and return the equivalent location
     in text2.  e.g. "The cat" vs "The big cat", 1->1, 5->8
 
     Args:
       diffs: Array of diff tuples.
       loc: Location within text1.
 
     Returns:
       Location within text2.
     """
     chars1 = 0
     chars2 = 0
     last_chars1 = 0
     last_chars2 = 0
     for x in xrange(len(diffs)):
       (op, text) = diffs[x]
       if op != self.DIFF_INSERT:  # Equality or deletion.
         chars1 += len(text)
       if op != self.DIFF_DELETE:  # Equality or insertion.
         chars2 += len(text)
       if chars1 > loc:  # Overshot the location.
         break
       last_chars1 = chars1
       last_chars2 = chars2
 
     if len(diffs) != x and diffs[x][0] == self.DIFF_DELETE:
       # The location was deleted.
       return last_chars2
     # Add the remaining len(character).
     return last_chars2 + (loc - last_chars1)
 
   def diff_prettyHtml(self, diffs):
     """Convert a diff array into a pretty HTML report.
 
     Args:
       diffs: Array of diff tuples.
 
     Returns:
       HTML representation.
     """
     html = []
     for (op, data) in diffs:
       text = (data.replace("&", "&amp;").replace("<", "&lt;")
                  .replace(">", "&gt;").replace("\n", "&para;<br>"))
       if op == self.DIFF_INSERT:
         html.append("<ins style=\"background:#e6ffe6;\">%s</ins>" % text)
       elif op == self.DIFF_DELETE:
         html.append("<del style=\"background:#ffe6e6;\">%s</del>" % text)
       elif op == self.DIFF_EQUAL:
         html.append("<span>%s</span>" % text)
     return "".join(html)
 
   def diff_text1(self, diffs):
     """Compute and return the source text (all equalities and deletions).
 
     Args:
       diffs: Array of diff tuples.
 
     Returns:
       Source text.
     """
     text = []
     for (op, data) in diffs:
       if op != self.DIFF_INSERT:
         text.append(data)
     return "".join(text)
 
   def diff_text2(self, diffs):
     """Compute and return the destination text (all equalities and insertions).
 
     Args:
       diffs: Array of diff tuples.
 
     Returns:
       Destination text.
     """
     text = []
     for (op, data) in diffs:
       if op != self.DIFF_DELETE:
         text.append(data)
     return "".join(text)
 
   def diff_levenshtein(self, diffs):
     """Compute the Levenshtein distance; the number of inserted, deleted or
     substituted characters.
 
     Args:
       diffs: Array of diff tuples.
 
     Returns:
       Number of changes.
     """
     levenshtein = 0
     insertions = 0
     deletions = 0
     for (op, data) in diffs:
       if op == self.DIFF_INSERT:
         insertions += len(data)
       elif op == self.DIFF_DELETE:
         deletions += len(data)
       elif op == self.DIFF_EQUAL:
         # A deletion and an insertion is one substitution.
         levenshtein += max(insertions, deletions)
         insertions = 0
         deletions = 0
     levenshtein += max(insertions, deletions)
     return levenshtein
 
   def diff_toDelta(self, diffs):
     """Crush the diff into an encoded string which describes the operations
     required to transform text1 into text2.
     E.g. =3\t-2\t+ing  -> Keep 3 chars, delete 2 chars, insert 'ing'.
     Operations are tab-separated.  Inserted text is escaped using %xx notation.
 
     Args:
       diffs: Array of diff tuples.
 
     Returns:
       Delta text.
     """
     text = []
     for (op, data) in diffs:
       if op == self.DIFF_INSERT:
         # High ascii will raise UnicodeDecodeError.  Use Unicode instead.
         data = data.encode("utf-8")
         text.append("+" + urllib.quote(data, "!~*'();/?:@&=+$,# "))
       elif op == self.DIFF_DELETE:
         text.append("-%d" % len(data))
       elif op == self.DIFF_EQUAL:
         text.append("=%d" % len(data))
     return "\t".join(text)
 
   def diff_fromDelta(self, text1, delta):
     """Given the original text1, and an encoded string which describes the
     operations required to transform text1 into text2, compute the full diff.
 
     Args:
       text1: Source string for the diff.
       delta: Delta text.
 
     Returns:
       Array of diff tuples.
 
     Raises:
       ValueError: If invalid input.
     """
     if type(delta) == unicode:
       # Deltas should be composed of a subset of ascii chars, Unicode not
       # required.  If this encode raises UnicodeEncodeError, delta is invalid.
       delta = delta.encode("ascii")
     diffs = []
     pointer = 0  # Cursor in text1
     tokens = delta.split("\t")
     for token in tokens:
       if token == "":
         # Blank tokens are ok (from a trailing \t).
         continue
       # Each token begins with a one character parameter which specifies the
       # operation of this token (delete, insert, equality).
       param = token[1:]
       if token[0] == "+":
         param = urllib.unquote(param).decode("utf-8")
         diffs.append((self.DIFF_INSERT, param))
       elif token[0] == "-" or token[0] == "=":
         try:
           n = int(param)
         except ValueError:
           raise ValueError("Invalid number in diff_fromDelta: " + param)
         if n < 0:
           raise ValueError("Negative number in diff_fromDelta: " + param)
         text = text1[pointer : pointer + n]
         pointer += n
         if token[0] == "=":
           diffs.append((self.DIFF_EQUAL, text))
         else:
           diffs.append((self.DIFF_DELETE, text))
       else:
         # Anything else is an error.
         raise ValueError("Invalid diff operation in diff_fromDelta: " +
             token[0])
     if pointer != len(text1):
       raise ValueError(
           "Delta length (%d) does not equal source text length (%d)." %
          (pointer, len(text1)))
     return diffs
 
   #  MATCH FUNCTIONS
 
   def match_main(self, text, pattern, loc):
     """Locate the best instance of 'pattern' in 'text' near 'loc'.
 
     Args:
       text: The text to search.
       pattern: The pattern to search for.
       loc: The location to search around.
 
     Returns:
       Best match index or -1.
     """
     # Check for null inputs.
-    if text == None or pattern == None:
+    if text is None or pattern is None:
       raise ValueError("Null inputs. (match_main)")
 
     loc = max(0, min(loc, len(text)))
     if text == pattern:
       # Shortcut (potentially not guaranteed by the algorithm)
       return 0
     elif not text:
       # Nothing to match.
       return -1
     elif text[loc:loc + len(pattern)] == pattern:
       # Perfect match at the perfect spot!  (Includes case of null pattern)
       return loc
     else:
       # Do a fuzzy compare.
       match = self.match_bitap(text, pattern, loc)
       return match
 
   def match_bitap(self, text, pattern, loc):
     """Locate the best instance of 'pattern' in 'text' near 'loc' using the
     Bitap algorithm.
 
     Args:
       text: The text to search.
       pattern: The pattern to search for.
       loc: The location to search around.
 
     Returns:
       Best match index or -1.
     """
     # Python doesn't have a maxint limit, so ignore this check.
     #if self.Match_MaxBits != 0 and len(pattern) > self.Match_MaxBits:
     #  raise ValueError("Pattern too long for this application.")
 
     # Initialise the alphabet.
     s = self.match_alphabet(pattern)
 
     def match_bitapScore(e, x):
       """Compute and return the score for a match with e errors and x location.
       Accesses loc and pattern through being a closure.
 
       Args:
         e: Number of errors in match.
         x: Location of match.
 
       Returns:
         Overall score for match (0.0 = good, 1.0 = bad).
       """
       accuracy = float(e) / len(pattern)
       proximity = abs(loc - x)
       if not self.Match_Distance:
         # Dodge divide by zero error.
         return proximity and 1.0 or accuracy
       return accuracy + (proximity / float(self.Match_Distance))
 
     # Highest score beyond which we give up.
     score_threshold = self.Match_Threshold
     # Is there a nearby exact match? (speedup)
     best_loc = text.find(pattern, loc)
     if best_loc != -1:
       score_threshold = min(match_bitapScore(0, best_loc), score_threshold)
       # What about in the other direction? (speedup)
       best_loc = text.rfind(pattern, loc + len(pattern))
       if best_loc != -1:
         score_threshold = min(match_bitapScore(0, best_loc), score_threshold)
 
     # Initialise the bit arrays.
     matchmask = 1 << (len(pattern) - 1)
     best_loc = -1
 
     bin_max = len(pattern) + len(text)
     # Empty initialization added to appease pychecker.
     last_rd = None
     for d in xrange(len(pattern)):
       # Scan for the best match each iteration allows for one more error.
       # Run a binary search to determine how far from 'loc' we can stray at
       # this error level.
       bin_min = 0
       bin_mid = bin_max
       while bin_min < bin_mid:
         if match_bitapScore(d, loc + bin_mid) <= score_threshold:
           bin_min = bin_mid
         else:
           bin_max = bin_mid
         bin_mid = (bin_max - bin_min) // 2 + bin_min
 
       # Use the result from this iteration as the maximum for the next.
       bin_max = bin_mid
       start = max(1, loc - bin_mid + 1)
       finish = min(loc + bin_mid, len(text)) + len(pattern)
 
       rd = [0] * (finish + 2)
       rd[finish + 1] = (1 << d) - 1
       for j in xrange(finish, start - 1, -1):
         if len(text) <= j - 1:
           # Out of range.
           charMatch = 0
         else:
           charMatch = s.get(text[j - 1], 0)
         if d == 0:  # First pass: exact match.
           rd[j] = ((rd[j + 1] << 1) | 1) & charMatch
         else:  # Subsequent passes: fuzzy match.
           rd[j] = (((rd[j + 1] << 1) | 1) & charMatch) | (
               ((last_rd[j + 1] | last_rd[j]) << 1) | 1) | last_rd[j + 1]
         if rd[j] & matchmask:
           score = match_bitapScore(d, j - 1)
           # This match will almost certainly be better than any existing match.
           # But check anyway.
           if score <= score_threshold:
             # Told you so.
             score_threshold = score
             best_loc = j - 1
             if best_loc > loc:
               # When passing loc, don't exceed our current distance from loc.
               start = max(1, 2 * loc - best_loc)
             else:
               # Already passed loc, downhill from here on in.
               break
       # No hope for a (better) match at greater error levels.
       if match_bitapScore(d + 1, loc) > score_threshold:
         break
       last_rd = rd
     return best_loc
 
   def match_alphabet(self, pattern):
     """Initialise the alphabet for the Bitap algorithm.
 
     Args:
       pattern: The text to encode.
 
     Returns:
       Hash of character locations.
     """
     s = {}
     for char in pattern:
       s[char] = 0
     for i in xrange(len(pattern)):
       s[pattern[i]] |= 1 << (len(pattern) - i - 1)
     return s
 
   #  PATCH FUNCTIONS
 
   def patch_addContext(self, patch, text):
     """Increase the context until it is unique,
     but don't let the pattern expand beyond Match_MaxBits.
 
     Args:
       patch: The patch to grow.
       text: Source text.
     """
     if len(text) == 0:
       return
     pattern = text[patch.start2 : patch.start2 + patch.length1]
     padding = 0
 
     # Look for the first and last matches of pattern in text.  If two different
     # matches are found, increase the pattern length.
     while (text.find(pattern) != text.rfind(pattern) and (self.Match_MaxBits ==
         0 or len(pattern) < self.Match_MaxBits - self.Patch_Margin -
         self.Patch_Margin)):
       padding += self.Patch_Margin
       pattern = text[max(0, patch.start2 - padding) :
                      patch.start2 + patch.length1 + padding]
     # Add one chunk for good luck.
     padding += self.Patch_Margin
 
     # Add the prefix.
     prefix = text[max(0, patch.start2 - padding) : patch.start2]
     if prefix:
       patch.diffs[:0] = [(self.DIFF_EQUAL, prefix)]
     # Add the suffix.
     suffix = text[patch.start2 + patch.length1 :
                   patch.start2 + patch.length1 + padding]
     if suffix:
       patch.diffs.append((self.DIFF_EQUAL, suffix))
 
     # Roll back the start points.
     patch.start1 -= len(prefix)
     patch.start2 -= len(prefix)
     # Extend lengths.
     patch.length1 += len(prefix) + len(suffix)
     patch.length2 += len(prefix) + len(suffix)
 
   def patch_make(self, a, b=None, c=None):
     """Compute a list of patches to turn text1 into text2.
     Use diffs if provided, otherwise compute it ourselves.
     There are four ways to call this function, depending on what data is
     available to the caller:
     Method 1:
     a = text1, b = text2
     Method 2:
     a = diffs
     Method 3 (optimal):
     a = text1, b = diffs
     Method 4 (deprecated, use method 3):
     a = text1, b = text2, c = diffs
 
     Args:
       a: text1 (methods 1,3,4) or Array of diff tuples for text1 to
           text2 (method 2).
       b: text2 (methods 1,4) or Array of diff tuples for text1 to
           text2 (method 3) or undefined (method 2).
       c: Array of diff tuples for text1 to text2 (method 4) or
           undefined (methods 1,2,3).
 
     Returns:
       Array of Patch objects.
     """
     text1 = None
     diffs = None
     # Note that texts may arrive as 'str' or 'unicode'.
     if isinstance(a, basestring) and isinstance(b, basestring) and c is None:
       # Method 1: text1, text2
       # Compute diffs from text1 and text2.
       text1 = a
       diffs = self.diff_main(text1, b, True)
       if len(diffs) > 2:
         self.diff_cleanupSemantic(diffs)
         self.diff_cleanupEfficiency(diffs)
     elif isinstance(a, list) and b is None and c is None:
       # Method 2: diffs
       # Compute text1 from diffs.
       diffs = a
       text1 = self.diff_text1(diffs)
     elif isinstance(a, basestring) and isinstance(b, list) and c is None:
       # Method 3: text1, diffs
       text1 = a
       diffs = b
     elif (isinstance(a, basestring) and isinstance(b, basestring) and
           isinstance(c, list)):
       # Method 4: text1, text2, diffs
       # text2 is not used.
       text1 = a
       diffs = c
     else:
       raise ValueError("Unknown call format to patch_make.")
 
     if not diffs:
       return []  # Get rid of the None case.
     patches = []
     patch = patch_obj()
     char_count1 = 0  # Number of characters into the text1 string.
     char_count2 = 0  # Number of characters into the text2 string.
     prepatch_text = text1  # Recreate the patches to determine context info.
     postpatch_text = text1
     for x in xrange(len(diffs)):
       (diff_type, diff_text) = diffs[x]
       if len(patch.diffs) == 0 and diff_type != self.DIFF_EQUAL:
         # A new patch starts here.
         patch.start1 = char_count1
         patch.start2 = char_count2
       if diff_type == self.DIFF_INSERT:
         # Insertion
         patch.diffs.append(diffs[x])
         patch.length2 += len(diff_text)
         postpatch_text = (postpatch_text[:char_count2] + diff_text +
                           postpatch_text[char_count2:])
       elif diff_type == self.DIFF_DELETE:
         # Deletion.
         patch.length1 += len(diff_text)
         patch.diffs.append(diffs[x])
         postpatch_text = (postpatch_text[:char_count2] +
                           postpatch_text[char_count2 + len(diff_text):])
       elif (diff_type == self.DIFF_EQUAL and
             len(diff_text) <= 2 * self.Patch_Margin and
             len(patch.diffs) != 0 and len(diffs) != x + 1):
         # Small equality inside a patch.
         patch.diffs.append(diffs[x])
         patch.length1 += len(diff_text)
         patch.length2 += len(diff_text)
 
       if (diff_type == self.DIFF_EQUAL and
           len(diff_text) >= 2 * self.Patch_Margin):
         # Time for a new patch.
         if len(patch.diffs) != 0:
           self.patch_addContext(patch, prepatch_text)
           patches.append(patch)
           patch = patch_obj()
           # Unlike Unidiff, our patch lists have a rolling context.
           # http://code.google.com/p/google-diff-match-patch/wiki/Unidiff
           # Update prepatch text & pos to reflect the application of the
           # just completed patch.
           prepatch_text = postpatch_text
           char_count1 = char_count2
 
       # Update the current character count.
       if diff_type != self.DIFF_INSERT:
         char_count1 += len(diff_text)
       if diff_type != self.DIFF_DELETE:
         char_count2 += len(diff_text)
 
     # Pick up the leftover patch if not empty.
     if len(patch.diffs) != 0:
       self.patch_addContext(patch, prepatch_text)
       patches.append(patch)
     return patches
 
   def patch_deepCopy(self, patches):
     """Given an array of patches, return another array that is identical.
 
     Args:
       patches: Array of Patch objects.
 
     Returns:
       Array of Patch objects.
     """
     patchesCopy = []
     for patch in patches:
       patchCopy = patch_obj()
       # No need to deep copy the tuples since they are immutable.
       patchCopy.diffs = patch.diffs[:]
       patchCopy.start1 = patch.start1
       patchCopy.start2 = patch.start2
       patchCopy.length1 = patch.length1
       patchCopy.length2 = patch.length2
       patchesCopy.append(patchCopy)
     return patchesCopy
 
   def patch_apply(self, patches, text):
     """Merge a set of patches onto the text.  Return a patched text, as well
     as a list of true/false values indicating which patches were applied.
 
     Args:
       patches: Array of Patch objects.
       text: Old text.
 
     Returns:
       Two element Array, containing the new text and an array of boolean values.
     """
     if not patches:
       return (text, [])
 
     # Deep copy the patches so that no changes are made to originals.
     patches = self.patch_deepCopy(patches)
 
     nullPadding = self.patch_addPadding(patches)
     text = nullPadding + text + nullPadding
     self.patch_splitMax(patches)
 
     # delta keeps track of the offset between the expected and actual location
     # of the previous patch.  If there are patches expected at positions 10 and
     # 20, but the first patch was found at 12, delta is 2 and the second patch
     # has an effective expected position of 22.
     delta = 0
     results = []
     for patch in patches:
       expected_loc = patch.start2 + delta
       text1 = self.diff_text1(patch.diffs)
       end_loc = -1
       if len(text1) > self.Match_MaxBits:
         # patch_splitMax will only provide an oversized pattern in the case of
         # a monster delete.
         start_loc = self.match_main(text, text1[:self.Match_MaxBits],
                                     expected_loc)
         if start_loc != -1:
           end_loc = self.match_main(text, text1[-self.Match_MaxBits:],
               expected_loc + len(text1) - self.Match_MaxBits)
           if end_loc == -1 or start_loc >= end_loc:
             # Can't find valid trailing context.  Drop this patch.
             start_loc = -1
       else:
         start_loc = self.match_main(text, text1, expected_loc)
       if start_loc == -1:
         # No match found.  :(
         results.append(False)
         # Subtract the delta for this failed patch from subsequent patches.
         delta -= patch.length2 - patch.length1
       else:
         # Found a match.  :)
         results.append(True)
         delta = start_loc - expected_loc
         if end_loc == -1:
           text2 = text[start_loc : start_loc + len(text1)]
         else:
           text2 = text[start_loc : end_loc + self.Match_MaxBits]
         if text1 == text2:
           # Perfect match, just shove the replacement text in.
           text = (text[:start_loc] + self.diff_text2(patch.diffs) +
                       text[start_loc + len(text1):])
         else:
           # Imperfect match.
           # Run a diff to get a framework of equivalent indices.
           diffs = self.diff_main(text1, text2, False)
           if (len(text1) > self.Match_MaxBits and
               self.diff_levenshtein(diffs) / float(len(text1)) >
               self.Patch_DeleteThreshold):
             # The end points match, but the content is unacceptably bad.
             results[-1] = False
           else:
             self.diff_cleanupSemanticLossless(diffs)
             index1 = 0
             for (op, data) in patch.diffs:
               if op != self.DIFF_EQUAL:
                 index2 = self.diff_xIndex(diffs, index1)
               if op == self.DIFF_INSERT:  # Insertion
                 text = text[:start_loc + index2] + data + text[start_loc +
                                                                index2:]
               elif op == self.DIFF_DELETE:  # Deletion
                 text = text[:start_loc + index2] + text[start_loc +
                     self.diff_xIndex(diffs, index1 + len(data)):]
               if op != self.DIFF_DELETE:
                 index1 += len(data)
     # Strip the padding off.
     text = text[len(nullPadding):-len(nullPadding)]
     return (text, results)
 
   def patch_addPadding(self, patches):
     """Add some padding on text start and end so that edges can match
     something.  Intended to be called only from within patch_apply.
 
     Args:
       patches: Array of Patch objects.
 
     Returns:
       The padding string added to each side.
     """
     paddingLength = self.Patch_Margin
     nullPadding = ""
     for x in xrange(1, paddingLength + 1):
       nullPadding += chr(x)
 
     # Bump all the patches forward.
     for patch in patches:
       patch.start1 += paddingLength
       patch.start2 += paddingLength
 
     # Add some padding on start of first diff.
     patch = patches[0]
     diffs = patch.diffs
     if not diffs or diffs[0][0] != self.DIFF_EQUAL:
       # Add nullPadding equality.
       diffs.insert(0, (self.DIFF_EQUAL, nullPadding))
       patch.start1 -= paddingLength  # Should be 0.
       patch.start2 -= paddingLength  # Should be 0.
       patch.length1 += paddingLength
       patch.length2 += paddingLength
     elif paddingLength > len(diffs[0][1]):
       # Grow first equality.
       extraLength = paddingLength - len(diffs[0][1])
       newText = nullPadding[len(diffs[0][1]):] + diffs[0][1]
       diffs[0] = (diffs[0][0], newText)
       patch.start1 -= extraLength
       patch.start2 -= extraLength
       patch.length1 += extraLength
       patch.length2 += extraLength
 
     # Add some padding on end of last diff.
     patch = patches[-1]
     diffs = patch.diffs
     if not diffs or diffs[-1][0] != self.DIFF_EQUAL:
       # Add nullPadding equality.
       diffs.append((self.DIFF_EQUAL, nullPadding))
       patch.length1 += paddingLength
       patch.length2 += paddingLength
     elif paddingLength > len(diffs[-1][1]):
       # Grow last equality.
       extraLength = paddingLength - len(diffs[-1][1])
       newText = diffs[-1][1] + nullPadding[:extraLength]
       diffs[-1] = (diffs[-1][0], newText)
       patch.length1 += extraLength
       patch.length2 += extraLength
 
     return nullPadding
 
   def patch_splitMax(self, patches):
     """Look through the patches and break up any which are longer than the
     maximum limit of the match algorithm.
     Intended to be called only from within patch_apply.
 
     Args:
       patches: Array of Patch objects.
     """
     patch_size = self.Match_MaxBits
     if patch_size == 0:
       # Python has the option of not splitting strings due to its ability
       # to handle integers of arbitrary precision.
       return
     for x in xrange(len(patches)):
       if patches[x].length1 <= patch_size:
         continue
       bigpatch = patches[x]
       # Remove the big old patch.
       del patches[x]
       x -= 1
       start1 = bigpatch.start1
       start2 = bigpatch.start2
       precontext = ''
       while len(bigpatch.diffs) != 0:
         # Create one of several smaller patches.
         patch = patch_obj()
         empty = True
         patch.start1 = start1 - len(precontext)
         patch.start2 = start2 - len(precontext)
         if precontext:
           patch.length1 = patch.length2 = len(precontext)
           patch.diffs.append((self.DIFF_EQUAL, precontext))
 
         while (len(bigpatch.diffs) != 0 and
                patch.length1 < patch_size - self.Patch_Margin):
           (diff_type, diff_text) = bigpatch.diffs[0]
           if diff_type == self.DIFF_INSERT:
             # Insertions are harmless.
             patch.length2 += len(diff_text)
             start2 += len(diff_text)
             patch.diffs.append(bigpatch.diffs.pop(0))
             empty = False
           elif (diff_type == self.DIFF_DELETE and len(patch.diffs) == 1 and
               patch.diffs[0][0] == self.DIFF_EQUAL and
               len(diff_text) > 2 * patch_size):
             # This is a large deletion.  Let it pass in one chunk.
             patch.length1 += len(diff_text)
             start1 += len(diff_text)
             empty = False
             patch.diffs.append((diff_type, diff_text))
             del bigpatch.diffs[0]
           else:
             # Deletion or equality.  Only take as much as we can stomach.
             diff_text = diff_text[:patch_size - patch.length1 -
                                   self.Patch_Margin]
             patch.length1 += len(diff_text)
             start1 += len(diff_text)
             if diff_type == self.DIFF_EQUAL:
               patch.length2 += len(diff_text)
               start2 += len(diff_text)
             else:
               empty = False
 
             patch.diffs.append((diff_type, diff_text))
             if diff_text == bigpatch.diffs[0][1]:
               del bigpatch.diffs[0]
             else:
               bigpatch.diffs[0] = (bigpatch.diffs[0][0],
                                    bigpatch.diffs[0][1][len(diff_text):])
 
         # Compute the head context for the next patch.
         precontext = self.diff_text2(patch.diffs)
         precontext = precontext[-self.Patch_Margin:]
         # Append the end context for this patch.
         postcontext = self.diff_text1(bigpatch.diffs)[:self.Patch_Margin]
         if postcontext:
           patch.length1 += len(postcontext)
           patch.length2 += len(postcontext)
           if len(patch.diffs) != 0 and patch.diffs[-1][0] == self.DIFF_EQUAL:
             patch.diffs[-1] = (self.DIFF_EQUAL, patch.diffs[-1][1] +
                                postcontext)
           else:
             patch.diffs.append((self.DIFF_EQUAL, postcontext))
 
         if not empty:
           x += 1
           patches.insert(x, patch)
 
   def patch_toText(self, patches):
     """Take a list of patches and return a textual representation.
 
     Args:
       patches: Array of Patch objects.
 
     Returns:
       Text representation of patches.
     """
     text = []
     for patch in patches:
       text.append(str(patch))
     return "".join(text)
 
   def patch_fromText(self, textline):
     """Parse a textual representation of patches and return a list of patch
     objects.
 
     Args:
       textline: Text representation of patches.
 
     Returns:
       Array of Patch objects.
 
     Raises:
       ValueError: If invalid input.
     """
     if type(textline) == unicode:
       # Patches should be composed of a subset of ascii chars, Unicode not
       # required.  If this encode raises UnicodeEncodeError, patch is invalid.
       textline = textline.encode("ascii")
     patches = []
     if not textline:
       return patches
     text = textline.split('\n')
     while len(text) != 0:
       m = re.match("^@@ -(\d+),?(\d*) \+(\d+),?(\d*) @@$", text[0])
       if not m:
         raise ValueError("Invalid patch string: " + text[0])
       patch = patch_obj()
       patches.append(patch)
       patch.start1 = int(m.group(1))
       if m.group(2) == '':
         patch.start1 -= 1
         patch.length1 = 1
       elif m.group(2) == '0':
         patch.length1 = 0
       else:
         patch.start1 -= 1
         patch.length1 = int(m.group(2))
 
       patch.start2 = int(m.group(3))
       if m.group(4) == '':
         patch.start2 -= 1
         patch.length2 = 1
       elif m.group(4) == '0':
         patch.length2 = 0
       else:
         patch.start2 -= 1
         patch.length2 = int(m.group(4))
 
       del text[0]
 
       while len(text) != 0:
         if text[0]:
           sign = text[0][0]
         else:
           sign = ''
         line = urllib.unquote(text[0][1:])
         line = line.decode("utf-8")
         if sign == '+':
           # Insertion.
           patch.diffs.append((self.DIFF_INSERT, line))
         elif sign == '-':
           # Deletion.
           patch.diffs.append((self.DIFF_DELETE, line))
         elif sign == ' ':
           # Minor equality.
           patch.diffs.append((self.DIFF_EQUAL, line))
         elif sign == '@':
           # Start of next patch.
           break
         elif sign == '':
           # Blank line?  Whatever.
           pass
         else:
           # WTF?
           raise ValueError("Invalid patch mode: '%s'\n%s" % (sign, line))
         del text[0]
     return patches
 
 
 class patch_obj:
   """Class representing one patch operation.
   """
 
   def __init__(self):
     """Initializes with an empty list of diffs.
     """
     self.diffs = []
     self.start1 = None
     self.start2 = None
     self.length1 = 0
     self.length2 = 0
 
   def __str__(self):
     """Emmulate GNU diff's format.
     Header: @@ -382,8 +481,9 @@
     Indicies are printed as 1-based, not 0-based.
 
     Returns:
       The GNU diff string.
     """
     if self.length1 == 0:
       coords1 = str(self.start1) + ",0"
     elif self.length1 == 1:
       coords1 = str(self.start1 + 1)
     else:
       coords1 = str(self.start1 + 1) + "," + str(self.length1)
     if self.length2 == 0:
       coords2 = str(self.start2) + ",0"
     elif self.length2 == 1:
       coords2 = str(self.start2 + 1)
     else:
       coords2 = str(self.start2 + 1) + "," + str(self.length2)
     text = ["@@ -", coords1, " +", coords2, " @@\n"]
     # Escape the body of the patch with %xx notation.
     for (op, data) in self.diffs:
       if op == diff_match_patch.DIFF_INSERT:
         text.append("+")
       elif op == diff_match_patch.DIFF_DELETE:
         text.append("-")
       elif op == diff_match_patch.DIFF_EQUAL:
         text.append(" ")
       # High ascii will raise UnicodeDecodeError.  Use Unicode instead.
       data = data.encode("utf-8")
       text.append(urllib.quote(data, "!~*'();/?:@&=+$,# ") + "\n")
     return "".join(text)
  No newline at end of file

rhodecode/model/pull_request.py

@@ -1,1739 +1,1737 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2012-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 pull request model for RhodeCode
 """
 
 
 import json
 import logging
 import datetime
 import urllib
 import collections
 
 from pyramid.threadlocal import get_current_request
 
 from rhodecode import events
 from rhodecode.translation import lazy_ugettext#, _
 from rhodecode.lib import helpers as h, hooks_utils, diffs
 from rhodecode.lib import audit_logger
 from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.markup_renderer import (
     DEFAULT_COMMENTS_RENDERER, RstTemplateRenderer)
 from rhodecode.lib.utils2 import safe_unicode, safe_str, md5_safe
 from rhodecode.lib.vcs.backends.base import (
     Reference, MergeResponse, MergeFailureReason, UpdateFailureReason)
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.exceptions import (
     CommitDoesNotExistError, EmptyRepositoryError)
 from rhodecode.model import BaseModel
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import (
     or_, PullRequest, PullRequestReviewers, ChangesetStatus,
     PullRequestVersion, ChangesetComment, Repository, RepoReviewRule)
 from rhodecode.model.meta import Session
 from rhodecode.model.notification import NotificationModel, \
     EmailNotificationModel
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.settings import VcsSettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 # Data structure to hold the response data when updating commits during a pull
 # request update.
 UpdateResponse = collections.namedtuple('UpdateResponse', [
     'executed', 'reason', 'new', 'old', 'changes',
     'source_changed', 'target_changed'])
 
 
 class PullRequestModel(BaseModel):
 
     cls = PullRequest
 
     DIFF_CONTEXT = diffs.DEFAULT_CONTEXT
 
     MERGE_STATUS_MESSAGES = {
         MergeFailureReason.NONE: lazy_ugettext(
             'This pull request can be automatically merged.'),
         MergeFailureReason.UNKNOWN: lazy_ugettext(
             'This pull request cannot be merged because of an unhandled'
             ' exception.'),
         MergeFailureReason.MERGE_FAILED: lazy_ugettext(
             'This pull request cannot be merged because of merge conflicts.'),
         MergeFailureReason.PUSH_FAILED: lazy_ugettext(
             'This pull request could not be merged because push to target'
             ' failed.'),
         MergeFailureReason.TARGET_IS_NOT_HEAD: lazy_ugettext(
             'This pull request cannot be merged because the target is not a'
             ' head.'),
         MergeFailureReason.HG_SOURCE_HAS_MORE_BRANCHES: lazy_ugettext(
             'This pull request cannot be merged because the source contains'
             ' more branches than the target.'),
         MergeFailureReason.HG_TARGET_HAS_MULTIPLE_HEADS: lazy_ugettext(
             'This pull request cannot be merged because the target has'
             ' multiple heads.'),
         MergeFailureReason.TARGET_IS_LOCKED: lazy_ugettext(
             'This pull request cannot be merged because the target repository'
             ' is locked.'),
         MergeFailureReason._DEPRECATED_MISSING_COMMIT: lazy_ugettext(
             'This pull request cannot be merged because the target or the '
             'source reference is missing.'),
         MergeFailureReason.MISSING_TARGET_REF: lazy_ugettext(
             'This pull request cannot be merged because the target '
             'reference is missing.'),
         MergeFailureReason.MISSING_SOURCE_REF: lazy_ugettext(
             'This pull request cannot be merged because the source '
             'reference is missing.'),
         MergeFailureReason.SUBREPO_MERGE_FAILED: lazy_ugettext(
             'This pull request cannot be merged because of conflicts related '
             'to sub repositories.'),
     }
 
     UPDATE_STATUS_MESSAGES = {
         UpdateFailureReason.NONE: lazy_ugettext(
             'Pull request update successful.'),
         UpdateFailureReason.UNKNOWN: lazy_ugettext(
             'Pull request update failed because of an unknown error.'),
         UpdateFailureReason.NO_CHANGE: lazy_ugettext(
             'No update needed because the source and target have not changed.'),
         UpdateFailureReason.WRONG_REF_TYPE: lazy_ugettext(
             'Pull request cannot be updated because the reference type is '
             'not supported for an update. Only Branch, Tag or Bookmark is allowed.'),
         UpdateFailureReason.MISSING_TARGET_REF: lazy_ugettext(
             'This pull request cannot be updated because the target '
             'reference is missing.'),
         UpdateFailureReason.MISSING_SOURCE_REF: lazy_ugettext(
             'This pull request cannot be updated because the source '
             'reference is missing.'),
     }
 
     def __get_pull_request(self, pull_request):
         return self._get_instance((
             PullRequest, PullRequestVersion), pull_request)
 
     def _check_perms(self, perms, pull_request, user, api=False):
         if not api:
             return h.HasRepoPermissionAny(*perms)(
                 user=user, repo_name=pull_request.target_repo.repo_name)
         else:
             return h.HasRepoPermissionAnyApi(*perms)(
                 user=user, repo_name=pull_request.target_repo.repo_name)
 
     def check_user_read(self, pull_request, user, api=False):
         _perms = ('repository.admin', 'repository.write', 'repository.read',)
         return self._check_perms(_perms, pull_request, user, api)
 
     def check_user_merge(self, pull_request, user, api=False):
         _perms = ('repository.admin', 'repository.write', 'hg.admin',)
         return self._check_perms(_perms, pull_request, user, api)
 
     def check_user_update(self, pull_request, user, api=False):
         owner = user.user_id == pull_request.user_id
         return self.check_user_merge(pull_request, user, api) or owner
 
     def check_user_delete(self, pull_request, user):
         owner = user.user_id == pull_request.user_id
         _perms = ('repository.admin',)
         return self._check_perms(_perms, pull_request, user) or owner
 
     def check_user_change_status(self, pull_request, user, api=False):
         reviewer = user.user_id in [x.user_id for x in
                                     pull_request.reviewers]
         return self.check_user_update(pull_request, user, api) or reviewer
 
     def check_user_comment(self, pull_request, user):
         owner = user.user_id == pull_request.user_id
         return self.check_user_read(pull_request, user) or owner
 
     def get(self, pull_request):
         return self.__get_pull_request(pull_request)
 
     def _prepare_get_all_query(self, repo_name, source=False, statuses=None,
                                opened_by=None, order_by=None,
                                order_dir='desc'):
         repo = None
         if repo_name:
             repo = self._get_repo(repo_name)
 
         q = PullRequest.query()
 
         # source or target
         if repo and source:
             q = q.filter(PullRequest.source_repo == repo)
         elif repo:
             q = q.filter(PullRequest.target_repo == repo)
 
         # closed,opened
         if statuses:
             q = q.filter(PullRequest.status.in_(statuses))
 
         # opened by filter
         if opened_by:
             q = q.filter(PullRequest.user_id.in_(opened_by))
 
         if order_by:
             order_map = {
                 'name_raw': PullRequest.pull_request_id,
                 'title': PullRequest.title,
                 'updated_on_raw': PullRequest.updated_on,
                 'target_repo': PullRequest.target_repo_id
             }
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_all(self, repo_name, source=False, statuses=None,
                   opened_by=None):
         """
         Count the number of pull requests for a specific repository.
 
         :param repo_name: target or source repo
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :returns: int number of pull requests
         """
         q = self._prepare_get_all_query(
             repo_name, source=source, statuses=statuses, opened_by=opened_by)
 
         return q.count()
 
     def get_all(self, repo_name, source=False, statuses=None, opened_by=None,
                 offset=0, length=None, order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository.
 
         :param repo_name: target or source repo
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
         q = self._prepare_get_all_query(
             repo_name, source=source, statuses=statuses, opened_by=opened_by,
             order_by=order_by, order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def count_awaiting_review(self, repo_name, source=False, statuses=None,
                               opened_by=None):
         """
         Count the number of pull requests for a specific repository that are
         awaiting review.
 
         :param repo_name: target or source repo
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :returns: int number of pull requests
         """
         pull_requests = self.get_awaiting_review(
             repo_name, source=source, statuses=statuses, opened_by=opened_by)
 
         return len(pull_requests)
 
     def get_awaiting_review(self, repo_name, source=False, statuses=None,
                             opened_by=None, offset=0, length=None,
                             order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository that are awaiting
         review.
 
         :param repo_name: target or source repo
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
         pull_requests = self.get_all(
             repo_name, source=source, statuses=statuses, opened_by=opened_by,
             order_by=order_by, order_dir=order_dir)
 
         _filtered_pull_requests = []
         for pr in pull_requests:
             status = pr.calculated_review_status()
             if status in [ChangesetStatus.STATUS_NOT_REVIEWED,
                           ChangesetStatus.STATUS_UNDER_REVIEW]:
                 _filtered_pull_requests.append(pr)
         if length:
             return _filtered_pull_requests[offset:offset+length]
         else:
             return _filtered_pull_requests
 
     def count_awaiting_my_review(self, repo_name, source=False, statuses=None,
                                  opened_by=None, user_id=None):
         """
         Count the number of pull requests for a specific repository that are
         awaiting review from a specific user.
 
         :param repo_name: target or source repo
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :param user_id: reviewer user of the pull request
         :returns: int number of pull requests
         """
         pull_requests = self.get_awaiting_my_review(
             repo_name, source=source, statuses=statuses, opened_by=opened_by,
             user_id=user_id)
 
         return len(pull_requests)
 
     def get_awaiting_my_review(self, repo_name, source=False, statuses=None,
                                opened_by=None, user_id=None, offset=0,
                                length=None, order_by=None, order_dir='desc'):
         """
         Get all pull requests for a specific repository that are awaiting
         review from a specific user.
 
         :param repo_name: target or source repo
         :param source: boolean flag to specify if repo_name refers to source
         :param statuses: list of pull request statuses
         :param opened_by: author user of the pull request
         :param user_id: reviewer user of the pull request
         :param offset: pagination offset
         :param length: length of returned list
         :param order_by: order of the returned list
         :param order_dir: 'asc' or 'desc' ordering direction
         :returns: list of pull requests
         """
         pull_requests = self.get_all(
             repo_name, source=source, statuses=statuses, opened_by=opened_by,
             order_by=order_by, order_dir=order_dir)
 
         _my = PullRequestModel().get_not_reviewed(user_id)
         my_participation = []
         for pr in pull_requests:
             if pr in _my:
                 my_participation.append(pr)
         _filtered_pull_requests = my_participation
         if length:
             return _filtered_pull_requests[offset:offset+length]
         else:
             return _filtered_pull_requests
 
     def get_not_reviewed(self, user_id):
         return [
             x.pull_request for x in PullRequestReviewers.query().filter(
                 PullRequestReviewers.user_id == user_id).all()
         ]
 
     def _prepare_participating_query(self, user_id=None, statuses=None,
                                      order_by=None, order_dir='desc'):
         q = PullRequest.query()
         if user_id:
             reviewers_subquery = Session().query(
                 PullRequestReviewers.pull_request_id).filter(
                 PullRequestReviewers.user_id == user_id).subquery()
             user_filter = or_(
                 PullRequest.user_id == user_id,
                 PullRequest.pull_request_id.in_(reviewers_subquery)
             )
             q = PullRequest.query().filter(user_filter)
 
         # closed,opened
         if statuses:
             q = q.filter(PullRequest.status.in_(statuses))
 
         if order_by:
             order_map = {
                 'name_raw': PullRequest.pull_request_id,
                 'title': PullRequest.title,
                 'updated_on_raw': PullRequest.updated_on,
                 'target_repo': PullRequest.target_repo_id
             }
             if order_dir == 'asc':
                 q = q.order_by(order_map[order_by].asc())
             else:
                 q = q.order_by(order_map[order_by].desc())
 
         return q
 
     def count_im_participating_in(self, user_id=None, statuses=None):
         q = self._prepare_participating_query(user_id, statuses=statuses)
         return q.count()
 
     def get_im_participating_in(
             self, user_id=None, statuses=None, offset=0,
             length=None, order_by=None, order_dir='desc'):
         """
         Get all Pull requests that i'm participating in, or i have opened
         """
 
         q = self._prepare_participating_query(
             user_id, statuses=statuses, order_by=order_by,
             order_dir=order_dir)
 
         if length:
             pull_requests = q.limit(length).offset(offset).all()
         else:
             pull_requests = q.all()
 
         return pull_requests
 
     def get_versions(self, pull_request):
         """
         returns version of pull request sorted by ID descending
         """
         return PullRequestVersion.query()\
             .filter(PullRequestVersion.pull_request == pull_request)\
             .order_by(PullRequestVersion.pull_request_version_id.asc())\
             .all()
 
     def get_pr_version(self, pull_request_id, version=None):
         at_version = None
 
         if version and version == 'latest':
             pull_request_ver = PullRequest.get(pull_request_id)
             pull_request_obj = pull_request_ver
             _org_pull_request_obj = pull_request_obj
             at_version = 'latest'
         elif version:
             pull_request_ver = PullRequestVersion.get_or_404(version)
             pull_request_obj = pull_request_ver
             _org_pull_request_obj = pull_request_ver.pull_request
             at_version = pull_request_ver.pull_request_version_id
         else:
             _org_pull_request_obj = pull_request_obj = PullRequest.get_or_404(
                 pull_request_id)
 
         pull_request_display_obj = PullRequest.get_pr_display_object(
             pull_request_obj, _org_pull_request_obj)
 
         return _org_pull_request_obj, pull_request_obj, \
                pull_request_display_obj, at_version
 
     def create(self, created_by, source_repo, source_ref, target_repo,
                target_ref, revisions, reviewers, title, description=None,
                description_renderer=None,
                reviewer_data=None, translator=None, auth_user=None):
         translator = translator or get_current_request().translate
 
         created_by_user = self._get_user(created_by)
         auth_user = auth_user or created_by_user.AuthUser()
         source_repo = self._get_repo(source_repo)
         target_repo = self._get_repo(target_repo)
 
         pull_request = PullRequest()
         pull_request.source_repo = source_repo
         pull_request.source_ref = source_ref
         pull_request.target_repo = target_repo
         pull_request.target_ref = target_ref
         pull_request.revisions = revisions
         pull_request.title = title
         pull_request.description = description
         pull_request.description_renderer = description_renderer
         pull_request.author = created_by_user
         pull_request.reviewer_data = reviewer_data
 
         Session().add(pull_request)
         Session().flush()
 
         reviewer_ids = set()
         # members / reviewers
         for reviewer_object in reviewers:
             user_id, reasons, mandatory, rules = reviewer_object
             user = self._get_user(user_id)
 
             # skip duplicates
             if user.user_id in reviewer_ids:
                 continue
 
             reviewer_ids.add(user.user_id)
 
             reviewer = PullRequestReviewers()
             reviewer.user = user
             reviewer.pull_request = pull_request
             reviewer.reasons = reasons
             reviewer.mandatory = mandatory
 
             # NOTE(marcink): pick only first rule for now
             rule_id = list(rules)[0] if rules else None
             rule = RepoReviewRule.get(rule_id) if rule_id else None
             if rule:
                 review_group = rule.user_group_vote_rule(user_id)
                 # we check if this particular reviewer is member of a voting group
                 if review_group:
                     # NOTE(marcink):
                     # can be that user is member of more but we pick the first same,
                     # same as default reviewers algo
                     review_group = review_group[0]
 
                     rule_data = {
                         'rule_name':
                             rule.review_rule_name,
                         'rule_user_group_entry_id':
                             review_group.repo_review_rule_users_group_id,
                         'rule_user_group_name':
                             review_group.users_group.users_group_name,
                         'rule_user_group_members':
                             [x.user.username for x in review_group.users_group.members],
                         'rule_user_group_members_id':
                             [x.user.user_id for x in review_group.users_group.members],
                     }
                     # e.g {'vote_rule': -1, 'mandatory': True}
                     rule_data.update(review_group.rule_data())
 
                     reviewer.rule_data = rule_data
 
             Session().add(reviewer)
             Session().flush()
 
         # Set approval status to "Under Review" for all commits which are
         # part of this pull request.
         ChangesetStatusModel().set_status(
             repo=target_repo,
             status=ChangesetStatus.STATUS_UNDER_REVIEW,
             user=created_by_user,
             pull_request=pull_request
         )
         # we commit early at this point. This has to do with a fact
         # that before queries do some row-locking. And because of that
         # we need to commit and finish transation before below validate call
         # that for large repos could be long resulting in long row locks
         Session().commit()
 
         # prepare workspace, and run initial merge simulation
         MergeCheck.validate(
             pull_request, auth_user=auth_user, translator=translator)
 
         self.notify_reviewers(pull_request, reviewer_ids)
         self._trigger_pull_request_hook(
             pull_request, created_by_user, 'create')
 
         creation_data = pull_request.get_api_data(with_merge_state=False)
         self._log_audit_action(
             'repo.pull_request.create', {'data': creation_data},
             auth_user, pull_request)
 
         return pull_request
 
     def _trigger_pull_request_hook(self, pull_request, user, action):
         pull_request = self.__get_pull_request(pull_request)
         target_scm = pull_request.target_repo.scm_instance()
         if action == 'create':
             trigger_hook = hooks_utils.trigger_log_create_pull_request_hook
         elif action == 'merge':
             trigger_hook = hooks_utils.trigger_log_merge_pull_request_hook
         elif action == 'close':
             trigger_hook = hooks_utils.trigger_log_close_pull_request_hook
         elif action == 'review_status_change':
             trigger_hook = hooks_utils.trigger_log_review_pull_request_hook
         elif action == 'update':
             trigger_hook = hooks_utils.trigger_log_update_pull_request_hook
         else:
             return
 
         trigger_hook(
             username=user.username,
             repo_name=pull_request.target_repo.repo_name,
             repo_alias=target_scm.alias,
             pull_request=pull_request)
 
     def _get_commit_ids(self, pull_request):
         """
         Return the commit ids of the merged pull request.
 
         This method is not dealing correctly yet with the lack of autoupdates
         nor with the implicit target updates.
         For example: if a commit in the source repo is already in the target it
         will be reported anyways.
         """
         merge_rev = pull_request.merge_rev
         if merge_rev is None:
             raise ValueError('This pull request was not merged yet')
 
         commit_ids = list(pull_request.revisions)
         if merge_rev not in commit_ids:
             commit_ids.append(merge_rev)
 
         return commit_ids
 
     def merge_repo(self, pull_request, user, extras):
         log.debug("Merging pull request %s", pull_request.pull_request_id)
         extras['user_agent'] = 'internal-merge'
         merge_state = self._merge_pull_request(pull_request, user, extras)
         if merge_state.executed:
             log.debug(
                 "Merge was successful, updating the pull request comments.")
             self._comment_and_close_pr(pull_request, user, merge_state)
 
             self._log_audit_action(
                 'repo.pull_request.merge',
                 {'merge_state': merge_state.__dict__},
                 user, pull_request)
 
         else:
             log.warn("Merge failed, not updating the pull request.")
         return merge_state
 
     def _merge_pull_request(self, pull_request, user, extras, merge_msg=None):
         target_vcs = pull_request.target_repo.scm_instance()
         source_vcs = pull_request.source_repo.scm_instance()
 
         message = safe_unicode(merge_msg or vcs_settings.MERGE_MESSAGE_TMPL).format(
             pr_id=pull_request.pull_request_id,
             pr_title=pull_request.title,
             source_repo=source_vcs.name,
             source_ref_name=pull_request.source_ref_parts.name,
             target_repo=target_vcs.name,
             target_ref_name=pull_request.target_ref_parts.name,
         )
 
         workspace_id = self._workspace_id(pull_request)
         repo_id = pull_request.target_repo.repo_id
         use_rebase = self._use_rebase_for_merging(pull_request)
         close_branch = self._close_branch_before_merging(pull_request)
 
         target_ref = self._refresh_reference(
             pull_request.target_ref_parts, target_vcs)
 
         callback_daemon, extras = prepare_callback_daemon(
             extras, protocol=vcs_settings.HOOKS_PROTOCOL,
             host=vcs_settings.HOOKS_HOST,
             use_direct_calls=vcs_settings.HOOKS_DIRECT_CALLS)
 
         with callback_daemon:
             # TODO: johbo: Implement a clean way to run a config_override
             # for a single call.
             target_vcs.config.set(
                 'rhodecode', 'RC_SCM_DATA', json.dumps(extras))
 
             user_name = user.short_contact
             merge_state = target_vcs.merge(
                 repo_id, workspace_id, target_ref, source_vcs,
                 pull_request.source_ref_parts,
                 user_name=user_name, user_email=user.email,
                 message=message, use_rebase=use_rebase,
                 close_branch=close_branch)
         return merge_state
 
     def _comment_and_close_pr(self, pull_request, user, merge_state, close_msg=None):
         pull_request.merge_rev = merge_state.merge_ref.commit_id
         pull_request.updated_on = datetime.datetime.now()
         close_msg = close_msg or 'Pull request merged and closed'
 
         CommentsModel().create(
             text=safe_unicode(close_msg),
             repo=pull_request.target_repo.repo_id,
             user=user.user_id,
             pull_request=pull_request.pull_request_id,
             f_path=None,
             line_no=None,
             closing_pr=True
         )
 
         Session().add(pull_request)
         Session().flush()
         # TODO: paris: replace invalidation with less radical solution
         ScmModel().mark_for_invalidation(
             pull_request.target_repo.repo_name)
         self._trigger_pull_request_hook(pull_request, user, 'merge')
 
     def has_valid_update_type(self, pull_request):
         source_ref_type = pull_request.source_ref_parts.type
         return source_ref_type in ['book', 'branch', 'tag']
 
     def update_commits(self, pull_request):
         """
         Get the updated list of commits for the pull request
         and return the new pull request version and the list
         of commits processed by this update action
         """
         pull_request = self.__get_pull_request(pull_request)
         source_ref_type = pull_request.source_ref_parts.type
         source_ref_name = pull_request.source_ref_parts.name
         source_ref_id = pull_request.source_ref_parts.commit_id
 
         target_ref_type = pull_request.target_ref_parts.type
         target_ref_name = pull_request.target_ref_parts.name
         target_ref_id = pull_request.target_ref_parts.commit_id
 
         if not self.has_valid_update_type(pull_request):
             log.debug(
                 "Skipping update of pull request %s due to ref type: %s",
                 pull_request, source_ref_type)
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.WRONG_REF_TYPE,
                 old=pull_request, new=None, changes=None,
                 source_changed=False, target_changed=False)
 
         # source repo
         source_repo = pull_request.source_repo.scm_instance()
         try:
             source_commit = source_repo.get_commit(commit_id=source_ref_name)
         except CommitDoesNotExistError:
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.MISSING_SOURCE_REF,
                 old=pull_request, new=None, changes=None,
                 source_changed=False, target_changed=False)
 
         source_changed = source_ref_id != source_commit.raw_id
 
         # target repo
         target_repo = pull_request.target_repo.scm_instance()
         try:
             target_commit = target_repo.get_commit(commit_id=target_ref_name)
         except CommitDoesNotExistError:
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.MISSING_TARGET_REF,
                 old=pull_request, new=None, changes=None,
                 source_changed=False, target_changed=False)
         target_changed = target_ref_id != target_commit.raw_id
 
         if not (source_changed or target_changed):
             log.debug("Nothing changed in pull request %s", pull_request)
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.NO_CHANGE,
                 old=pull_request, new=None, changes=None,
                 source_changed=target_changed, target_changed=source_changed)
 
         change_in_found = 'target repo' if target_changed else 'source repo'
         log.debug('Updating pull request because of change in %s detected',
                   change_in_found)
 
         # Finally there is a need for an update, in case of source change
         # we create a new version, else just an update
         if source_changed:
             pull_request_version = self._create_version_from_snapshot(pull_request)
             self._link_comments_to_version(pull_request_version)
         else:
             try:
                 ver = pull_request.versions[-1]
             except IndexError:
                 ver = None
 
             pull_request.pull_request_version_id = \
                 ver.pull_request_version_id if ver else None
             pull_request_version = pull_request
 
         try:
             if target_ref_type in ('tag', 'branch', 'book'):
                 target_commit = target_repo.get_commit(target_ref_name)
             else:
                 target_commit = target_repo.get_commit(target_ref_id)
         except CommitDoesNotExistError:
             return UpdateResponse(
                 executed=False,
                 reason=UpdateFailureReason.MISSING_TARGET_REF,
                 old=pull_request, new=None, changes=None,
                 source_changed=source_changed, target_changed=target_changed)
 
         # re-compute commit ids
         old_commit_ids = pull_request.revisions
         pre_load = ["author", "branch", "date", "message"]
         commit_ranges = target_repo.compare(
             target_commit.raw_id, source_commit.raw_id, source_repo, merge=True,
             pre_load=pre_load)
 
         ancestor = target_repo.get_common_ancestor(
             target_commit.raw_id, source_commit.raw_id, source_repo)
 
         pull_request.source_ref = '%s:%s:%s' % (
             source_ref_type, source_ref_name, source_commit.raw_id)
         pull_request.target_ref = '%s:%s:%s' % (
             target_ref_type, target_ref_name, ancestor)
 
         pull_request.revisions = [
             commit.raw_id for commit in reversed(commit_ranges)]
         pull_request.updated_on = datetime.datetime.now()
         Session().add(pull_request)
         new_commit_ids = pull_request.revisions
 
         old_diff_data, new_diff_data = self._generate_update_diffs(
             pull_request, pull_request_version)
 
         # calculate commit and file changes
         changes = self._calculate_commit_id_changes(
             old_commit_ids, new_commit_ids)
         file_changes = self._calculate_file_changes(
             old_diff_data, new_diff_data)
 
         # set comments as outdated if DIFFS changed
         CommentsModel().outdate_comments(
             pull_request, old_diff_data=old_diff_data,
             new_diff_data=new_diff_data)
 
         commit_changes = (changes.added or changes.removed)
         file_node_changes = (
             file_changes.added or file_changes.modified or file_changes.removed)
         pr_has_changes = commit_changes or file_node_changes
 
         # Add an automatic comment to the pull request, in case
         # anything has changed
         if pr_has_changes:
             update_comment = CommentsModel().create(
                 text=self._render_update_message(changes, file_changes),
                 repo=pull_request.target_repo,
                 user=pull_request.author,
                 pull_request=pull_request,
                 send_email=False, renderer=DEFAULT_COMMENTS_RENDERER)
 
             # Update status to "Under Review" for added commits
             for commit_id in changes.added:
                 ChangesetStatusModel().set_status(
                     repo=pull_request.source_repo,
                     status=ChangesetStatus.STATUS_UNDER_REVIEW,
                     comment=update_comment,
                     user=pull_request.author,
                     pull_request=pull_request,
                     revision=commit_id)
 
         log.debug(
             'Updated pull request %s, added_ids: %s, common_ids: %s, '
             'removed_ids: %s', pull_request.pull_request_id,
             changes.added, changes.common, changes.removed)
         log.debug(
             'Updated pull request with the following file changes: %s',
             file_changes)
 
         log.info(
             "Updated pull request %s from commit %s to commit %s, "
             "stored new version %s of this pull request.",
             pull_request.pull_request_id, source_ref_id,
             pull_request.source_ref_parts.commit_id,
             pull_request_version.pull_request_version_id)
         Session().commit()
         self._trigger_pull_request_hook(
             pull_request, pull_request.author, 'update')
 
         return UpdateResponse(
             executed=True, reason=UpdateFailureReason.NONE,
             old=pull_request, new=pull_request_version, changes=changes,
             source_changed=source_changed, target_changed=target_changed)
 
     def _create_version_from_snapshot(self, pull_request):
         version = PullRequestVersion()
         version.title = pull_request.title
         version.description = pull_request.description
         version.status = pull_request.status
         version.created_on = datetime.datetime.now()
         version.updated_on = pull_request.updated_on
         version.user_id = pull_request.user_id
         version.source_repo = pull_request.source_repo
         version.source_ref = pull_request.source_ref
         version.target_repo = pull_request.target_repo
         version.target_ref = pull_request.target_ref
 
         version._last_merge_source_rev = pull_request._last_merge_source_rev
         version._last_merge_target_rev = pull_request._last_merge_target_rev
         version.last_merge_status = pull_request.last_merge_status
         version.shadow_merge_ref = pull_request.shadow_merge_ref
         version.merge_rev = pull_request.merge_rev
         version.reviewer_data = pull_request.reviewer_data
 
         version.revisions = pull_request.revisions
         version.pull_request = pull_request
         Session().add(version)
         Session().flush()
 
         return version
 
     def _generate_update_diffs(self, pull_request, pull_request_version):
 
         diff_context = (
             self.DIFF_CONTEXT +
             CommentsModel.needed_extra_diff_context())
         hide_whitespace_changes = False
         source_repo = pull_request_version.source_repo
         source_ref_id = pull_request_version.source_ref_parts.commit_id
         target_ref_id = pull_request_version.target_ref_parts.commit_id
         old_diff = self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes=hide_whitespace_changes, diff_context=diff_context)
 
         source_repo = pull_request.source_repo
         source_ref_id = pull_request.source_ref_parts.commit_id
         target_ref_id = pull_request.target_ref_parts.commit_id
 
         new_diff = self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes=hide_whitespace_changes, diff_context=diff_context)
 
         old_diff_data = diffs.DiffProcessor(old_diff)
         old_diff_data.prepare()
         new_diff_data = diffs.DiffProcessor(new_diff)
         new_diff_data.prepare()
 
         return old_diff_data, new_diff_data
 
     def _link_comments_to_version(self, pull_request_version):
         """
         Link all unlinked comments of this pull request to the given version.
 
         :param pull_request_version: The `PullRequestVersion` to which
             the comments shall be linked.
 
         """
         pull_request = pull_request_version.pull_request
         comments = ChangesetComment.query()\
             .filter(
                 # TODO: johbo: Should we query for the repo at all here?
                 # Pending decision on how comments of PRs are to be related
                 # to either the source repo, the target repo or no repo at all.
                 ChangesetComment.repo_id == pull_request.target_repo.repo_id,
                 ChangesetComment.pull_request == pull_request,
                 ChangesetComment.pull_request_version == None)\
             .order_by(ChangesetComment.comment_id.asc())
 
         # TODO: johbo: Find out why this breaks if it is done in a bulk
         # operation.
         for comment in comments:
             comment.pull_request_version_id = (
                 pull_request_version.pull_request_version_id)
             Session().add(comment)
 
     def _calculate_commit_id_changes(self, old_ids, new_ids):
         added = [x for x in new_ids if x not in old_ids]
         common = [x for x in new_ids if x in old_ids]
         removed = [x for x in old_ids if x not in new_ids]
         total = new_ids
         return ChangeTuple(added, common, removed, total)
 
     def _calculate_file_changes(self, old_diff_data, new_diff_data):
 
         old_files = OrderedDict()
         for diff_data in old_diff_data.parsed_diff:
             old_files[diff_data['filename']] = md5_safe(diff_data['raw_diff'])
 
         added_files = []
         modified_files = []
         removed_files = []
         for diff_data in new_diff_data.parsed_diff:
             new_filename = diff_data['filename']
             new_hash = md5_safe(diff_data['raw_diff'])
 
             old_hash = old_files.get(new_filename)
             if not old_hash:
                 # file is not present in old diff, means it's added
                 added_files.append(new_filename)
             else:
                 if new_hash != old_hash:
                     modified_files.append(new_filename)
                 # now remove a file from old, since we have seen it already
                 del old_files[new_filename]
 
         # removed files is when there are present in old, but not in NEW,
         # since we remove old files that are present in new diff, left-overs
         # if any should be the removed files
         removed_files.extend(old_files.keys())
 
         return FileChangeTuple(added_files, modified_files, removed_files)
 
     def _render_update_message(self, changes, file_changes):
         """
         render the message using DEFAULT_COMMENTS_RENDERER (RST renderer),
         so it's always looking the same disregarding on which default
         renderer system is using.
 
         :param changes: changes named tuple
         :param file_changes: file changes named tuple
 
         """
         new_status = ChangesetStatus.get_status_lbl(
             ChangesetStatus.STATUS_UNDER_REVIEW)
 
         changed_files = (
             file_changes.added + file_changes.modified + file_changes.removed)
 
         params = {
             'under_review_label': new_status,
             'added_commits': changes.added,
             'removed_commits': changes.removed,
             'changed_files': changed_files,
             'added_files': file_changes.added,
             'modified_files': file_changes.modified,
             'removed_files': file_changes.removed,
         }
         renderer = RstTemplateRenderer()
         return renderer.render('pull_request_update.mako', **params)
 
     def edit(self, pull_request, title, description, description_renderer, user):
         pull_request = self.__get_pull_request(pull_request)
         old_data = pull_request.get_api_data(with_merge_state=False)
         if pull_request.is_closed():
             raise ValueError('This pull request is closed')
         if title:
             pull_request.title = title
         pull_request.description = description
         pull_request.updated_on = datetime.datetime.now()
         pull_request.description_renderer = description_renderer
         Session().add(pull_request)
         self._log_audit_action(
             'repo.pull_request.edit', {'old_data': old_data},
             user, pull_request)
 
     def update_reviewers(self, pull_request, reviewer_data, user):
         """
         Update the reviewers in the pull request
 
         :param pull_request: the pr to update
         :param reviewer_data: list of tuples
             [(user, ['reason1', 'reason2'], mandatory_flag, [rules])]
         """
         pull_request = self.__get_pull_request(pull_request)
         if pull_request.is_closed():
             raise ValueError('This pull request is closed')
 
         reviewers = {}
         for user_id, reasons, mandatory, rules in reviewer_data:
             if isinstance(user_id, (int, basestring)):
                 user_id = self._get_user(user_id).user_id
             reviewers[user_id] = {
                 'reasons': reasons, 'mandatory': mandatory}
 
         reviewers_ids = set(reviewers.keys())
         current_reviewers = PullRequestReviewers.query()\
             .filter(PullRequestReviewers.pull_request ==
                     pull_request).all()
         current_reviewers_ids = set([x.user.user_id for x in current_reviewers])
 
         ids_to_add = reviewers_ids.difference(current_reviewers_ids)
         ids_to_remove = current_reviewers_ids.difference(reviewers_ids)
 
         log.debug("Adding %s reviewers", ids_to_add)
         log.debug("Removing %s reviewers", ids_to_remove)
         changed = False
         for uid in ids_to_add:
             changed = True
             _usr = self._get_user(uid)
             reviewer = PullRequestReviewers()
             reviewer.user = _usr
             reviewer.pull_request = pull_request
             reviewer.reasons = reviewers[uid]['reasons']
             # NOTE(marcink): mandatory shouldn't be changed now
             # reviewer.mandatory = reviewers[uid]['reasons']
             Session().add(reviewer)
             self._log_audit_action(
                 'repo.pull_request.reviewer.add', {'data': reviewer.get_dict()},
                 user, pull_request)
 
         for uid in ids_to_remove:
             changed = True
             reviewers = PullRequestReviewers.query()\
                 .filter(PullRequestReviewers.user_id == uid,
                         PullRequestReviewers.pull_request == pull_request)\
                 .all()
             # use .all() in case we accidentally added the same person twice
             # this CAN happen due to the lack of DB checks
             for obj in reviewers:
                 old_data = obj.get_dict()
                 Session().delete(obj)
                 self._log_audit_action(
                     'repo.pull_request.reviewer.delete',
                     {'old_data': old_data}, user, pull_request)
 
         if changed:
             pull_request.updated_on = datetime.datetime.now()
             Session().add(pull_request)
 
         self.notify_reviewers(pull_request, ids_to_add)
         return ids_to_add, ids_to_remove
 
     def get_url(self, pull_request, request=None, permalink=False):
         if not request:
             request = get_current_request()
 
         if permalink:
             return request.route_url(
                 'pull_requests_global',
                 pull_request_id=pull_request.pull_request_id,)
         else:
             return request.route_url('pullrequest_show',
                 repo_name=safe_str(pull_request.target_repo.repo_name),
                 pull_request_id=pull_request.pull_request_id,)
 
     def get_shadow_clone_url(self, pull_request, request=None):
         """
         Returns qualified url pointing to the shadow repository. If this pull
         request is closed there is no shadow repository and ``None`` will be
         returned.
         """
         if pull_request.is_closed():
             return None
         else:
             pr_url = urllib.unquote(self.get_url(pull_request, request=request))
             return safe_unicode('{pr_url}/repository'.format(pr_url=pr_url))
 
     def notify_reviewers(self, pull_request, reviewers_ids):
         # notification to reviewers
         if not reviewers_ids:
             return
 
         pull_request_obj = pull_request
         # get the current participants of this pull request
         recipients = reviewers_ids
         notification_type = EmailNotificationModel.TYPE_PULL_REQUEST
 
         pr_source_repo = pull_request_obj.source_repo
         pr_target_repo = pull_request_obj.target_repo
 
         pr_url = h.route_url('pullrequest_show',
             repo_name=pr_target_repo.repo_name,
             pull_request_id=pull_request_obj.pull_request_id,)
 
         # set some variables for email notification
         pr_target_repo_url = h.route_url(
             'repo_summary', repo_name=pr_target_repo.repo_name)
 
         pr_source_repo_url = h.route_url(
             'repo_summary', repo_name=pr_source_repo.repo_name)
 
         # pull request specifics
         pull_request_commits = [
             (x.raw_id, x.message)
             for x in map(pr_source_repo.get_commit, pull_request.revisions)]
 
         kwargs = {
             'user': pull_request.author,
             'pull_request': pull_request_obj,
             'pull_request_commits': pull_request_commits,
 
             'pull_request_target_repo': pr_target_repo,
             'pull_request_target_repo_url': pr_target_repo_url,
 
             'pull_request_source_repo': pr_source_repo,
             'pull_request_source_repo_url': pr_source_repo_url,
 
             'pull_request_url': pr_url,
         }
 
         # pre-generate the subject for notification itself
         (subject,
          _h, _e,  # we don't care about those
          body_plaintext) = EmailNotificationModel().render_email(
             notification_type, **kwargs)
 
         # create notification objects, and emails
         NotificationModel().create(
             created_by=pull_request.author,
             notification_subject=subject,
             notification_body=body_plaintext,
             notification_type=notification_type,
             recipients=recipients,
             email_kwargs=kwargs,
         )
 
     def delete(self, pull_request, user):
         pull_request = self.__get_pull_request(pull_request)
         old_data = pull_request.get_api_data(with_merge_state=False)
         self._cleanup_merge_workspace(pull_request)
         self._log_audit_action(
             'repo.pull_request.delete', {'old_data': old_data},
             user, pull_request)
         Session().delete(pull_request)
 
     def close_pull_request(self, pull_request, user):
         pull_request = self.__get_pull_request(pull_request)
         self._cleanup_merge_workspace(pull_request)
         pull_request.status = PullRequest.STATUS_CLOSED
         pull_request.updated_on = datetime.datetime.now()
         Session().add(pull_request)
         self._trigger_pull_request_hook(
             pull_request, pull_request.author, 'close')
 
         pr_data = pull_request.get_api_data(with_merge_state=False)
         self._log_audit_action(
             'repo.pull_request.close', {'data': pr_data}, user, pull_request)
 
     def close_pull_request_with_comment(
             self, pull_request, user, repo, message=None, auth_user=None):
 
         pull_request_review_status = pull_request.calculated_review_status()
 
         if pull_request_review_status == ChangesetStatus.STATUS_APPROVED:
             # approved only if we have voting consent
             status = ChangesetStatus.STATUS_APPROVED
         else:
             status = ChangesetStatus.STATUS_REJECTED
         status_lbl = ChangesetStatus.get_status_lbl(status)
 
         default_message = (
             'Closing with status change {transition_icon} {status}.'
         ).format(transition_icon='>', status=status_lbl)
         text = message or default_message
 
         # create a comment, and link it to new status
         comment = CommentsModel().create(
             text=text,
             repo=repo.repo_id,
             user=user.user_id,
             pull_request=pull_request.pull_request_id,
             status_change=status_lbl,
             status_change_type=status,
             closing_pr=True,
             auth_user=auth_user,
         )
 
         # calculate old status before we change it
         old_calculated_status = pull_request.calculated_review_status()
         ChangesetStatusModel().set_status(
             repo.repo_id,
             status,
             user.user_id,
             comment=comment,
             pull_request=pull_request.pull_request_id
         )
 
         Session().flush()
         events.trigger(events.PullRequestCommentEvent(pull_request, comment))
         # we now calculate the status of pull request again, and based on that
         # calculation trigger status change. This might happen in cases
         # that non-reviewer admin closes a pr, which means his vote doesn't
         # change the status, while if he's a reviewer this might change it.
         calculated_status = pull_request.calculated_review_status()
         if old_calculated_status != calculated_status:
             self._trigger_pull_request_hook(
                 pull_request, user, 'review_status_change')
 
         # finally close the PR
         PullRequestModel().close_pull_request(
             pull_request.pull_request_id, user)
 
         return comment, status
 
     def merge_status(self, pull_request, translator=None,
                      force_shadow_repo_refresh=False):
         _ = translator or get_current_request().translate
 
         if not self._is_merge_enabled(pull_request):
             return False, _('Server-side pull request merging is disabled.')
         if pull_request.is_closed():
             return False, _('This pull request is closed.')
         merge_possible, msg = self._check_repo_requirements(
             target=pull_request.target_repo, source=pull_request.source_repo,
             translator=_)
         if not merge_possible:
             return merge_possible, msg
 
         try:
             resp = self._try_merge(
                 pull_request,
                 force_shadow_repo_refresh=force_shadow_repo_refresh)
             log.debug("Merge response: %s", resp)
             status = resp.possible, self.merge_status_message(
                 resp.failure_reason)
         except NotImplementedError:
             status = False, _('Pull request merging is not supported.')
 
         return status
 
     def _check_repo_requirements(self, target, source, translator):
         """
         Check if `target` and `source` have compatible requirements.
 
         Currently this is just checking for largefiles.
         """
         _ = translator
         target_has_largefiles = self._has_largefiles(target)
         source_has_largefiles = self._has_largefiles(source)
         merge_possible = True
         message = u''
 
         if target_has_largefiles != source_has_largefiles:
             merge_possible = False
             if source_has_largefiles:
                 message = _(
                     'Target repository large files support is disabled.')
             else:
                 message = _(
                     'Source repository large files support is disabled.')
 
         return merge_possible, message
 
     def _has_largefiles(self, repo):
         largefiles_ui = VcsSettingsModel(repo=repo).get_ui_settings(
             'extensions', 'largefiles')
         return largefiles_ui and largefiles_ui[0].active
 
     def _try_merge(self, pull_request, force_shadow_repo_refresh=False):
         """
         Try to merge the pull request and return the merge status.
         """
         log.debug(
             "Trying out if the pull request %s can be merged. Force_refresh=%s",
             pull_request.pull_request_id, force_shadow_repo_refresh)
         target_vcs = pull_request.target_repo.scm_instance()
 
         # Refresh the target reference.
         try:
             target_ref = self._refresh_reference(
                 pull_request.target_ref_parts, target_vcs)
         except CommitDoesNotExistError:
             merge_state = MergeResponse(
                 False, False, None, MergeFailureReason.MISSING_TARGET_REF)
             return merge_state
 
         target_locked = pull_request.target_repo.locked
         if target_locked and target_locked[0]:
             log.debug("The target repository is locked.")
             merge_state = MergeResponse(
                 False, False, None, MergeFailureReason.TARGET_IS_LOCKED)
         elif force_shadow_repo_refresh or self._needs_merge_state_refresh(
                 pull_request, target_ref):
             log.debug("Refreshing the merge status of the repository.")
             merge_state = self._refresh_merge_state(
                 pull_request, target_vcs, target_ref)
         else:
             possible = pull_request.\
                 last_merge_status == MergeFailureReason.NONE
             merge_state = MergeResponse(
                 possible, False, None, pull_request.last_merge_status)
 
         return merge_state
 
     def _refresh_reference(self, reference, vcs_repository):
         if reference.type in ('branch', 'book'):
             name_or_id = reference.name
         else:
             name_or_id = reference.commit_id
         refreshed_commit = vcs_repository.get_commit(name_or_id)
         refreshed_reference = Reference(
             reference.type, reference.name, refreshed_commit.raw_id)
         return refreshed_reference
 
     def _needs_merge_state_refresh(self, pull_request, target_reference):
         return not(
             pull_request.revisions and
             pull_request.revisions[0] == pull_request._last_merge_source_rev and
             target_reference.commit_id == pull_request._last_merge_target_rev)
 
     def _refresh_merge_state(self, pull_request, target_vcs, target_reference):
         workspace_id = self._workspace_id(pull_request)
         source_vcs = pull_request.source_repo.scm_instance()
         repo_id = pull_request.target_repo.repo_id
         use_rebase = self._use_rebase_for_merging(pull_request)
         close_branch = self._close_branch_before_merging(pull_request)
         merge_state = target_vcs.merge(
             repo_id, workspace_id,
             target_reference, source_vcs, pull_request.source_ref_parts,
             dry_run=True, use_rebase=use_rebase,
             close_branch=close_branch)
 
         # Do not store the response if there was an unknown error.
         if merge_state.failure_reason != MergeFailureReason.UNKNOWN:
             pull_request._last_merge_source_rev = \
                 pull_request.source_ref_parts.commit_id
             pull_request._last_merge_target_rev = target_reference.commit_id
             pull_request.last_merge_status = merge_state.failure_reason
             pull_request.shadow_merge_ref = merge_state.merge_ref
             Session().add(pull_request)
             Session().commit()
 
         return merge_state
 
     def _workspace_id(self, pull_request):
         workspace_id = 'pr-%s' % pull_request.pull_request_id
         return workspace_id
 
     def merge_status_message(self, status_code):
         """
         Return a human friendly error message for the given merge status code.
         """
         return self.MERGE_STATUS_MESSAGES[status_code]
 
     def generate_repo_data(self, repo, commit_id=None, branch=None,
                            bookmark=None, translator=None):
         from rhodecode.model.repo import RepoModel
 
         all_refs, selected_ref = \
             self._get_repo_pullrequest_sources(
                 repo.scm_instance(), commit_id=commit_id,
                 branch=branch, bookmark=bookmark, translator=translator)
 
         refs_select2 = []
         for element in all_refs:
             children = [{'id': x[0], 'text': x[1]} for x in element[0]]
             refs_select2.append({'text': element[1], 'children': children})
 
         return {
             'user': {
                 'user_id': repo.user.user_id,
                 'username': repo.user.username,
                 'firstname': repo.user.first_name,
                 'lastname': repo.user.last_name,
                 'gravatar_link': h.gravatar_url(repo.user.email, 14),
             },
             'name': repo.repo_name,
             'link': RepoModel().get_url(repo),
             'description': h.chop_at_smart(repo.description_safe, '\n'),
             'refs': {
                 'all_refs': all_refs,
                 'selected_ref': selected_ref,
                 'select2_refs': refs_select2
             }
         }
 
     def generate_pullrequest_title(self, source, source_ref, target):
         return u'{source}#{at_ref} to {target}'.format(
             source=source,
             at_ref=source_ref,
             target=target,
         )
 
     def _cleanup_merge_workspace(self, pull_request):
         # Merging related cleanup
         repo_id = pull_request.target_repo.repo_id
         target_scm = pull_request.target_repo.scm_instance()
         workspace_id = self._workspace_id(pull_request)
 
         try:
             target_scm.cleanup_merge_workspace(repo_id, workspace_id)
         except NotImplementedError:
             pass
 
     def _get_repo_pullrequest_sources(
             self, repo, commit_id=None, branch=None, bookmark=None,
             translator=None):
         """
         Return a structure with repo's interesting commits, suitable for
         the selectors in pullrequest controller
 
         :param commit_id: a commit that must be in the list somehow
             and selected by default
         :param branch: a branch that must be in the list and selected
             by default - even if closed
         :param bookmark: a bookmark that must be in the list and selected
         """
         _ = translator or get_current_request().translate
 
         commit_id = safe_str(commit_id) if commit_id else None
         branch = safe_str(branch) if branch else None
         bookmark = safe_str(bookmark) if bookmark else None
 
         selected = None
 
         # order matters: first source that has commit_id in it will be selected
         sources = []
         sources.append(('book', repo.bookmarks.items(), _('Bookmarks'), bookmark))
         sources.append(('branch', repo.branches.items(), _('Branches'), branch))
 
         if commit_id:
             ref_commit = (h.short_id(commit_id), commit_id)
             sources.append(('rev', [ref_commit], _('Commit IDs'), commit_id))
 
         sources.append(
             ('branch', repo.branches_closed.items(), _('Closed Branches'), branch),
         )
 
         groups = []
         for group_key, ref_list, group_name, match in sources:
             group_refs = []
             for ref_name, ref_id in ref_list:
                 ref_key = '%s:%s:%s' % (group_key, ref_name, ref_id)
                 group_refs.append((ref_key, ref_name))
 
                 if not selected:
                     if set([commit_id, match]) & set([ref_id, ref_name]):
                         selected = ref_key
 
             if group_refs:
                 groups.append((group_refs, group_name))
 
         if not selected:
             ref = commit_id or branch or bookmark
             if ref:
                 raise CommitDoesNotExistError(
                     'No commit refs could be found matching: %s' % ref)
             elif repo.DEFAULT_BRANCH_NAME in repo.branches:
                 selected = 'branch:%s:%s' % (
                     repo.DEFAULT_BRANCH_NAME,
                     repo.branches[repo.DEFAULT_BRANCH_NAME]
                 )
             elif repo.commit_ids:
                 # make the user select in this case
                 selected = None
             else:
                 raise EmptyRepositoryError()
         return groups, selected
 
     def get_diff(self, source_repo, source_ref_id, target_ref_id,
                  hide_whitespace_changes, diff_context):
 
         return self._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes=hide_whitespace_changes, diff_context=diff_context)
 
     def _get_diff_from_pr_or_version(
             self, source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes, diff_context):
 
         target_commit = source_repo.get_commit(
             commit_id=safe_str(target_ref_id))
         source_commit = source_repo.get_commit(
             commit_id=safe_str(source_ref_id))
         if isinstance(source_repo, Repository):
             vcs_repo = source_repo.scm_instance()
         else:
             vcs_repo = source_repo
 
         # TODO: johbo: In the context of an update, we cannot reach
         # the old commit anymore with our normal mechanisms. It needs
         # some sort of special support in the vcs layer to avoid this
         # workaround.
         if (source_commit.raw_id == vcs_repo.EMPTY_COMMIT_ID and
                 vcs_repo.alias == 'git'):
             source_commit.raw_id = safe_str(source_ref_id)
 
         log.debug('calculating diff between '
                   'source_ref:%s and target_ref:%s for repo `%s`',
                   target_ref_id, source_ref_id,
                   safe_unicode(vcs_repo.path))
 
         vcs_diff = vcs_repo.get_diff(
             commit1=target_commit, commit2=source_commit,
             ignore_whitespace=hide_whitespace_changes, context=diff_context)
         return vcs_diff
 
     def _is_merge_enabled(self, pull_request):
         return self._get_general_setting(
             pull_request, 'rhodecode_pr_merge_enabled')
 
     def _use_rebase_for_merging(self, pull_request):
         repo_type = pull_request.target_repo.repo_type
         if repo_type == 'hg':
             return self._get_general_setting(
                 pull_request, 'rhodecode_hg_use_rebase_for_merging')
         elif repo_type == 'git':
             return self._get_general_setting(
                 pull_request, 'rhodecode_git_use_rebase_for_merging')
 
         return False
 
     def _close_branch_before_merging(self, pull_request):
         repo_type = pull_request.target_repo.repo_type
         if repo_type == 'hg':
             return self._get_general_setting(
                 pull_request, 'rhodecode_hg_close_branch_before_merging')
         elif repo_type == 'git':
             return self._get_general_setting(
                 pull_request, 'rhodecode_git_close_branch_before_merging')
 
         return False
 
     def _get_general_setting(self, pull_request, settings_key, default=False):
         settings_model = VcsSettingsModel(repo=pull_request.target_repo)
         settings = settings_model.get_general_settings()
         return settings.get(settings_key, default)
 
     def _log_audit_action(self, action, action_data, user, pull_request):
         audit_logger.store(
             action=action,
             action_data=action_data,
             user=user,
             repo=pull_request.target_repo)
 
     def get_reviewer_functions(self):
         """
         Fetches functions for validation and fetching default reviewers.
         If available we use the EE package, else we fallback to CE
         package functions
         """
         try:
             from rc_reviewers.utils import get_default_reviewers_data
             from rc_reviewers.utils import validate_default_reviewers
         except ImportError:
-            from rhodecode.apps.repository.utils import \
-                get_default_reviewers_data
-            from rhodecode.apps.repository.utils import \
-                validate_default_reviewers
+            from rhodecode.apps.repository.utils import get_default_reviewers_data
+            from rhodecode.apps.repository.utils import validate_default_reviewers
 
         return get_default_reviewers_data, validate_default_reviewers
 
 
 class MergeCheck(object):
     """
     Perform Merge Checks and returns a check object which stores information
     about merge errors, and merge conditions
     """
     TODO_CHECK = 'todo'
     PERM_CHECK = 'perm'
     REVIEW_CHECK = 'review'
     MERGE_CHECK = 'merge'
 
     def __init__(self):
         self.review_status = None
         self.merge_possible = None
         self.merge_msg = ''
         self.failed = None
         self.errors = []
         self.error_details = OrderedDict()
 
     def push_error(self, error_type, message, error_key, details):
         self.failed = True
         self.errors.append([error_type, message])
         self.error_details[error_key] = dict(
             details=details,
             error_type=error_type,
             message=message
         )
 
     @classmethod
     def validate(cls, pull_request, auth_user, translator, fail_early=False,
                  force_shadow_repo_refresh=False):
         _ = translator
         merge_check = cls()
 
         # permissions to merge
         user_allowed_to_merge = PullRequestModel().check_user_merge(
             pull_request, auth_user)
         if not user_allowed_to_merge:
             log.debug("MergeCheck: cannot merge, approval is pending.")
 
             msg = _('User `{}` not allowed to perform merge.').format(auth_user.username)
             merge_check.push_error('error', msg, cls.PERM_CHECK, auth_user.username)
             if fail_early:
                 return merge_check
 
         # permission to merge into the target branch
         target_commit_id = pull_request.target_ref_parts.commit_id
         if pull_request.target_ref_parts.type == 'branch':
             branch_name = pull_request.target_ref_parts.name
         else:
             # for mercurial we can always figure out the branch from the commit
             # in case of bookmark
             target_commit = pull_request.target_repo.get_commit(target_commit_id)
             branch_name = target_commit.branch
 
         rule, branch_perm = auth_user.get_rule_and_branch_permission(
             pull_request.target_repo.repo_name, branch_name)
         if branch_perm and branch_perm == 'branch.none':
             msg = _('Target branch `{}` changes rejected by rule {}.').format(
                 branch_name, rule)
             merge_check.push_error('error', msg, cls.PERM_CHECK, auth_user.username)
             if fail_early:
                 return merge_check
 
         # review status, must be always present
         review_status = pull_request.calculated_review_status()
         merge_check.review_status = review_status
 
         status_approved = review_status == ChangesetStatus.STATUS_APPROVED
         if not status_approved:
             log.debug("MergeCheck: cannot merge, approval is pending.")
 
             msg = _('Pull request reviewer approval is pending.')
 
             merge_check.push_error(
                 'warning', msg, cls.REVIEW_CHECK, review_status)
 
             if fail_early:
                 return merge_check
 
         # left over TODOs
         todos = CommentsModel().get_unresolved_todos(pull_request)
         if todos:
             log.debug("MergeCheck: cannot merge, {} "
                       "unresolved todos left.".format(len(todos)))
 
             if len(todos) == 1:
                 msg = _('Cannot merge, {} TODO still not resolved.').format(
                     len(todos))
             else:
                 msg = _('Cannot merge, {} TODOs still not resolved.').format(
                     len(todos))
 
             merge_check.push_error('warning', msg, cls.TODO_CHECK, todos)
 
             if fail_early:
                 return merge_check
 
         # merge possible, here is the filesystem simulation + shadow repo
         merge_status, msg = PullRequestModel().merge_status(
             pull_request, translator=translator,
             force_shadow_repo_refresh=force_shadow_repo_refresh)
         merge_check.merge_possible = merge_status
         merge_check.merge_msg = msg
         if not merge_status:
             log.debug(
                 "MergeCheck: cannot merge, pull request merge not possible.")
             merge_check.push_error('warning', msg, cls.MERGE_CHECK, None)
 
             if fail_early:
                 return merge_check
 
         log.debug('MergeCheck: is failed: %s', merge_check.failed)
         return merge_check
 
     @classmethod
     def get_merge_conditions(cls, pull_request, translator):
         _ = translator
         merge_details = {}
 
         model = PullRequestModel()
         use_rebase = model._use_rebase_for_merging(pull_request)
 
         if use_rebase:
             merge_details['merge_strategy'] = dict(
                 details={},
                 message=_('Merge strategy: rebase')
             )
         else:
             merge_details['merge_strategy'] = dict(
                 details={},
                 message=_('Merge strategy: explicit merge commit')
             )
 
         close_branch = model._close_branch_before_merging(pull_request)
         if close_branch:
             repo_type = pull_request.target_repo.repo_type
             if repo_type == 'hg':
                 close_msg = _('Source branch will be closed after merge.')
             elif repo_type == 'git':
                 close_msg = _('Source branch will be deleted after merge.')
 
             merge_details['close_branch'] = dict(
                 details={},
                 message=close_msg
             )
 
         return merge_details
 
 ChangeTuple = collections.namedtuple(
     'ChangeTuple', ['added', 'common', 'removed', 'total'])
 
 FileChangeTuple = collections.namedtuple(
     'FileChangeTuple', ['added', 'modified', 'removed'])

rhodecode/model/settings.py

@@ -1,838 +1,837 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import hashlib
 import logging
 from collections import namedtuple
 from functools import wraps
 import bleach
 
 from rhodecode.lib import rc_cache
 from rhodecode.lib.utils2 import (
     Optional, AttributeDict, safe_str, remove_prefix, str2bool)
 from rhodecode.lib.vcs.backends import base
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     RepoRhodeCodeUi, RepoRhodeCodeSetting, RhodeCodeUi, RhodeCodeSetting, CacheKey)
 from rhodecode.model.meta import Session
 
 
 log = logging.getLogger(__name__)
 
 
 UiSetting = namedtuple(
     'UiSetting', ['section', 'key', 'value', 'active'])
 
 SOCIAL_PLUGINS_LIST = ['github', 'bitbucket', 'twitter', 'google']
 
 
 class SettingNotFound(Exception):
     def __init__(self, setting_id):
         msg = 'Setting `{}` is not found'.format(setting_id)
         super(SettingNotFound, self).__init__(msg)
 
 
 class SettingsModel(BaseModel):
     BUILTIN_HOOKS = (
         RhodeCodeUi.HOOK_REPO_SIZE, RhodeCodeUi.HOOK_PUSH,
         RhodeCodeUi.HOOK_PRE_PUSH, RhodeCodeUi.HOOK_PRETX_PUSH,
         RhodeCodeUi.HOOK_PULL, RhodeCodeUi.HOOK_PRE_PULL,
         RhodeCodeUi.HOOK_PUSH_KEY,)
     HOOKS_SECTION = 'hooks'
 
     def __init__(self, sa=None, repo=None):
         self.repo = repo
         self.UiDbModel = RepoRhodeCodeUi if repo else RhodeCodeUi
         self.SettingsDbModel = (
             RepoRhodeCodeSetting if repo else RhodeCodeSetting)
         super(SettingsModel, self).__init__(sa)
 
     def get_ui_by_key(self, key):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_key == key)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.scalar()
 
     def get_ui_by_section(self, section):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_section == section)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.all()
 
     def get_ui_by_section_and_key(self, section, key):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_section == section)
         q = q.filter(self.UiDbModel.ui_key == key)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.scalar()
 
     def get_ui(self, section=None, key=None):
         q = self.UiDbModel.query()
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
 
         if section:
             q = q.filter(self.UiDbModel.ui_section == section)
         if key:
             q = q.filter(self.UiDbModel.ui_key == key)
 
         # TODO: mikhail: add caching
         result = [
             UiSetting(
                 section=safe_str(r.ui_section), key=safe_str(r.ui_key),
                 value=safe_str(r.ui_value), active=r.ui_active
             )
             for r in q.all()
         ]
         return result
 
     def get_builtin_hooks(self):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_key.in_(self.BUILTIN_HOOKS))
         return self._get_hooks(q)
 
     def get_custom_hooks(self):
         q = self.UiDbModel.query()
         q = q.filter(~self.UiDbModel.ui_key.in_(self.BUILTIN_HOOKS))
         return self._get_hooks(q)
 
     def create_ui_section_value(self, section, val, key=None, active=True):
         new_ui = self.UiDbModel()
         new_ui.ui_section = section
         new_ui.ui_value = val
         new_ui.ui_active = active
 
         if self.repo:
             repo = self._get_repo(self.repo)
             repository_id = repo.repo_id
             new_ui.repository_id = repository_id
 
         if not key:
             # keys are unique so they need appended info
             if self.repo:
                 key = hashlib.sha1(
                     '{}{}{}'.format(section, val, repository_id)).hexdigest()
             else:
                 key = hashlib.sha1('{}{}'.format(section, val)).hexdigest()
 
         new_ui.ui_key = key
 
         Session().add(new_ui)
         return new_ui
 
     def create_or_update_hook(self, key, value):
         ui = (
             self.get_ui_by_section_and_key(self.HOOKS_SECTION, key) or
             self.UiDbModel())
         ui.ui_section = self.HOOKS_SECTION
         ui.ui_active = True
         ui.ui_key = key
         ui.ui_value = value
 
         if self.repo:
             repo = self._get_repo(self.repo)
             repository_id = repo.repo_id
             ui.repository_id = repository_id
 
         Session().add(ui)
         return ui
 
     def delete_ui(self, id_):
         ui = self.UiDbModel.get(id_)
         if not ui:
             raise SettingNotFound(id_)
         Session().delete(ui)
 
     def get_setting_by_name(self, name):
         q = self._get_settings_query()
         q = q.filter(self.SettingsDbModel.app_settings_name == name)
         return q.scalar()
 
     def create_or_update_setting(
             self, name, val=Optional(''), type_=Optional('unicode')):
         """
         Creates or updates RhodeCode setting. If updates is triggered it will
         only update parameters that are explicityl set Optional instance will
         be skipped
 
         :param name:
         :param val:
         :param type_:
         :return:
         """
 
         res = self.get_setting_by_name(name)
         repo = self._get_repo(self.repo) if self.repo else None
 
         if not res:
             val = Optional.extract(val)
             type_ = Optional.extract(type_)
 
             args = (
                 (repo.repo_id, name, val, type_)
                 if repo else (name, val, type_))
             res = self.SettingsDbModel(*args)
 
         else:
             if self.repo:
                 res.repository_id = repo.repo_id
 
             res.app_settings_name = name
             if not isinstance(type_, Optional):
                 # update if set
                 res.app_settings_type = type_
             if not isinstance(val, Optional):
                 # update if set
                 res.app_settings_value = val
 
         Session().add(res)
         return res
 
     def invalidate_settings_cache(self):
         invalidation_namespace = CacheKey.SETTINGS_INVALIDATION_NAMESPACE
         CacheKey.set_invalidate(invalidation_namespace)
 
     def get_all_settings(self, cache=False):
         region = rc_cache.get_or_create_region('sql_cache_short')
         invalidation_namespace = CacheKey.SETTINGS_INVALIDATION_NAMESPACE
 
         @region.conditional_cache_on_arguments(condition=cache)
         def _get_all_settings(name, key):
             q = self._get_settings_query()
             if not q:
                 raise Exception('Could not get application settings !')
 
             settings = {
                 'rhodecode_' + result.app_settings_name: result.app_settings_value
                 for result in q
             }
             return settings
 
         repo = self._get_repo(self.repo) if self.repo else None
         key = "settings_repo.{}".format(repo.repo_id) if repo else "settings_app"
 
         inv_context_manager = rc_cache.InvalidationContext(
             uid='cache_settings', invalidation_namespace=invalidation_namespace)
         with inv_context_manager as invalidation_context:
             # check for stored invalidation signal, and maybe purge the cache
             # before computing it again
             if invalidation_context.should_invalidate():
                 # NOTE:(marcink) we flush the whole sql_cache_short region, because it
                 # reads different settings etc. It's little too much but those caches
                 # are anyway very short lived and it's a safest way.
                 region = rc_cache.get_or_create_region('sql_cache_short')
                 region.invalidate()
 
             result = _get_all_settings('rhodecode_settings', key)
             log.debug('Fetching app settings for key: %s took: %.3fs', key,
                       inv_context_manager.compute_time)
 
         return result
 
     def get_auth_settings(self):
         q = self._get_settings_query()
         q = q.filter(
             self.SettingsDbModel.app_settings_name.startswith('auth_'))
         rows = q.all()
         auth_settings = {
             row.app_settings_name: row.app_settings_value for row in rows}
         return auth_settings
 
     def get_auth_plugins(self):
         auth_plugins = self.get_setting_by_name("auth_plugins")
         return auth_plugins.app_settings_value
 
     def get_default_repo_settings(self, strip_prefix=False):
         q = self._get_settings_query()
         q = q.filter(
             self.SettingsDbModel.app_settings_name.startswith('default_'))
         rows = q.all()
 
         result = {}
         for row in rows:
             key = row.app_settings_name
             if strip_prefix:
                 key = remove_prefix(key, prefix='default_')
             result.update({key: row.app_settings_value})
         return result
 
     def get_repo(self):
         repo = self._get_repo(self.repo)
         if not repo:
             raise Exception(
                 'Repository `{}` cannot be found inside the database'.format(
                     self.repo))
         return repo
 
     def _filter_by_repo(self, model, query):
         if self.repo:
             repo = self.get_repo()
             query = query.filter(model.repository_id == repo.repo_id)
         return query
 
     def _get_hooks(self, query):
         query = query.filter(self.UiDbModel.ui_section == self.HOOKS_SECTION)
         query = self._filter_by_repo(RepoRhodeCodeUi, query)
         return query.all()
 
     def _get_settings_query(self):
         q = self.SettingsDbModel.query()
         return self._filter_by_repo(RepoRhodeCodeSetting, q)
 
     def list_enabled_social_plugins(self, settings):
         enabled = []
         for plug in SOCIAL_PLUGINS_LIST:
             if str2bool(settings.get('rhodecode_auth_{}_enabled'.format(plug)
                                      )):
                 enabled.append(plug)
         return enabled
 
 
 def assert_repo_settings(func):
     @wraps(func)
     def _wrapper(self, *args, **kwargs):
         if not self.repo_settings:
             raise Exception('Repository is not specified')
         return func(self, *args, **kwargs)
     return _wrapper
 
 
 class IssueTrackerSettingsModel(object):
     INHERIT_SETTINGS = 'inherit_issue_tracker_settings'
     SETTINGS_PREFIX = 'issuetracker_'
 
     def __init__(self, sa=None, repo=None):
         self.global_settings = SettingsModel(sa=sa)
         self.repo_settings = SettingsModel(sa=sa, repo=repo) if repo else None
 
     @property
     def inherit_global_settings(self):
         if not self.repo_settings:
             return True
         setting = self.repo_settings.get_setting_by_name(self.INHERIT_SETTINGS)
         return setting.app_settings_value if setting else True
 
     @inherit_global_settings.setter
     def inherit_global_settings(self, value):
         if self.repo_settings:
             settings = self.repo_settings.create_or_update_setting(
                 self.INHERIT_SETTINGS, value, type_='bool')
             Session().add(settings)
 
     def _get_keyname(self, key, uid, prefix=''):
         return '{0}{1}{2}_{3}'.format(
             prefix, self.SETTINGS_PREFIX, key, uid)
 
     def _make_dict_for_settings(self, qs):
         prefix_match = self._get_keyname('pat', '', 'rhodecode_')
 
         issuetracker_entries = {}
         # create keys
         for k, v in qs.items():
             if k.startswith(prefix_match):
                 uid = k[len(prefix_match):]
                 issuetracker_entries[uid] = None
 
         # populate
         for uid in issuetracker_entries:
             issuetracker_entries[uid] = AttributeDict({
                 'pat': qs.get(
                     self._get_keyname('pat', uid, 'rhodecode_')),
                 'url': bleach.clean(
                     qs.get(self._get_keyname('url', uid, 'rhodecode_')) or ''),
                 'pref': bleach.clean(
                     qs.get(self._get_keyname('pref', uid, 'rhodecode_')) or ''),
                 'desc': qs.get(
                     self._get_keyname('desc', uid, 'rhodecode_')),
             })
         return issuetracker_entries
 
     def get_global_settings(self, cache=False):
         """
         Returns list of global issue tracker settings
         """
         defaults = self.global_settings.get_all_settings(cache=cache)
         settings = self._make_dict_for_settings(defaults)
         return settings
 
     def get_repo_settings(self, cache=False):
         """
         Returns list of issue tracker settings per repository
         """
         if not self.repo_settings:
             raise Exception('Repository is not specified')
         all_settings = self.repo_settings.get_all_settings(cache=cache)
         settings = self._make_dict_for_settings(all_settings)
         return settings
 
     def get_settings(self, cache=False):
         if self.inherit_global_settings:
             return self.get_global_settings(cache=cache)
         else:
             return self.get_repo_settings(cache=cache)
 
     def delete_entries(self, uid):
         if self.repo_settings:
             all_patterns = self.get_repo_settings()
             settings_model = self.repo_settings
         else:
             all_patterns = self.get_global_settings()
             settings_model = self.global_settings
         entries = all_patterns.get(uid, [])
 
         for del_key in entries:
             setting_name = self._get_keyname(del_key, uid)
             entry = settings_model.get_setting_by_name(setting_name)
             if entry:
                 Session().delete(entry)
 
         Session().commit()
 
     def create_or_update_setting(
             self, name, val=Optional(''), type_=Optional('unicode')):
         if self.repo_settings:
             setting = self.repo_settings.create_or_update_setting(
                 name, val, type_)
         else:
             setting = self.global_settings.create_or_update_setting(
                 name, val, type_)
         return setting
 
 
 class VcsSettingsModel(object):
 
     INHERIT_SETTINGS = 'inherit_vcs_settings'
     GENERAL_SETTINGS = (
         'use_outdated_comments',
         'pr_merge_enabled',
         'hg_use_rebase_for_merging',
         'hg_close_branch_before_merging',
         'git_use_rebase_for_merging',
         'git_close_branch_before_merging',
         'diff_cache',
     )
 
     HOOKS_SETTINGS = (
         ('hooks', 'changegroup.repo_size'),
         ('hooks', 'changegroup.push_logger'),
         ('hooks', 'outgoing.pull_logger'),)
     HG_SETTINGS = (
         ('extensions', 'largefiles'),
         ('phases', 'publish'),
         ('extensions', 'evolve'),)
     GIT_SETTINGS = (
         ('vcs_git_lfs', 'enabled'),)
     GLOBAL_HG_SETTINGS = (
         ('extensions', 'largefiles'),
         ('largefiles', 'usercache'),
         ('phases', 'publish'),
         ('extensions', 'hgsubversion'),
         ('extensions', 'evolve'),)
     GLOBAL_GIT_SETTINGS = (
         ('vcs_git_lfs', 'enabled'),
         ('vcs_git_lfs', 'store_location'))
 
     GLOBAL_SVN_SETTINGS = (
         ('vcs_svn_proxy', 'http_requests_enabled'),
         ('vcs_svn_proxy', 'http_server_url'))
 
     SVN_BRANCH_SECTION = 'vcs_svn_branch'
     SVN_TAG_SECTION = 'vcs_svn_tag'
     SSL_SETTING = ('web', 'push_ssl')
     PATH_SETTING = ('paths', '/')
 
     def __init__(self, sa=None, repo=None):
         self.global_settings = SettingsModel(sa=sa)
         self.repo_settings = SettingsModel(sa=sa, repo=repo) if repo else None
         self._ui_settings = (
             self.HG_SETTINGS + self.GIT_SETTINGS + self.HOOKS_SETTINGS)
         self._svn_sections = (self.SVN_BRANCH_SECTION, self.SVN_TAG_SECTION)
 
     @property
     @assert_repo_settings
     def inherit_global_settings(self):
         setting = self.repo_settings.get_setting_by_name(self.INHERIT_SETTINGS)
         return setting.app_settings_value if setting else True
 
     @inherit_global_settings.setter
     @assert_repo_settings
     def inherit_global_settings(self, value):
         self.repo_settings.create_or_update_setting(
             self.INHERIT_SETTINGS, value, type_='bool')
 
     def get_global_svn_branch_patterns(self):
         return self.global_settings.get_ui_by_section(self.SVN_BRANCH_SECTION)
 
     @assert_repo_settings
     def get_repo_svn_branch_patterns(self):
         return self.repo_settings.get_ui_by_section(self.SVN_BRANCH_SECTION)
 
     def get_global_svn_tag_patterns(self):
         return self.global_settings.get_ui_by_section(self.SVN_TAG_SECTION)
 
     @assert_repo_settings
     def get_repo_svn_tag_patterns(self):
         return self.repo_settings.get_ui_by_section(self.SVN_TAG_SECTION)
 
     def get_global_settings(self):
         return self._collect_all_settings(global_=True)
 
     @assert_repo_settings
     def get_repo_settings(self):
         return self._collect_all_settings(global_=False)
 
     @assert_repo_settings
     def create_or_update_repo_settings(
             self, data, inherit_global_settings=False):
         from rhodecode.model.scm import ScmModel
 
         self.inherit_global_settings = inherit_global_settings
 
         repo = self.repo_settings.get_repo()
         if not inherit_global_settings:
             if repo.repo_type == 'svn':
                 self.create_repo_svn_settings(data)
             else:
                 self.create_or_update_repo_hook_settings(data)
                 self.create_or_update_repo_pr_settings(data)
 
             if repo.repo_type == 'hg':
                 self.create_or_update_repo_hg_settings(data)
 
             if repo.repo_type == 'git':
                 self.create_or_update_repo_git_settings(data)
 
         ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
 
     @assert_repo_settings
     def create_or_update_repo_hook_settings(self, data):
         for section, key in self.HOOKS_SETTINGS:
             data_key = self._get_form_ui_key(section, key)
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
 
             active = data.get(data_key)
             repo_setting = self.repo_settings.get_ui_by_section_and_key(
                 section, key)
             if not repo_setting:
                 global_setting = self.global_settings.\
                     get_ui_by_section_and_key(section, key)
                 self.repo_settings.create_ui_section_value(
                     section, global_setting.ui_value, key=key, active=active)
             else:
                 repo_setting.ui_active = active
                 Session().add(repo_setting)
 
     def update_global_hook_settings(self, data):
         for section, key in self.HOOKS_SETTINGS:
             data_key = self._get_form_ui_key(section, key)
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
             active = data.get(data_key)
             repo_setting = self.global_settings.get_ui_by_section_and_key(
                 section, key)
             repo_setting.ui_active = active
             Session().add(repo_setting)
 
     @assert_repo_settings
     def create_or_update_repo_pr_settings(self, data):
         return self._create_or_update_general_settings(
             self.repo_settings, data)
 
     def create_or_update_global_pr_settings(self, data):
         return self._create_or_update_general_settings(
             self.global_settings, data)
 
     @assert_repo_settings
     def create_repo_svn_settings(self, data):
         return self._create_svn_settings(self.repo_settings, data)
 
     @assert_repo_settings
     def create_or_update_repo_hg_settings(self, data):
         largefiles, phases, evolve = \
             self.HG_SETTINGS
         largefiles_key, phases_key, evolve_key = \
             self._get_settings_keys(self.HG_SETTINGS, data)
 
         self._create_or_update_ui(
             self.repo_settings, *largefiles, value='',
             active=data[largefiles_key])
         self._create_or_update_ui(
             self.repo_settings, *evolve, value='',
             active=data[evolve_key])
         self._create_or_update_ui(
             self.repo_settings, *phases, value=safe_str(data[phases_key]))
 
-
     def create_or_update_global_hg_settings(self, data):
         largefiles, largefiles_store, phases, hgsubversion, evolve \
             = self.GLOBAL_HG_SETTINGS
         largefiles_key, largefiles_store_key, phases_key, subversion_key, evolve_key \
             = self._get_settings_keys(self.GLOBAL_HG_SETTINGS, data)
 
         self._create_or_update_ui(
             self.global_settings, *largefiles, value='',
             active=data[largefiles_key])
         self._create_or_update_ui(
             self.global_settings, *largefiles_store,
             value=data[largefiles_store_key])
         self._create_or_update_ui(
             self.global_settings, *phases, value=safe_str(data[phases_key]))
         self._create_or_update_ui(
             self.global_settings, *hgsubversion, active=data[subversion_key])
         self._create_or_update_ui(
             self.global_settings, *evolve, value='',
             active=data[evolve_key])
 
     def create_or_update_repo_git_settings(self, data):
         # NOTE(marcink): # comma make unpack work properly
         lfs_enabled, \
             = self.GIT_SETTINGS
 
         lfs_enabled_key, \
             = self._get_settings_keys(self.GIT_SETTINGS, data)
 
         self._create_or_update_ui(
             self.repo_settings, *lfs_enabled, value=data[lfs_enabled_key],
             active=data[lfs_enabled_key])
 
     def create_or_update_global_git_settings(self, data):
         lfs_enabled, lfs_store_location \
             = self.GLOBAL_GIT_SETTINGS
         lfs_enabled_key, lfs_store_location_key \
             = self._get_settings_keys(self.GLOBAL_GIT_SETTINGS, data)
 
         self._create_or_update_ui(
             self.global_settings, *lfs_enabled, value=data[lfs_enabled_key],
             active=data[lfs_enabled_key])
         self._create_or_update_ui(
             self.global_settings, *lfs_store_location,
             value=data[lfs_store_location_key])
 
     def create_or_update_global_svn_settings(self, data):
         # branch/tags patterns
         self._create_svn_settings(self.global_settings, data)
 
         http_requests_enabled, http_server_url = self.GLOBAL_SVN_SETTINGS
         http_requests_enabled_key, http_server_url_key = self._get_settings_keys(
             self.GLOBAL_SVN_SETTINGS, data)
 
         self._create_or_update_ui(
             self.global_settings, *http_requests_enabled,
             value=safe_str(data[http_requests_enabled_key]))
         self._create_or_update_ui(
             self.global_settings, *http_server_url,
             value=data[http_server_url_key])
 
     def update_global_ssl_setting(self, value):
         self._create_or_update_ui(
             self.global_settings, *self.SSL_SETTING, value=value)
 
     def update_global_path_setting(self, value):
         self._create_or_update_ui(
             self.global_settings, *self.PATH_SETTING, value=value)
 
     @assert_repo_settings
     def delete_repo_svn_pattern(self, id_):
         ui = self.repo_settings.UiDbModel.get(id_)
         if ui and ui.repository.repo_name == self.repo_settings.repo:
             # only delete if it's the same repo as initialized settings
             self.repo_settings.delete_ui(id_)
         else:
             # raise error as if we wouldn't find this option
             self.repo_settings.delete_ui(-1)
 
     def delete_global_svn_pattern(self, id_):
         self.global_settings.delete_ui(id_)
 
     @assert_repo_settings
     def get_repo_ui_settings(self, section=None, key=None):
         global_uis = self.global_settings.get_ui(section, key)
         repo_uis = self.repo_settings.get_ui(section, key)
         filtered_repo_uis = self._filter_ui_settings(repo_uis)
         filtered_repo_uis_keys = [
             (s.section, s.key) for s in filtered_repo_uis]
 
         def _is_global_ui_filtered(ui):
             return (
                 (ui.section, ui.key) in filtered_repo_uis_keys
                 or ui.section in self._svn_sections)
 
         filtered_global_uis = [
             ui for ui in global_uis if not _is_global_ui_filtered(ui)]
 
         return filtered_global_uis + filtered_repo_uis
 
     def get_global_ui_settings(self, section=None, key=None):
         return self.global_settings.get_ui(section, key)
 
     def get_ui_settings_as_config_obj(self, section=None, key=None):
         config = base.Config()
 
         ui_settings = self.get_ui_settings(section=section, key=key)
 
         for entry in ui_settings:
             config.set(entry.section, entry.key, entry.value)
 
         return config
 
     def get_ui_settings(self, section=None, key=None):
         if not self.repo_settings or self.inherit_global_settings:
             return self.get_global_ui_settings(section, key)
         else:
             return self.get_repo_ui_settings(section, key)
 
     def get_svn_patterns(self, section=None):
         if not self.repo_settings:
             return self.get_global_ui_settings(section)
         else:
             return self.get_repo_ui_settings(section)
 
     @assert_repo_settings
     def get_repo_general_settings(self):
         global_settings = self.global_settings.get_all_settings()
         repo_settings = self.repo_settings.get_all_settings()
         filtered_repo_settings = self._filter_general_settings(repo_settings)
         global_settings.update(filtered_repo_settings)
         return global_settings
 
     def get_global_general_settings(self):
         return self.global_settings.get_all_settings()
 
     def get_general_settings(self):
         if not self.repo_settings or self.inherit_global_settings:
             return self.get_global_general_settings()
         else:
             return self.get_repo_general_settings()
 
     def get_repos_location(self):
         return self.global_settings.get_ui_by_key('/').ui_value
 
     def _filter_ui_settings(self, settings):
         filtered_settings = [
             s for s in settings if self._should_keep_setting(s)]
         return filtered_settings
 
     def _should_keep_setting(self, setting):
         keep = (
             (setting.section, setting.key) in self._ui_settings or
             setting.section in self._svn_sections)
         return keep
 
     def _filter_general_settings(self, settings):
         keys = ['rhodecode_{}'.format(key) for key in self.GENERAL_SETTINGS]
         return {
             k: settings[k]
             for k in settings if k in keys}
 
     def _collect_all_settings(self, global_=False):
         settings = self.global_settings if global_ else self.repo_settings
         result = {}
 
         for section, key in self._ui_settings:
             ui = settings.get_ui_by_section_and_key(section, key)
             result_key = self._get_form_ui_key(section, key)
 
             if ui:
                 if section in ('hooks', 'extensions'):
                     result[result_key] = ui.ui_active
                 elif result_key in ['vcs_git_lfs_enabled']:
                     result[result_key] = ui.ui_active
                 else:
                     result[result_key] = ui.ui_value
 
         for name in self.GENERAL_SETTINGS:
             setting = settings.get_setting_by_name(name)
             if setting:
                 result_key = 'rhodecode_{}'.format(name)
                 result[result_key] = setting.app_settings_value
 
         return result
 
     def _get_form_ui_key(self, section, key):
         return '{section}_{key}'.format(
             section=section, key=key.replace('.', '_'))
 
     def _create_or_update_ui(
             self, settings, section, key, value=None, active=None):
         ui = settings.get_ui_by_section_and_key(section, key)
         if not ui:
             active = True if active is None else active
             settings.create_ui_section_value(
                 section, value, key=key, active=active)
         else:
             if active is not None:
                 ui.ui_active = active
             if value is not None:
                 ui.ui_value = value
             Session().add(ui)
 
     def _create_svn_settings(self, settings, data):
         svn_settings = {
             'new_svn_branch': self.SVN_BRANCH_SECTION,
             'new_svn_tag': self.SVN_TAG_SECTION
         }
         for key in svn_settings:
             if data.get(key):
                 settings.create_ui_section_value(svn_settings[key], data[key])
 
     def _create_or_update_general_settings(self, settings, data):
         for name in self.GENERAL_SETTINGS:
             data_key = 'rhodecode_{}'.format(name)
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
             setting = settings.create_or_update_setting(
                 name, data[data_key], 'bool')
             Session().add(setting)
 
     def _get_settings_keys(self, settings, data):
         data_keys = [self._get_form_ui_key(*s) for s in settings]
         for data_key in data_keys:
             if data_key not in data:
                 raise ValueError(
                     'The given data does not contain {} key'.format(data_key))
         return data_keys
 
     def create_largeobjects_dirs_if_needed(self, repo_store_path):
         """
         This is subscribed to the `pyramid.events.ApplicationCreated` event. It
         does a repository scan if enabled in the settings.
         """
 
         from rhodecode.lib.vcs.backends.hg import largefiles_store
         from rhodecode.lib.vcs.backends.git import lfs_store
 
         paths = [
             largefiles_store(repo_store_path),
             lfs_store(repo_store_path)]
 
         for path in paths:
             if os.path.isdir(path):
                 continue
             if os.path.isfile(path):
                 continue
             # not a file nor dir, we try to create it
             try:
                 os.makedirs(path)
             except Exception:
                 log.warning('Failed to create largefiles dir:%s', path)

rhodecode/tests/vcs/test_repository.py

@@ -1,553 +1,553 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import datetime
 from urllib2 import URLError
 
 import mock
 import pytest
 
 from rhodecode.lib.vcs import backends
 from rhodecode.lib.vcs.backends.base import (
     Config, BaseInMemoryCommit, Reference, MergeResponse, MergeFailureReason)
 from rhodecode.lib.vcs.exceptions import VCSError, RepositoryError
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.tests.vcs.conftest import BackendTestMixin
 from rhodecode.tests import repo_id_generator
 
 
 @pytest.mark.usefixtures("vcs_repository_support")
 class TestRepositoryBase(BackendTestMixin):
     recreate_repo_per_test = False
 
     def test_init_accepts_unicode_path(self, tmpdir):
         path = unicode(tmpdir.join(u'unicode ä'))
         self.Backend(path, create=True)
 
     def test_init_accepts_str_path(self, tmpdir):
         path = str(tmpdir.join('str ä'))
         self.Backend(path, create=True)
 
     def test_init_fails_if_path_does_not_exist(self, tmpdir):
         path = unicode(tmpdir.join('i-do-not-exist'))
         with pytest.raises(VCSError):
             self.Backend(path)
 
     def test_init_fails_if_path_is_not_a_valid_repository(self, tmpdir):
         path = unicode(tmpdir.mkdir(u'unicode ä'))
         with pytest.raises(VCSError):
             self.Backend(path)
 
     def test_has_commits_attribute(self):
         self.repo.commit_ids
 
     def test_name(self):
         assert self.repo.name.startswith('vcs-test')
 
     @pytest.mark.backends("hg", "git")
     def test_has_default_branch_name(self):
         assert self.repo.DEFAULT_BRANCH_NAME is not None
 
     @pytest.mark.backends("svn")
     def test_has_no_default_branch_name(self):
         assert self.repo.DEFAULT_BRANCH_NAME is None
 
     def test_has_empty_commit(self):
         assert self.repo.EMPTY_COMMIT_ID is not None
         assert self.repo.EMPTY_COMMIT is not None
 
     def test_empty_changeset_is_deprecated(self):
         def get_empty_changeset(repo):
             return repo.EMPTY_CHANGESET
         pytest.deprecated_call(get_empty_changeset, self.repo)
 
     def test_bookmarks(self):
         assert len(self.repo.bookmarks) == 0
 
     # TODO: Cover two cases: Local repo path, remote URL
     def test_check_url(self):
         config = Config()
         assert self.Backend.check_url(self.repo.path, config)
 
     def test_check_url_invalid(self):
         config = Config()
         with pytest.raises(URLError):
             self.Backend.check_url(self.repo.path + "invalid", config)
 
     def test_get_contact(self):
         assert self.repo.contact
 
     def test_get_description(self):
         assert self.repo.description
 
     def test_get_hook_location(self):
         assert len(self.repo.get_hook_location()) != 0
 
     def test_last_change(self, local_dt_to_utc):
         assert self.repo.last_change >= local_dt_to_utc(
             datetime.datetime(2010, 1, 1, 21, 0))
 
     def test_last_change_in_empty_repository(self, vcsbackend, local_dt_to_utc):
         delta = datetime.timedelta(seconds=1)
 
         start = local_dt_to_utc(datetime.datetime.now())
         empty_repo = vcsbackend.create_repo()
         now = local_dt_to_utc(datetime.datetime.now())
         assert empty_repo.last_change >= start - delta
         assert empty_repo.last_change <= now + delta
 
     def test_repo_equality(self):
         assert self.repo == self.repo
 
     def test_repo_equality_broken_object(self):
         import copy
         _repo = copy.copy(self.repo)
         delattr(_repo, 'path')
         assert self.repo != _repo
 
     def test_repo_equality_other_object(self):
         class dummy(object):
             path = self.repo.path
         assert self.repo != dummy()
 
     def test_get_commit_is_implemented(self):
         self.repo.get_commit()
 
     def test_get_commits_is_implemented(self):
         commit_iter = iter(self.repo.get_commits())
         commit = next(commit_iter)
         assert commit.idx == 0
 
     def test_supports_iteration(self):
         repo_iter = iter(self.repo)
         commit = next(repo_iter)
         assert commit.idx == 0
 
     def test_in_memory_commit(self):
         imc = self.repo.in_memory_commit
         assert isinstance(imc, BaseInMemoryCommit)
 
     @pytest.mark.backends("hg")
     def test__get_url_unicode(self):
         url = u'/home/repos/malmö'
         assert self.repo._get_url(url)
 
 
 @pytest.mark.usefixtures("vcs_repository_support")
 class TestDeprecatedRepositoryAPI(BackendTestMixin):
     recreate_repo_per_test = False
 
     def test_revisions_is_deprecated(self):
         def get_revisions(repo):
             return repo.revisions
         pytest.deprecated_call(get_revisions, self.repo)
 
     def test_get_changeset_is_deprecated(self):
         pytest.deprecated_call(self.repo.get_changeset)
 
     def test_get_changesets_is_deprecated(self):
         pytest.deprecated_call(self.repo.get_changesets)
 
     def test_in_memory_changeset_is_deprecated(self):
         def get_imc(repo):
             return repo.in_memory_changeset
         pytest.deprecated_call(get_imc, self.repo)
 
 
 # TODO: these tests are incomplete, must check the resulting compare result for
 # correcteness
 class TestRepositoryCompare:
 
     @pytest.mark.parametrize('merge', [True, False])
     def test_compare_commits_of_same_repository(self, vcsbackend, merge):
         target_repo = vcsbackend.create_repo(number_of_commits=5)
         target_repo.compare(
             target_repo[1].raw_id, target_repo[3].raw_id, target_repo,
             merge=merge)
 
     @pytest.mark.xfail_backends('svn')
     @pytest.mark.parametrize('merge', [True, False])
     def test_compare_cloned_repositories(self, vcsbackend, merge):
         target_repo = vcsbackend.create_repo(number_of_commits=5)
         source_repo = vcsbackend.clone_repo(target_repo)
         assert target_repo != source_repo
 
         vcsbackend.add_file(source_repo, 'newfile', 'somecontent')
         source_commit = source_repo.get_commit()
 
         target_repo.compare(
             target_repo[1].raw_id, source_repo[3].raw_id, source_repo,
             merge=merge)
 
     @pytest.mark.xfail_backends('svn')
     @pytest.mark.parametrize('merge', [True, False])
     def test_compare_unrelated_repositories(self, vcsbackend, merge):
         orig = vcsbackend.create_repo(number_of_commits=5)
         unrelated = vcsbackend.create_repo(number_of_commits=5)
         assert orig != unrelated
 
         orig.compare(
             orig[1].raw_id, unrelated[3].raw_id, unrelated, merge=merge)
 
 
 class TestRepositoryGetCommonAncestor:
 
     def test_get_common_ancestor_from_same_repo_existing(self, vcsbackend):
         target_repo = vcsbackend.create_repo(number_of_commits=5)
 
         expected_ancestor = target_repo[2].raw_id
 
         assert target_repo.get_common_ancestor(
             commit_id1=target_repo[2].raw_id,
             commit_id2=target_repo[4].raw_id,
             repo2=target_repo
         ) == expected_ancestor
 
         assert target_repo.get_common_ancestor(
             commit_id1=target_repo[4].raw_id,
             commit_id2=target_repo[2].raw_id,
             repo2=target_repo
         ) == expected_ancestor
 
     @pytest.mark.xfail_backends("svn")
     def test_get_common_ancestor_from_cloned_repo_existing(self, vcsbackend):
         target_repo = vcsbackend.create_repo(number_of_commits=5)
         source_repo = vcsbackend.clone_repo(target_repo)
         assert target_repo != source_repo
 
         vcsbackend.add_file(source_repo, 'newfile', 'somecontent')
         source_commit = source_repo.get_commit()
 
         expected_ancestor = target_repo[4].raw_id
 
         assert target_repo.get_common_ancestor(
             commit_id1=target_repo[4].raw_id,
             commit_id2=source_commit.raw_id,
             repo2=source_repo
         ) == expected_ancestor
 
         assert target_repo.get_common_ancestor(
             commit_id1=source_commit.raw_id,
             commit_id2=target_repo[4].raw_id,
             repo2=target_repo
         ) == expected_ancestor
 
     @pytest.mark.xfail_backends("svn")
     def test_get_common_ancestor_from_unrelated_repo_missing(self, vcsbackend):
         original = vcsbackend.create_repo(number_of_commits=5)
         unrelated = vcsbackend.create_repo(number_of_commits=5)
         assert original != unrelated
 
         assert original.get_common_ancestor(
             commit_id1=original[0].raw_id,
             commit_id2=unrelated[0].raw_id,
             repo2=unrelated
-        ) == None
+        ) is None
 
         assert original.get_common_ancestor(
             commit_id1=original[-1].raw_id,
             commit_id2=unrelated[-1].raw_id,
             repo2=unrelated
-        ) == None
+        ) is None
 
 
 @pytest.mark.backends("git", "hg")
 class TestRepositoryMerge(object):
     def prepare_for_success(self, vcsbackend):
         self.target_repo = vcsbackend.create_repo(number_of_commits=1)
         self.source_repo = vcsbackend.clone_repo(self.target_repo)
         vcsbackend.add_file(self.target_repo, 'README_MERGE1', 'Version 1')
         vcsbackend.add_file(self.source_repo, 'README_MERGE2', 'Version 2')
         imc = self.source_repo.in_memory_commit
         imc.add(FileNode('file_x', content=self.source_repo.name))
         imc.commit(
             message=u'Automatic commit from repo merge test',
             author=u'Automatic')
         self.target_commit = self.target_repo.get_commit()
         self.source_commit = self.source_repo.get_commit()
         # This only works for Git and Mercurial
         default_branch = self.target_repo.DEFAULT_BRANCH_NAME
         self.target_ref = Reference(
             'branch', default_branch, self.target_commit.raw_id)
         self.source_ref = Reference(
             'branch', default_branch, self.source_commit.raw_id)
         self.workspace_id = 'test-merge'
         self.repo_id = repo_id_generator(self.target_repo.path)
 
     def prepare_for_conflict(self, vcsbackend):
         self.target_repo = vcsbackend.create_repo(number_of_commits=1)
         self.source_repo = vcsbackend.clone_repo(self.target_repo)
         vcsbackend.add_file(self.target_repo, 'README_MERGE', 'Version 1')
         vcsbackend.add_file(self.source_repo, 'README_MERGE', 'Version 2')
         self.target_commit = self.target_repo.get_commit()
         self.source_commit = self.source_repo.get_commit()
         # This only works for Git and Mercurial
         default_branch = self.target_repo.DEFAULT_BRANCH_NAME
         self.target_ref = Reference(
             'branch', default_branch, self.target_commit.raw_id)
         self.source_ref = Reference(
             'branch', default_branch, self.source_commit.raw_id)
         self.workspace_id = 'test-merge'
         self.repo_id = repo_id_generator(self.target_repo.path)
 
     def test_merge_success(self, vcsbackend):
         self.prepare_for_success(vcsbackend)
 
         merge_response = self.target_repo.merge(
             self.repo_id, self.workspace_id, self.target_ref, self.source_repo,
             self.source_ref,
             'test user', '[e-mail]', 'merge message 1',
             dry_run=False)
         expected_merge_response = MergeResponse(
             True, True, merge_response.merge_ref,
             MergeFailureReason.NONE)
         assert merge_response == expected_merge_response
 
         target_repo = backends.get_backend(vcsbackend.alias)(
             self.target_repo.path)
         target_commits = list(target_repo.get_commits())
         commit_ids = [c.raw_id for c in target_commits[:-1]]
         assert self.source_ref.commit_id in commit_ids
         assert self.target_ref.commit_id in commit_ids
 
         merge_commit = target_commits[-1]
         assert merge_commit.raw_id == merge_response.merge_ref.commit_id
         assert merge_commit.message.strip() == 'merge message 1'
         assert merge_commit.author == 'test user <[e-mail]>'
 
         # We call it twice so to make sure we can handle updates
         target_ref = Reference(
             self.target_ref.type, self.target_ref.name,
             merge_response.merge_ref.commit_id)
 
         merge_response = target_repo.merge(
             self.repo_id, self.workspace_id, target_ref, self.source_repo, self.source_ref,
             'test user', '[e-mail]', 'merge message 2',
             dry_run=False)
         expected_merge_response = MergeResponse(
             True, True, merge_response.merge_ref,
             MergeFailureReason.NONE)
         assert merge_response == expected_merge_response
 
         target_repo = backends.get_backend(
             vcsbackend.alias)(self.target_repo.path)
         merge_commit = target_repo.get_commit(
             merge_response.merge_ref.commit_id)
         assert merge_commit.message.strip() == 'merge message 1'
         assert merge_commit.author == 'test user <[e-mail]>'
 
     def test_merge_success_dry_run(self, vcsbackend):
         self.prepare_for_success(vcsbackend)
 
         merge_response = self.target_repo.merge(
             self.repo_id, self.workspace_id, self.target_ref, self.source_repo,
             self.source_ref, dry_run=True)
 
         # We call it twice so to make sure we can handle updates
         merge_response_update = self.target_repo.merge(
             self.repo_id, self.workspace_id, self.target_ref, self.source_repo,
             self.source_ref, dry_run=True)
 
         # Multiple merges may differ in their commit id. Therefore we set the
         # commit id to `None` before comparing the merge responses.
         merge_response = merge_response._replace(
             merge_ref=merge_response.merge_ref._replace(commit_id=None))
         merge_response_update = merge_response_update._replace(
             merge_ref=merge_response_update.merge_ref._replace(commit_id=None))
 
         assert merge_response == merge_response_update
         assert merge_response.possible is True
         assert merge_response.executed is False
         assert merge_response.merge_ref
         assert merge_response.failure_reason is MergeFailureReason.NONE
 
     @pytest.mark.parametrize('dry_run', [True, False])
     def test_merge_conflict(self, vcsbackend, dry_run):
         self.prepare_for_conflict(vcsbackend)
         expected_merge_response = MergeResponse(
             False, False, None, MergeFailureReason.MERGE_FAILED)
 
         merge_response = self.target_repo.merge(
             self.repo_id,  self.workspace_id, self.target_ref,
             self.source_repo, self.source_ref,
             'test_user', '[e-mail]', 'test message', dry_run=dry_run)
         assert merge_response == expected_merge_response
 
         # We call it twice so to make sure we can handle updates
         merge_response = self.target_repo.merge(
             self.repo_id, self.workspace_id, self.target_ref, self.source_repo,
             self.source_ref,
             'test_user', '[e-mail]', 'test message', dry_run=dry_run)
         assert merge_response == expected_merge_response
 
     def test_merge_target_is_not_head(self, vcsbackend):
         self.prepare_for_success(vcsbackend)
         expected_merge_response = MergeResponse(
             False, False, None, MergeFailureReason.TARGET_IS_NOT_HEAD)
 
         target_ref = Reference(
             self.target_ref.type, self.target_ref.name, '0' * 40)
 
         merge_response = self.target_repo.merge(
             self.repo_id, self.workspace_id, target_ref, self.source_repo,
             self.source_ref, dry_run=True)
 
         assert merge_response == expected_merge_response
 
     def test_merge_missing_source_reference(self, vcsbackend):
         self.prepare_for_success(vcsbackend)
         expected_merge_response = MergeResponse(
             False, False, None, MergeFailureReason.MISSING_SOURCE_REF)
 
         source_ref = Reference(
             self.source_ref.type, 'not_existing', self.source_ref.commit_id)
 
         merge_response = self.target_repo.merge(
             self.repo_id, self.workspace_id, self.target_ref,
             self.source_repo, source_ref,
             dry_run=True)
 
         assert merge_response == expected_merge_response
 
     def test_merge_raises_exception(self, vcsbackend):
         self.prepare_for_success(vcsbackend)
         expected_merge_response = MergeResponse(
             False, False, None, MergeFailureReason.UNKNOWN)
 
         with mock.patch.object(self.target_repo, '_merge_repo',
                                side_effect=RepositoryError()):
             merge_response = self.target_repo.merge(
                 self.repo_id, self.workspace_id, self.target_ref,
                 self.source_repo, self.source_ref,
                 dry_run=True)
 
         assert merge_response == expected_merge_response
 
     def test_merge_invalid_user_name(self, vcsbackend):
         repo = vcsbackend.create_repo(number_of_commits=1)
         ref = Reference('branch', 'master', 'not_used')
         workspace_id = 'test-errors-in-merge'
         repo_id = repo_id_generator(workspace_id)
         with pytest.raises(ValueError):
             repo.merge(repo_id,  workspace_id, ref, self, ref)
 
     def test_merge_invalid_user_email(self, vcsbackend):
         repo = vcsbackend.create_repo(number_of_commits=1)
         ref = Reference('branch', 'master', 'not_used')
         workspace_id = 'test-errors-in-merge'
         repo_id = repo_id_generator(workspace_id)
         with pytest.raises(ValueError):
             repo.merge(
                 repo_id,  workspace_id, ref, self, ref, 'user name')
 
     def test_merge_invalid_message(self, vcsbackend):
         repo = vcsbackend.create_repo(number_of_commits=1)
         ref = Reference('branch', 'master', 'not_used')
         workspace_id = 'test-errors-in-merge'
         repo_id = repo_id_generator(workspace_id)
         with pytest.raises(ValueError):
             repo.merge(
                 repo_id, workspace_id, ref, self, ref,
                 'user name', '[e-mail]')
 
 
 @pytest.mark.usefixtures("vcs_repository_support")
 class TestRepositoryStrip(BackendTestMixin):
     recreate_repo_per_test = True
 
     @classmethod
     def _get_commits(cls):
         commits = [
             {
                 'message': 'Initial commit',
                 'author': 'Joe Doe <[e-mail]>',
                 'date': datetime.datetime(2010, 1, 1, 20),
                 'branch': 'master',
                 'added': [
                     FileNode('foobar', content='foobar'),
                     FileNode('foobar2', content='foobar2'),
                 ],
             },
         ]
         for x in xrange(10):
             commit_data = {
                 'message': 'Changed foobar - commit%s' % x,
                 'author': 'Jane Doe <[e-mail]>',
                 'date': datetime.datetime(2010, 1, 1, 21, x),
                 'branch': 'master',
                 'changed': [
                     FileNode('foobar', 'FOOBAR - %s' % x),
                 ],
             }
             commits.append(commit_data)
         return commits
 
     @pytest.mark.backends("git", "hg")
     def test_strip_commit(self):
         tip = self.repo.get_commit()
         assert tip.idx == 10
         self.repo.strip(tip.raw_id, self.repo.DEFAULT_BRANCH_NAME)
 
         tip = self.repo.get_commit()
         assert tip.idx == 9
 
     @pytest.mark.backends("git", "hg")
     def test_strip_multiple_commits(self):
         tip = self.repo.get_commit()
         assert tip.idx == 10
 
         old = self.repo.get_commit(commit_idx=5)
         self.repo.strip(old.raw_id, self.repo.DEFAULT_BRANCH_NAME)
 
         tip = self.repo.get_commit()
         assert tip.idx == 4
 
 
 @pytest.mark.backends('hg', 'git')
 class TestRepositoryPull(object):
 
     def test_pull(self, vcsbackend):
         source_repo = vcsbackend.repo
         target_repo = vcsbackend.create_repo()
         assert len(source_repo.commit_ids) > len(target_repo.commit_ids)
 
         target_repo.pull(source_repo.path)
         # Note: Get a fresh instance, avoids caching trouble
         target_repo = vcsbackend.backend(target_repo.path)
         assert len(source_repo.commit_ids) == len(target_repo.commit_ids)
 
     def test_pull_wrong_path(self, vcsbackend):
         target_repo = vcsbackend.create_repo()
         with pytest.raises(RepositoryError):
             target_repo.pull(target_repo.path + "wrong")
 
     def test_pull_specific_commits(self, vcsbackend):
         source_repo = vcsbackend.repo
         target_repo = vcsbackend.create_repo()
 
         second_commit = source_repo[1].raw_id
         if vcsbackend.alias == 'git':
             second_commit_ref = 'refs/test-refs/a'
             source_repo.set_refs(second_commit_ref, second_commit)
 
         target_repo.pull(source_repo.path, commit_ids=[second_commit])
         target_repo = vcsbackend.backend(target_repo.path)
         assert 2 == len(target_repo.commit_ids)
         assert second_commit == target_repo.get_commit().raw_id