tests: fixing tests for pull-requests and changelog(commits)

rhodecode/api/tests/test_comment_commit.py

@@ -1,81 +1,81 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
 
 from rhodecode.model.db import ChangesetStatus
 from rhodecode.api.tests.utils import (
     build_data, api_call, assert_error, assert_ok)
 
 
 @pytest.mark.usefixtures("testuser_api", "app")
 class TestCommentCommit(object):
     def test_api_comment_commit_on_empty_repo(self, backend):
         repo = backend.create_repo()
         id_, params = build_data(
             self.apikey, 'comment_commit', repoid=repo.repo_name,
             commit_id='tip', message='message', status_change=None)
         response = api_call(self.app, params)
         expected = 'There are no commits yet'
         assert_error(id_, expected, given=response.body)
 
     @pytest.mark.parametrize("commit_id, expected_err", [
-        ('abcabca', {'hg': 'Commit {commit} does not exist for {repo}',
-                     'git': 'Commit {commit} does not exist for {repo}',
+        ('abcabca', {'hg': 'Commit {commit} does not exist for `{repo}`',
+                     'git': 'Commit {commit} does not exist for `{repo}`',
                      'svn': 'Commit id {commit} not understood.'}),
-        ('idontexist', {'hg': 'Commit {commit} does not exist for {repo}',
-                        'git': 'Commit {commit} does not exist for {repo}',
+        ('idontexist', {'hg': 'Commit {commit} does not exist for `{repo}`',
+                        'git': 'Commit {commit} does not exist for `{repo}`',
                         'svn': 'Commit id {commit} not understood.'}),
     ])
     def test_api_comment_commit_wrong_hash(self, backend, commit_id, expected_err):
         repo_name = backend.repo.repo_name
         id_, params = build_data(
             self.apikey, 'comment_commit', repoid=repo_name,
             commit_id=commit_id, message='message', status_change=None)
         response = api_call(self.app, params)
 
         expected_err = expected_err[backend.alias]
         expected_err = expected_err.format(
-            repo=backend.repo.scm_instance(), commit=commit_id)
+            repo=backend.repo.scm_instance().name, commit=commit_id)
         assert_error(id_, expected_err, given=response.body)
 
     @pytest.mark.parametrize("status_change, message, commit_id", [
         (None, 'Hallo', 'tip'),
         (ChangesetStatus.STATUS_APPROVED, 'Approved', 'tip'),
         (ChangesetStatus.STATUS_REJECTED, 'Rejected', 'tip'),
     ])
     def test_api_comment_commit(
             self, backend, status_change, message, commit_id,
             no_notifications):
 
         commit_id = backend.repo.scm_instance().get_commit(commit_id).raw_id
 
         id_, params = build_data(
             self.apikey, 'comment_commit', repoid=backend.repo_name,
             commit_id=commit_id, message=message, status=status_change)
         response = api_call(self.app, params)
         repo = backend.repo.scm_instance()
         expected = {
             'msg': 'Commented on commit `%s` for repository `%s`' % (
                 repo.get_commit().raw_id, backend.repo_name),
             'status_change': status_change,
             'success': True
         }
         assert_ok(id_, expected, given=response.body)

rhodecode/apps/repository/tests/test_pull_requests_list.py

@@ -1,83 +1,84 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
 from rhodecode.model.db import Repository
 
 
 def route_path(name, params=None, **kwargs):
     import urllib
 
     base_url = {
         'pullrequest_show_all': '/{repo_name}/pull-request',
         'pullrequest_show_all_data': '/{repo_name}/pull-request-data',
     }[name].format(**kwargs)
 
     if params:
         base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
     return base_url
 
 
 @pytest.mark.backends("git", "hg")
 @pytest.mark.usefixtures('autologin_user', 'app')
 class TestPullRequestList(object):
 
     @pytest.mark.parametrize('params, expected_title', [
-        ({'source': 0, 'closed': 1}, 'Closed Pull Requests'),
-        ({'source': 0, 'my': 1}, 'opened by me'),
-        ({'source': 0, 'awaiting_review': 1}, 'awaiting review'),
-        ({'source': 0, 'awaiting_my_review': 1}, 'awaiting my review'),
-        ({'source': 1}, 'Pull Requests from'),
+        ({'source': 0, 'closed': 1}, 'Closed'),
+        ({'source': 0, 'my': 1}, 'Opened by me'),
+        ({'source': 0, 'awaiting_review': 1}, 'Awaiting review'),
+        ({'source': 0, 'awaiting_my_review': 1}, 'Awaiting my review'),
+        ({'source': 1}, 'From this repo'),
     ])
     def test_showing_list_page(self, backend, pr_util, params, expected_title):
         pull_request = pr_util.create_pull_request()
 
         response = self.app.get(
             route_path('pullrequest_show_all',
                        repo_name=pull_request.target_repo.repo_name,
                        params=params))
 
         assert_response = response.assert_response()
-        assert_response.element_equals_to('.panel-title', expected_title)
-        element = assert_response.get_element('.panel-title')
-        element_text = assert_response._element_to_string(element)
+
+        element = assert_response.get_element('.title .active')
+        element_text = element.text_content()
+        assert expected_title == element_text
 
     def test_showing_list_page_data(self, backend, pr_util, xhr_header):
         pull_request = pr_util.create_pull_request()
         response = self.app.get(
             route_path('pullrequest_show_all_data',
                        repo_name=pull_request.target_repo.repo_name),
             extra_environ=xhr_header)
 
         assert response.json['recordsTotal'] == 1
         assert response.json['data'][0]['description'] == 'Description'
 
     def test_description_is_escaped_on_index_page(self, backend, pr_util, xhr_header):
         xss_description = "<script>alert('Hi!')</script>"
         pull_request = pr_util.create_pull_request(description=xss_description)
 
         response = self.app.get(
             route_path('pullrequest_show_all_data',
                        repo_name=pull_request.target_repo.repo_name),
             extra_environ=xhr_header)
 
         assert response.json['recordsTotal'] == 1
         assert response.json['data'][0]['description'] == \
                "&lt;script&gt;alert(&#39;Hi!&#39;)&lt;/script&gt;"

rhodecode/apps/repository/tests/test_repo_changelog.py

@@ -1,213 +1,218 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import re
 
 import pytest
 
 from rhodecode.apps.repository.views.repo_changelog import DEFAULT_CHANGELOG_SIZE
 from rhodecode.tests import TestController
 
 MATCH_HASH = re.compile(r'<span class="commit_hash">r(\d+):[\da-f]+</span>')
 
 
 def route_path(name, params=None, **kwargs):
     import urllib
 
     base_url = {
         'repo_changelog': '/{repo_name}/changelog',
         'repo_commits': '/{repo_name}/commits',
         'repo_commits_file': '/{repo_name}/commits/{commit_id}/{f_path}',
         'repo_commits_elements': '/{repo_name}/commits_elements',
     }[name].format(**kwargs)
 
     if params:
         base_url = '{}?{}'.format(base_url, urllib.urlencode(params))
     return base_url
 
 
 def assert_commits_on_page(response, indexes):
     found_indexes = [int(idx) for idx in MATCH_HASH.findall(response.body)]
     assert found_indexes == indexes
 
 
 class TestChangelogController(TestController):
 
     def test_commits_page(self, backend):
         self.log_user()
         response = self.app.get(
             route_path('repo_commits', repo_name=backend.repo_name))
 
         first_idx = -1
         last_idx = -DEFAULT_CHANGELOG_SIZE
         self.assert_commit_range_on_page(
             response, first_idx, last_idx, backend)
 
     def test_changelog(self, backend):
         self.log_user()
         response = self.app.get(
             route_path('repo_changelog', repo_name=backend.repo_name))
 
         first_idx = -1
         last_idx = -DEFAULT_CHANGELOG_SIZE
         self.assert_commit_range_on_page(
             response, first_idx, last_idx, backend)
 
     @pytest.mark.backends("hg", "git")
     def test_changelog_filtered_by_branch(self, backend):
         self.log_user()
         self.app.get(
             route_path('repo_changelog', repo_name=backend.repo_name,
                        params=dict(branch=backend.default_branch_name)),
             status=200)
 
     @pytest.mark.backends("hg", "git")
     def test_commits_filtered_by_branch(self, backend):
         self.log_user()
         self.app.get(
             route_path('repo_commits', repo_name=backend.repo_name,
                        params=dict(branch=backend.default_branch_name)),
             status=200)
 
     @pytest.mark.backends("svn")
     def test_changelog_filtered_by_branch_svn(self, autologin_user, backend):
         repo = backend['svn-simple-layout']
         response = self.app.get(
             route_path('repo_changelog', repo_name=repo.repo_name,
                        params=dict(branch='trunk')),
             status=200)
 
         assert_commits_on_page(response, indexes=[15, 12, 7, 3, 2, 1])
 
     def test_commits_filtered_by_wrong_branch(self, backend):
         self.log_user()
         branch = 'wrong-branch-name'
         response = self.app.get(
             route_path('repo_commits', repo_name=backend.repo_name,
                        params=dict(branch=branch)),
             status=302)
         expected_url = '/{repo}/commits/{branch}'.format(
             repo=backend.repo_name, branch=branch)
         assert expected_url in response.location
         response = response.follow()
         expected_warning = 'Branch {} is not found.'.format(branch)
         assert expected_warning in response.body
 
     @pytest.mark.xfail_backends("svn", reason="Depends on branch support")
     def test_changelog_filtered_by_branch_with_merges(
             self, autologin_user, backend):
 
         # Note: The changelog of branch "b" does not contain the commit "a1"
         # although this is a parent of commit "b1". And branch "b" has commits
         # which have a smaller index than commit "a1".
         commits = [
             {'message': 'a'},
             {'message': 'b', 'branch': 'b'},
             {'message': 'a1', 'parents': ['a']},
             {'message': 'b1', 'branch': 'b', 'parents': ['b', 'a1']},
         ]
         backend.create_repo(commits)
 
         self.app.get(
             route_path('repo_changelog', repo_name=backend.repo_name,
                        params=dict(branch='b')),
             status=200)
 
     @pytest.mark.backends("hg")
     def test_commits_closed_branches(self, autologin_user, backend):
         repo = backend['closed_branch']
         response = self.app.get(
             route_path('repo_commits', repo_name=repo.repo_name,
                        params=dict(branch='experimental')),
             status=200)
 
         assert_commits_on_page(response, indexes=[3, 1])
 
     def test_changelog_pagination(self, backend):
         self.log_user()
         # pagination, walk up to page 6
         changelog_url = route_path(
             'repo_commits', repo_name=backend.repo_name)
 
         for page in range(1, 7):
             response = self.app.get(changelog_url, {'page': page})
 
         first_idx = -DEFAULT_CHANGELOG_SIZE * (page - 1) - 1
         last_idx = -DEFAULT_CHANGELOG_SIZE * page
         self.assert_commit_range_on_page(response, first_idx, last_idx, backend)
 
     def assert_commit_range_on_page(
             self, response, first_idx, last_idx, backend):
         input_template = (
-            """<input class="commit-range" id="%(raw_id)s" """
+            """<input class="commit-range" """ 
+            """data-commit-id="%(raw_id)s" data-commit-idx="%(idx)s" id="%(raw_id)s" """
             """name="%(raw_id)s" type="checkbox" value="1" />"""
         )
+
         commit_span_template = """<span class="commit_hash">r%s:%s</span>"""
         repo = backend.repo
 
         first_commit_on_page = repo.get_commit(commit_idx=first_idx)
         response.mustcontain(
-            input_template % {'raw_id': first_commit_on_page.raw_id})
+            input_template % {'raw_id': first_commit_on_page.raw_id,
+                              'idx': first_commit_on_page.idx})
+
         response.mustcontain(commit_span_template % (
             first_commit_on_page.idx, first_commit_on_page.short_id)
         )
 
         last_commit_on_page = repo.get_commit(commit_idx=last_idx)
         response.mustcontain(
-            input_template % {'raw_id': last_commit_on_page.raw_id})
+            input_template % {'raw_id': last_commit_on_page.raw_id,
+                              'idx': last_commit_on_page.idx})
         response.mustcontain(commit_span_template % (
             last_commit_on_page.idx, last_commit_on_page.short_id)
         )
 
         first_commit_of_next_page = repo.get_commit(commit_idx=last_idx - 1)
         first_span_of_next_page = commit_span_template % (
             first_commit_of_next_page.idx, first_commit_of_next_page.short_id)
         assert first_span_of_next_page not in response
 
     @pytest.mark.parametrize('test_path', [
         'vcs/exceptions.py',
         '/vcs/exceptions.py',
         '//vcs/exceptions.py'
     ])
     def test_commits_with_filenode(self, backend, test_path):
         self.log_user()
         response = self.app.get(
             route_path('repo_commits_file', repo_name=backend.repo_name,
                        commit_id='tip', f_path=test_path),
             )
 
         # history commits messages
         response.mustcontain('Added exceptions module, this time for real')
         response.mustcontain('Added not implemented hg backend test case')
         response.mustcontain('Added BaseChangeset class')
 
     def test_commits_with_filenode_that_is_dirnode(self, backend):
         self.log_user()
         self.app.get(
             route_path('repo_commits_file', repo_name=backend.repo_name,
                        commit_id='tip', f_path='/tests'),
             status=302)
 
     def test_commits_with_filenode_not_existing(self, backend):
         self.log_user()
         self.app.get(
             route_path('repo_commits_file', repo_name=backend.repo_name,
                        commit_id='tip', f_path='wrong_path'),
             status=302)

rhodecode/lib/vcs/backends/git/commit.py

@@ -1,535 +1,533 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 GIT commit module
 """
 
 import re
 import stat
 from itertools import chain
 from StringIO import StringIO
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode.lib.datelib import utcdate_fromtimestamp
 from rhodecode.lib.utils import safe_unicode, safe_str
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.lib.vcs.conf import settings
 from rhodecode.lib.vcs.backends import base
 from rhodecode.lib.vcs.exceptions import CommitError, NodeDoesNotExistError
 from rhodecode.lib.vcs.nodes import (
     FileNode, DirNode, NodeKind, RootNode, SubModuleNode,
     ChangedFileNodesGenerator, AddedFileNodesGenerator,
     RemovedFileNodesGenerator, LargeFileNode)
 from rhodecode.lib.vcs.compat import configparser
 
 
 class GitCommit(base.BaseCommit):
     """
     Represents state of the repository at single commit id.
     """
     _author_property = 'author'
     _committer_property = 'committer'
     _date_property = 'commit_time'
     _date_tz_property = 'commit_timezone'
     _message_property = 'message'
     _parents_property = 'parents'
 
     _filter_pre_load = [
         # done through a more complex tree walk on parents
         "affected_files",
         # based on repository cached property
         "branch",
         # done through subprocess not remote call
         "children",
         # done through a more complex tree walk on parents
         "status",
         # mercurial specific property not supported here
         "_file_paths",
         # mercurial specific property not supported here
         'obsolete',
         # mercurial specific property not supported here
         'phase',
         # mercurial specific property not supported here
         'hidden'
     ]
 
     def __init__(self, repository, raw_id, idx, pre_load=None):
         self.repository = repository
         self._remote = repository._remote
         # TODO: johbo: Tweak of raw_id should not be necessary
         self.raw_id = safe_str(raw_id)
         self.idx = idx
 
         self._set_bulk_properties(pre_load)
 
         # caches
         self._stat_modes = {}  # stat info for paths
         self._paths = {}  # path processed with parse_tree
         self.nodes = {}
         self._submodules = None
 
     def _set_bulk_properties(self, pre_load):
         if not pre_load:
             return
         pre_load = [entry for entry in pre_load
                     if entry not in self._filter_pre_load]
         if not pre_load:
             return
 
         result = self._remote.bulk_request(self.raw_id, pre_load)
         for attr, value in result.items():
             if attr in ["author", "message"]:
                 if value:
                     value = safe_unicode(value)
             elif attr == "date":
                 value = utcdate_fromtimestamp(*value)
             elif attr == "parents":
                 value = self._make_commits(value)
             self.__dict__[attr] = value
 
     @LazyProperty
     def _commit(self):
         return self._remote[self.raw_id]
 
     @LazyProperty
     def _tree_id(self):
         return self._remote[self._commit['tree']]['id']
 
     @LazyProperty
     def id(self):
         return self.raw_id
 
     @LazyProperty
     def short_id(self):
         return self.raw_id[:12]
 
     @LazyProperty
     def message(self):
         return safe_unicode(
             self._remote.commit_attribute(self.id, self._message_property))
 
     @LazyProperty
     def committer(self):
         return safe_unicode(
             self._remote.commit_attribute(self.id, self._committer_property))
 
     @LazyProperty
     def author(self):
         return safe_unicode(
             self._remote.commit_attribute(self.id, self._author_property))
 
     @LazyProperty
     def date(self):
         unix_ts, tz = self._remote.get_object_attrs(
             self.raw_id, self._date_property, self._date_tz_property)
         return utcdate_fromtimestamp(unix_ts, tz)
 
     @LazyProperty
     def status(self):
         """
         Returns modified, added, removed, deleted files for current commit
         """
         return self.changed, self.added, self.removed
 
     @LazyProperty
     def tags(self):
         tags = [safe_unicode(name) for name,
                 commit_id in self.repository.tags.iteritems()
                 if commit_id == self.raw_id]
         return tags
 
     @LazyProperty
     def branch(self):
         for name, commit_id in self.repository.branches.iteritems():
             if commit_id == self.raw_id:
                 return safe_unicode(name)
         return None
 
     def _get_id_for_path(self, path):
         path = safe_str(path)
         if path in self._paths:
             return self._paths[path]
 
         tree_id = self._tree_id
 
         path = path.strip('/')
         if path == '':
             data = [tree_id, "tree"]
             self._paths[''] = data
             return data
 
         parts = path.split('/')
         dirs, name = parts[:-1], parts[-1]
         cur_dir = ''
 
         # initially extract things from root dir
         tree_items = self._remote.tree_items(tree_id)
         self._process_tree_items(tree_items, cur_dir)
 
         for dir in dirs:
             if cur_dir:
                 cur_dir = '/'.join((cur_dir, dir))
             else:
                 cur_dir = dir
             dir_id = None
             for item, stat_, id_, type_ in tree_items:
                 if item == dir:
                     dir_id = id_
                     break
             if dir_id:
                 if type_ != "tree":
                     raise CommitError('%s is not a directory' % cur_dir)
                 # update tree
                 tree_items = self._remote.tree_items(dir_id)
             else:
                 raise CommitError('%s have not been found' % cur_dir)
 
             # cache all items from the given traversed tree
             self._process_tree_items(tree_items, cur_dir)
 
         if path not in self._paths:
             raise self.no_node_at_path(path)
 
         return self._paths[path]
 
     def _process_tree_items(self, items, cur_dir):
         for item, stat_, id_, type_ in items:
             if cur_dir:
                 name = '/'.join((cur_dir, item))
             else:
                 name = item
             self._paths[name] = [id_, type_]
             self._stat_modes[name] = stat_
 
     def _get_kind(self, path):
         path_id, type_ = self._get_id_for_path(path)
         if type_ == 'blob':
             return NodeKind.FILE
         elif type_ == 'tree':
             return NodeKind.DIR
         elif type == 'link':
             return NodeKind.SUBMODULE
         return None
 
     def _get_filectx(self, path):
         path = self._fix_path(path)
         if self._get_kind(path) != NodeKind.FILE:
             raise CommitError(
-                "File does not exist for commit %s at  '%s'" %
-                (self.raw_id, path))
+                "File does not exist for commit %s at  '%s'" % (self.raw_id, path))
         return path
 
     def _get_file_nodes(self):
         return chain(*(t[2] for t in self.walk()))
 
     @LazyProperty
     def parents(self):
         """
         Returns list of parent commits.
         """
         parent_ids = self._remote.commit_attribute(
             self.id, self._parents_property)
         return self._make_commits(parent_ids)
 
     @LazyProperty
     def children(self):
         """
         Returns list of child commits.
         """
         rev_filter = settings.GIT_REV_FILTER
         output, __ = self.repository.run_git_command(
             ['rev-list', '--children'] + rev_filter)
 
         child_ids = []
         pat = re.compile(r'^%s' % self.raw_id)
         for l in output.splitlines():
             if pat.match(l):
                 found_ids = l.split(' ')[1:]
                 child_ids.extend(found_ids)
         return self._make_commits(child_ids)
 
     def _make_commits(self, commit_ids, pre_load=None):
         return [
             self.repository.get_commit(commit_id=commit_id, pre_load=pre_load,
                                        translate_tag=False)
             for commit_id in commit_ids]
 
     def get_file_mode(self, path):
         """
         Returns stat mode of the file at the given `path`.
         """
         path = safe_str(path)
         # ensure path is traversed
         self._get_id_for_path(path)
         return self._stat_modes[path]
 
     def is_link(self, path):
         return stat.S_ISLNK(self.get_file_mode(path))
 
     def get_file_content(self, path):
         """
         Returns content of the file at given `path`.
         """
         id_, _ = self._get_id_for_path(path)
         return self._remote.blob_as_pretty_string(id_)
 
     def get_file_size(self, path):
         """
         Returns size of the file at given `path`.
         """
         id_, _ = self._get_id_for_path(path)
         return self._remote.blob_raw_length(id_)
 
     def get_path_history(self, path, limit=None, pre_load=None):
         """
         Returns history of file as reversed list of `GitCommit` objects for
         which file at given `path` has been modified.
 
         TODO: This function now uses an underlying 'git' command which works
         quickly but ideally we should replace with an algorithm.
         """
         self._get_filectx(path)
         f_path = safe_str(path)
 
         # optimize for n==1, rev-list is much faster for that use-case
         if limit == 1:
             cmd = ['rev-list', '-1', self.raw_id, '--', f_path]
         else:
             cmd = ['log']
             if limit:
                 cmd.extend(['-n', str(safe_int(limit, 0))])
             cmd.extend(['--pretty=format: %H', '-s', self.raw_id, '--', f_path])
 
         output, __ = self.repository.run_git_command(cmd)
         commit_ids = re.findall(r'[0-9a-fA-F]{40}', output)
 
         return [
             self.repository.get_commit(commit_id=commit_id, pre_load=pre_load)
             for commit_id in commit_ids]
 
     def get_file_annotate(self, path, pre_load=None):
         """
         Returns a generator of four element tuples with
             lineno, commit_id, commit lazy loader and line
 
         TODO: This function now uses os underlying 'git' command which is
         generally not good. Should be replaced with algorithm iterating
         commits.
         """
         cmd = ['blame', '-l', '--root', '-r', self.raw_id, '--', path]
         # -l     ==> outputs long shas (and we need all 40 characters)
         # --root ==> doesn't put '^' character for bounderies
         # -r commit_id ==> blames for the given commit
         output, __ = self.repository.run_git_command(cmd)
 
         for i, blame_line in enumerate(output.split('\n')[:-1]):
             line_no = i + 1
             commit_id, line = re.split(r' ', blame_line, 1)
             yield (
                 line_no, commit_id,
                 lambda: self.repository.get_commit(commit_id=commit_id,
                                                    pre_load=pre_load),
                 line)
 
     def get_nodes(self, path):
         if self._get_kind(path) != NodeKind.DIR:
             raise CommitError(
-                "Directory does not exist for commit %s at "
-                " '%s'" % (self.raw_id, path))
+                "Directory does not exist for commit %s at '%s'" % (self.raw_id, path))
         path = self._fix_path(path)
         id_, _ = self._get_id_for_path(path)
         tree_id = self._remote[id_]['id']
         dirnodes = []
         filenodes = []
         alias = self.repository.alias
         for name, stat_, id_, type_ in self._remote.tree_items(tree_id):
             if type_ == 'link':
                 url = self._get_submodule_url('/'.join((path, name)))
                 dirnodes.append(SubModuleNode(
                     name, url=url, commit=id_, alias=alias))
                 continue
 
             if path != '':
                 obj_path = '/'.join((path, name))
             else:
                 obj_path = name
             if obj_path not in self._stat_modes:
                 self._stat_modes[obj_path] = stat_
 
             if type_ == 'tree':
                 dirnodes.append(DirNode(obj_path, commit=self))
             elif type_ == 'blob':
                 filenodes.append(FileNode(obj_path, commit=self, mode=stat_))
             else:
                 raise CommitError(
                     "Requested object should be Tree or Blob, is %s", type_)
 
         nodes = dirnodes + filenodes
         for node in nodes:
             if node.path not in self.nodes:
                 self.nodes[node.path] = node
         nodes.sort()
         return nodes
 
     def get_node(self, path, pre_load=None):
         if isinstance(path, unicode):
             path = path.encode('utf-8')
         path = self._fix_path(path)
         if path not in self.nodes:
             try:
                 id_, type_ = self._get_id_for_path(path)
             except CommitError:
                 raise NodeDoesNotExistError(
                     "Cannot find one of parents' directories for a given "
                     "path: %s" % path)
 
             if type_ == 'link':
                 url = self._get_submodule_url(path)
                 node = SubModuleNode(path, url=url, commit=id_,
                                      alias=self.repository.alias)
             elif type_ == 'tree':
                 if path == '':
                     node = RootNode(commit=self)
                 else:
                     node = DirNode(path, commit=self)
             elif type_ == 'blob':
                 node = FileNode(path, commit=self, pre_load=pre_load)
             else:
                 raise self.no_node_at_path(path)
 
             # cache node
             self.nodes[path] = node
         return self.nodes[path]
 
     def get_largefile_node(self, path):
         id_, _ = self._get_id_for_path(path)
         pointer_spec = self._remote.is_large_file(id_)
 
         if pointer_spec:
             # content of that file regular FileNode is the hash of largefile
             file_id = pointer_spec.get('oid_hash')
             if self._remote.in_largefiles_store(file_id):
                 lf_path = self._remote.store_path(file_id)
                 return LargeFileNode(lf_path, commit=self, org_path=path)
 
     @LazyProperty
     def affected_files(self):
         """
         Gets a fast accessible file changes for given commit
         """
         added, modified, deleted = self._changes_cache
         return list(added.union(modified).union(deleted))
 
     @LazyProperty
     def _changes_cache(self):
         added = set()
         modified = set()
         deleted = set()
         _r = self._remote
 
         parents = self.parents
         if not self.parents:
             parents = [base.EmptyCommit()]
         for parent in parents:
             if isinstance(parent, base.EmptyCommit):
                 oid = None
             else:
                 oid = parent.raw_id
             changes = _r.tree_changes(oid, self.raw_id)
             for (oldpath, newpath), (_, _), (_, _) in changes:
                 if newpath and oldpath:
                     modified.add(newpath)
                 elif newpath and not oldpath:
                     added.add(newpath)
                 elif not newpath and oldpath:
                     deleted.add(oldpath)
         return added, modified, deleted
 
     def _get_paths_for_status(self, status):
         """
         Returns sorted list of paths for given ``status``.
 
         :param status: one of: *added*, *modified* or *deleted*
         """
         added, modified, deleted = self._changes_cache
         return sorted({
             'added': list(added),
             'modified': list(modified),
             'deleted': list(deleted)}[status]
         )
 
     @LazyProperty
     def added(self):
         """
         Returns list of added ``FileNode`` objects.
         """
         if not self.parents:
             return list(self._get_file_nodes())
         return AddedFileNodesGenerator(
             [n for n in self._get_paths_for_status('added')], self)
 
     @LazyProperty
     def changed(self):
         """
         Returns list of modified ``FileNode`` objects.
         """
         if not self.parents:
             return []
         return ChangedFileNodesGenerator(
             [n for n in self._get_paths_for_status('modified')], self)
 
     @LazyProperty
     def removed(self):
         """
         Returns list of removed ``FileNode`` objects.
         """
         if not self.parents:
             return []
         return RemovedFileNodesGenerator(
             [n for n in self._get_paths_for_status('deleted')], self)
 
     def _get_submodule_url(self, submodule_path):
         git_modules_path = '.gitmodules'
 
         if self._submodules is None:
             self._submodules = {}
 
             try:
                 submodules_node = self.get_node(git_modules_path)
             except NodeDoesNotExistError:
                 return None
 
             content = submodules_node.content
 
             # ConfigParser fails if there are whitespaces
             content = '\n'.join(l.strip() for l in content.split('\n'))
 
             parser = configparser.ConfigParser()
             parser.readfp(StringIO(content))
 
             for section in parser.sections():
                 path = parser.get(section, 'path')
                 url = parser.get(section, 'url')
                 if path and url:
                     self._submodules[path.strip('/')] = url
 
         return self._submodules.get(submodule_path.strip('/'))

rhodecode/lib/vcs/backends/git/repository.py

@@ -1,1037 +1,1037 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 GIT repository module
 """
 
 import logging
 import os
 import re
 import time
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 from zope.cachedescriptors.property import CachedProperty
 
 from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib.datelib import (
     utcdate_fromtimestamp, makedate, date_astimestamp)
 from rhodecode.lib.utils import safe_unicode, safe_str
 from rhodecode.lib.vcs import connection, path as vcspath
 from rhodecode.lib.vcs.backends.base import (
     BaseRepository, CollectionGenerator, Config, MergeResponse,
     MergeFailureReason, Reference)
 from rhodecode.lib.vcs.backends.git.commit import GitCommit
 from rhodecode.lib.vcs.backends.git.diff import GitDiff
 from rhodecode.lib.vcs.backends.git.inmemory import GitInMemoryCommit
 from rhodecode.lib.vcs.exceptions import (
     CommitDoesNotExistError, EmptyRepositoryError,
     RepositoryError, TagAlreadyExistError, TagDoesNotExistError, VCSError)
 
 
 SHA_PATTERN = re.compile(r'^[[0-9a-fA-F]{12}|[0-9a-fA-F]{40}]$')
 
 log = logging.getLogger(__name__)
 
 
 class GitRepository(BaseRepository):
     """
     Git repository backend.
     """
     DEFAULT_BRANCH_NAME = 'master'
 
     contact = BaseRepository.DEFAULT_CONTACT
 
     def __init__(self, repo_path, config=None, create=False, src_url=None,
                  do_workspace_checkout=False, with_wire=None, bare=False):
 
         self.path = safe_str(os.path.abspath(repo_path))
         self.config = config if config else self.get_default_config()
         self.with_wire = with_wire
 
         self._init_repo(create, src_url, do_workspace_checkout, bare)
 
         # caches
         self._commit_ids = {}
 
         # dependent that trigger re-computation of  commit_ids
         self._commit_ids_ver = 0
 
     @LazyProperty
     def _remote(self):
         return connection.Git(self.path, self.config, with_wire=self.with_wire)
 
     @LazyProperty
     def bare(self):
         return self._remote.bare()
 
     @LazyProperty
     def head(self):
         return self._remote.head()
 
     @CachedProperty('_commit_ids_ver')
     def commit_ids(self):
         """
         Returns list of commit ids, in ascending order.  Being lazy
         attribute allows external tools to inject commit ids from cache.
         """
         commit_ids = self._get_all_commit_ids()
         self._rebuild_cache(commit_ids)
         return commit_ids
 
     def _rebuild_cache(self, commit_ids):
         self._commit_ids = dict((commit_id, index)
                                 for index, commit_id in enumerate(commit_ids))
 
     def run_git_command(self, cmd, **opts):
         """
         Runs given ``cmd`` as git command and returns tuple
         (stdout, stderr).
 
         :param cmd: git command to be executed
         :param opts: env options to pass into Subprocess command
         """
         if not isinstance(cmd, list):
             raise ValueError('cmd must be a list, got %s instead' % type(cmd))
 
         skip_stderr_log = opts.pop('skip_stderr_log', False)
         out, err = self._remote.run_git_command(cmd, **opts)
         if err and not skip_stderr_log:
             log.debug('Stderr output of git command "%s":\n%s', cmd, err)
         return out, err
 
     @staticmethod
     def check_url(url, config):
         """
         Function will check given url and try to verify if it's a valid
         link. Sometimes it may happened that git will issue basic
         auth request that can cause whole API to hang when used from python
         or other external calls.
 
         On failures it'll raise urllib2.HTTPError, exception is also thrown
         when the return code is non 200
         """
         # check first if it's not an url
         if os.path.isdir(url) or url.startswith('file:'):
             return True
 
         if '+' in url.split('://', 1)[0]:
             url = url.split('+', 1)[1]
 
         # Request the _remote to verify the url
         return connection.Git.check_url(url, config.serialize())
 
     @staticmethod
     def is_valid_repository(path):
         if os.path.isdir(os.path.join(path, '.git')):
             return True
         # check case of bare repository
         try:
             GitRepository(path)
             return True
         except VCSError:
             pass
         return False
 
     def _init_repo(self, create, src_url=None, do_workspace_checkout=False,
                    bare=False):
         if create and os.path.exists(self.path):
             raise RepositoryError(
                 "Cannot create repository at %s, location already exist"
                 % self.path)
 
         if bare and do_workspace_checkout:
             raise RepositoryError("Cannot update a bare repository")
         try:
 
             if src_url:
                 # check URL before any actions
                 GitRepository.check_url(src_url, self.config)
 
             if create:
                 os.makedirs(self.path, mode=0o755)
 
                 if bare:
                     self._remote.init_bare()
                 else:
                     self._remote.init()
 
                 if src_url and bare:
                     # bare repository only allows a fetch and checkout is not allowed
                     self.fetch(src_url, commit_ids=None)
                 elif src_url:
                     self.pull(src_url, commit_ids=None,
                               update_after=do_workspace_checkout)
 
             else:
                 if not self._remote.assert_correct_path():
                     raise RepositoryError(
                         'Path "%s" does not contain a Git repository' %
                         (self.path,))
 
         # TODO: johbo: check if we have to translate the OSError here
         except OSError as err:
             raise RepositoryError(err)
 
     def _get_all_commit_ids(self, filters=None):
         # we must check if this repo is not empty, since later command
         # fails if it is. And it's cheaper to ask than throw the subprocess
         # errors
 
         head = self._remote.head(show_exc=False)
         if not head:
             return []
 
         rev_filter = ['--branches', '--tags']
         extra_filter = []
 
         if filters:
             if filters.get('since'):
                 extra_filter.append('--since=%s' % (filters['since']))
             if filters.get('until'):
                 extra_filter.append('--until=%s' % (filters['until']))
             if filters.get('branch_name'):
                 rev_filter = ['--tags']
                 extra_filter.append(filters['branch_name'])
         rev_filter.extend(extra_filter)
 
             # if filters.get('start') or filters.get('end'):
             #     # skip is offset, max-count is limit
             #     if filters.get('start'):
             #         extra_filter += ' --skip=%s' % filters['start']
             #     if filters.get('end'):
             #         extra_filter += ' --max-count=%s' % (filters['end'] - (filters['start'] or 0))
 
         cmd = ['rev-list', '--reverse', '--date-order'] + rev_filter
         try:
             output, __ = self.run_git_command(cmd)
         except RepositoryError:
             # Can be raised for empty repositories
             return []
         return output.splitlines()
 
     def _lookup_commit(self, commit_id_or_idx, translate_tag=True):
         def is_null(value):
             return len(value) == commit_id_or_idx.count('0')
 
         if commit_id_or_idx in (None, '', 'tip', 'HEAD', 'head', -1):
             return self.commit_ids[-1]
 
         is_bstr = isinstance(commit_id_or_idx, (str, unicode))
         if ((is_bstr and commit_id_or_idx.isdigit() and len(commit_id_or_idx) < 12)
             or isinstance(commit_id_or_idx, int) or is_null(commit_id_or_idx)):
             try:
                 commit_id_or_idx = self.commit_ids[int(commit_id_or_idx)]
             except Exception:
-                msg = "Commit %s does not exist for %s" % (commit_id_or_idx, self.name)
+                msg = "Commit {} does not exist for `{}`".format(commit_id_or_idx, self.name)
                 raise CommitDoesNotExistError(msg)
 
         elif is_bstr:
             # check full path ref, eg. refs/heads/master
             ref_id = self._refs.get(commit_id_or_idx)
             if ref_id:
                 return ref_id
 
             # check branch name
             branch_ids = self.branches.values()
             ref_id = self._refs.get('refs/heads/%s' % commit_id_or_idx)
             if ref_id:
                 return ref_id
 
             # check tag name
             ref_id = self._refs.get('refs/tags/%s' % commit_id_or_idx)
             if ref_id:
                 return ref_id
 
             if (not SHA_PATTERN.match(commit_id_or_idx) or
                     commit_id_or_idx not in self.commit_ids):
-                msg = "Commit %s does not exist for %s" % (commit_id_or_idx, self.name)
+                msg = "Commit {} does not exist for `{}`".format(commit_id_or_idx, self.name)
                 raise CommitDoesNotExistError(msg)
 
         # Ensure we return full id
         if not SHA_PATTERN.match(str(commit_id_or_idx)):
             raise CommitDoesNotExistError(
                 "Given commit id %s not recognized" % commit_id_or_idx)
         return commit_id_or_idx
 
     def get_hook_location(self):
         """
         returns absolute path to location where hooks are stored
         """
         loc = os.path.join(self.path, 'hooks')
         if not self.bare:
             loc = os.path.join(self.path, '.git', 'hooks')
         return loc
 
     @LazyProperty
     def last_change(self):
         """
         Returns last change made on this repository as
         `datetime.datetime` object.
         """
         try:
             return self.get_commit().date
         except RepositoryError:
             tzoffset = makedate()[1]
             return utcdate_fromtimestamp(self._get_fs_mtime(), tzoffset)
 
     def _get_fs_mtime(self):
         idx_loc = '' if self.bare else '.git'
         # fallback to filesystem
         in_path = os.path.join(self.path, idx_loc, "index")
         he_path = os.path.join(self.path, idx_loc, "HEAD")
         if os.path.exists(in_path):
             return os.stat(in_path).st_mtime
         else:
             return os.stat(he_path).st_mtime
 
     @LazyProperty
     def description(self):
         description = self._remote.get_description()
         return safe_unicode(description or self.DEFAULT_DESCRIPTION)
 
     def _get_refs_entries(self, prefix='', reverse=False, strip_prefix=True):
         if self.is_empty():
             return OrderedDict()
 
         result = []
         for ref, sha in self._refs.iteritems():
             if ref.startswith(prefix):
                 ref_name = ref
                 if strip_prefix:
                     ref_name = ref[len(prefix):]
                 result.append((safe_unicode(ref_name), sha))
 
         def get_name(entry):
             return entry[0]
 
         return OrderedDict(sorted(result, key=get_name, reverse=reverse))
 
     def _get_branches(self):
         return self._get_refs_entries(prefix='refs/heads/', strip_prefix=True)
 
     @LazyProperty
     def branches(self):
         return self._get_branches()
 
     @LazyProperty
     def branches_closed(self):
         return {}
 
     @LazyProperty
     def bookmarks(self):
         return {}
 
     @LazyProperty
     def branches_all(self):
         all_branches = {}
         all_branches.update(self.branches)
         all_branches.update(self.branches_closed)
         return all_branches
 
     @LazyProperty
     def tags(self):
         return self._get_tags()
 
     def _get_tags(self):
         return self._get_refs_entries(
             prefix='refs/tags/', strip_prefix=True, reverse=True)
 
     def tag(self, name, user, commit_id=None, message=None, date=None,
             **kwargs):
         # TODO: fix this method to apply annotated tags correct with message
         """
         Creates and returns a tag for the given ``commit_id``.
 
         :param name: name for new tag
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param commit_id: commit id for which new tag would be created
         :param message: message of the tag's commit
         :param date: date of tag's commit
 
         :raises TagAlreadyExistError: if tag with same name already exists
         """
         if name in self.tags:
             raise TagAlreadyExistError("Tag %s already exists" % name)
         commit = self.get_commit(commit_id=commit_id)
         message = message or "Added tag %s for commit %s" % (
             name, commit.raw_id)
         self._remote.set_refs('refs/tags/%s' % name, commit._commit['id'])
 
         self._refs = self._get_refs()
         self.tags = self._get_tags()
         return commit
 
     def remove_tag(self, name, user, message=None, date=None):
         """
         Removes tag with the given ``name``.
 
         :param name: name of the tag to be removed
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param message: message of the tag's removal commit
         :param date: date of tag's removal commit
 
         :raises TagDoesNotExistError: if tag with given name does not exists
         """
         if name not in self.tags:
             raise TagDoesNotExistError("Tag %s does not exist" % name)
         tagpath = vcspath.join(
             self._remote.get_refs_path(), 'refs', 'tags', name)
         try:
             os.remove(tagpath)
             self._refs = self._get_refs()
             self.tags = self._get_tags()
         except OSError as e:
             raise RepositoryError(e.strerror)
 
     def _get_refs(self):
         return self._remote.get_refs()
 
     @LazyProperty
     def _refs(self):
         return self._get_refs()
 
     @property
     def _ref_tree(self):
         node = tree = {}
         for ref, sha in self._refs.iteritems():
             path = ref.split('/')
             for bit in path[:-1]:
                 node = node.setdefault(bit, {})
             node[path[-1]] = sha
             node = tree
         return tree
 
     def get_remote_ref(self, ref_name):
         ref_key = 'refs/remotes/origin/{}'.format(safe_str(ref_name))
         try:
             return self._refs[ref_key]
         except Exception:
             return
 
     def get_commit(self, commit_id=None, commit_idx=None, pre_load=None, translate_tag=True):
         """
         Returns `GitCommit` object representing commit from git repository
         at the given `commit_id` or head (most recent commit) if None given.
         """
         if self.is_empty():
             raise EmptyRepositoryError("There are no commits yet")
 
         if commit_id is not None:
             self._validate_commit_id(commit_id)
             try:
                 # we have cached idx, use it without contacting the remote
                 idx = self._commit_ids[commit_id]
                 return GitCommit(self, commit_id, idx, pre_load=pre_load)
             except KeyError:
                 pass
 
         elif commit_idx is not None:
             self._validate_commit_idx(commit_idx)
             try:
                 _commit_id = self.commit_ids[commit_idx]
                 if commit_idx < 0:
                     commit_idx = self.commit_ids.index(_commit_id)
                 return GitCommit(self, _commit_id, commit_idx, pre_load=pre_load)
             except IndexError:
                 commit_id = commit_idx
         else:
             commit_id = "tip"
 
         commit_id = self._lookup_commit(commit_id)
         remote_idx = None
         if translate_tag:
             # Need to call remote to translate id for tagging scenario
             remote_data = self._remote.get_object(commit_id)
             commit_id = remote_data["commit_id"]
             remote_idx = remote_data["idx"]
 
         try:
             idx = self._commit_ids[commit_id]
         except KeyError:
             idx = remote_idx or 0
 
         return GitCommit(self, commit_id, idx, pre_load=pre_load)
 
     def get_commits(
             self, start_id=None, end_id=None, start_date=None, end_date=None,
             branch_name=None, show_hidden=False, pre_load=None, translate_tags=True):
         """
         Returns generator of `GitCommit` objects from start to end (both
         are inclusive), in ascending date order.
 
         :param start_id: None, str(commit_id)
         :param end_id: None, str(commit_id)
         :param start_date: if specified, commits with commit date less than
           ``start_date`` would be filtered out from returned set
         :param end_date: if specified, commits with commit date greater than
           ``end_date`` would be filtered out from returned set
         :param branch_name: if specified, commits not reachable from given
           branch would be filtered out from returned set
         :param show_hidden: Show hidden commits such as obsolete or hidden from
             Mercurial evolve
         :raise BranchDoesNotExistError: If given `branch_name` does not
             exist.
         :raise CommitDoesNotExistError: If commits for given `start` or
           `end` could not be found.
 
         """
         if self.is_empty():
             raise EmptyRepositoryError("There are no commits yet")
 
         self._validate_branch_name(branch_name)
 
         if start_id is not None:
             self._validate_commit_id(start_id)
         if end_id is not None:
             self._validate_commit_id(end_id)
 
         start_raw_id = self._lookup_commit(start_id)
         start_pos = self._commit_ids[start_raw_id] if start_id else None
         end_raw_id = self._lookup_commit(end_id)
         end_pos = max(0, self._commit_ids[end_raw_id]) if end_id else None
 
         if None not in [start_id, end_id] and start_pos > end_pos:
             raise RepositoryError(
                 "Start commit '%s' cannot be after end commit '%s'" %
                 (start_id, end_id))
 
         if end_pos is not None:
             end_pos += 1
 
         filter_ = []
         if branch_name:
             filter_.append({'branch_name': branch_name})
         if start_date and not end_date:
             filter_.append({'since': start_date})
         if end_date and not start_date:
             filter_.append({'until': end_date})
         if start_date and end_date:
             filter_.append({'since': start_date})
             filter_.append({'until': end_date})
 
         # if start_pos or end_pos:
         #     filter_.append({'start': start_pos})
         #     filter_.append({'end': end_pos})
 
         if filter_:
             revfilters = {
                 'branch_name': branch_name,
                 'since': start_date.strftime('%m/%d/%y %H:%M:%S') if start_date else None,
                 'until': end_date.strftime('%m/%d/%y %H:%M:%S') if end_date else None,
                 'start': start_pos,
                 'end': end_pos,
             }
             commit_ids = self._get_all_commit_ids(filters=revfilters)
 
             # pure python stuff, it's slow due to walker walking whole repo
             # def get_revs(walker):
             #     for walker_entry in walker:
             #         yield walker_entry.commit.id
             # revfilters = {}
             # commit_ids = list(reversed(list(get_revs(self._repo.get_walker(**revfilters)))))
         else:
             commit_ids = self.commit_ids
 
         if start_pos or end_pos:
             commit_ids = commit_ids[start_pos: end_pos]
 
         return CollectionGenerator(self, commit_ids, pre_load=pre_load,
                                    translate_tag=translate_tags)
 
     def get_diff(
             self, commit1, commit2, path='', ignore_whitespace=False,
             context=3, path1=None):
         """
         Returns (git like) *diff*, as plain text. Shows changes introduced by
         ``commit2`` since ``commit1``.
 
         :param commit1: Entry point from which diff is shown. Can be
           ``self.EMPTY_COMMIT`` - in this case, patch showing all
           the changes since empty state of the repository until ``commit2``
         :param commit2: Until which commits changes should be shown.
         :param ignore_whitespace: If set to ``True``, would not show whitespace
           changes. Defaults to ``False``.
         :param context: How many lines before/after changed lines should be
           shown. Defaults to ``3``.
         """
         self._validate_diff_commits(commit1, commit2)
         if path1 is not None and path1 != path:
             raise ValueError("Diff of two different paths not supported.")
 
         flags = [
             '-U%s' % context, '--full-index', '--binary', '-p',
             '-M', '--abbrev=40']
         if ignore_whitespace:
             flags.append('-w')
 
         if commit1 == self.EMPTY_COMMIT:
             cmd = ['show'] + flags + [commit2.raw_id]
         else:
             cmd = ['diff'] + flags + [commit1.raw_id, commit2.raw_id]
 
         if path:
             cmd.extend(['--', path])
 
         stdout, __ = self.run_git_command(cmd)
         # If we used 'show' command, strip first few lines (until actual diff
         # starts)
         if commit1 == self.EMPTY_COMMIT:
             lines = stdout.splitlines()
             x = 0
             for line in lines:
                 if line.startswith('diff'):
                     break
                 x += 1
             # Append new line just like 'diff' command do
             stdout = '\n'.join(lines[x:]) + '\n'
         return GitDiff(stdout)
 
     def strip(self, commit_id, branch_name):
         commit = self.get_commit(commit_id=commit_id)
         if commit.merge:
             raise Exception('Cannot reset to merge commit')
 
         # parent is going to be the new head now
         commit = commit.parents[0]
         self._remote.set_refs('refs/heads/%s' % branch_name, commit.raw_id)
 
         self._commit_ids_ver = time.time()
         # we updated _commit_ids_ver so accessing self.commit_ids will re-compute it
         return len(self.commit_ids)
 
     def get_common_ancestor(self, commit_id1, commit_id2, repo2):
         if commit_id1 == commit_id2:
             return commit_id1
 
         if self != repo2:
             commits = self._remote.get_missing_revs(
                 commit_id1, commit_id2, repo2.path)
             if commits:
                 commit = repo2.get_commit(commits[-1])
                 if commit.parents:
                     ancestor_id = commit.parents[0].raw_id
                 else:
                     ancestor_id = None
             else:
                 # no commits from other repo, ancestor_id is the commit_id2
                 ancestor_id = commit_id2
         else:
             output, __ = self.run_git_command(
                 ['merge-base', commit_id1, commit_id2])
             ancestor_id = re.findall(r'[0-9a-fA-F]{40}', output)[0]
 
         return ancestor_id
 
     def compare(self, commit_id1, commit_id2, repo2, merge, pre_load=None):
         repo1 = self
         ancestor_id = None
 
         if commit_id1 == commit_id2:
             commits = []
         elif repo1 != repo2:
             missing_ids = self._remote.get_missing_revs(commit_id1, commit_id2,
                                                         repo2.path)
             commits = [
                 repo2.get_commit(commit_id=commit_id, pre_load=pre_load)
                 for commit_id in reversed(missing_ids)]
         else:
             output, __ = repo1.run_git_command(
                 ['log', '--reverse', '--pretty=format: %H', '-s',
                  '%s..%s' % (commit_id1, commit_id2)])
             commits = [
                 repo1.get_commit(commit_id=commit_id, pre_load=pre_load)
                 for commit_id in re.findall(r'[0-9a-fA-F]{40}', output)]
 
         return commits
 
     @LazyProperty
     def in_memory_commit(self):
         """
         Returns ``GitInMemoryCommit`` object for this repository.
         """
         return GitInMemoryCommit(self)
 
     def pull(self, url, commit_ids=None, update_after=False):
         """
         Pull changes from external location. Pull is different in GIT
         that fetch since it's doing a checkout
 
         :param commit_ids: Optional. Can be set to a list of commit ids
            which shall be pulled from the other repository.
         """
         refs = None
         if commit_ids is not None:
             remote_refs = self._remote.get_remote_refs(url)
             refs = [ref for ref in remote_refs if remote_refs[ref] in commit_ids]
         self._remote.pull(url, refs=refs, update_after=update_after)
         self._remote.invalidate_vcs_cache()
 
     def fetch(self, url, commit_ids=None):
         """
         Fetch all git objects from external location.
         """
         self._remote.sync_fetch(url, refs=commit_ids)
         self._remote.invalidate_vcs_cache()
 
     def push(self, url):
         refs = None
         self._remote.sync_push(url, refs=refs)
 
     def set_refs(self, ref_name, commit_id):
         self._remote.set_refs(ref_name, commit_id)
 
     def remove_ref(self, ref_name):
         self._remote.remove_ref(ref_name)
 
     def _update_server_info(self):
         """
         runs gits update-server-info command in this repo instance
         """
         self._remote.update_server_info()
 
     def _current_branch(self):
         """
         Return the name of the current branch.
 
         It only works for non bare repositories (i.e. repositories with a
         working copy)
         """
         if self.bare:
             raise RepositoryError('Bare git repos do not have active branches')
 
         if self.is_empty():
             return None
 
         stdout, _ = self.run_git_command(['rev-parse', '--abbrev-ref', 'HEAD'])
         return stdout.strip()
 
     def _checkout(self, branch_name, create=False, force=False):
         """
         Checkout a branch in the working directory.
 
         It tries to create the branch if create is True, failing if the branch
         already exists.
 
         It only works for non bare repositories (i.e. repositories with a
         working copy)
         """
         if self.bare:
             raise RepositoryError('Cannot checkout branches in a bare git repo')
 
         cmd = ['checkout']
         if force:
             cmd.append('-f')
         if create:
             cmd.append('-b')
         cmd.append(branch_name)
         self.run_git_command(cmd, fail_on_stderr=False)
 
     def _identify(self):
         """
         Return the current state of the working directory.
         """
         if self.bare:
             raise RepositoryError('Bare git repos do not have active branches')
 
         if self.is_empty():
             return None
 
         stdout, _ = self.run_git_command(['rev-parse', 'HEAD'])
         return stdout.strip()
 
     def _local_clone(self, clone_path, branch_name, source_branch=None):
         """
         Create a local clone of the current repo.
         """
         # N.B.(skreft): the --branch option is required as otherwise the shallow
         # clone will only fetch the active branch.
         cmd = ['clone', '--branch', branch_name,
                self.path, os.path.abspath(clone_path)]
 
         self.run_git_command(cmd, fail_on_stderr=False)
 
         # if we get the different source branch, make sure we also fetch it for
         # merge conditions
         if source_branch and source_branch != branch_name:
             # check if the ref exists.
             shadow_repo = GitRepository(os.path.abspath(clone_path))
             if shadow_repo.get_remote_ref(source_branch):
                 cmd = ['fetch', self.path, source_branch]
                 self.run_git_command(cmd, fail_on_stderr=False)
 
     def _local_fetch(self, repository_path, branch_name, use_origin=False):
         """
         Fetch a branch from a local repository.
         """
         repository_path = os.path.abspath(repository_path)
         if repository_path == self.path:
             raise ValueError('Cannot fetch from the same repository')
 
         if use_origin:
             branch_name = '+{branch}:refs/heads/{branch}'.format(
                 branch=branch_name)
 
         cmd = ['fetch', '--no-tags', '--update-head-ok',
                repository_path, branch_name]
         self.run_git_command(cmd, fail_on_stderr=False)
 
     def _local_reset(self, branch_name):
         branch_name = '{}'.format(branch_name)
         cmd = ['reset', '--hard', branch_name, '--']
         self.run_git_command(cmd, fail_on_stderr=False)
 
     def _last_fetch_heads(self):
         """
         Return the last fetched heads that need merging.
 
         The algorithm is defined at
         https://github.com/git/git/blob/v2.1.3/git-pull.sh#L283
         """
         if not self.bare:
             fetch_heads_path = os.path.join(self.path, '.git', 'FETCH_HEAD')
         else:
             fetch_heads_path = os.path.join(self.path, 'FETCH_HEAD')
 
         heads = []
         with open(fetch_heads_path) as f:
             for line in f:
                 if '    not-for-merge   ' in line:
                     continue
                 line = re.sub('\t.*', '', line, flags=re.DOTALL)
                 heads.append(line)
 
         return heads
 
     def _get_shadow_instance(self, shadow_repository_path, enable_hooks=False):
         return GitRepository(shadow_repository_path)
 
     def _local_pull(self, repository_path, branch_name, ff_only=True):
         """
         Pull a branch from a local repository.
         """
         if self.bare:
             raise RepositoryError('Cannot pull into a bare git repository')
         # N.B.(skreft): The --ff-only option is to make sure this is a
         # fast-forward (i.e., we are only pulling new changes and there are no
         # conflicts with our current branch)
         # Additionally, that option needs to go before --no-tags, otherwise git
         # pull complains about it being an unknown flag.
         cmd = ['pull']
         if ff_only:
             cmd.append('--ff-only')
         cmd.extend(['--no-tags', repository_path, branch_name])
         self.run_git_command(cmd, fail_on_stderr=False)
 
     def _local_merge(self, merge_message, user_name, user_email, heads):
         """
         Merge the given head into the checked out branch.
 
         It will force a merge commit.
 
         Currently it raises an error if the repo is empty, as it is not possible
         to create a merge commit in an empty repo.
 
         :param merge_message: The message to use for the merge commit.
         :param heads: the heads to merge.
         """
         if self.bare:
             raise RepositoryError('Cannot merge into a bare git repository')
 
         if not heads:
             return
 
         if self.is_empty():
             # TODO(skreft): do somehting more robust in this case.
             raise RepositoryError(
                 'Do not know how to merge into empty repositories yet')
 
         # N.B.(skreft): the --no-ff option is used to enforce the creation of a
         # commit message. We also specify the user who is doing the merge.
         cmd = ['-c', 'user.name="%s"' % safe_str(user_name),
                '-c', 'user.email=%s' % safe_str(user_email),
                'merge', '--no-ff', '-m', safe_str(merge_message)]
         cmd.extend(heads)
         try:
             output = self.run_git_command(cmd, fail_on_stderr=False)
         except RepositoryError:
             # Cleanup any merge leftovers
             self.run_git_command(['merge', '--abort'], fail_on_stderr=False)
             raise
 
     def _local_push(
             self, source_branch, repository_path, target_branch,
             enable_hooks=False, rc_scm_data=None):
         """
         Push the source_branch to the given repository and target_branch.
 
         Currently it if the target_branch is not master and the target repo is
         empty, the push will work, but then GitRepository won't be able to find
         the pushed branch or the commits. As the HEAD will be corrupted (i.e.,
         pointing to master, which does not exist).
 
         It does not run the hooks in the target repo.
         """
         # TODO(skreft): deal with the case in which the target repo is empty,
         # and the target_branch is not master.
         target_repo = GitRepository(repository_path)
         if (not target_repo.bare and
                 target_repo._current_branch() == target_branch):
             # Git prevents pushing to the checked out branch, so simulate it by
             # pulling into the target repository.
             target_repo._local_pull(self.path, source_branch)
         else:
             cmd = ['push', os.path.abspath(repository_path),
                    '%s:%s' % (source_branch, target_branch)]
             gitenv = {}
             if rc_scm_data:
                 gitenv.update({'RC_SCM_DATA': rc_scm_data})
 
             if not enable_hooks:
                 gitenv['RC_SKIP_HOOKS'] = '1'
             self.run_git_command(cmd, fail_on_stderr=False, extra_env=gitenv)
 
     def _get_new_pr_branch(self, source_branch, target_branch):
         prefix = 'pr_%s-%s_' % (source_branch, target_branch)
         pr_branches = []
         for branch in self.branches:
             if branch.startswith(prefix):
                 pr_branches.append(int(branch[len(prefix):]))
 
         if not pr_branches:
             branch_id = 0
         else:
             branch_id = max(pr_branches) + 1
 
         return '%s%d' % (prefix, branch_id)
 
     def _maybe_prepare_merge_workspace(
             self, repo_id, workspace_id, target_ref, source_ref):
         shadow_repository_path = self._get_shadow_repository_path(
             repo_id, workspace_id)
         if not os.path.exists(shadow_repository_path):
             self._local_clone(
                 shadow_repository_path, target_ref.name, source_ref.name)
             log.debug(
                 'Prepared shadow repository in %s', shadow_repository_path)
 
         return shadow_repository_path
 
     def _merge_repo(self, repo_id, workspace_id, target_ref,
                     source_repo, source_ref, merge_message,
                     merger_name, merger_email, dry_run=False,
                     use_rebase=False, close_branch=False):
 
         log.debug('Executing merge_repo with %s strategy, dry_run mode:%s',
                   'rebase' if use_rebase else 'merge', dry_run)
         if target_ref.commit_id != self.branches[target_ref.name]:
             log.warning('Target ref %s commit mismatch %s vs %s', target_ref,
                         target_ref.commit_id, self.branches[target_ref.name])
             return MergeResponse(
                 False, False, None, MergeFailureReason.TARGET_IS_NOT_HEAD,
                 metadata={'target_ref': target_ref})
 
         shadow_repository_path = self._maybe_prepare_merge_workspace(
             repo_id, workspace_id, target_ref, source_ref)
         shadow_repo = self._get_shadow_instance(shadow_repository_path)
 
         # checkout source, if it's different. Otherwise we could not
         # fetch proper commits for merge testing
         if source_ref.name != target_ref.name:
             if shadow_repo.get_remote_ref(source_ref.name):
                 shadow_repo._checkout(source_ref.name, force=True)
 
         # checkout target, and fetch changes
         shadow_repo._checkout(target_ref.name, force=True)
 
         # fetch/reset pull the target, in case it is changed
         # this handles even force changes
         shadow_repo._local_fetch(self.path, target_ref.name, use_origin=True)
         shadow_repo._local_reset(target_ref.name)
 
         # Need to reload repo to invalidate the cache, or otherwise we cannot
         # retrieve the last target commit.
         shadow_repo = self._get_shadow_instance(shadow_repository_path)
         if target_ref.commit_id != shadow_repo.branches[target_ref.name]:
             log.warning('Shadow Target ref %s commit mismatch %s vs %s',
                         target_ref, target_ref.commit_id,
                         shadow_repo.branches[target_ref.name])
             return MergeResponse(
                 False, False, None, MergeFailureReason.TARGET_IS_NOT_HEAD,
                 metadata={'target_ref': target_ref})
 
         # calculate new branch
         pr_branch = shadow_repo._get_new_pr_branch(
             source_ref.name, target_ref.name)
         log.debug('using pull-request merge branch: `%s`', pr_branch)
         # checkout to temp branch, and fetch changes
         shadow_repo._checkout(pr_branch, create=True)
         try:
             shadow_repo._local_fetch(source_repo.path, source_ref.name)
         except RepositoryError:
             log.exception('Failure when doing local fetch on '
                           'shadow repo: %s', shadow_repo)
             return MergeResponse(
                 False, False, None, MergeFailureReason.MISSING_SOURCE_REF,
                 metadata={'source_ref': source_ref})
 
         merge_ref = None
         merge_failure_reason = MergeFailureReason.NONE
         metadata = {}
         try:
             shadow_repo._local_merge(merge_message, merger_name, merger_email,
                                      [source_ref.commit_id])
             merge_possible = True
 
             # Need to reload repo to invalidate the cache, or otherwise we
             # cannot retrieve the merge commit.
             shadow_repo = GitRepository(shadow_repository_path)
             merge_commit_id = shadow_repo.branches[pr_branch]
 
             # Set a reference pointing to the merge commit. This reference may
             # be used to easily identify the last successful merge commit in
             # the shadow repository.
             shadow_repo.set_refs('refs/heads/pr-merge', merge_commit_id)
             merge_ref = Reference('branch', 'pr-merge', merge_commit_id)
         except RepositoryError:
             log.exception('Failure when doing local merge on git shadow repo')
             merge_possible = False
             merge_failure_reason = MergeFailureReason.MERGE_FAILED
 
         if merge_possible and not dry_run:
             try:
                 shadow_repo._local_push(
                     pr_branch, self.path, target_ref.name, enable_hooks=True,
                     rc_scm_data=self.config.get('rhodecode', 'RC_SCM_DATA'))
                 merge_succeeded = True
             except RepositoryError:
                 log.exception(
                     'Failure when doing local push from the shadow '
                     'repository to the target repository at %s.', self.path)
                 merge_succeeded = False
                 merge_failure_reason = MergeFailureReason.PUSH_FAILED
                 metadata['target'] = 'git shadow repo'
                 metadata['merge_commit'] = pr_branch
         else:
             merge_succeeded = False
 
         return MergeResponse(
             merge_possible, merge_succeeded, merge_ref, merge_failure_reason,
             metadata=metadata)

rhodecode/lib/vcs/backends/hg/repository.py

@@ -1,949 +1,949 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2014-2019 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 HG repository module
 """
 import os
 import logging
 import binascii
 import time
 import urllib
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
 from zope.cachedescriptors.property import CachedProperty
 
 from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib.datelib import (
     date_to_timestamp_plus_offset, utcdate_fromtimestamp, makedate)
 from rhodecode.lib.utils import safe_unicode, safe_str
 from rhodecode.lib.vcs import connection, exceptions
 from rhodecode.lib.vcs.backends.base import (
     BaseRepository, CollectionGenerator, Config, MergeResponse,
     MergeFailureReason, Reference, BasePathPermissionChecker)
 from rhodecode.lib.vcs.backends.hg.commit import MercurialCommit
 from rhodecode.lib.vcs.backends.hg.diff import MercurialDiff
 from rhodecode.lib.vcs.backends.hg.inmemory import MercurialInMemoryCommit
 from rhodecode.lib.vcs.exceptions import (
     EmptyRepositoryError, RepositoryError, TagAlreadyExistError,
     TagDoesNotExistError, CommitDoesNotExistError, SubrepoMergeError)
 from rhodecode.lib.vcs.compat import configparser
 
 hexlify = binascii.hexlify
 nullid = "\0" * 20
 
 log = logging.getLogger(__name__)
 
 
 class MercurialRepository(BaseRepository):
     """
     Mercurial repository backend
     """
     DEFAULT_BRANCH_NAME = 'default'
 
     def __init__(self, repo_path, config=None, create=False, src_url=None,
                  do_workspace_checkout=False, with_wire=None, bare=False):
         """
         Raises RepositoryError if repository could not be find at the given
         ``repo_path``.
 
         :param repo_path: local path of the repository
         :param config: config object containing the repo configuration
         :param create=False: if set to True, would try to create repository if
            it does not exist rather than raising exception
         :param src_url=None: would try to clone repository from given location
         :param do_workspace_checkout=False: sets update of working copy after
            making a clone
         :param bare: not used, compatible with other VCS
         """
 
         self.path = safe_str(os.path.abspath(repo_path))
         # mercurial since 4.4.X requires certain configuration to be present
         # because sometimes we init the repos with config we need to meet
         # special requirements
         self.config = config if config else self.get_default_config(
             default=[('extensions', 'largefiles', '1')])
         self.with_wire = with_wire
 
         self._init_repo(create, src_url, do_workspace_checkout)
 
         # caches
         self._commit_ids = {}
 
         # dependent that trigger re-computation of  commit_ids
         self._commit_ids_ver = 0
 
     @LazyProperty
     def _remote(self):
         return connection.Hg(self.path, self.config, with_wire=self.with_wire)
 
     @CachedProperty('_commit_ids_ver')
     def commit_ids(self):
         """
         Returns list of commit ids, in ascending order.  Being lazy
         attribute allows external tools to inject shas from cache.
         """
         commit_ids = self._get_all_commit_ids()
         self._rebuild_cache(commit_ids)
         return commit_ids
 
     def _rebuild_cache(self, commit_ids):
         self._commit_ids = dict((commit_id, index)
                                 for index, commit_id in enumerate(commit_ids))
 
     @LazyProperty
     def branches(self):
         return self._get_branches()
 
     @LazyProperty
     def branches_closed(self):
         return self._get_branches(active=False, closed=True)
 
     @LazyProperty
     def branches_all(self):
         all_branches = {}
         all_branches.update(self.branches)
         all_branches.update(self.branches_closed)
         return all_branches
 
     def _get_branches(self, active=True, closed=False):
         """
         Gets branches for this repository
         Returns only not closed active branches by default
 
         :param active: return also active branches
         :param closed: return also closed branches
 
         """
         if self.is_empty():
             return {}
 
         def get_name(ctx):
             return ctx[0]
 
         _branches = [(safe_unicode(n), hexlify(h),) for n, h in
                      self._remote.branches(active, closed).items()]
 
         return OrderedDict(sorted(_branches, key=get_name, reverse=False))
 
     @LazyProperty
     def tags(self):
         """
         Gets tags for this repository
         """
         return self._get_tags()
 
     def _get_tags(self):
         if self.is_empty():
             return {}
 
         def get_name(ctx):
             return ctx[0]
 
         _tags = [(safe_unicode(n), hexlify(h),) for n, h in
                  self._remote.tags().items()]
 
         return OrderedDict(sorted(_tags, key=get_name, reverse=True))
 
     def tag(self, name, user, commit_id=None, message=None, date=None, **kwargs):
         """
         Creates and returns a tag for the given ``commit_id``.
 
         :param name: name for new tag
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param commit_id: commit id for which new tag would be created
         :param message: message of the tag's commit
         :param date: date of tag's commit
 
         :raises TagAlreadyExistError: if tag with same name already exists
         """
         if name in self.tags:
             raise TagAlreadyExistError("Tag %s already exists" % name)
 
         commit = self.get_commit(commit_id=commit_id)
         local = kwargs.setdefault('local', False)
 
         if message is None:
             message = "Added tag %s for commit %s" % (name, commit.short_id)
 
         date, tz = date_to_timestamp_plus_offset(date)
 
         self._remote.tag(name, commit.raw_id, message, local, user, date, tz)
         self._remote.invalidate_vcs_cache()
 
         # Reinitialize tags
         self.tags = self._get_tags()
         tag_id = self.tags[name]
 
         return self.get_commit(commit_id=tag_id)
 
     def remove_tag(self, name, user, message=None, date=None):
         """
         Removes tag with the given `name`.
 
         :param name: name of the tag to be removed
         :param user: full username, i.e.: "Joe Doe <[e-mail]>"
         :param message: message of the tag's removal commit
         :param date: date of tag's removal commit
 
         :raises TagDoesNotExistError: if tag with given name does not exists
         """
         if name not in self.tags:
             raise TagDoesNotExistError("Tag %s does not exist" % name)
 
         if message is None:
             message = "Removed tag %s" % name
         local = False
 
         date, tz = date_to_timestamp_plus_offset(date)
 
         self._remote.tag(name, nullid, message, local, user, date, tz)
         self._remote.invalidate_vcs_cache()
         self.tags = self._get_tags()
 
     @LazyProperty
     def bookmarks(self):
         """
         Gets bookmarks for this repository
         """
         return self._get_bookmarks()
 
     def _get_bookmarks(self):
         if self.is_empty():
             return {}
 
         def get_name(ctx):
             return ctx[0]
 
         _bookmarks = [
             (safe_unicode(n), hexlify(h)) for n, h in
             self._remote.bookmarks().items()]
 
         return OrderedDict(sorted(_bookmarks, key=get_name))
 
     def _get_all_commit_ids(self):
         return self._remote.get_all_commit_ids('visible')
 
     def get_diff(
             self, commit1, commit2, path='', ignore_whitespace=False,
             context=3, path1=None):
         """
         Returns (git like) *diff*, as plain text. Shows changes introduced by
         `commit2` since `commit1`.
 
         :param commit1: Entry point from which diff is shown. Can be
           ``self.EMPTY_COMMIT`` - in this case, patch showing all
           the changes since empty state of the repository until `commit2`
         :param commit2: Until which commit changes should be shown.
         :param ignore_whitespace: If set to ``True``, would not show whitespace
           changes. Defaults to ``False``.
         :param context: How many lines before/after changed lines should be
           shown. Defaults to ``3``.
         """
         self._validate_diff_commits(commit1, commit2)
         if path1 is not None and path1 != path:
             raise ValueError("Diff of two different paths not supported.")
 
         if path:
             file_filter = [self.path, path]
         else:
             file_filter = None
 
         diff = self._remote.diff(
             commit1.raw_id, commit2.raw_id, file_filter=file_filter,
             opt_git=True, opt_ignorews=ignore_whitespace,
             context=context)
         return MercurialDiff(diff)
 
     def strip(self, commit_id, branch=None):
         self._remote.strip(commit_id, update=False, backup="none")
 
         self._remote.invalidate_vcs_cache()
         self._commit_ids_ver = time.time()
         # we updated _commit_ids_ver so accessing self.commit_ids will re-compute it
         return len(self.commit_ids)
 
     def verify(self):
         verify = self._remote.verify()
 
         self._remote.invalidate_vcs_cache()
         return verify
 
     def get_common_ancestor(self, commit_id1, commit_id2, repo2):
         if commit_id1 == commit_id2:
             return commit_id1
 
         ancestors = self._remote.revs_from_revspec(
             "ancestor(id(%s), id(%s))", commit_id1, commit_id2,
             other_path=repo2.path)
         return repo2[ancestors[0]].raw_id if ancestors else None
 
     def compare(self, commit_id1, commit_id2, repo2, merge, pre_load=None):
         if commit_id1 == commit_id2:
             commits = []
         else:
             if merge:
                 indexes = self._remote.revs_from_revspec(
                     "ancestors(id(%s)) - ancestors(id(%s)) - id(%s)",
                     commit_id2, commit_id1, commit_id1, other_path=repo2.path)
             else:
                 indexes = self._remote.revs_from_revspec(
                     "id(%s)..id(%s) - id(%s)", commit_id1, commit_id2,
                     commit_id1, other_path=repo2.path)
 
             commits = [repo2.get_commit(commit_idx=idx, pre_load=pre_load)
                        for idx in indexes]
 
         return commits
 
     @staticmethod
     def check_url(url, config):
         """
         Function will check given url and try to verify if it's a valid
         link. Sometimes it may happened that mercurial will issue basic
         auth request that can cause whole API to hang when used from python
         or other external calls.
 
         On failures it'll raise urllib2.HTTPError, exception is also thrown
         when the return code is non 200
         """
         # check first if it's not an local url
         if os.path.isdir(url) or url.startswith('file:'):
             return True
 
         # Request the _remote to verify the url
         return connection.Hg.check_url(url, config.serialize())
 
     @staticmethod
     def is_valid_repository(path):
         return os.path.isdir(os.path.join(path, '.hg'))
 
     def _init_repo(self, create, src_url=None, do_workspace_checkout=False):
         """
         Function will check for mercurial repository in given path. If there
         is no repository in that path it will raise an exception unless
         `create` parameter is set to True - in that case repository would
         be created.
 
         If `src_url` is given, would try to clone repository from the
         location at given clone_point. Additionally it'll make update to
         working copy accordingly to `do_workspace_checkout` flag.
         """
         if create and os.path.exists(self.path):
             raise RepositoryError(
                 "Cannot create repository at %s, location already exist"
                 % self.path)
 
         if src_url:
             url = str(self._get_url(src_url))
             MercurialRepository.check_url(url, self.config)
 
             self._remote.clone(url, self.path, do_workspace_checkout)
 
             # Don't try to create if we've already cloned repo
             create = False
 
         if create:
             os.makedirs(self.path, mode=0o755)
 
         self._remote.localrepository(create)
 
     @LazyProperty
     def in_memory_commit(self):
         return MercurialInMemoryCommit(self)
 
     @LazyProperty
     def description(self):
         description = self._remote.get_config_value(
             'web', 'description', untrusted=True)
         return safe_unicode(description or self.DEFAULT_DESCRIPTION)
 
     @LazyProperty
     def contact(self):
         contact = (
             self._remote.get_config_value("web", "contact") or
             self._remote.get_config_value("ui", "username"))
         return safe_unicode(contact or self.DEFAULT_CONTACT)
 
     @LazyProperty
     def last_change(self):
         """
         Returns last change made on this repository as
         `datetime.datetime` object.
         """
         try:
             return self.get_commit().date
         except RepositoryError:
             tzoffset = makedate()[1]
             return utcdate_fromtimestamp(self._get_fs_mtime(), tzoffset)
 
     def _get_fs_mtime(self):
         # fallback to filesystem
         cl_path = os.path.join(self.path, '.hg', "00changelog.i")
         st_path = os.path.join(self.path, '.hg', "store")
         if os.path.exists(cl_path):
             return os.stat(cl_path).st_mtime
         else:
             return os.stat(st_path).st_mtime
 
     def _get_url(self, url):
         """
         Returns normalized url. If schema is not given, would fall
         to filesystem
         (``file:///``) schema.
         """
         url = url.encode('utf8')
         if url != 'default' and '://' not in url:
             url = "file:" + urllib.pathname2url(url)
         return url
 
     def get_hook_location(self):
         """
         returns absolute path to location where hooks are stored
         """
         return os.path.join(self.path, '.hg', '.hgrc')
 
     def get_commit(self, commit_id=None, commit_idx=None, pre_load=None, translate_tag=None):
         """
         Returns ``MercurialCommit`` object representing repository's
         commit at the given `commit_id` or `commit_idx`.
         """
         if self.is_empty():
             raise EmptyRepositoryError("There are no commits yet")
 
         if commit_id is not None:
             self._validate_commit_id(commit_id)
             try:
                 # we have cached idx, use it without contacting the remote
                 idx = self._commit_ids[commit_id]
                 return MercurialCommit(self, commit_id, idx, pre_load=pre_load)
             except KeyError:
                 pass
 
         elif commit_idx is not None:
             self._validate_commit_idx(commit_idx)
             try:
                 _commit_id = self.commit_ids[commit_idx]
                 if commit_idx < 0:
                     commit_idx = self.commit_ids.index(_commit_id)
 
                 return MercurialCommit(self, _commit_id, commit_idx, pre_load=pre_load)
             except IndexError:
                 commit_id = commit_idx
         else:
             commit_id = "tip"
 
         if isinstance(commit_id, unicode):
             commit_id = safe_str(commit_id)
 
         try:
             raw_id, idx = self._remote.lookup(commit_id, both=True)
         except CommitDoesNotExistError:
-            msg = "Commit {} does not exist for {}".format(
+            msg = "Commit {} does not exist for `{}`".format(
                 *map(safe_str, [commit_id, self.name]))
             raise CommitDoesNotExistError(msg)
 
         return MercurialCommit(self, raw_id, idx, pre_load=pre_load)
 
     def get_commits(
             self, start_id=None, end_id=None, start_date=None, end_date=None,
             branch_name=None, show_hidden=False, pre_load=None, translate_tags=None):
         """
         Returns generator of ``MercurialCommit`` objects from start to end
         (both are inclusive)
 
         :param start_id: None, str(commit_id)
         :param end_id: None, str(commit_id)
         :param start_date: if specified, commits with commit date less than
           ``start_date`` would be filtered out from returned set
         :param end_date: if specified, commits with commit date greater than
           ``end_date`` would be filtered out from returned set
         :param branch_name: if specified, commits not reachable from given
           branch would be filtered out from returned set
         :param show_hidden: Show hidden commits such as obsolete or hidden from
             Mercurial evolve
         :raise BranchDoesNotExistError: If given ``branch_name`` does not
             exist.
         :raise CommitDoesNotExistError: If commit for given ``start`` or
           ``end`` could not be found.
         """
         # actually we should check now if it's not an empty repo
         if self.is_empty():
             raise EmptyRepositoryError("There are no commits yet")
         self._validate_branch_name(branch_name)
 
         branch_ancestors = False
         if start_id is not None:
             self._validate_commit_id(start_id)
             c_start = self.get_commit(commit_id=start_id)
             start_pos = self._commit_ids[c_start.raw_id]
         else:
             start_pos = None
 
         if end_id is not None:
             self._validate_commit_id(end_id)
             c_end = self.get_commit(commit_id=end_id)
             end_pos = max(0, self._commit_ids[c_end.raw_id])
         else:
             end_pos = None
 
         if None not in [start_id, end_id] and start_pos > end_pos:
             raise RepositoryError(
                 "Start commit '%s' cannot be after end commit '%s'" %
                 (start_id, end_id))
 
         if end_pos is not None:
             end_pos += 1
 
         commit_filter = []
 
         if branch_name and not branch_ancestors:
             commit_filter.append('branch("%s")' % (branch_name,))
         elif branch_name and branch_ancestors:
             commit_filter.append('ancestors(branch("%s"))' % (branch_name,))
 
         if start_date and not end_date:
             commit_filter.append('date(">%s")' % (start_date,))
         if end_date and not start_date:
             commit_filter.append('date("<%s")' % (end_date,))
         if start_date and end_date:
             commit_filter.append(
                 'date(">%s") and date("<%s")' % (start_date, end_date))
 
         if not show_hidden:
             commit_filter.append('not obsolete()')
             commit_filter.append('not hidden()')
 
         # TODO: johbo: Figure out a simpler way for this solution
         collection_generator = CollectionGenerator
         if commit_filter:
             commit_filter = ' and '.join(map(safe_str, commit_filter))
             revisions = self._remote.rev_range([commit_filter])
             collection_generator = MercurialIndexBasedCollectionGenerator
         else:
             revisions = self.commit_ids
 
         if start_pos or end_pos:
             revisions = revisions[start_pos:end_pos]
 
         return collection_generator(self, revisions, pre_load=pre_load)
 
     def pull(self, url, commit_ids=None):
         """
         Pull changes from external location.
 
         :param commit_ids: Optional. Can be set to a list of commit ids
            which shall be pulled from the other repository.
         """
         url = self._get_url(url)
         self._remote.pull(url, commit_ids=commit_ids)
         self._remote.invalidate_vcs_cache()
 
     def fetch(self, url, commit_ids=None):
         """
         Backward compatibility with GIT fetch==pull
         """
         return self.pull(url, commit_ids=commit_ids)
 
     def push(self, url):
         url = self._get_url(url)
         self._remote.sync_push(url)
 
     def _local_clone(self, clone_path):
         """
         Create a local clone of the current repo.
         """
         self._remote.clone(self.path, clone_path, update_after_clone=True,
                            hooks=False)
 
     def _update(self, revision, clean=False):
         """
         Update the working copy to the specified revision.
         """
         log.debug('Doing checkout to commit: `%s` for %s', revision, self)
         self._remote.update(revision, clean=clean)
 
     def _identify(self):
         """
         Return the current state of the working directory.
         """
         return self._remote.identify().strip().rstrip('+')
 
     def _heads(self, branch=None):
         """
         Return the commit ids of the repository heads.
         """
         return self._remote.heads(branch=branch).strip().split(' ')
 
     def _ancestor(self, revision1, revision2):
         """
         Return the common ancestor of the two revisions.
         """
         return self._remote.ancestor(revision1, revision2)
 
     def _local_push(
             self, revision, repository_path, push_branches=False,
             enable_hooks=False):
         """
         Push the given revision to the specified repository.
 
         :param push_branches: allow to create branches in the target repo.
         """
         self._remote.push(
             [revision], repository_path, hooks=enable_hooks,
             push_branches=push_branches)
 
     def _local_merge(self, target_ref, merge_message, user_name, user_email,
                      source_ref, use_rebase=False, dry_run=False):
         """
         Merge the given source_revision into the checked out revision.
 
         Returns the commit id of the merge and a boolean indicating if the
         commit needs to be pushed.
         """
         self._update(target_ref.commit_id, clean=True)
 
         ancestor = self._ancestor(target_ref.commit_id, source_ref.commit_id)
         is_the_same_branch = self._is_the_same_branch(target_ref, source_ref)
 
         if ancestor == source_ref.commit_id:
             # Nothing to do, the changes were already integrated
             return target_ref.commit_id, False
 
         elif ancestor == target_ref.commit_id and is_the_same_branch:
             # In this case we should force a commit message
             return source_ref.commit_id, True
 
         if use_rebase:
             try:
                 bookmark_name = 'rcbook%s%s' % (source_ref.commit_id,
                                                 target_ref.commit_id)
                 self.bookmark(bookmark_name, revision=source_ref.commit_id)
                 self._remote.rebase(
                     source=source_ref.commit_id, dest=target_ref.commit_id)
                 self._remote.invalidate_vcs_cache()
                 self._update(bookmark_name, clean=True)
                 return self._identify(), True
             except RepositoryError:
                 # The rebase-abort may raise another exception which 'hides'
                 # the original one, therefore we log it here.
                 log.exception('Error while rebasing shadow repo during merge.')
 
                 # Cleanup any rebase leftovers
                 self._remote.invalidate_vcs_cache()
                 self._remote.rebase(abort=True)
                 self._remote.invalidate_vcs_cache()
                 self._remote.update(clean=True)
                 raise
         else:
             try:
                 self._remote.merge(source_ref.commit_id)
                 self._remote.invalidate_vcs_cache()
                 self._remote.commit(
                     message=safe_str(merge_message),
                     username=safe_str('%s <%s>' % (user_name, user_email)))
                 self._remote.invalidate_vcs_cache()
                 return self._identify(), True
             except RepositoryError:
                 # Cleanup any merge leftovers
                 self._remote.update(clean=True)
                 raise
 
     def _local_close(self, target_ref, user_name, user_email,
                      source_ref, close_message=''):
         """
         Close the branch of the given source_revision
 
         Returns the commit id of the close and a boolean indicating if the
         commit needs to be pushed.
         """
         self._update(source_ref.commit_id)
         message = close_message or "Closing branch: `{}`".format(source_ref.name)
         try:
             self._remote.commit(
                 message=safe_str(message),
                 username=safe_str('%s <%s>' % (user_name, user_email)),
                 close_branch=True)
             self._remote.invalidate_vcs_cache()
             return self._identify(), True
         except RepositoryError:
             # Cleanup any commit leftovers
             self._remote.update(clean=True)
             raise
 
     def _is_the_same_branch(self, target_ref, source_ref):
         return (
             self._get_branch_name(target_ref) ==
             self._get_branch_name(source_ref))
 
     def _get_branch_name(self, ref):
         if ref.type == 'branch':
             return ref.name
         return self._remote.ctx_branch(ref.commit_id)
 
     def _maybe_prepare_merge_workspace(
             self, repo_id, workspace_id, unused_target_ref, unused_source_ref):
         shadow_repository_path = self._get_shadow_repository_path(
             repo_id, workspace_id)
         if not os.path.exists(shadow_repository_path):
             self._local_clone(shadow_repository_path)
             log.debug(
                 'Prepared shadow repository in %s', shadow_repository_path)
 
         return shadow_repository_path
 
     def _merge_repo(self, repo_id, workspace_id, target_ref,
                     source_repo, source_ref, merge_message,
                     merger_name, merger_email, dry_run=False,
                     use_rebase=False, close_branch=False):
 
         log.debug('Executing merge_repo with %s strategy, dry_run mode:%s',
                   'rebase' if use_rebase else 'merge', dry_run)
         if target_ref.commit_id not in self._heads():
             return MergeResponse(
                 False, False, None, MergeFailureReason.TARGET_IS_NOT_HEAD,
                 metadata={'target_ref': target_ref})
 
         try:
             if target_ref.type == 'branch' and len(self._heads(target_ref.name)) != 1:
                 heads = '\n,'.join(self._heads(target_ref.name))
                 metadata = {
                     'target_ref': target_ref,
                     'source_ref': source_ref,
                     'heads': heads
                 }
                 return MergeResponse(
                     False, False, None,
                     MergeFailureReason.HG_TARGET_HAS_MULTIPLE_HEADS,
                     metadata=metadata)
         except CommitDoesNotExistError:
             log.exception('Failure when looking up branch heads on hg target')
             return MergeResponse(
                 False, False, None, MergeFailureReason.MISSING_TARGET_REF,
                 metadata={'target_ref': target_ref})
 
         shadow_repository_path = self._maybe_prepare_merge_workspace(
             repo_id, workspace_id, target_ref, source_ref)
         shadow_repo = self._get_shadow_instance(shadow_repository_path)
 
         log.debug('Pulling in target reference %s', target_ref)
         self._validate_pull_reference(target_ref)
         shadow_repo._local_pull(self.path, target_ref)
 
         try:
             log.debug('Pulling in source reference %s', source_ref)
             source_repo._validate_pull_reference(source_ref)
             shadow_repo._local_pull(source_repo.path, source_ref)
         except CommitDoesNotExistError:
             log.exception('Failure when doing local pull on hg shadow repo')
             return MergeResponse(
                 False, False, None, MergeFailureReason.MISSING_SOURCE_REF,
                 metadata={'source_ref': source_ref})
 
         merge_ref = None
         merge_commit_id = None
         close_commit_id = None
         merge_failure_reason = MergeFailureReason.NONE
         metadata = {}
 
         # enforce that close branch should be used only in case we source from
         # an actual Branch
         close_branch = close_branch and source_ref.type == 'branch'
 
         # don't allow to close branch if source and target are the same
         close_branch = close_branch and source_ref.name != target_ref.name
 
         needs_push_on_close = False
         if close_branch and not use_rebase and not dry_run:
             try:
                 close_commit_id, needs_push_on_close = shadow_repo._local_close(
                     target_ref, merger_name, merger_email, source_ref)
                 merge_possible = True
             except RepositoryError:
                 log.exception('Failure when doing close branch on '
                               'shadow repo: %s', shadow_repo)
                 merge_possible = False
                 merge_failure_reason = MergeFailureReason.MERGE_FAILED
         else:
             merge_possible = True
 
         needs_push = False
         if merge_possible:
             try:
                 merge_commit_id, needs_push = shadow_repo._local_merge(
                     target_ref, merge_message, merger_name, merger_email,
                     source_ref, use_rebase=use_rebase, dry_run=dry_run)
                 merge_possible = True
 
                 # read the state of the close action, if it
                 # maybe required a push
                 needs_push = needs_push or needs_push_on_close
 
                 # Set a bookmark pointing to the merge commit. This bookmark
                 # may be used to easily identify the last successful merge
                 # commit in the shadow repository.
                 shadow_repo.bookmark('pr-merge', revision=merge_commit_id)
                 merge_ref = Reference('book', 'pr-merge', merge_commit_id)
             except SubrepoMergeError:
                 log.exception(
                     'Subrepo merge error during local merge on hg shadow repo.')
                 merge_possible = False
                 merge_failure_reason = MergeFailureReason.SUBREPO_MERGE_FAILED
                 needs_push = False
             except RepositoryError:
                 log.exception('Failure when doing local merge on hg shadow repo')
                 merge_possible = False
                 merge_failure_reason = MergeFailureReason.MERGE_FAILED
                 needs_push = False
 
         if merge_possible and not dry_run:
             if needs_push:
                 # In case the target is a bookmark, update it, so after pushing
                 # the bookmarks is also updated in the target.
                 if target_ref.type == 'book':
                     shadow_repo.bookmark(
                         target_ref.name, revision=merge_commit_id)
                 try:
                     shadow_repo_with_hooks = self._get_shadow_instance(
                         shadow_repository_path,
                         enable_hooks=True)
                     # This is the actual merge action, we push from shadow
                     # into origin.
                     # Note: the push_branches option will push any new branch
                     # defined in the source repository to the target. This may
                     # be dangerous as branches are permanent in Mercurial.
                     # This feature was requested in issue #441.
                     shadow_repo_with_hooks._local_push(
                         merge_commit_id, self.path, push_branches=True,
                         enable_hooks=True)
 
                     # maybe we also need to push the close_commit_id
                     if close_commit_id:
                         shadow_repo_with_hooks._local_push(
                             close_commit_id, self.path, push_branches=True,
                             enable_hooks=True)
                     merge_succeeded = True
                 except RepositoryError:
                     log.exception(
                         'Failure when doing local push from the shadow '
                         'repository to the target repository at %s.', self.path)
                     merge_succeeded = False
                     merge_failure_reason = MergeFailureReason.PUSH_FAILED
                     metadata['target'] = 'hg shadow repo'
                     metadata['merge_commit'] = merge_commit_id
             else:
                 merge_succeeded = True
         else:
             merge_succeeded = False
 
         return MergeResponse(
             merge_possible, merge_succeeded, merge_ref, merge_failure_reason,
             metadata=metadata)
 
     def _get_shadow_instance(self, shadow_repository_path, enable_hooks=False):
         config = self.config.copy()
         if not enable_hooks:
             config.clear_section('hooks')
         return MercurialRepository(shadow_repository_path, config)
 
     def _validate_pull_reference(self, reference):
         if not (reference.name in self.bookmarks or
                 reference.name in self.branches or
                 self.get_commit(reference.commit_id)):
             raise CommitDoesNotExistError(
                 'Unknown branch, bookmark or commit id')
 
     def _local_pull(self, repository_path, reference):
         """
         Fetch a branch, bookmark or commit from a local repository.
         """
         repository_path = os.path.abspath(repository_path)
         if repository_path == self.path:
             raise ValueError('Cannot pull from the same repository')
 
         reference_type_to_option_name = {
             'book': 'bookmark',
             'branch': 'branch',
         }
         option_name = reference_type_to_option_name.get(
             reference.type, 'revision')
 
         if option_name == 'revision':
             ref = reference.commit_id
         else:
             ref = reference.name
 
         options = {option_name: [ref]}
         self._remote.pull_cmd(repository_path, hooks=False, **options)
         self._remote.invalidate_vcs_cache()
 
     def bookmark(self, bookmark, revision=None):
         if isinstance(bookmark, unicode):
             bookmark = safe_str(bookmark)
         self._remote.bookmark(bookmark, revision=revision)
         self._remote.invalidate_vcs_cache()
 
     def get_path_permissions(self, username):
         hgacl_file = os.path.join(self.path, '.hg/hgacl')
 
         def read_patterns(suffix):
             svalue = None
             for section, option in [
                 ('narrowacl', username + suffix),
                 ('narrowacl', 'default' + suffix),
                 ('narrowhgacl', username + suffix),
                 ('narrowhgacl', 'default' + suffix)
             ]:
                 try:
                     svalue = hgacl.get(section, option)
                     break  # stop at the first value we find
                 except configparser.NoOptionError:
                     pass
             if not svalue:
                 return None
             result = ['/']
             for pattern in svalue.split():
                 result.append(pattern)
                 if '*' not in pattern and '?' not in pattern:
                     result.append(pattern + '/*')
             return result
 
         if os.path.exists(hgacl_file):
             try:
                 hgacl = configparser.RawConfigParser()
                 hgacl.read(hgacl_file)
 
                 includes = read_patterns('.includes')
                 excludes = read_patterns('.excludes')
                 return BasePathPermissionChecker.create_from_patterns(
                     includes, excludes)
             except BaseException as e:
                 msg = 'Cannot read ACL settings from {} on {}: {}'.format(
                     hgacl_file, self.name, e)
                 raise exceptions.RepositoryRequirementError(msg)
         else:
             return None
 
 
 class MercurialIndexBasedCollectionGenerator(CollectionGenerator):
 
     def _commit_factory(self, commit_id):
         return self.repo.get_commit(
             commit_idx=commit_id, pre_load=self.pre_load)