commits: updated logic of in-memory-commits, fixed tests and re-architectured a bit how commit_ids are calculated and updated.

- it's all part of recent updates on performance, and new commit pages

rhodecode/lib/vcs/backends/base.py

@@ -33,6 +33,8 b' import collections'
 import warnings
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
+from zope.cachedescriptors.property import CachedProperty
+
 from pyramid import compat
 
 from rhodecode.translation import lazy_ugettext
@@ -261,6 +263,7 b' class BaseRepository(object):'
     EMPTY_COMMIT_ID = '0' * 40
 
     path = None
+    _commit_ids_ver = 0
 
     def __init__(self, repo_path, config=None, create=False, **kwargs):
         """
@@ -404,6 +407,15 b' class BaseRepository(object):'
     # COMMITS
     # ==========================================================================
 
+    @CachedProperty('_commit_ids_ver')
+    def commit_ids(self):
+        raise NotImplementedError
+
+    def append_commit_id(self, commit_id):
+        if commit_id not in self.commit_ids:
+            self._rebuild_cache(self.commit_ids + [commit_id])
+            self._commit_ids_ver = time.time()
+
     def get_commit(self, commit_id=None, commit_idx=None, pre_load=None, translate_tag=None):
         """
         Returns instance of `BaseCommit` class. If `commit_id` and `commit_idx`
@@ -1509,9 +1521,7 b' class BaseInMemoryCommit(object):'
                 "Cannot remove node at %s from "
                 "following parents: %s" % (not_removed, parents))
 
-    def commit(
-            self, message, author, parents=None, branch=None, date=None,
-            **kwargs):
+    def commit(self, message, author, parents=None, branch=None, date=None, **kwargs):
         """
         Performs in-memory commit (doesn't check workdir in any way) and
         returns newly created :class:`BaseCommit`. Updates repository's

rhodecode/lib/vcs/backends/git/inmemory.py

@@ -29,8 +29,7 b' from rhodecode.lib.vcs.backends import b'
 
 class GitInMemoryCommit(base.BaseInMemoryCommit):
 
-    def commit(self, message, author, parents=None, branch=None, date=None,
-               **kwargs):
+    def commit(self, message, author, parents=None, branch=None, date=None, **kwargs):
         """
         Performs in-memory commit (doesn't check workdir in any way) and
         returns newly created `GitCommit`. Updates repository's
@@ -94,13 +93,12 b' class GitInMemoryCommit(base.BaseInMemor'
             commit_data, branch, commit_tree, updated, removed)
 
         # Update vcs repository object
-        if commit_id not in self.repository.commit_ids:
-            self.repository.commit_ids.append(commit_id)
-            self.repository._rebuild_cache(self.repository.commit_ids)
+        self.repository.append_commit_id(commit_id)
 
         # invalidate parsed refs after commit
         self.repository._refs = self.repository._get_refs()
         self.repository.branches = self.repository._get_branches()
-        tip = self.repository.get_commit()
+        tip = self.repository.get_commit(commit_id)
+
         self.reset()
         return tip

rhodecode/lib/vcs/backends/git/repository.py

@@ -25,8 +25,10 b' GIT repository module'
 import logging
 import os
 import re
+import time
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
+from zope.cachedescriptors.property import CachedProperty
 
 from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib.datelib import (
@@ -69,6 +71,9 b' class GitRepository(BaseRepository):'
         # caches
         self._commit_ids = {}
 
+        # dependent that trigger re-computation of  commit_ids
+        self._commit_ids_ver = 0
+
     @LazyProperty
     def _remote(self):
         return connection.Git(self.path, self.config, with_wire=self.with_wire)
@@ -81,7 +86,7 b' class GitRepository(BaseRepository):'
     def head(self):
         return self._remote.head()
 
-    @LazyProperty
+    @CachedProperty('_commit_ids_ver')
     def commit_ids(self):
         """
         Returns list of commit ids, in ascending order.  Being lazy
@@ -608,8 +613,9 b' class GitRepository(BaseRepository):'
         commit = commit.parents[0]
         self._remote.set_refs('refs/heads/%s' % branch_name, commit.raw_id)
 
-        self.commit_ids = self._get_all_commit_ids()
-        self._rebuild_cache(self.commit_ids)
+        self._commit_ids_ver = time.time()
+        # we updated _commit_ids_ver so accessing self.commit_ids will re-compute it
+        return len(self.commit_ids)
 
     def get_common_ancestor(self, commit_id1, commit_id2, repo2):
         if commit_id1 == commit_id2:

rhodecode/lib/vcs/backends/hg/inmemory.py

@@ -30,8 +30,7 b' from rhodecode.lib.vcs.exceptions import'
 
 class MercurialInMemoryCommit(BaseInMemoryCommit):
 
-    def commit(self, message, author, parents=None, branch=None, date=None,
-               **kwargs):
+    def commit(self, message, author, parents=None, branch=None, date=None, **kwargs):
         """
         Performs in-memory commit (doesn't check workdir in any way) and
         returns newly created `MercurialCommit`. Updates repository's
@@ -88,11 +87,9 b' class MercurialInMemoryCommit(BaseInMemo'
             commit_time=date, commit_timezone=tz, user=author,
             files=self.get_paths(), extra=kwargs, removed=removed,
             updated=updated)
-        if commit_id not in self.repository.commit_ids:
-            self.repository.commit_ids.append(commit_id)
-            self.repository._rebuild_cache(self.repository.commit_ids)
+        self.repository.append_commit_id(commit_id)
 
         self.repository.branches = self.repository._get_branches()
-        tip = self.repository.get_commit()
+        tip = self.repository.get_commit(commit_id)
         self.reset()
         return tip

rhodecode/lib/vcs/backends/hg/repository.py

@@ -24,9 +24,11 b' HG repository module'
 import os
 import logging
 import binascii
+import time
 import urllib
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
+from zope.cachedescriptors.property import CachedProperty
 
 from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib.datelib import (
@@ -85,11 +87,14 b' class MercurialRepository(BaseRepository'
         # caches
         self._commit_ids = {}
 
+        # dependent that trigger re-computation of  commit_ids
+        self._commit_ids_ver = 0
+
     @LazyProperty
     def _remote(self):
         return connection.Hg(self.path, self.config, with_wire=self.with_wire)
 
-    @LazyProperty
+    @CachedProperty('_commit_ids_ver')
     def commit_ids(self):
         """
         Returns list of commit ids, in ascending order.  Being lazy
@@ -157,8 +162,7 b' class MercurialRepository(BaseRepository'
 
         return OrderedDict(sorted(_tags, key=get_name, reverse=True))
 
-    def tag(self, name, user, commit_id=None, message=None, date=None,
-            **kwargs):
+    def tag(self, name, user, commit_id=None, message=None, date=None, **kwargs):
         """
         Creates and returns a tag for the given ``commit_id``.
 
@@ -172,6 +176,7 b' class MercurialRepository(BaseRepository'
         """
         if name in self.tags:
             raise TagAlreadyExistError("Tag %s already exists" % name)
+
         commit = self.get_commit(commit_id=commit_id)
         local = kwargs.setdefault('local', False)
 
@@ -180,8 +185,7 b' class MercurialRepository(BaseRepository'
 
         date, tz = date_to_timestamp_plus_offset(date)
 
-        self._remote.tag(
-            name, commit.raw_id, message, local, user, date, tz)
+        self._remote.tag(name, commit.raw_id, message, local, user, date, tz)
         self._remote.invalidate_vcs_cache()
 
         # Reinitialize tags
@@ -203,6 +207,7 b' class MercurialRepository(BaseRepository'
         """
         if name not in self.tags:
             raise TagDoesNotExistError("Tag %s does not exist" % name)
+
         if message is None:
             message = "Removed tag %s" % name
         local = False
@@ -271,8 +276,9 b' class MercurialRepository(BaseRepository'
         self._remote.strip(commit_id, update=False, backup="none")
 
         self._remote.invalidate_vcs_cache()
-        self.commit_ids = self._get_all_commit_ids()
-        self._rebuild_cache(self.commit_ids)
+        self._commit_ids_ver = time.time()
+        # we updated _commit_ids_ver so accessing self.commit_ids will re-compute it
+        return len(self.commit_ids)
 
     def verify(self):
         verify = self._remote.verify()
@@ -450,7 +456,8 b' class MercurialRepository(BaseRepository'
         try:
             raw_id, idx = self._remote.lookup(commit_id, both=True)
         except CommitDoesNotExistError:
-            msg = "Commit %s does not exist for %s" % (commit_id, self.name)
+            msg = "Commit {} does not exist for {}".format(
+                *map(safe_str, [commit_id, self.name]))
             raise CommitDoesNotExistError(msg)
 
         return MercurialCommit(self, raw_id, idx, pre_load=pre_load)

rhodecode/lib/vcs/backends/svn/inmemory.py

@@ -30,8 +30,7 b' from rhodecode.lib.vcs.backends import b'
 
 class SubversionInMemoryCommit(base.BaseInMemoryCommit):
 
-    def commit(self, message, author, parents=None, branch=None, date=None,
-               **kwargs):
+    def commit(self, message, author, parents=None, branch=None, date=None, **kwargs):
         if branch not in (None, self.repository.DEFAULT_BRANCH_NAME):
             raise NotImplementedError("Branches are not yet supported")
 
@@ -74,8 +73,7 b' class SubversionInMemoryCommit(base.Base'
         # we should not add the commit_id, if it is already evaluated, it
         # will not be evaluated again.
         commit_id = str(svn_rev)
-        if commit_id not in self.repository.commit_ids:
-            self.repository.commit_ids.append(commit_id)
+        self.repository.append_commit_id(commit_id)
         tip = self.repository.get_commit()
         self.reset()
         return tip

rhodecode/lib/vcs/backends/svn/repository.py

@@ -27,6 +27,7 b' import os'
 import urllib
 
 from zope.cachedescriptors.property import Lazy as LazyProperty
+from zope.cachedescriptors.property import CachedProperty
 
 from rhodecode.lib.compat import OrderedDict
 from rhodecode.lib.datelib import date_astimestamp
@@ -75,6 +76,9 b' class SubversionRepository(base.BaseRepo'
 
         self._init_repo(create, src_url)
 
+        # dependent that trigger re-computation of  commit_ids
+        self._commit_ids_ver = 0
+
     @LazyProperty
     def _remote(self):
         return connection.Svn(self.path, self.config)
@@ -93,11 +97,14 b' class SubversionRepository(base.BaseRepo'
         else:
             self._check_path()
 
-    @LazyProperty
+    @CachedProperty('_commit_ids_ver')
     def commit_ids(self):
         head = self._remote.lookup(None)
         return [str(r) for r in xrange(1, head + 1)]
 
+    def _rebuild_cache(self, commit_ids):
+        pass
+
     def run_svn_command(self, cmd, **opts):
         """
         Runs given ``cmd`` as svn command and returns tuple

rhodecode/tests/vcs/test_archives.py

@@ -97,7 +97,7 b' class TestArchives(BackendTestMixin):'
         metafile = out.read('.archival.txt')
 
         raw_id = self.tip.raw_id
-        assert 'rev:%s' % raw_id in metafile
+        assert 'commit_id:%s' % raw_id in metafile
 
         for x in range(5):
             node_path = '%d/file_%d.txt' % (x, x)

rhodecode/tests/vcs/test_hg_vcsserver_cache_invalidation.py

@@ -70,10 +70,13 b' class TestMercurialRemoteRepoInvalidatio'
             tags[name] = raw_id
 
         repo = backend_hg.repo.scm_instance()
+
         with patch.object(repo, '_remote') as remote:
+            repo.tags = tags
             remote.lookup.return_value = ('commit-id', 'commit-idx')
             remote.tags.return_value = tags
             remote._get_tags.return_value = tags
+            remote.is_empty.return_value = False
             remote.tag.side_effect = add_tag
 
             # Invoke method.

rhodecode/tests/vcs/test_inmemory.py

@@ -79,9 +79,17 b' class TestInMemoryCommit(BackendTestMixi'
         self.commit()
         self.assert_succesful_commit(nodes)
 
-    @pytest.mark.skip_backends(
-        'svn', reason="Svn does not support commits on branches.")
-    def test_add_on_branch(self, nodes):
+    @pytest.mark.backends("hg")
+    def test_add_on_branch_hg(self, nodes):
+        for node in nodes:
+            self.imc.add(node)
+        self.commit(branch=u'stable')
+        self.assert_succesful_commit(nodes)
+
+    @pytest.mark.backends("git")
+    def test_add_on_branch_git(self, nodes):
+        self.repo._checkout('stable', create=True)
+
         for node in nodes:
             self.imc.add(node)
         self.commit(branch=u'stable')