caches: use repo.lru based Dict cache. This LRUDict uses Timing Algo to not have to use locking

for consistency.

vcsserver/lib/memory_lru_dict.py

@@ -0,0 +1,65 b''
+# -*- coding: utf-8 -*-
+
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2018 RhodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+
+import logging
+
+from repoze.lru import LRUCache
+
+from vcsserver.utils import safe_str
+
+log = logging.getLogger(__name__)
+
+
+class LRUDict(LRUCache):
+    """
+    Wrapper to provide partial dict access
+    """
+
+    def __setitem__(self, key, value):
+        return self.put(key, value)
+
+    def __getitem__(self, key):
+        return self.get(key)
+
+    def __contains__(self, key):
+        return bool(self.get(key))
+
+    def __delitem__(self, key):
+        del self.data[key]
+
+    def keys(self):
+        return self.data.keys()
+
+
+class LRUDictDebug(LRUDict):
+    """
+    Wrapper to provide some debug options
+    """
+    def _report_keys(self):
+        elems_cnt = '%s/%s' % (len(self.keys()), self.size)
+        # trick for pformat print it more nicely
+        fmt = '\n'
+        for cnt, elem in enumerate(self.keys()):
+            fmt += '%s - %s\n' % (cnt+1, safe_str(elem))
+        log.debug('current LRU keys (%s):%s' % (elems_cnt, fmt))
+
+    def __getitem__(self, key):
+        self._report_keys()
+        return self.get(key)

pkgs/python-packages.nix

@@ -287,17 +287,6 b' self: super: {'
       license = [ pkgs.lib.licenses.bsdOriginal ];
     };
   };
-  "lru-dict" = super.buildPythonPackage {
-    name = "lru-dict-1.1.6";
-    doCheck = false;
-    src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/00/a5/32ed6e10246cd341ca8cc205acea5d208e4053f48a4dced2b1b31d45ba3f/lru-dict-1.1.6.tar.gz";
-      sha256 = "1k2lhd4dpl6xa6iialbwx4l6bkdzxmzhygms39pvf19x1rk5fm1n";
-    };
-    meta = {
-      license = [ pkgs.lib.licenses.mit ];
-    };
-  };
   "mako" = super.buildPythonPackage {
     name = "mako-1.0.7";
     doCheck = false;
@@ -691,7 +680,6 b' self: super: {'
       self."dulwich"
       self."hgsubversion"
       self."hg-evolve"
-      self."lru-dict"
       self."mako"
       self."markupsafe"
       self."mercurial"

requirements.txt

@@ -8,7 +8,6 b' decorator==4.1.2'
 dulwich==0.13.0
 hgsubversion==1.9.2
 hg-evolve==8.0.1
-lru-dict==1.1.6
 mako==1.0.7
 markupsafe==1.0.0
 mercurial==4.6.2

vcsserver/lib/rc_cache/backends.py

@@ -18,7 +18,7 b''
 import logging
 
 from dogpile.cache.backends import memory as memory_backend
-from lru import LRU as LRUDict
+from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
 
 
 _default_max_size = 1024
@@ -31,21 +31,21 b' class LRUMemoryBackend(memory_backend.Me'
 
     def __init__(self, arguments):
         max_size = arguments.pop('max_size', _default_max_size)
-        callback = None
-        if arguments.pop('log_max_size_reached', None):
-            def evicted(key, value):
-                log.debug(
-                    'LRU: evicting key `%s` due to max size %s reach', key, max_size)
-            callback = evicted
 
-        arguments['cache_dict'] = LRUDict(max_size, callback=callback)
+        LRUDictClass = LRUDict
+        if arguments.pop('log_key_count', None):
+            LRUDictClass = LRUDictDebug
+
+        arguments['cache_dict'] = LRUDictClass(max_size)
         super(LRUMemoryBackend, self).__init__(arguments)
 
     def delete(self, key):
-        if self._cache.has_key(key):
+        try:
             del self._cache[key]
+        except KeyError:
+            # we don't care if key isn't there at deletion
+            pass
 
     def delete_multi(self, keys):
         for key in keys:
-            if self._cache.has_key(key):
-                del self._cache[key]
+            self.delete(key)