# HG changeset patch # User Marcin Kuzminski # Date 2018-08-06 15:10:40 # Node ID 23957c07b227516acfa03cb01fa9f8f29edaf763 # Parent c93a0ad3e7dfd22469135ed0d4522cefef3dfa47 caches: use repo.lru based Dict cache. This LRUDict uses Timing Algo to not have to use locking for consistency. diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -287,17 +287,6 @@ self: super: { license = [ pkgs.lib.licenses.bsdOriginal ]; }; }; - "lru-dict" = super.buildPythonPackage { - name = "lru-dict-1.1.6"; - doCheck = false; - src = fetchurl { - url = "https://files.pythonhosted.org/packages/00/a5/32ed6e10246cd341ca8cc205acea5d208e4053f48a4dced2b1b31d45ba3f/lru-dict-1.1.6.tar.gz"; - sha256 = "1k2lhd4dpl6xa6iialbwx4l6bkdzxmzhygms39pvf19x1rk5fm1n"; - }; - meta = { - license = [ pkgs.lib.licenses.mit ]; - }; - }; "mako" = super.buildPythonPackage { name = "mako-1.0.7"; doCheck = false; @@ -691,7 +680,6 @@ self: super: { self."dulwich" self."hgsubversion" self."hg-evolve" - self."lru-dict" self."mako" self."markupsafe" self."mercurial" diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ decorator==4.1.2 dulwich==0.13.0 hgsubversion==1.9.2 hg-evolve==8.0.1 -lru-dict==1.1.6 mako==1.0.7 markupsafe==1.0.0 mercurial==4.6.2 diff --git a/vcsserver/lib/memory_lru_dict.py b/vcsserver/lib/memory_lru_dict.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/memory_lru_dict.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + +import logging + +from repoze.lru import LRUCache + +from vcsserver.utils import safe_str + +log = logging.getLogger(__name__) + + +class LRUDict(LRUCache): + """ + Wrapper to provide partial dict access + """ + + def __setitem__(self, key, value): + return self.put(key, value) + + def __getitem__(self, key): + return self.get(key) + + def __contains__(self, key): + return bool(self.get(key)) + + def __delitem__(self, key): + del self.data[key] + + def keys(self): + return self.data.keys() + + +class LRUDictDebug(LRUDict): + """ + Wrapper to provide some debug options + """ + def _report_keys(self): + elems_cnt = '%s/%s' % (len(self.keys()), self.size) + # trick for pformat print it more nicely + fmt = '\n' + for cnt, elem in enumerate(self.keys()): + fmt += '%s - %s\n' % (cnt+1, safe_str(elem)) + log.debug('current LRU keys (%s):%s' % (elems_cnt, fmt)) + + def __getitem__(self, key): + self._report_keys() + return self.get(key) diff --git a/vcsserver/lib/rc_cache/backends.py b/vcsserver/lib/rc_cache/backends.py --- a/vcsserver/lib/rc_cache/backends.py +++ b/vcsserver/lib/rc_cache/backends.py @@ -18,7 +18,7 @@ import logging from dogpile.cache.backends import memory as memory_backend -from lru import LRU as LRUDict +from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug _default_max_size = 1024 @@ -31,21 +31,21 @@ class LRUMemoryBackend(memory_backend.Me def __init__(self, arguments): max_size = arguments.pop('max_size', _default_max_size) - callback = None - if arguments.pop('log_max_size_reached', None): - def evicted(key, value): - log.debug( - 'LRU: evicting key `%s` due to max size %s reach', key, max_size) - callback = evicted - arguments['cache_dict'] = LRUDict(max_size, callback=callback) + LRUDictClass = LRUDict + if arguments.pop('log_key_count', None): + LRUDictClass = LRUDictDebug + + arguments['cache_dict'] = LRUDictClass(max_size) super(LRUMemoryBackend, self).__init__(arguments) def delete(self, key): - if self._cache.has_key(key): + try: del self._cache[key] + except KeyError: + # we don't care if key isn't there at deletion + pass def delete_multi(self, keys): for key in keys: - if self._cache.has_key(key): - del self._cache[key] + self.delete(key)