##// END OF EJS Templates
caches: use repo.lru based Dict cache. This LRUDict uses Timing Algo to not have to use locking...
marcink -
r2945:ec5716e4 default
parent child
Show More
@@ -791,17 +791,6 self: super: {
791 license = [ pkgs.lib.licenses.bsdOriginal ];
791 license = [ pkgs.lib.licenses.bsdOriginal ];
792 };
792 };
793 };
793 };
794 "lru-dict" = super.buildPythonPackage {
795 name = "lru-dict-1.1.6";
796 doCheck = false;
797 src = fetchurl {
798 url = "https://files.pythonhosted.org/packages/00/a5/32ed6e10246cd341ca8cc205acea5d208e4053f48a4dced2b1b31d45ba3f/lru-dict-1.1.6.tar.gz";
799 sha256 = "1k2lhd4dpl6xa6iialbwx4l6bkdzxmzhygms39pvf19x1rk5fm1n";
800 };
801 meta = {
802 license = [ pkgs.lib.licenses.mit ];
803 };
804 };
805 "lxml" = super.buildPythonPackage {
794 "lxml" = super.buildPythonPackage {
806 name = "lxml-3.7.3";
795 name = "lxml-3.7.3";
807 doCheck = false;
796 doCheck = false;
@@ -1690,7 +1679,6 self: super: {
1690 self."billiard"
1679 self."billiard"
1691 self."kombu"
1680 self."kombu"
1692 self."lxml"
1681 self."lxml"
1693 self."lru-dict"
1694 self."mako"
1682 self."mako"
1695 self."markdown"
1683 self."markdown"
1696 self."markupsafe"
1684 self."markupsafe"
@@ -31,7 +31,6 jinja2==2.9.6
31 billiard==3.5.0.3
31 billiard==3.5.0.3
32 kombu==4.2.0
32 kombu==4.2.0
33 lxml==3.7.3
33 lxml==3.7.3
34 lru-dict==1.1.6
35 mako==1.0.7
34 mako==1.0.7
36 markdown==2.6.11
35 markdown==2.6.11
37 markupsafe==1.0.0
36 markupsafe==1.0.0
@@ -25,15 +25,8 and the state of LRU dict.
25 inrae.cache is licensed under LRUDict is licensed under ZPL license
25 inrae.cache is licensed under LRUDict is licensed under ZPL license
26 This software is Copyright (c) Zope Corporation (tm) and
26 This software is Copyright (c) Zope Corporation (tm) and
27 Contributors. All rights reserved.
27 Contributors. All rights reserved.
28
28 """
29 TODO: marcink, we might think of replacing the LRUDict with lru-dict library
30 written in C.
31
29
32 eg difference in speed:
33
34 LRUDictC Time : 0.00025 s, Memory : 110592 Kb
35 LRUDict Time : 0.00369 s, Memory : 147456 Kb
36 """
37 import logging
30 import logging
38
31
39 from repoze.lru import LRUCache
32 from repoze.lru import LRUCache
@@ -44,8 +37,10 log = logging.getLogger(__name__)
44
37
45
38
46 class LRUDict(LRUCache):
39 class LRUDict(LRUCache):
47 """ Wrapper to provide partial dict access
48 """
40 """
41 Wrapper to provide partial dict access
42 """
43
49 def __setitem__(self, key, value):
44 def __setitem__(self, key, value):
50 return self.put(key, value)
45 return self.put(key, value)
51
46
@@ -28,7 +28,8 from dogpile.cache.backends import file
28 from dogpile.cache.backends import redis as redis_backend
28 from dogpile.cache.backends import redis as redis_backend
29 from dogpile.cache.backends.file import NO_VALUE, compat, FileLock
29 from dogpile.cache.backends.file import NO_VALUE, compat, FileLock
30 from dogpile.cache.util import memoized_property
30 from dogpile.cache.util import memoized_property
31 from lru import LRU as LRUDict
31
32 from rhodecode.lib.memory_lru_dict import LRUDict, LRUDictDebug
32
33
33
34
34 _default_max_size = 1024
35 _default_max_size = 1024
@@ -41,14 +42,12 class LRUMemoryBackend(memory_backend.Me
41
42
42 def __init__(self, arguments):
43 def __init__(self, arguments):
43 max_size = arguments.pop('max_size', _default_max_size)
44 max_size = arguments.pop('max_size', _default_max_size)
44 callback = None
45 if arguments.pop('log_max_size_reached', None):
46 def evicted(key, value):
47 log.debug(
48 'LRU: evicting key `%s` due to max size %s reach', key, max_size)
49 callback = evicted
50
45
51 arguments['cache_dict'] = LRUDict(max_size, callback=callback)
46 LRUDictClass = LRUDict
47 if arguments.pop('log_key_count', None):
48 LRUDictClass = LRUDictDebug
49
50 arguments['cache_dict'] = LRUDictClass(max_size)
52 super(LRUMemoryBackend, self).__init__(arguments)
51 super(LRUMemoryBackend, self).__init__(arguments)
53
52
54 def delete(self, key):
53 def delete(self, key):
@@ -111,9 +110,9 class CustomLockFactory(FileLock):
111 # waited to much time on a lock, better fail than loop for ever
110 # waited to much time on a lock, better fail than loop for ever
112 log.error('Failed to acquire lock on %s file', self.filename)
111 log.error('Failed to acquire lock on %s file', self.filename)
113 raise
112 raise
114
113 timeout = 0.03
115 log.debug('Failed to acquire lock, retry in 0.03')
114 log.debug('Failed to acquire lock, retry in %ss', timeout)
116 gevent.sleep(0.03)
115 gevent.sleep(timeout)
117
116
118 fcntl.flock = gevent_flock
117 fcntl.flock = gevent_flock
119 return fcntl
118 return fcntl
@@ -182,8 +182,8 setup(
182 'rc-ssh-wrapper=rhodecode.apps.ssh_support.lib.ssh_wrapper:main',
182 'rc-ssh-wrapper=rhodecode.apps.ssh_support.lib.ssh_wrapper:main',
183 ],
183 ],
184 'beaker.backends': [
184 'beaker.backends': [
185 'memorylru_base=rhodecode.lib.memory_lru_debug:MemoryLRUNamespaceManagerBase',
185 'memorylru_base=rhodecode.lib.memory_lru_dict:MemoryLRUNamespaceManagerBase',
186 'memorylru_debug=rhodecode.lib.memory_lru_debug:MemoryLRUNamespaceManagerDebug'
186 'memorylru_debug=rhodecode.lib.memory_lru_dict:MemoryLRUNamespaceManagerDebug'
187 ]
187 ]
188 },
188 },
189 )
189 )
General Comments 0
You need to be logged in to leave comments. Login now