##// END OF EJS Templates
caches: use LRUDict inside our lru helper so we can import it later on when required.
marcink -
r2844:ae068608 default
parent child Browse files
Show More
@@ -1,96 +1,115 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2018 RhodeCode GmbH
3 # Copyright (C) 2010-2018 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 """
21 """
22 Custom LRU memory manager for debugging purposes. It allows to track the keys
22 Custom LRU memory manager for debugging purposes. It allows to track the keys
23 and the state of LRU dict.
23 and the state of LRU dict.
24
24
25 inrae.cache is licensed under LRUDict is licensed under ZPL license
25 inrae.cache is licensed under LRUDict is licensed under ZPL license
26 This software is Copyright (c) Zope Corporation (tm) and
26 This software is Copyright (c) Zope Corporation (tm) and
27 Contributors. All rights reserved.
27 Contributors. All rights reserved.
28
28
29 TODO: marcink, we might think of replacing the LRUDict with lru-dict library
29 TODO: marcink, we might think of replacing the LRUDict with lru-dict library
30 written in C.
30 written in C.
31
31
32 eg difference in speed:
32 eg difference in speed:
33
33
34 LRUDictC Time : 0.00025 s, Memory : 110592 Kb
34 LRUDictC Time : 0.00025 s, Memory : 110592 Kb
35 LRUDict Time : 0.00369 s, Memory : 147456 Kb
35 LRUDict Time : 0.00369 s, Memory : 147456 Kb
36 """
36 """
37 import logging
37 import logging
38
38
39 from infrae.cache.beakerext.lru import LRUDict
39 from repoze.lru import LRUCache
40 from beaker.container import MemoryNamespaceManager, AbstractDictionaryNSManager
40 from beaker.container import MemoryNamespaceManager, AbstractDictionaryNSManager
41 from rhodecode.lib.utils2 import safe_str
41 from rhodecode.lib.utils2 import safe_str
42
42
43 log = logging.getLogger(__name__)
43 log = logging.getLogger(__name__)
44
44
45
45
46 class LRUDict(LRUCache):
47 """ Wrapper to provide partial dict access
48 """
49 def __setitem__(self, key, value):
50 return self.put(key, value)
51
52 def __getitem__(self, key):
53 return self.get(key)
54
55 def __contains__(self, key):
56 return bool(self.get(key))
57
58 def __delitem__(self, key):
59 del self.data[key]
60
61 def keys(self):
62 return self.data.keys()
63
64
46 class LRUDictDebug(LRUDict):
65 class LRUDictDebug(LRUDict):
47 """
66 """
48 Wrapper to provide some debug options
67 Wrapper to provide some debug options
49 """
68 """
50 def _report_keys(self):
69 def _report_keys(self):
51 elems_cnt = '%s/%s' % (len(self.keys()), self.size)
70 elems_cnt = '%s/%s' % (len(self.keys()), self.size)
52 # trick for pformat print it more nicely
71 # trick for pformat print it more nicely
53 fmt = '\n'
72 fmt = '\n'
54 for cnt, elem in enumerate(self.keys()):
73 for cnt, elem in enumerate(self.keys()):
55 fmt += '%s - %s\n' % (cnt+1, safe_str(elem))
74 fmt += '%s - %s\n' % (cnt+1, safe_str(elem))
56 log.debug('current LRU keys (%s):%s' % (elems_cnt, fmt))
75 log.debug('current LRU keys (%s):%s' % (elems_cnt, fmt))
57
76
58 def __getitem__(self, key):
77 def __getitem__(self, key):
59 self._report_keys()
78 self._report_keys()
60 return self.get(key)
79 return self.get(key)
61
80
62
81
63 class MemoryLRUNamespaceManagerBase(MemoryNamespaceManager):
82 class MemoryLRUNamespaceManagerBase(MemoryNamespaceManager):
64 default_max_items = 10000
83 default_max_items = 10000
65
84
66 def _get_factory(self, max_items):
85 def _get_factory(self, max_items):
67
86
68 def Factory():
87 def Factory():
69 return LRUDict(int(max_items))
88 return LRUDict(int(max_items))
70 return Factory
89 return Factory
71
90
72 def __init__(self, namespace, **kwargs):
91 def __init__(self, namespace, **kwargs):
73 AbstractDictionaryNSManager.__init__(self, namespace)
92 AbstractDictionaryNSManager.__init__(self, namespace)
74 if 'max_items' in kwargs:
93 if 'max_items' in kwargs:
75 max_items = kwargs['max_items']
94 max_items = kwargs['max_items']
76 else:
95 else:
77 max_items = self.default_max_items
96 max_items = self.default_max_items
78
97
79 Factory = self._get_factory(max_items)
98 Factory = self._get_factory(max_items)
80
99
81 self.dictionary = MemoryNamespaceManager.namespaces.get(
100 self.dictionary = MemoryNamespaceManager.namespaces.get(
82 self.namespace, Factory)
101 self.namespace, Factory)
83
102
84
103
85 class MemoryLRUNamespaceManagerDebug(MemoryLRUNamespaceManagerBase):
104 class MemoryLRUNamespaceManagerDebug(MemoryLRUNamespaceManagerBase):
86 """
105 """
87 A memory namespace manager that return with LRU dicts backend,
106 A memory namespace manager that return with LRU dicts backend,
88 special debug for testing
107 special debug for testing
89 """
108 """
90 default_max_items = 10000
109 default_max_items = 10000
91
110
92 def _get_factory(self, max_items):
111 def _get_factory(self, max_items):
93
112
94 def Factory():
113 def Factory():
95 return LRUDictDebug(int(max_items))
114 return LRUDictDebug(int(max_items))
96 return Factory
115 return Factory
General Comments 0
You need to be logged in to leave comments. Login now