caches: use delete_multi for clearing out cache keys.

- This is a safer and more efficient way of purging many keys.
- For file based cache it only opens the DB once

rhodecode/lib/rc_cache/utils.py

@@ -1,107 +1,106 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2015-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import os
 import logging
 from dogpile.cache import make_region
 
 from rhodecode.lib.utils import safe_str, sha1
 from . import region_meta
 
 log = logging.getLogger(__name__)
 
 
 def get_default_cache_settings(settings, prefixes=None):
     prefixes = prefixes or []
     cache_settings = {}
     for key in settings.keys():
         for prefix in prefixes:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 val = settings[key]
                 if isinstance(val, basestring):
                     val = val.strip()
                 cache_settings[name] = val
     return cache_settings
 
 
 def compute_key_from_params(*args):
     """
     Helper to compute key from given params to be used in cache manager
     """
     return sha1("_".join(map(safe_str, args)))
 
 
 def key_generator(namespace, fn):
     fname = fn.__name__
 
     def generate_key(*args):
         namespace_pref = namespace or 'default'
         arg_key = compute_key_from_params(*args)
         final_key = "{}:{}_{}".format(namespace_pref, fname, arg_key)
 
         return final_key
 
     return generate_key
 
 
 def get_or_create_region(region_name, region_namespace=None):
     from rhodecode.lib.rc_cache.backends import FileNamespaceBackend
     region_obj = region_meta.dogpile_cache_regions.get(region_name)
     if not region_obj:
         raise EnvironmentError(
             'Region `{}` not in configured: {}.'.format(
                 region_name, region_meta.dogpile_cache_regions.keys()))
 
     region_uid_name = '{}:{}'.format(region_name, region_namespace)
     if isinstance(region_obj.actual_backend, FileNamespaceBackend):
         region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
         if region_exist:
             log.debug('Using already configured region: %s', region_namespace)
             return region_exist
         cache_dir = region_meta.dogpile_config_defaults['cache_dir']
         expiration_time = region_obj.expiration_time
 
         if not os.path.isdir(cache_dir):
             os.makedirs(cache_dir)
         new_region = make_region(
             name=region_uid_name, function_key_generator=key_generator
         )
         namespace_filename = os.path.join(
             cache_dir, "{}.cache.dbm".format(region_namespace))
         # special type that allows 1db per namespace
         new_region.configure(
             backend='dogpile.cache.rc.file_namespace',
             expiration_time=expiration_time,
             arguments={"filename": namespace_filename}
         )
 
         # create and save in region caches
         log.debug('configuring new region: %s',region_uid_name)
         region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
 
     return region_obj
 
 
 def clear_cache_namespace(cache_region, cache_namespace_uid):
     region = get_or_create_region(cache_region, cache_namespace_uid)
     cache_keys = region.backend.list_keys(prefix=cache_namespace_uid)
-    for k in cache_keys:
-        region.delete(k)
+    region.delete_multi(cache_keys)
     return len(cache_keys)

rhodecode/tests/lib/test_caches.py

@@ -1,109 +1,108 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2018 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import time
 
 import pytest
 
 from rhodecode.lib import rc_cache
 
 
 @pytest.mark.usefixtures( 'app')
 class TestCaches(object):
 
     def test_cache_decorator_init_not_configured(self):
         with pytest.raises(EnvironmentError):
             rc_cache.get_or_create_region('dontexist')
 
     @pytest.mark.parametrize('region_name', [
         'cache_perms', u'cache_perms',
     ])
     def test_cache_decorator_init(self, region_name):
         namespace = region_name
         cache_region = rc_cache.get_or_create_region(
             region_name, region_namespace=namespace)
         assert cache_region
 
     @pytest.mark.parametrize('example_input', [
         ('',),
         (u'/ac',),
         (u'/ac', 1, 2, object()),
         (u'/ęćc', 1, 2, object()),
         ('/ąac',),
         (u'/ac', ),
     ])
     def test_cache_manager_create_key(self, example_input):
         key = rc_cache.utils.compute_key_from_params(*example_input)
         assert key
 
     @pytest.mark.parametrize('example_namespace', [
         'namespace', None
     ])
     @pytest.mark.parametrize('example_input', [
         ('',),
         (u'/ac',),
         (u'/ac', 1, 2, object()),
         (u'/ęćc', 1, 2, object()),
         ('/ąac',),
         (u'/ac', ),
     ])
     def test_cache_keygen(self, example_input, example_namespace):
         def func_wrapped():
             return 1
         func = rc_cache.utils.key_generator(example_namespace, func_wrapped)
         key = func(*example_input)
         assert key
 
     def test_store_value_in_cache(self):
         cache_region = rc_cache.get_or_create_region('cache_perms')
         # make sure we empty the cache now
-        for key in cache_region.backend.list_keys():
-            cache_region.delete(key)
+        cache_region.delete_multi(cache_region.backend.list_keys())
 
         assert cache_region.backend.list_keys() == []
 
         @cache_region.cache_on_arguments(expiration_time=5)
         def compute(key):
             return time.time()
 
         for x in range(10):
             compute(x)
 
         assert len(set(cache_region.backend.list_keys())) == 10
 
     def test_store_and_get_value_from_region(self):
         cache_region = rc_cache.get_or_create_region('cache_perms')
         # make sure we empty the cache now
         for key in cache_region.backend.list_keys():
             cache_region.delete(key)
         assert cache_region.backend.list_keys() == []
 
         @cache_region.cache_on_arguments(expiration_time=5)
         def compute(key):
             return time.time()
 
         result = set()
         for x in range(10):
             ret = compute('x')
             result.add(ret)
 
         # once computed we have only one value (the same from cache)
         # after executing it 10x
         assert len(result) == 1