diff --git a/vcsserver/base.py b/vcsserver/base.py --- a/vcsserver/base.py +++ b/vcsserver/base.py @@ -23,7 +23,7 @@ import urlparse from vcsserver import exceptions from vcsserver.exceptions import NoContentException from vcsserver.hgcompat import (archival) -from vcsserver.lib.rc_cache import region_meta + log = logging.getLogger(__name__) @@ -37,7 +37,7 @@ class RepoFactory(object): repo_type = None def __init__(self): - self._cache_region = region_meta.dogpile_cache_regions['repo_object'] + pass def _create_config(self, path, config): config = {} diff --git a/vcsserver/git.py b/vcsserver/git.py --- a/vcsserver/git.py +++ b/vcsserver/git.py @@ -185,7 +185,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def assert_correct_path(self, wire): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _assert_correct_path(_context_uid, _repo_id): try: repo_init = self._factory.repo_libgit2(wire) @@ -217,7 +218,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def blob_raw_length(self, wire, sha): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _blob_raw_length(_repo_id, _sha): repo_init = self._factory.repo_libgit2(wire) @@ -248,7 +250,8 @@ class GitRemote(RemoteBase): def is_large_file(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _is_large_file(_repo_id, _sha): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -264,7 +267,8 @@ class GitRemote(RemoteBase): def is_binary(self, wire, tree_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _is_binary(_repo_id, _tree_id): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -306,7 +310,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def bulk_request(self, wire, rev, pre_load): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _bulk_request(_repo_id, _rev, _pre_load): result = {} for attr in pre_load: @@ -409,7 +414,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def branch(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _branch(_context_uid, _repo_id, _commit_id): regex = re.compile('^refs/heads') @@ -424,7 +430,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def commit_branches(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _commit_branches(_context_uid, _repo_id, _commit_id): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -690,7 +697,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def get_object(self, wire, sha, maybe_unreachable=False): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_object(_context_uid, _repo_id, _sha): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -748,7 +756,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def get_refs(self, wire): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_refs(_context_uid, _repo_id): repo_init = self._factory.repo_libgit2(wire) @@ -762,7 +771,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def get_branch_pointers(self, wire): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_branch_pointers(_context_uid, _repo_id): repo_init = self._factory.repo_libgit2(wire) @@ -776,7 +786,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def head(self, wire, show_exc=True): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _head(_context_uid, _repo_id, _show_exc): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -801,7 +812,8 @@ class GitRemote(RemoteBase): def revision(self, wire, rev): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _revision(_context_uid, _repo_id, _rev): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -819,7 +831,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def date(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _date(_repo_id, _commit_id): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -838,7 +851,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def author(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _author(_repo_id, _commit_id): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -862,7 +876,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def message(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _message(_repo_id, _commit_id): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -873,7 +888,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def parents(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _parents(_repo_id, _commit_id): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -889,7 +905,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def children(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _children(_repo_id, _commit_id): output, __ = self.run_git_command( wire, ['rev-list', '--all', '--children']) @@ -948,7 +965,8 @@ class GitRemote(RemoteBase): def tree_and_type_for_path(self, wire, commit_id, path): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path): repo_init = self._factory.repo_libgit2(wire) @@ -965,7 +983,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def tree_items(self, wire, tree_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _tree_items(_repo_id, _tree_id): repo_init = self._factory.repo_libgit2(wire) @@ -1066,7 +1085,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def node_history(self, wire, commit_id, path, limit): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit): # optimize for n==1, rev-list is much faster for that use-case if limit == 1: @@ -1108,7 +1128,8 @@ class GitRemote(RemoteBase): def get_all_commit_ids(self, wire): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_all_commit_ids(_context_uid, _repo_id): cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags'] diff --git a/vcsserver/hg.py b/vcsserver/hg.py --- a/vcsserver/hg.py +++ b/vcsserver/hg.py @@ -209,7 +209,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def bookmarks(self, wire): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _bookmarks(_context_uid, _repo_id): repo = self._factory.repo(wire) return dict(repo._bookmarks) @@ -219,7 +220,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def branches(self, wire, normal, closed): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _branches(_context_uid, _repo_id, _normal, _closed): repo = self._factory.repo(wire) iter_branches = repo.branchmap().iterbranches() @@ -237,7 +239,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def bulk_request(self, wire, commit_id, pre_load): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _bulk_request(_repo_id, _commit_id, _pre_load): result = {} for attr in pre_load: @@ -254,7 +257,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_branch(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_branch(_repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -264,7 +268,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_date(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_date(_repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -280,7 +285,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_files(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_files(_repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -297,7 +303,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_parents(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_parents(_repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -309,7 +316,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_children(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_children(_repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -321,7 +329,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_phase(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_phase(_context_uid, _repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -332,7 +341,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_obsolete(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_obsolete(_context_uid, _repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -342,7 +352,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_hidden(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_hidden(_context_uid, _repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -455,7 +466,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def node_history(self, wire, revision, path, limit): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _node_history(_context_uid, _repo_id, _revision, _path, _limit): repo = self._factory.repo(wire) @@ -485,7 +497,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def node_history_untill(self, wire, revision, path, limit): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _node_history_until(_context_uid, _repo_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, revision) @@ -523,7 +536,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def fctx_flags(self, wire, commit_id, path): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _fctx_flags(_repo_id, _commit_id, _path): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -535,7 +549,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def fctx_size(self, wire, commit_id, path): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _fctx_size(_repo_id, _revision, _path): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -546,7 +561,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def get_all_commit_ids(self, wire, name): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_all_commit_ids(_context_uid, _repo_id, _name): repo = self._factory.repo(wire) repo = repo.filtered(name) @@ -562,7 +578,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def is_large_file(self, wire, commit_id, path): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _is_large_file(_context_uid, _repo_id, _commit_id, _path): return largefiles.lfutil.isstandin(path) @@ -572,7 +589,8 @@ class HgRemote(RemoteBase): def is_binary(self, wire, revision, path): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _is_binary(_repo_id, _sha, _path): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, revision) @@ -610,7 +628,8 @@ class HgRemote(RemoteBase): def lookup(self, wire, revision, both): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _lookup(_context_uid, _repo_id, _revision, _both): repo = self._factory.repo(wire) @@ -668,7 +687,8 @@ class HgRemote(RemoteBase): def rev_range(self, wire, commit_filter): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _rev_range(_context_uid, _repo_id, _filter): repo = self._factory.repo(wire) revisions = [rev for rev in revrange(repo, commit_filter)] @@ -743,7 +763,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def tags(self, wire): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _tags(_context_uid, _repo_id): repo = self._factory.repo(wire) return repo.tags() diff --git a/vcsserver/http_main.py b/vcsserver/http_main.py --- a/vcsserver/http_main.py +++ b/vcsserver/http_main.py @@ -394,7 +394,7 @@ class HTTPApplication(object): else: call_args = args[1:] - log.debug('method requested:%s with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s', + log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s', method, call_args, kwargs, context_uid, repo_state_uid) return payload, remote, method, args, kwargs diff --git a/vcsserver/lib/rc_cache/__init__.py b/vcsserver/lib/rc_cache/__init__.py --- a/vcsserver/lib/rc_cache/__init__.py +++ b/vcsserver/lib/rc_cache/__init__.py @@ -38,7 +38,9 @@ register_backend( log = logging.getLogger(__name__) from . import region_meta -from .utils import (get_default_cache_settings, backend_key_generator, make_region) +from .utils import ( + get_default_cache_settings, backend_key_generator, get_or_create_region, + clear_cache_namespace, make_region) def configure_dogpile_cache(settings): diff --git a/vcsserver/lib/rc_cache/utils.py b/vcsserver/lib/rc_cache/utils.py --- a/vcsserver/lib/rc_cache/utils.py +++ b/vcsserver/lib/rc_cache/utils.py @@ -16,17 +16,16 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import os +import time import logging import functools -import time - -from decorator import decorate from dogpile.cache import CacheRegion from dogpile.cache.util import compat from vcsserver.utils import safe_str, sha1 +from vcsserver.lib.rc_cache import region_meta log = logging.getLogger(__name__) @@ -50,6 +49,61 @@ class RhodeCodeCacheRegion(CacheRegion): if function_key_generator is None: function_key_generator = self.function_key_generator + # workaround for py2 and cython problems, this block should be removed + # once we've migrated to py3 + if 'cython' == 'cython': + def decorator(fn): + if to_str is compat.string_type: + # backwards compatible + key_generator = function_key_generator(namespace, fn) + else: + key_generator = function_key_generator(namespace, fn, to_str=to_str) + + @functools.wraps(fn) + def decorate(*arg, **kw): + key = key_generator(*arg, **kw) + + @functools.wraps(fn) + def creator(): + return fn(*arg, **kw) + + if not condition: + return creator() + + timeout = expiration_time() if expiration_time_is_callable \ + else expiration_time + + return self.get_or_create(key, creator, timeout, should_cache_fn) + + def invalidate(*arg, **kw): + key = key_generator(*arg, **kw) + self.delete(key) + + def set_(value, *arg, **kw): + key = key_generator(*arg, **kw) + self.set(key, value) + + def get(*arg, **kw): + key = key_generator(*arg, **kw) + return self.get(key) + + def refresh(*arg, **kw): + key = key_generator(*arg, **kw) + value = fn(*arg, **kw) + self.set(key, value) + return value + + decorate.set = set_ + decorate.invalidate = invalidate + decorate.refresh = refresh + decorate.get = get + decorate.original = fn + decorate.key_generator = key_generator + decorate.__wrapped__ = fn + + return decorate + return decorator + def get_or_create_for_user_func(key_generator, user_func, *arg, **kw): if not condition: @@ -104,7 +158,7 @@ class RhodeCodeCacheRegion(CacheRegion): user_func.original = user_func # Use `decorate` to preserve the signature of :param:`user_func`. - return decorate(user_func, functools.partial( + return decorator.decorate(user_func, functools.partial( get_or_create_for_user_func, key_generator)) return cache_decorator @@ -156,3 +210,54 @@ def key_generator(backend, namespace, fn return final_key return generate_key + + +def get_or_create_region(region_name, region_namespace=None): + from vcsserver.lib.rc_cache.backends import FileNamespaceBackend + region_obj = region_meta.dogpile_cache_regions.get(region_name) + if not region_obj: + raise EnvironmentError( + 'Region `{}` not in configured: {}.'.format( + region_name, region_meta.dogpile_cache_regions.keys())) + + region_uid_name = '{}:{}'.format(region_name, region_namespace) + if isinstance(region_obj.actual_backend, FileNamespaceBackend): + region_exist = region_meta.dogpile_cache_regions.get(region_namespace) + if region_exist: + log.debug('Using already configured region: %s', region_namespace) + return region_exist + cache_dir = region_meta.dogpile_config_defaults['cache_dir'] + expiration_time = region_obj.expiration_time + + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + new_region = make_region( + name=region_uid_name, + function_key_generator=backend_key_generator(region_obj.actual_backend) + ) + namespace_filename = os.path.join( + cache_dir, "{}.cache.dbm".format(region_namespace)) + # special type that allows 1db per namespace + new_region.configure( + backend='dogpile.cache.rc.file_namespace', + expiration_time=expiration_time, + arguments={"filename": namespace_filename} + ) + + # create and save in region caches + log.debug('configuring new region: %s', region_uid_name) + region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region + + return region_obj + + +def clear_cache_namespace(cache_region, cache_namespace_uid, invalidate=False): + region = get_or_create_region(cache_region, cache_namespace_uid) + cache_keys = region.backend.list_keys(prefix=cache_namespace_uid) + num_delete_keys = len(cache_keys) + if invalidate: + region.invalidate(hard=False) + else: + if num_delete_keys: + region.delete_multi(cache_keys) + return num_delete_keys diff --git a/vcsserver/svn.py b/vcsserver/svn.py --- a/vcsserver/svn.py +++ b/vcsserver/svn.py @@ -201,7 +201,8 @@ class SvnRemote(RemoteBase): def revision_properties(self, wire, revision): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _revision_properties(_repo_id, _revision): repo = self._factory.repo(wire) fs_ptr = svn.repos.fs(repo) @@ -255,7 +256,8 @@ class SvnRemote(RemoteBase): @reraise_safe_exceptions def node_history(self, wire, path, revision, limit): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit): cross_copies = False repo = self._factory.repo(wire) @@ -276,7 +278,8 @@ class SvnRemote(RemoteBase): def node_properties(self, wire, path, revision): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _node_properties(_repo_id, _path, _revision): repo = self._factory.repo(wire) fsobj = svn.repos.fs(repo) @@ -314,7 +317,8 @@ class SvnRemote(RemoteBase): def get_node_type(self, wire, path, revision=None): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_node_type(_repo_id, _path, _revision): repo = self._factory.repo(wire) fs_ptr = svn.repos.fs(repo) @@ -328,7 +332,8 @@ class SvnRemote(RemoteBase): def get_nodes(self, wire, path, revision=None): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_nodes(_repo_id, _path, _revision): repo = self._factory.repo(wire) fsobj = svn.repos.fs(repo) @@ -355,7 +360,8 @@ class SvnRemote(RemoteBase): def get_file_size(self, wire, path, revision=None): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_file_size(_repo_id, _path, _revision): repo = self._factory.repo(wire) fsobj = svn.repos.fs(repo) @@ -470,7 +476,8 @@ class SvnRemote(RemoteBase): def is_binary(self, wire, rev, path): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self.region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _is_binary(_repo_id, _rev, _path): raw_bytes = self.get_file_content(wire, path, rev) return raw_bytes and '\0' in raw_bytes diff --git a/vcsserver/vcs_base.py b/vcsserver/vcs_base.py --- a/vcsserver/vcs_base.py +++ b/vcsserver/vcs_base.py @@ -15,13 +15,15 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from vcsserver.lib import rc_cache class RemoteBase(object): EMPTY_COMMIT = '0' * 40 - @property - def region(self): - return self._factory._cache_region + def region(self, wire): + repo_id = wire.get('repo_id', '') + cache_namespace_uid = 'cache_repo.{}'.format(repo_id) + return rc_cache.get_or_create_region('repo_object', cache_namespace_uid) def _cache_on(self, wire): context = wire.get('context', '') @@ -30,3 +32,14 @@ class RemoteBase(object): cache = wire.get('cache', True) cache_on = context and cache return cache_on, context_uid, repo_id + + def vcsserver_invalidate_cache(self, wire, delete): + from vcsserver.lib import rc_cache + repo_id = wire.get('repo_id', '') + + if delete: + cache_namespace_uid = 'cache_repo.{}'.format(repo_id) + rc_cache.clear_cache_namespace( + 'repo_object', cache_namespace_uid, invalidate=True) + + return {'invalidated': {'repo_id': repo_id, 'delete': delete}}