diff --git a/.bumpversion.cfg b/.bumpversion.cfg --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.26.0 +current_version = 4.27.0 message = release: Bump version {current_version} to {new_version} [bumpversion:file:vcsserver/VERSION] diff --git a/.release.cfg b/.release.cfg --- a/.release.cfg +++ b/.release.cfg @@ -5,12 +5,10 @@ done = false done = true [task:fixes_on_stable] -done = true [task:pip2nix_generated] -done = true [release] -state = prepared -version = 4.26.0 +state = in_progress +version = 4.27.0 diff --git a/pkgs/python-packages.nix b/pkgs/python-packages.nix --- a/pkgs/python-packages.nix +++ b/pkgs/python-packages.nix @@ -784,7 +784,7 @@ self: super: { }; }; "rhodecode-vcsserver" = super.buildPythonPackage { - name = "rhodecode-vcsserver-4.26.0"; + name = "rhodecode-vcsserver-4.27.0"; buildInputs = [ self."pytest" self."py" diff --git a/vcsserver/VERSION b/vcsserver/VERSION --- a/vcsserver/VERSION +++ b/vcsserver/VERSION @@ -1,1 +1,1 @@ -4.26.0 \ No newline at end of file +4.27.0 \ No newline at end of file diff --git a/vcsserver/base.py b/vcsserver/base.py --- a/vcsserver/base.py +++ b/vcsserver/base.py @@ -23,7 +23,7 @@ import urlparse from vcsserver import exceptions from vcsserver.exceptions import NoContentException from vcsserver.hgcompat import (archival) -from vcsserver.lib.rc_cache import region_meta + log = logging.getLogger(__name__) @@ -37,7 +37,7 @@ class RepoFactory(object): repo_type = None def __init__(self): - self._cache_region = region_meta.dogpile_cache_regions['repo_object'] + pass def _create_config(self, path, config): config = {} diff --git a/vcsserver/git.py b/vcsserver/git.py --- a/vcsserver/git.py +++ b/vcsserver/git.py @@ -185,7 +185,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def assert_correct_path(self, wire): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _assert_correct_path(_context_uid, _repo_id): try: repo_init = self._factory.repo_libgit2(wire) @@ -217,7 +218,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def blob_raw_length(self, wire, sha): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _blob_raw_length(_repo_id, _sha): repo_init = self._factory.repo_libgit2(wire) @@ -248,7 +250,8 @@ class GitRemote(RemoteBase): def is_large_file(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _is_large_file(_repo_id, _sha): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -264,7 +267,8 @@ class GitRemote(RemoteBase): def is_binary(self, wire, tree_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _is_binary(_repo_id, _tree_id): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -306,7 +310,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def bulk_request(self, wire, rev, pre_load): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _bulk_request(_repo_id, _rev, _pre_load): result = {} for attr in pre_load: @@ -409,7 +414,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def branch(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _branch(_context_uid, _repo_id, _commit_id): regex = re.compile('^refs/heads') @@ -424,7 +430,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def commit_branches(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _commit_branches(_context_uid, _repo_id, _commit_id): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -445,11 +452,22 @@ class GitRemote(RemoteBase): # TODO: this is quite complex, check if that can be simplified @reraise_safe_exceptions def commit(self, wire, commit_data, branch, commit_tree, updated, removed): + # Defines the root tree + class _Root(object): + def __repr__(self): + return 'ROOT TREE' + ROOT = _Root() + repo = self._factory.repo(wire) object_store = repo.object_store # Create tree and populates it with blobs - commit_tree = commit_tree and repo[commit_tree] or objects.Tree() + + if commit_tree and repo[commit_tree]: + git_commit = repo[commit_data['parents'][0]] + commit_tree = repo[git_commit.tree] # root tree + else: + commit_tree = objects.Tree() for node in updated: # Compute subdirs if needed @@ -508,21 +526,34 @@ class GitRemote(RemoteBase): for node_path in removed: paths = node_path.split('/') - tree = commit_tree - trees = [tree] + tree = commit_tree # start with top-level + trees = [{'tree': tree, 'path': ROOT}] # Traverse deep into the forest... + # resolve final tree by iterating the path. + # e.g a/b/c.txt will get + # - root as tree then + # - 'a' as tree, + # - 'b' as tree, + # - stop at c as blob. for path in paths: try: obj = repo[tree[path][1]] if isinstance(obj, objects.Tree): - trees.append(obj) + trees.append({'tree': obj, 'path': path}) tree = obj except KeyError: break + #PROBLEM: + """ + We're not editing same reference tree object + """ # Cut down the blob and all rotten trees on the way back... - for path, tree in reversed(zip(paths, trees)): - del tree[path] - if tree: + for path, tree_data in reversed(zip(paths, trees)): + tree = tree_data['tree'] + tree.__delitem__(path) + # This operation edits the tree, we need to mark new commit back + + if len(tree) > 0: # This tree still has elements - don't remove it or any # of it's parents break @@ -532,7 +563,7 @@ class GitRemote(RemoteBase): # Create commit commit = objects.Commit() commit.tree = commit_tree.id - for k, v in commit_data.iteritems(): + for k, v in commit_data.items(): setattr(commit, k, v) object_store.add_object(commit) @@ -588,7 +619,7 @@ class GitRemote(RemoteBase): if refs and not update_after: # mikhail: explicitly set the head to the last ref. - repo['HEAD'] = remote_refs[refs[-1]] + repo["HEAD"] = remote_refs[refs[-1]] if update_after: # we want to checkout HEAD @@ -690,7 +721,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def get_object(self, wire, sha, maybe_unreachable=False): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_object(_context_uid, _repo_id, _sha): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -748,7 +780,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def get_refs(self, wire): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_refs(_context_uid, _repo_id): repo_init = self._factory.repo_libgit2(wire) @@ -762,7 +795,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def get_branch_pointers(self, wire): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_branch_pointers(_context_uid, _repo_id): repo_init = self._factory.repo_libgit2(wire) @@ -776,7 +810,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def head(self, wire, show_exc=True): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _head(_context_uid, _repo_id, _show_exc): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -801,7 +836,8 @@ class GitRemote(RemoteBase): def revision(self, wire, rev): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _revision(_context_uid, _repo_id, _rev): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -819,7 +855,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def date(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _date(_repo_id, _commit_id): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -838,7 +875,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def author(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _author(_repo_id, _commit_id): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -862,7 +900,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def message(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _message(_repo_id, _commit_id): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -873,7 +912,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def parents(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _parents(_repo_id, _commit_id): repo_init = self._factory.repo_libgit2(wire) with repo_init as repo: @@ -889,7 +929,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def children(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _children(_repo_id, _commit_id): output, __ = self.run_git_command( wire, ['rev-list', '--all', '--children']) @@ -948,7 +989,8 @@ class GitRemote(RemoteBase): def tree_and_type_for_path(self, wire, commit_id, path): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path): repo_init = self._factory.repo_libgit2(wire) @@ -965,7 +1007,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def tree_items(self, wire, tree_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _tree_items(_repo_id, _tree_id): repo_init = self._factory.repo_libgit2(wire) @@ -1066,7 +1109,8 @@ class GitRemote(RemoteBase): @reraise_safe_exceptions def node_history(self, wire, commit_id, path, limit): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit): # optimize for n==1, rev-list is much faster for that use-case if limit == 1: @@ -1108,7 +1152,8 @@ class GitRemote(RemoteBase): def get_all_commit_ids(self, wire): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_all_commit_ids(_context_uid, _repo_id): cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags'] @@ -1193,6 +1238,13 @@ class GitRemote(RemoteBase): } @reraise_safe_exceptions + def set_head_ref(self, wire, head_name): + log.debug('Setting refs/head to `%s`', head_name) + cmd = ['symbolic-ref', 'HEAD', 'refs/heads/%s' % head_name] + output, __ = self.run_git_command(wire, cmd) + return [head_name] + output.splitlines() + + @reraise_safe_exceptions def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path, archive_dir_name, commit_id): @@ -1220,7 +1272,10 @@ class GitRemote(RemoteBase): file_path = fn.path mode = fn.mode is_link = stat.S_ISLNK(mode) - yield ArchiveNode(file_path, mode, is_link, repo[fn.id].read_raw) + if mode == pygit2.GIT_FILEMODE_COMMIT: + log.debug('Skipping path %s as a commit node', file_path) + continue + yield ArchiveNode(file_path, mode, is_link, repo[fn.hex].read_raw) return archive_repo(file_walker, archive_dest_path, kind, mtime, archive_at_path, archive_dir_name, commit_id) diff --git a/vcsserver/hg.py b/vcsserver/hg.py --- a/vcsserver/hg.py +++ b/vcsserver/hg.py @@ -209,7 +209,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def bookmarks(self, wire): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _bookmarks(_context_uid, _repo_id): repo = self._factory.repo(wire) return dict(repo._bookmarks) @@ -219,7 +220,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def branches(self, wire, normal, closed): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _branches(_context_uid, _repo_id, _normal, _closed): repo = self._factory.repo(wire) iter_branches = repo.branchmap().iterbranches() @@ -237,7 +239,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def bulk_request(self, wire, commit_id, pre_load): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _bulk_request(_repo_id, _commit_id, _pre_load): result = {} for attr in pre_load: @@ -254,7 +257,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_branch(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_branch(_repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -264,7 +268,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_date(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_date(_repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -280,7 +285,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_files(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_files(_repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -297,7 +303,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_parents(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_parents(_repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -309,7 +316,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_children(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_children(_repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -321,7 +329,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_phase(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_phase(_context_uid, _repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -332,7 +341,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_obsolete(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_obsolete(_context_uid, _repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -342,7 +352,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def ctx_hidden(self, wire, commit_id): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _ctx_hidden(_context_uid, _repo_id, _commit_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -455,7 +466,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def node_history(self, wire, revision, path, limit): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _node_history(_context_uid, _repo_id, _revision, _path, _limit): repo = self._factory.repo(wire) @@ -485,7 +497,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def node_history_untill(self, wire, revision, path, limit): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _node_history_until(_context_uid, _repo_id): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, revision) @@ -523,7 +536,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def fctx_flags(self, wire, commit_id, path): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _fctx_flags(_repo_id, _commit_id, _path): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -535,7 +549,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def fctx_size(self, wire, commit_id, path): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _fctx_size(_repo_id, _revision, _path): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, commit_id) @@ -546,7 +561,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def get_all_commit_ids(self, wire, name): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_all_commit_ids(_context_uid, _repo_id, _name): repo = self._factory.repo(wire) repo = repo.filtered(name) @@ -562,7 +578,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def is_large_file(self, wire, commit_id, path): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _is_large_file(_context_uid, _repo_id, _commit_id, _path): return largefiles.lfutil.isstandin(path) @@ -572,7 +589,8 @@ class HgRemote(RemoteBase): def is_binary(self, wire, revision, path): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _is_binary(_repo_id, _sha, _path): repo = self._factory.repo(wire) ctx = self._get_ctx(repo, revision) @@ -610,7 +628,8 @@ class HgRemote(RemoteBase): def lookup(self, wire, revision, both): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _lookup(_context_uid, _repo_id, _revision, _both): repo = self._factory.repo(wire) @@ -668,7 +687,8 @@ class HgRemote(RemoteBase): def rev_range(self, wire, commit_filter): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _rev_range(_context_uid, _repo_id, _filter): repo = self._factory.repo(wire) revisions = [rev for rev in revrange(repo, commit_filter)] @@ -743,7 +763,8 @@ class HgRemote(RemoteBase): @reraise_safe_exceptions def tags(self, wire): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _tags(_context_uid, _repo_id): repo = self._factory.repo(wire) return repo.tags() @@ -996,6 +1017,10 @@ class HgRemote(RemoteBase): } @reraise_safe_exceptions + def set_head_ref(self, wire, head_name): + pass + + @reraise_safe_exceptions def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path, archive_dir_name, commit_id): diff --git a/vcsserver/hooks.py b/vcsserver/hooks.py --- a/vcsserver/hooks.py +++ b/vcsserver/hooks.py @@ -49,7 +49,7 @@ class HooksHttpClient(object): try: connection.request('POST', '/', body) except Exception: - log.error('Connection failed on %s', connection) + log.error('Hooks calling Connection failed on %s', connection.__dict__) raise response = connection.getresponse() diff --git a/vcsserver/http_main.py b/vcsserver/http_main.py --- a/vcsserver/http_main.py +++ b/vcsserver/http_main.py @@ -394,7 +394,7 @@ class HTTPApplication(object): else: call_args = args[1:] - log.debug('method requested:%s with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s', + log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s', method, call_args, kwargs, context_uid, repo_state_uid) return payload, remote, method, args, kwargs diff --git a/vcsserver/lib/rc_cache/__init__.py b/vcsserver/lib/rc_cache/__init__.py --- a/vcsserver/lib/rc_cache/__init__.py +++ b/vcsserver/lib/rc_cache/__init__.py @@ -38,7 +38,9 @@ register_backend( log = logging.getLogger(__name__) from . import region_meta -from .utils import (get_default_cache_settings, backend_key_generator, make_region) +from .utils import ( + get_default_cache_settings, backend_key_generator, get_or_create_region, + clear_cache_namespace, make_region) def configure_dogpile_cache(settings): diff --git a/vcsserver/lib/rc_cache/backends.py b/vcsserver/lib/rc_cache/backends.py --- a/vcsserver/lib/rc_cache/backends.py +++ b/vcsserver/lib/rc_cache/backends.py @@ -32,6 +32,7 @@ from dogpile.cache.util import memoized_ from pyramid.settings import asbool from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug +from vcsserver.utils import safe_str _default_max_size = 1024 @@ -264,7 +265,7 @@ class BaseRedisBackend(redis_backend.Red def get_mutex(self, key): if self.distributed_lock: - lock_key = redis_backend.u('_lock_{0}').format(key) + lock_key = redis_backend.u('_lock_{0}').format(safe_str(key)) return get_mutex_lock(self.client, lock_key, self._lock_timeout, auto_renewal=self._lock_auto_renewal) else: diff --git a/vcsserver/lib/rc_cache/utils.py b/vcsserver/lib/rc_cache/utils.py --- a/vcsserver/lib/rc_cache/utils.py +++ b/vcsserver/lib/rc_cache/utils.py @@ -16,17 +16,16 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import os +import time import logging import functools -import time - -from decorator import decorate from dogpile.cache import CacheRegion from dogpile.cache.util import compat from vcsserver.utils import safe_str, sha1 +from vcsserver.lib.rc_cache import region_meta log = logging.getLogger(__name__) @@ -50,14 +49,69 @@ class RhodeCodeCacheRegion(CacheRegion): if function_key_generator is None: function_key_generator = self.function_key_generator + # workaround for py2 and cython problems, this block should be removed + # once we've migrated to py3 + if 'cython' == 'cython': + def decorator(fn): + if to_str is compat.string_type: + # backwards compatible + key_generator = function_key_generator(namespace, fn) + else: + key_generator = function_key_generator(namespace, fn, to_str=to_str) + + @functools.wraps(fn) + def decorate(*arg, **kw): + key = key_generator(*arg, **kw) + + @functools.wraps(fn) + def creator(): + return fn(*arg, **kw) + + if not condition: + return creator() + + timeout = expiration_time() if expiration_time_is_callable \ + else expiration_time + + return self.get_or_create(key, creator, timeout, should_cache_fn) + + def invalidate(*arg, **kw): + key = key_generator(*arg, **kw) + self.delete(key) + + def set_(value, *arg, **kw): + key = key_generator(*arg, **kw) + self.set(key, value) + + def get(*arg, **kw): + key = key_generator(*arg, **kw) + return self.get(key) + + def refresh(*arg, **kw): + key = key_generator(*arg, **kw) + value = fn(*arg, **kw) + self.set(key, value) + return value + + decorate.set = set_ + decorate.invalidate = invalidate + decorate.refresh = refresh + decorate.get = get + decorate.original = fn + decorate.key_generator = key_generator + decorate.__wrapped__ = fn + + return decorate + return decorator + def get_or_create_for_user_func(key_generator, user_func, *arg, **kw): if not condition: - log.debug('Calling un-cached func:%s', user_func.func_name) + log.debug('Calling un-cached method:%s', user_func.func_name) start = time.time() result = user_func(*arg, **kw) total = time.time() - start - log.debug('un-cached func:%s took %.4fs', user_func.func_name, total) + log.debug('un-cached method:%s took %.4fs', user_func.func_name, total) return result key = key_generator(*arg, **kw) @@ -65,7 +119,7 @@ class RhodeCodeCacheRegion(CacheRegion): timeout = expiration_time() if expiration_time_is_callable \ else expiration_time - log.debug('Calling cached fn:%s', user_func.func_name) + log.debug('Calling cached method:`%s`', user_func.func_name) return self.get_or_create(key, user_func, timeout, should_cache_fn, (arg, kw)) def cache_decorator(user_func): @@ -104,7 +158,7 @@ class RhodeCodeCacheRegion(CacheRegion): user_func.original = user_func # Use `decorate` to preserve the signature of :param:`user_func`. - return decorate(user_func, functools.partial( + return decorator.decorate(user_func, functools.partial( get_or_create_for_user_func, key_generator)) return cache_decorator @@ -156,3 +210,54 @@ def key_generator(backend, namespace, fn return final_key return generate_key + + +def get_or_create_region(region_name, region_namespace=None): + from vcsserver.lib.rc_cache.backends import FileNamespaceBackend + region_obj = region_meta.dogpile_cache_regions.get(region_name) + if not region_obj: + raise EnvironmentError( + 'Region `{}` not in configured: {}.'.format( + region_name, region_meta.dogpile_cache_regions.keys())) + + region_uid_name = '{}:{}'.format(region_name, region_namespace) + if isinstance(region_obj.actual_backend, FileNamespaceBackend): + region_exist = region_meta.dogpile_cache_regions.get(region_namespace) + if region_exist: + log.debug('Using already configured region: %s', region_namespace) + return region_exist + cache_dir = region_meta.dogpile_config_defaults['cache_dir'] + expiration_time = region_obj.expiration_time + + if not os.path.isdir(cache_dir): + os.makedirs(cache_dir) + new_region = make_region( + name=region_uid_name, + function_key_generator=backend_key_generator(region_obj.actual_backend) + ) + namespace_filename = os.path.join( + cache_dir, "{}.cache.dbm".format(region_namespace)) + # special type that allows 1db per namespace + new_region.configure( + backend='dogpile.cache.rc.file_namespace', + expiration_time=expiration_time, + arguments={"filename": namespace_filename} + ) + + # create and save in region caches + log.debug('configuring new region: %s', region_uid_name) + region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region + + return region_obj + + +def clear_cache_namespace(cache_region, cache_namespace_uid, invalidate=False): + region = get_or_create_region(cache_region, cache_namespace_uid) + cache_keys = region.backend.list_keys(prefix=cache_namespace_uid) + num_delete_keys = len(cache_keys) + if invalidate: + region.invalidate(hard=False) + else: + if num_delete_keys: + region.delete_multi(cache_keys) + return num_delete_keys diff --git a/vcsserver/svn.py b/vcsserver/svn.py --- a/vcsserver/svn.py +++ b/vcsserver/svn.py @@ -201,7 +201,8 @@ class SvnRemote(RemoteBase): def revision_properties(self, wire, revision): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _revision_properties(_repo_id, _revision): repo = self._factory.repo(wire) fs_ptr = svn.repos.fs(repo) @@ -255,7 +256,8 @@ class SvnRemote(RemoteBase): @reraise_safe_exceptions def node_history(self, wire, path, revision, limit): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit): cross_copies = False repo = self._factory.repo(wire) @@ -276,7 +278,8 @@ class SvnRemote(RemoteBase): def node_properties(self, wire, path, revision): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _node_properties(_repo_id, _path, _revision): repo = self._factory.repo(wire) fsobj = svn.repos.fs(repo) @@ -314,7 +317,8 @@ class SvnRemote(RemoteBase): def get_node_type(self, wire, path, revision=None): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_node_type(_repo_id, _path, _revision): repo = self._factory.repo(wire) fs_ptr = svn.repos.fs(repo) @@ -328,7 +332,8 @@ class SvnRemote(RemoteBase): def get_nodes(self, wire, path, revision=None): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_nodes(_repo_id, _path, _revision): repo = self._factory.repo(wire) fsobj = svn.repos.fs(repo) @@ -355,7 +360,8 @@ class SvnRemote(RemoteBase): def get_file_size(self, wire, path, revision=None): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _get_file_size(_repo_id, _path, _revision): repo = self._factory.repo(wire) fsobj = svn.repos.fs(repo) @@ -470,7 +476,8 @@ class SvnRemote(RemoteBase): def is_binary(self, wire, rev, path): cache_on, context_uid, repo_id = self._cache_on(wire) - @self.region.conditional_cache_on_arguments(condition=cache_on) + region = self._region(wire) + @region.conditional_cache_on_arguments(condition=cache_on) def _is_binary(_repo_id, _rev, _path): raw_bytes = self.get_file_content(wire, path, rev) return raw_bytes and '\0' in raw_bytes @@ -530,6 +537,10 @@ class SvnRemote(RemoteBase): } @reraise_safe_exceptions + def set_head_ref(self, wire, head_name): + pass + + @reraise_safe_exceptions def archive_repo(self, wire, archive_dest_path, kind, mtime, archive_at_path, archive_dir_name, commit_id): diff --git a/vcsserver/tests/test_git.py b/vcsserver/tests/test_git.py --- a/vcsserver/tests/test_git.py +++ b/vcsserver/tests/test_git.py @@ -113,7 +113,7 @@ class TestReraiseSafeExceptions(object): methods = inspect.getmembers(git_remote, predicate=inspect.ismethod) for method_name, method in methods: - if not method_name.startswith('_'): + if not method_name.startswith('_') and method_name not in ['vcsserver_invalidate_cache']: assert method.im_func.__code__ == decorator.__code__ @pytest.mark.parametrize('side_effect, expected_type', [ diff --git a/vcsserver/tests/test_hg.py b/vcsserver/tests/test_hg.py --- a/vcsserver/tests/test_hg.py +++ b/vcsserver/tests/test_hg.py @@ -50,7 +50,7 @@ class TestReraiseSafeExceptions(object): methods = inspect.getmembers(hg_remote, predicate=inspect.ismethod) decorator = hg.reraise_safe_exceptions(None) for method_name, method in methods: - if not method_name.startswith('_'): + if not method_name.startswith('_') and method_name not in ['vcsserver_invalidate_cache']: assert method.im_func.__code__ == decorator.__code__ @pytest.mark.parametrize('side_effect, expected_type', [ diff --git a/vcsserver/vcs_base.py b/vcsserver/vcs_base.py --- a/vcsserver/vcs_base.py +++ b/vcsserver/vcs_base.py @@ -15,13 +15,15 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from vcsserver.lib import rc_cache class RemoteBase(object): EMPTY_COMMIT = '0' * 40 - @property - def region(self): - return self._factory._cache_region + def _region(self, wire): + repo_id = wire.get('repo_id', '') + cache_namespace_uid = 'cache_repo.{}'.format(repo_id) + return rc_cache.get_or_create_region('repo_object', cache_namespace_uid) def _cache_on(self, wire): context = wire.get('context', '') @@ -30,3 +32,14 @@ class RemoteBase(object): cache = wire.get('cache', True) cache_on = context and cache return cache_on, context_uid, repo_id + + def vcsserver_invalidate_cache(self, wire, delete): + from vcsserver.lib import rc_cache + repo_id = wire.get('repo_id', '') + + if delete: + cache_namespace_uid = 'cache_repo.{}'.format(repo_id) + rc_cache.clear_cache_namespace( + 'repo_object', cache_namespace_uid, invalidate=True) + + return {'invalidated': {'repo_id': repo_id, 'delete': delete}}