##// END OF EJS Templates
caches: allow regional per repo caches, and invalidate caches via a remote call.
super-admin -
r961:da0b3be2 default
parent child Browse files
Show More
@@ -23,7 +23,7 b' import urlparse'
23 from vcsserver import exceptions
23 from vcsserver import exceptions
24 from vcsserver.exceptions import NoContentException
24 from vcsserver.exceptions import NoContentException
25 from vcsserver.hgcompat import (archival)
25 from vcsserver.hgcompat import (archival)
26 from vcsserver.lib.rc_cache import region_meta
26
27 log = logging.getLogger(__name__)
27 log = logging.getLogger(__name__)
28
28
29
29
@@ -37,7 +37,7 b' class RepoFactory(object):'
37 repo_type = None
37 repo_type = None
38
38
39 def __init__(self):
39 def __init__(self):
40 self._cache_region = region_meta.dogpile_cache_regions['repo_object']
40 pass
41
41
42 def _create_config(self, path, config):
42 def _create_config(self, path, config):
43 config = {}
43 config = {}
@@ -185,7 +185,8 b' class GitRemote(RemoteBase):'
185 @reraise_safe_exceptions
185 @reraise_safe_exceptions
186 def assert_correct_path(self, wire):
186 def assert_correct_path(self, wire):
187 cache_on, context_uid, repo_id = self._cache_on(wire)
187 cache_on, context_uid, repo_id = self._cache_on(wire)
188 @self.region.conditional_cache_on_arguments(condition=cache_on)
188 region = self.region(wire)
189 @region.conditional_cache_on_arguments(condition=cache_on)
189 def _assert_correct_path(_context_uid, _repo_id):
190 def _assert_correct_path(_context_uid, _repo_id):
190 try:
191 try:
191 repo_init = self._factory.repo_libgit2(wire)
192 repo_init = self._factory.repo_libgit2(wire)
@@ -217,7 +218,8 b' class GitRemote(RemoteBase):'
217 @reraise_safe_exceptions
218 @reraise_safe_exceptions
218 def blob_raw_length(self, wire, sha):
219 def blob_raw_length(self, wire, sha):
219 cache_on, context_uid, repo_id = self._cache_on(wire)
220 cache_on, context_uid, repo_id = self._cache_on(wire)
220 @self.region.conditional_cache_on_arguments(condition=cache_on)
221 region = self.region(wire)
222 @region.conditional_cache_on_arguments(condition=cache_on)
221 def _blob_raw_length(_repo_id, _sha):
223 def _blob_raw_length(_repo_id, _sha):
222
224
223 repo_init = self._factory.repo_libgit2(wire)
225 repo_init = self._factory.repo_libgit2(wire)
@@ -248,7 +250,8 b' class GitRemote(RemoteBase):'
248 def is_large_file(self, wire, commit_id):
250 def is_large_file(self, wire, commit_id):
249 cache_on, context_uid, repo_id = self._cache_on(wire)
251 cache_on, context_uid, repo_id = self._cache_on(wire)
250
252
251 @self.region.conditional_cache_on_arguments(condition=cache_on)
253 region = self.region(wire)
254 @region.conditional_cache_on_arguments(condition=cache_on)
252 def _is_large_file(_repo_id, _sha):
255 def _is_large_file(_repo_id, _sha):
253 repo_init = self._factory.repo_libgit2(wire)
256 repo_init = self._factory.repo_libgit2(wire)
254 with repo_init as repo:
257 with repo_init as repo:
@@ -264,7 +267,8 b' class GitRemote(RemoteBase):'
264 def is_binary(self, wire, tree_id):
267 def is_binary(self, wire, tree_id):
265 cache_on, context_uid, repo_id = self._cache_on(wire)
268 cache_on, context_uid, repo_id = self._cache_on(wire)
266
269
267 @self.region.conditional_cache_on_arguments(condition=cache_on)
270 region = self.region(wire)
271 @region.conditional_cache_on_arguments(condition=cache_on)
268 def _is_binary(_repo_id, _tree_id):
272 def _is_binary(_repo_id, _tree_id):
269 repo_init = self._factory.repo_libgit2(wire)
273 repo_init = self._factory.repo_libgit2(wire)
270 with repo_init as repo:
274 with repo_init as repo:
@@ -306,7 +310,8 b' class GitRemote(RemoteBase):'
306 @reraise_safe_exceptions
310 @reraise_safe_exceptions
307 def bulk_request(self, wire, rev, pre_load):
311 def bulk_request(self, wire, rev, pre_load):
308 cache_on, context_uid, repo_id = self._cache_on(wire)
312 cache_on, context_uid, repo_id = self._cache_on(wire)
309 @self.region.conditional_cache_on_arguments(condition=cache_on)
313 region = self.region(wire)
314 @region.conditional_cache_on_arguments(condition=cache_on)
310 def _bulk_request(_repo_id, _rev, _pre_load):
315 def _bulk_request(_repo_id, _rev, _pre_load):
311 result = {}
316 result = {}
312 for attr in pre_load:
317 for attr in pre_load:
@@ -409,7 +414,8 b' class GitRemote(RemoteBase):'
409 @reraise_safe_exceptions
414 @reraise_safe_exceptions
410 def branch(self, wire, commit_id):
415 def branch(self, wire, commit_id):
411 cache_on, context_uid, repo_id = self._cache_on(wire)
416 cache_on, context_uid, repo_id = self._cache_on(wire)
412 @self.region.conditional_cache_on_arguments(condition=cache_on)
417 region = self.region(wire)
418 @region.conditional_cache_on_arguments(condition=cache_on)
413 def _branch(_context_uid, _repo_id, _commit_id):
419 def _branch(_context_uid, _repo_id, _commit_id):
414 regex = re.compile('^refs/heads')
420 regex = re.compile('^refs/heads')
415
421
@@ -424,7 +430,8 b' class GitRemote(RemoteBase):'
424 @reraise_safe_exceptions
430 @reraise_safe_exceptions
425 def commit_branches(self, wire, commit_id):
431 def commit_branches(self, wire, commit_id):
426 cache_on, context_uid, repo_id = self._cache_on(wire)
432 cache_on, context_uid, repo_id = self._cache_on(wire)
427 @self.region.conditional_cache_on_arguments(condition=cache_on)
433 region = self.region(wire)
434 @region.conditional_cache_on_arguments(condition=cache_on)
428 def _commit_branches(_context_uid, _repo_id, _commit_id):
435 def _commit_branches(_context_uid, _repo_id, _commit_id):
429 repo_init = self._factory.repo_libgit2(wire)
436 repo_init = self._factory.repo_libgit2(wire)
430 with repo_init as repo:
437 with repo_init as repo:
@@ -690,7 +697,8 b' class GitRemote(RemoteBase):'
690 @reraise_safe_exceptions
697 @reraise_safe_exceptions
691 def get_object(self, wire, sha, maybe_unreachable=False):
698 def get_object(self, wire, sha, maybe_unreachable=False):
692 cache_on, context_uid, repo_id = self._cache_on(wire)
699 cache_on, context_uid, repo_id = self._cache_on(wire)
693 @self.region.conditional_cache_on_arguments(condition=cache_on)
700 region = self.region(wire)
701 @region.conditional_cache_on_arguments(condition=cache_on)
694 def _get_object(_context_uid, _repo_id, _sha):
702 def _get_object(_context_uid, _repo_id, _sha):
695 repo_init = self._factory.repo_libgit2(wire)
703 repo_init = self._factory.repo_libgit2(wire)
696 with repo_init as repo:
704 with repo_init as repo:
@@ -748,7 +756,8 b' class GitRemote(RemoteBase):'
748 @reraise_safe_exceptions
756 @reraise_safe_exceptions
749 def get_refs(self, wire):
757 def get_refs(self, wire):
750 cache_on, context_uid, repo_id = self._cache_on(wire)
758 cache_on, context_uid, repo_id = self._cache_on(wire)
751 @self.region.conditional_cache_on_arguments(condition=cache_on)
759 region = self.region(wire)
760 @region.conditional_cache_on_arguments(condition=cache_on)
752 def _get_refs(_context_uid, _repo_id):
761 def _get_refs(_context_uid, _repo_id):
753
762
754 repo_init = self._factory.repo_libgit2(wire)
763 repo_init = self._factory.repo_libgit2(wire)
@@ -762,7 +771,8 b' class GitRemote(RemoteBase):'
762 @reraise_safe_exceptions
771 @reraise_safe_exceptions
763 def get_branch_pointers(self, wire):
772 def get_branch_pointers(self, wire):
764 cache_on, context_uid, repo_id = self._cache_on(wire)
773 cache_on, context_uid, repo_id = self._cache_on(wire)
765 @self.region.conditional_cache_on_arguments(condition=cache_on)
774 region = self.region(wire)
775 @region.conditional_cache_on_arguments(condition=cache_on)
766 def _get_branch_pointers(_context_uid, _repo_id):
776 def _get_branch_pointers(_context_uid, _repo_id):
767
777
768 repo_init = self._factory.repo_libgit2(wire)
778 repo_init = self._factory.repo_libgit2(wire)
@@ -776,7 +786,8 b' class GitRemote(RemoteBase):'
776 @reraise_safe_exceptions
786 @reraise_safe_exceptions
777 def head(self, wire, show_exc=True):
787 def head(self, wire, show_exc=True):
778 cache_on, context_uid, repo_id = self._cache_on(wire)
788 cache_on, context_uid, repo_id = self._cache_on(wire)
779 @self.region.conditional_cache_on_arguments(condition=cache_on)
789 region = self.region(wire)
790 @region.conditional_cache_on_arguments(condition=cache_on)
780 def _head(_context_uid, _repo_id, _show_exc):
791 def _head(_context_uid, _repo_id, _show_exc):
781 repo_init = self._factory.repo_libgit2(wire)
792 repo_init = self._factory.repo_libgit2(wire)
782 with repo_init as repo:
793 with repo_init as repo:
@@ -801,7 +812,8 b' class GitRemote(RemoteBase):'
801 def revision(self, wire, rev):
812 def revision(self, wire, rev):
802
813
803 cache_on, context_uid, repo_id = self._cache_on(wire)
814 cache_on, context_uid, repo_id = self._cache_on(wire)
804 @self.region.conditional_cache_on_arguments(condition=cache_on)
815 region = self.region(wire)
816 @region.conditional_cache_on_arguments(condition=cache_on)
805 def _revision(_context_uid, _repo_id, _rev):
817 def _revision(_context_uid, _repo_id, _rev):
806 repo_init = self._factory.repo_libgit2(wire)
818 repo_init = self._factory.repo_libgit2(wire)
807 with repo_init as repo:
819 with repo_init as repo:
@@ -819,7 +831,8 b' class GitRemote(RemoteBase):'
819 @reraise_safe_exceptions
831 @reraise_safe_exceptions
820 def date(self, wire, commit_id):
832 def date(self, wire, commit_id):
821 cache_on, context_uid, repo_id = self._cache_on(wire)
833 cache_on, context_uid, repo_id = self._cache_on(wire)
822 @self.region.conditional_cache_on_arguments(condition=cache_on)
834 region = self.region(wire)
835 @region.conditional_cache_on_arguments(condition=cache_on)
823 def _date(_repo_id, _commit_id):
836 def _date(_repo_id, _commit_id):
824 repo_init = self._factory.repo_libgit2(wire)
837 repo_init = self._factory.repo_libgit2(wire)
825 with repo_init as repo:
838 with repo_init as repo:
@@ -838,7 +851,8 b' class GitRemote(RemoteBase):'
838 @reraise_safe_exceptions
851 @reraise_safe_exceptions
839 def author(self, wire, commit_id):
852 def author(self, wire, commit_id):
840 cache_on, context_uid, repo_id = self._cache_on(wire)
853 cache_on, context_uid, repo_id = self._cache_on(wire)
841 @self.region.conditional_cache_on_arguments(condition=cache_on)
854 region = self.region(wire)
855 @region.conditional_cache_on_arguments(condition=cache_on)
842 def _author(_repo_id, _commit_id):
856 def _author(_repo_id, _commit_id):
843 repo_init = self._factory.repo_libgit2(wire)
857 repo_init = self._factory.repo_libgit2(wire)
844 with repo_init as repo:
858 with repo_init as repo:
@@ -862,7 +876,8 b' class GitRemote(RemoteBase):'
862 @reraise_safe_exceptions
876 @reraise_safe_exceptions
863 def message(self, wire, commit_id):
877 def message(self, wire, commit_id):
864 cache_on, context_uid, repo_id = self._cache_on(wire)
878 cache_on, context_uid, repo_id = self._cache_on(wire)
865 @self.region.conditional_cache_on_arguments(condition=cache_on)
879 region = self.region(wire)
880 @region.conditional_cache_on_arguments(condition=cache_on)
866 def _message(_repo_id, _commit_id):
881 def _message(_repo_id, _commit_id):
867 repo_init = self._factory.repo_libgit2(wire)
882 repo_init = self._factory.repo_libgit2(wire)
868 with repo_init as repo:
883 with repo_init as repo:
@@ -873,7 +888,8 b' class GitRemote(RemoteBase):'
873 @reraise_safe_exceptions
888 @reraise_safe_exceptions
874 def parents(self, wire, commit_id):
889 def parents(self, wire, commit_id):
875 cache_on, context_uid, repo_id = self._cache_on(wire)
890 cache_on, context_uid, repo_id = self._cache_on(wire)
876 @self.region.conditional_cache_on_arguments(condition=cache_on)
891 region = self.region(wire)
892 @region.conditional_cache_on_arguments(condition=cache_on)
877 def _parents(_repo_id, _commit_id):
893 def _parents(_repo_id, _commit_id):
878 repo_init = self._factory.repo_libgit2(wire)
894 repo_init = self._factory.repo_libgit2(wire)
879 with repo_init as repo:
895 with repo_init as repo:
@@ -889,7 +905,8 b' class GitRemote(RemoteBase):'
889 @reraise_safe_exceptions
905 @reraise_safe_exceptions
890 def children(self, wire, commit_id):
906 def children(self, wire, commit_id):
891 cache_on, context_uid, repo_id = self._cache_on(wire)
907 cache_on, context_uid, repo_id = self._cache_on(wire)
892 @self.region.conditional_cache_on_arguments(condition=cache_on)
908 region = self.region(wire)
909 @region.conditional_cache_on_arguments(condition=cache_on)
893 def _children(_repo_id, _commit_id):
910 def _children(_repo_id, _commit_id):
894 output, __ = self.run_git_command(
911 output, __ = self.run_git_command(
895 wire, ['rev-list', '--all', '--children'])
912 wire, ['rev-list', '--all', '--children'])
@@ -948,7 +965,8 b' class GitRemote(RemoteBase):'
948 def tree_and_type_for_path(self, wire, commit_id, path):
965 def tree_and_type_for_path(self, wire, commit_id, path):
949
966
950 cache_on, context_uid, repo_id = self._cache_on(wire)
967 cache_on, context_uid, repo_id = self._cache_on(wire)
951 @self.region.conditional_cache_on_arguments(condition=cache_on)
968 region = self.region(wire)
969 @region.conditional_cache_on_arguments(condition=cache_on)
952 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
970 def _tree_and_type_for_path(_context_uid, _repo_id, _commit_id, _path):
953 repo_init = self._factory.repo_libgit2(wire)
971 repo_init = self._factory.repo_libgit2(wire)
954
972
@@ -965,7 +983,8 b' class GitRemote(RemoteBase):'
965 @reraise_safe_exceptions
983 @reraise_safe_exceptions
966 def tree_items(self, wire, tree_id):
984 def tree_items(self, wire, tree_id):
967 cache_on, context_uid, repo_id = self._cache_on(wire)
985 cache_on, context_uid, repo_id = self._cache_on(wire)
968 @self.region.conditional_cache_on_arguments(condition=cache_on)
986 region = self.region(wire)
987 @region.conditional_cache_on_arguments(condition=cache_on)
969 def _tree_items(_repo_id, _tree_id):
988 def _tree_items(_repo_id, _tree_id):
970
989
971 repo_init = self._factory.repo_libgit2(wire)
990 repo_init = self._factory.repo_libgit2(wire)
@@ -1066,7 +1085,8 b' class GitRemote(RemoteBase):'
1066 @reraise_safe_exceptions
1085 @reraise_safe_exceptions
1067 def node_history(self, wire, commit_id, path, limit):
1086 def node_history(self, wire, commit_id, path, limit):
1068 cache_on, context_uid, repo_id = self._cache_on(wire)
1087 cache_on, context_uid, repo_id = self._cache_on(wire)
1069 @self.region.conditional_cache_on_arguments(condition=cache_on)
1088 region = self.region(wire)
1089 @region.conditional_cache_on_arguments(condition=cache_on)
1070 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1090 def _node_history(_context_uid, _repo_id, _commit_id, _path, _limit):
1071 # optimize for n==1, rev-list is much faster for that use-case
1091 # optimize for n==1, rev-list is much faster for that use-case
1072 if limit == 1:
1092 if limit == 1:
@@ -1108,7 +1128,8 b' class GitRemote(RemoteBase):'
1108 def get_all_commit_ids(self, wire):
1128 def get_all_commit_ids(self, wire):
1109
1129
1110 cache_on, context_uid, repo_id = self._cache_on(wire)
1130 cache_on, context_uid, repo_id = self._cache_on(wire)
1111 @self.region.conditional_cache_on_arguments(condition=cache_on)
1131 region = self.region(wire)
1132 @region.conditional_cache_on_arguments(condition=cache_on)
1112 def _get_all_commit_ids(_context_uid, _repo_id):
1133 def _get_all_commit_ids(_context_uid, _repo_id):
1113
1134
1114 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
1135 cmd = ['rev-list', '--reverse', '--date-order', '--branches', '--tags']
@@ -209,7 +209,8 b' class HgRemote(RemoteBase):'
209 @reraise_safe_exceptions
209 @reraise_safe_exceptions
210 def bookmarks(self, wire):
210 def bookmarks(self, wire):
211 cache_on, context_uid, repo_id = self._cache_on(wire)
211 cache_on, context_uid, repo_id = self._cache_on(wire)
212 @self.region.conditional_cache_on_arguments(condition=cache_on)
212 region = self.region(wire)
213 @region.conditional_cache_on_arguments(condition=cache_on)
213 def _bookmarks(_context_uid, _repo_id):
214 def _bookmarks(_context_uid, _repo_id):
214 repo = self._factory.repo(wire)
215 repo = self._factory.repo(wire)
215 return dict(repo._bookmarks)
216 return dict(repo._bookmarks)
@@ -219,7 +220,8 b' class HgRemote(RemoteBase):'
219 @reraise_safe_exceptions
220 @reraise_safe_exceptions
220 def branches(self, wire, normal, closed):
221 def branches(self, wire, normal, closed):
221 cache_on, context_uid, repo_id = self._cache_on(wire)
222 cache_on, context_uid, repo_id = self._cache_on(wire)
222 @self.region.conditional_cache_on_arguments(condition=cache_on)
223 region = self.region(wire)
224 @region.conditional_cache_on_arguments(condition=cache_on)
223 def _branches(_context_uid, _repo_id, _normal, _closed):
225 def _branches(_context_uid, _repo_id, _normal, _closed):
224 repo = self._factory.repo(wire)
226 repo = self._factory.repo(wire)
225 iter_branches = repo.branchmap().iterbranches()
227 iter_branches = repo.branchmap().iterbranches()
@@ -237,7 +239,8 b' class HgRemote(RemoteBase):'
237 @reraise_safe_exceptions
239 @reraise_safe_exceptions
238 def bulk_request(self, wire, commit_id, pre_load):
240 def bulk_request(self, wire, commit_id, pre_load):
239 cache_on, context_uid, repo_id = self._cache_on(wire)
241 cache_on, context_uid, repo_id = self._cache_on(wire)
240 @self.region.conditional_cache_on_arguments(condition=cache_on)
242 region = self.region(wire)
243 @region.conditional_cache_on_arguments(condition=cache_on)
241 def _bulk_request(_repo_id, _commit_id, _pre_load):
244 def _bulk_request(_repo_id, _commit_id, _pre_load):
242 result = {}
245 result = {}
243 for attr in pre_load:
246 for attr in pre_load:
@@ -254,7 +257,8 b' class HgRemote(RemoteBase):'
254 @reraise_safe_exceptions
257 @reraise_safe_exceptions
255 def ctx_branch(self, wire, commit_id):
258 def ctx_branch(self, wire, commit_id):
256 cache_on, context_uid, repo_id = self._cache_on(wire)
259 cache_on, context_uid, repo_id = self._cache_on(wire)
257 @self.region.conditional_cache_on_arguments(condition=cache_on)
260 region = self.region(wire)
261 @region.conditional_cache_on_arguments(condition=cache_on)
258 def _ctx_branch(_repo_id, _commit_id):
262 def _ctx_branch(_repo_id, _commit_id):
259 repo = self._factory.repo(wire)
263 repo = self._factory.repo(wire)
260 ctx = self._get_ctx(repo, commit_id)
264 ctx = self._get_ctx(repo, commit_id)
@@ -264,7 +268,8 b' class HgRemote(RemoteBase):'
264 @reraise_safe_exceptions
268 @reraise_safe_exceptions
265 def ctx_date(self, wire, commit_id):
269 def ctx_date(self, wire, commit_id):
266 cache_on, context_uid, repo_id = self._cache_on(wire)
270 cache_on, context_uid, repo_id = self._cache_on(wire)
267 @self.region.conditional_cache_on_arguments(condition=cache_on)
271 region = self.region(wire)
272 @region.conditional_cache_on_arguments(condition=cache_on)
268 def _ctx_date(_repo_id, _commit_id):
273 def _ctx_date(_repo_id, _commit_id):
269 repo = self._factory.repo(wire)
274 repo = self._factory.repo(wire)
270 ctx = self._get_ctx(repo, commit_id)
275 ctx = self._get_ctx(repo, commit_id)
@@ -280,7 +285,8 b' class HgRemote(RemoteBase):'
280 @reraise_safe_exceptions
285 @reraise_safe_exceptions
281 def ctx_files(self, wire, commit_id):
286 def ctx_files(self, wire, commit_id):
282 cache_on, context_uid, repo_id = self._cache_on(wire)
287 cache_on, context_uid, repo_id = self._cache_on(wire)
283 @self.region.conditional_cache_on_arguments(condition=cache_on)
288 region = self.region(wire)
289 @region.conditional_cache_on_arguments(condition=cache_on)
284 def _ctx_files(_repo_id, _commit_id):
290 def _ctx_files(_repo_id, _commit_id):
285 repo = self._factory.repo(wire)
291 repo = self._factory.repo(wire)
286 ctx = self._get_ctx(repo, commit_id)
292 ctx = self._get_ctx(repo, commit_id)
@@ -297,7 +303,8 b' class HgRemote(RemoteBase):'
297 @reraise_safe_exceptions
303 @reraise_safe_exceptions
298 def ctx_parents(self, wire, commit_id):
304 def ctx_parents(self, wire, commit_id):
299 cache_on, context_uid, repo_id = self._cache_on(wire)
305 cache_on, context_uid, repo_id = self._cache_on(wire)
300 @self.region.conditional_cache_on_arguments(condition=cache_on)
306 region = self.region(wire)
307 @region.conditional_cache_on_arguments(condition=cache_on)
301 def _ctx_parents(_repo_id, _commit_id):
308 def _ctx_parents(_repo_id, _commit_id):
302 repo = self._factory.repo(wire)
309 repo = self._factory.repo(wire)
303 ctx = self._get_ctx(repo, commit_id)
310 ctx = self._get_ctx(repo, commit_id)
@@ -309,7 +316,8 b' class HgRemote(RemoteBase):'
309 @reraise_safe_exceptions
316 @reraise_safe_exceptions
310 def ctx_children(self, wire, commit_id):
317 def ctx_children(self, wire, commit_id):
311 cache_on, context_uid, repo_id = self._cache_on(wire)
318 cache_on, context_uid, repo_id = self._cache_on(wire)
312 @self.region.conditional_cache_on_arguments(condition=cache_on)
319 region = self.region(wire)
320 @region.conditional_cache_on_arguments(condition=cache_on)
313 def _ctx_children(_repo_id, _commit_id):
321 def _ctx_children(_repo_id, _commit_id):
314 repo = self._factory.repo(wire)
322 repo = self._factory.repo(wire)
315 ctx = self._get_ctx(repo, commit_id)
323 ctx = self._get_ctx(repo, commit_id)
@@ -321,7 +329,8 b' class HgRemote(RemoteBase):'
321 @reraise_safe_exceptions
329 @reraise_safe_exceptions
322 def ctx_phase(self, wire, commit_id):
330 def ctx_phase(self, wire, commit_id):
323 cache_on, context_uid, repo_id = self._cache_on(wire)
331 cache_on, context_uid, repo_id = self._cache_on(wire)
324 @self.region.conditional_cache_on_arguments(condition=cache_on)
332 region = self.region(wire)
333 @region.conditional_cache_on_arguments(condition=cache_on)
325 def _ctx_phase(_context_uid, _repo_id, _commit_id):
334 def _ctx_phase(_context_uid, _repo_id, _commit_id):
326 repo = self._factory.repo(wire)
335 repo = self._factory.repo(wire)
327 ctx = self._get_ctx(repo, commit_id)
336 ctx = self._get_ctx(repo, commit_id)
@@ -332,7 +341,8 b' class HgRemote(RemoteBase):'
332 @reraise_safe_exceptions
341 @reraise_safe_exceptions
333 def ctx_obsolete(self, wire, commit_id):
342 def ctx_obsolete(self, wire, commit_id):
334 cache_on, context_uid, repo_id = self._cache_on(wire)
343 cache_on, context_uid, repo_id = self._cache_on(wire)
335 @self.region.conditional_cache_on_arguments(condition=cache_on)
344 region = self.region(wire)
345 @region.conditional_cache_on_arguments(condition=cache_on)
336 def _ctx_obsolete(_context_uid, _repo_id, _commit_id):
346 def _ctx_obsolete(_context_uid, _repo_id, _commit_id):
337 repo = self._factory.repo(wire)
347 repo = self._factory.repo(wire)
338 ctx = self._get_ctx(repo, commit_id)
348 ctx = self._get_ctx(repo, commit_id)
@@ -342,7 +352,8 b' class HgRemote(RemoteBase):'
342 @reraise_safe_exceptions
352 @reraise_safe_exceptions
343 def ctx_hidden(self, wire, commit_id):
353 def ctx_hidden(self, wire, commit_id):
344 cache_on, context_uid, repo_id = self._cache_on(wire)
354 cache_on, context_uid, repo_id = self._cache_on(wire)
345 @self.region.conditional_cache_on_arguments(condition=cache_on)
355 region = self.region(wire)
356 @region.conditional_cache_on_arguments(condition=cache_on)
346 def _ctx_hidden(_context_uid, _repo_id, _commit_id):
357 def _ctx_hidden(_context_uid, _repo_id, _commit_id):
347 repo = self._factory.repo(wire)
358 repo = self._factory.repo(wire)
348 ctx = self._get_ctx(repo, commit_id)
359 ctx = self._get_ctx(repo, commit_id)
@@ -455,7 +466,8 b' class HgRemote(RemoteBase):'
455 @reraise_safe_exceptions
466 @reraise_safe_exceptions
456 def node_history(self, wire, revision, path, limit):
467 def node_history(self, wire, revision, path, limit):
457 cache_on, context_uid, repo_id = self._cache_on(wire)
468 cache_on, context_uid, repo_id = self._cache_on(wire)
458 @self.region.conditional_cache_on_arguments(condition=cache_on)
469 region = self.region(wire)
470 @region.conditional_cache_on_arguments(condition=cache_on)
459 def _node_history(_context_uid, _repo_id, _revision, _path, _limit):
471 def _node_history(_context_uid, _repo_id, _revision, _path, _limit):
460 repo = self._factory.repo(wire)
472 repo = self._factory.repo(wire)
461
473
@@ -485,7 +497,8 b' class HgRemote(RemoteBase):'
485 @reraise_safe_exceptions
497 @reraise_safe_exceptions
486 def node_history_untill(self, wire, revision, path, limit):
498 def node_history_untill(self, wire, revision, path, limit):
487 cache_on, context_uid, repo_id = self._cache_on(wire)
499 cache_on, context_uid, repo_id = self._cache_on(wire)
488 @self.region.conditional_cache_on_arguments(condition=cache_on)
500 region = self.region(wire)
501 @region.conditional_cache_on_arguments(condition=cache_on)
489 def _node_history_until(_context_uid, _repo_id):
502 def _node_history_until(_context_uid, _repo_id):
490 repo = self._factory.repo(wire)
503 repo = self._factory.repo(wire)
491 ctx = self._get_ctx(repo, revision)
504 ctx = self._get_ctx(repo, revision)
@@ -523,7 +536,8 b' class HgRemote(RemoteBase):'
523 @reraise_safe_exceptions
536 @reraise_safe_exceptions
524 def fctx_flags(self, wire, commit_id, path):
537 def fctx_flags(self, wire, commit_id, path):
525 cache_on, context_uid, repo_id = self._cache_on(wire)
538 cache_on, context_uid, repo_id = self._cache_on(wire)
526 @self.region.conditional_cache_on_arguments(condition=cache_on)
539 region = self.region(wire)
540 @region.conditional_cache_on_arguments(condition=cache_on)
527 def _fctx_flags(_repo_id, _commit_id, _path):
541 def _fctx_flags(_repo_id, _commit_id, _path):
528 repo = self._factory.repo(wire)
542 repo = self._factory.repo(wire)
529 ctx = self._get_ctx(repo, commit_id)
543 ctx = self._get_ctx(repo, commit_id)
@@ -535,7 +549,8 b' class HgRemote(RemoteBase):'
535 @reraise_safe_exceptions
549 @reraise_safe_exceptions
536 def fctx_size(self, wire, commit_id, path):
550 def fctx_size(self, wire, commit_id, path):
537 cache_on, context_uid, repo_id = self._cache_on(wire)
551 cache_on, context_uid, repo_id = self._cache_on(wire)
538 @self.region.conditional_cache_on_arguments(condition=cache_on)
552 region = self.region(wire)
553 @region.conditional_cache_on_arguments(condition=cache_on)
539 def _fctx_size(_repo_id, _revision, _path):
554 def _fctx_size(_repo_id, _revision, _path):
540 repo = self._factory.repo(wire)
555 repo = self._factory.repo(wire)
541 ctx = self._get_ctx(repo, commit_id)
556 ctx = self._get_ctx(repo, commit_id)
@@ -546,7 +561,8 b' class HgRemote(RemoteBase):'
546 @reraise_safe_exceptions
561 @reraise_safe_exceptions
547 def get_all_commit_ids(self, wire, name):
562 def get_all_commit_ids(self, wire, name):
548 cache_on, context_uid, repo_id = self._cache_on(wire)
563 cache_on, context_uid, repo_id = self._cache_on(wire)
549 @self.region.conditional_cache_on_arguments(condition=cache_on)
564 region = self.region(wire)
565 @region.conditional_cache_on_arguments(condition=cache_on)
550 def _get_all_commit_ids(_context_uid, _repo_id, _name):
566 def _get_all_commit_ids(_context_uid, _repo_id, _name):
551 repo = self._factory.repo(wire)
567 repo = self._factory.repo(wire)
552 repo = repo.filtered(name)
568 repo = repo.filtered(name)
@@ -562,7 +578,8 b' class HgRemote(RemoteBase):'
562 @reraise_safe_exceptions
578 @reraise_safe_exceptions
563 def is_large_file(self, wire, commit_id, path):
579 def is_large_file(self, wire, commit_id, path):
564 cache_on, context_uid, repo_id = self._cache_on(wire)
580 cache_on, context_uid, repo_id = self._cache_on(wire)
565 @self.region.conditional_cache_on_arguments(condition=cache_on)
581 region = self.region(wire)
582 @region.conditional_cache_on_arguments(condition=cache_on)
566 def _is_large_file(_context_uid, _repo_id, _commit_id, _path):
583 def _is_large_file(_context_uid, _repo_id, _commit_id, _path):
567 return largefiles.lfutil.isstandin(path)
584 return largefiles.lfutil.isstandin(path)
568
585
@@ -572,7 +589,8 b' class HgRemote(RemoteBase):'
572 def is_binary(self, wire, revision, path):
589 def is_binary(self, wire, revision, path):
573 cache_on, context_uid, repo_id = self._cache_on(wire)
590 cache_on, context_uid, repo_id = self._cache_on(wire)
574
591
575 @self.region.conditional_cache_on_arguments(condition=cache_on)
592 region = self.region(wire)
593 @region.conditional_cache_on_arguments(condition=cache_on)
576 def _is_binary(_repo_id, _sha, _path):
594 def _is_binary(_repo_id, _sha, _path):
577 repo = self._factory.repo(wire)
595 repo = self._factory.repo(wire)
578 ctx = self._get_ctx(repo, revision)
596 ctx = self._get_ctx(repo, revision)
@@ -610,7 +628,8 b' class HgRemote(RemoteBase):'
610 def lookup(self, wire, revision, both):
628 def lookup(self, wire, revision, both):
611 cache_on, context_uid, repo_id = self._cache_on(wire)
629 cache_on, context_uid, repo_id = self._cache_on(wire)
612
630
613 @self.region.conditional_cache_on_arguments(condition=cache_on)
631 region = self.region(wire)
632 @region.conditional_cache_on_arguments(condition=cache_on)
614 def _lookup(_context_uid, _repo_id, _revision, _both):
633 def _lookup(_context_uid, _repo_id, _revision, _both):
615
634
616 repo = self._factory.repo(wire)
635 repo = self._factory.repo(wire)
@@ -668,7 +687,8 b' class HgRemote(RemoteBase):'
668 def rev_range(self, wire, commit_filter):
687 def rev_range(self, wire, commit_filter):
669 cache_on, context_uid, repo_id = self._cache_on(wire)
688 cache_on, context_uid, repo_id = self._cache_on(wire)
670
689
671 @self.region.conditional_cache_on_arguments(condition=cache_on)
690 region = self.region(wire)
691 @region.conditional_cache_on_arguments(condition=cache_on)
672 def _rev_range(_context_uid, _repo_id, _filter):
692 def _rev_range(_context_uid, _repo_id, _filter):
673 repo = self._factory.repo(wire)
693 repo = self._factory.repo(wire)
674 revisions = [rev for rev in revrange(repo, commit_filter)]
694 revisions = [rev for rev in revrange(repo, commit_filter)]
@@ -743,7 +763,8 b' class HgRemote(RemoteBase):'
743 @reraise_safe_exceptions
763 @reraise_safe_exceptions
744 def tags(self, wire):
764 def tags(self, wire):
745 cache_on, context_uid, repo_id = self._cache_on(wire)
765 cache_on, context_uid, repo_id = self._cache_on(wire)
746 @self.region.conditional_cache_on_arguments(condition=cache_on)
766 region = self.region(wire)
767 @region.conditional_cache_on_arguments(condition=cache_on)
747 def _tags(_context_uid, _repo_id):
768 def _tags(_context_uid, _repo_id):
748 repo = self._factory.repo(wire)
769 repo = self._factory.repo(wire)
749 return repo.tags()
770 return repo.tags()
@@ -394,7 +394,7 b' class HTTPApplication(object):'
394 else:
394 else:
395 call_args = args[1:]
395 call_args = args[1:]
396
396
397 log.debug('method requested:%s with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
397 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
398 method, call_args, kwargs, context_uid, repo_state_uid)
398 method, call_args, kwargs, context_uid, repo_state_uid)
399
399
400 return payload, remote, method, args, kwargs
400 return payload, remote, method, args, kwargs
@@ -38,7 +38,9 b' register_backend('
38 log = logging.getLogger(__name__)
38 log = logging.getLogger(__name__)
39
39
40 from . import region_meta
40 from . import region_meta
41 from .utils import (get_default_cache_settings, backend_key_generator, make_region)
41 from .utils import (
42 get_default_cache_settings, backend_key_generator, get_or_create_region,
43 clear_cache_namespace, make_region)
42
44
43
45
44 def configure_dogpile_cache(settings):
46 def configure_dogpile_cache(settings):
@@ -16,17 +16,16 b''
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import time
19 import logging
20 import logging
20 import functools
21 import functools
21 import time
22
23 from decorator import decorate
24
22
25 from dogpile.cache import CacheRegion
23 from dogpile.cache import CacheRegion
26 from dogpile.cache.util import compat
24 from dogpile.cache.util import compat
27
25
28 from vcsserver.utils import safe_str, sha1
26 from vcsserver.utils import safe_str, sha1
29
27
28 from vcsserver.lib.rc_cache import region_meta
30
29
31 log = logging.getLogger(__name__)
30 log = logging.getLogger(__name__)
32
31
@@ -50,6 +49,61 b' class RhodeCodeCacheRegion(CacheRegion):'
50 if function_key_generator is None:
49 if function_key_generator is None:
51 function_key_generator = self.function_key_generator
50 function_key_generator = self.function_key_generator
52
51
52 # workaround for py2 and cython problems, this block should be removed
53 # once we've migrated to py3
54 if 'cython' == 'cython':
55 def decorator(fn):
56 if to_str is compat.string_type:
57 # backwards compatible
58 key_generator = function_key_generator(namespace, fn)
59 else:
60 key_generator = function_key_generator(namespace, fn, to_str=to_str)
61
62 @functools.wraps(fn)
63 def decorate(*arg, **kw):
64 key = key_generator(*arg, **kw)
65
66 @functools.wraps(fn)
67 def creator():
68 return fn(*arg, **kw)
69
70 if not condition:
71 return creator()
72
73 timeout = expiration_time() if expiration_time_is_callable \
74 else expiration_time
75
76 return self.get_or_create(key, creator, timeout, should_cache_fn)
77
78 def invalidate(*arg, **kw):
79 key = key_generator(*arg, **kw)
80 self.delete(key)
81
82 def set_(value, *arg, **kw):
83 key = key_generator(*arg, **kw)
84 self.set(key, value)
85
86 def get(*arg, **kw):
87 key = key_generator(*arg, **kw)
88 return self.get(key)
89
90 def refresh(*arg, **kw):
91 key = key_generator(*arg, **kw)
92 value = fn(*arg, **kw)
93 self.set(key, value)
94 return value
95
96 decorate.set = set_
97 decorate.invalidate = invalidate
98 decorate.refresh = refresh
99 decorate.get = get
100 decorate.original = fn
101 decorate.key_generator = key_generator
102 decorate.__wrapped__ = fn
103
104 return decorate
105 return decorator
106
53 def get_or_create_for_user_func(key_generator, user_func, *arg, **kw):
107 def get_or_create_for_user_func(key_generator, user_func, *arg, **kw):
54
108
55 if not condition:
109 if not condition:
@@ -104,7 +158,7 b' class RhodeCodeCacheRegion(CacheRegion):'
104 user_func.original = user_func
158 user_func.original = user_func
105
159
106 # Use `decorate` to preserve the signature of :param:`user_func`.
160 # Use `decorate` to preserve the signature of :param:`user_func`.
107 return decorate(user_func, functools.partial(
161 return decorator.decorate(user_func, functools.partial(
108 get_or_create_for_user_func, key_generator))
162 get_or_create_for_user_func, key_generator))
109
163
110 return cache_decorator
164 return cache_decorator
@@ -156,3 +210,54 b' def key_generator(backend, namespace, fn'
156 return final_key
210 return final_key
157
211
158 return generate_key
212 return generate_key
213
214
215 def get_or_create_region(region_name, region_namespace=None):
216 from vcsserver.lib.rc_cache.backends import FileNamespaceBackend
217 region_obj = region_meta.dogpile_cache_regions.get(region_name)
218 if not region_obj:
219 raise EnvironmentError(
220 'Region `{}` not in configured: {}.'.format(
221 region_name, region_meta.dogpile_cache_regions.keys()))
222
223 region_uid_name = '{}:{}'.format(region_name, region_namespace)
224 if isinstance(region_obj.actual_backend, FileNamespaceBackend):
225 region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
226 if region_exist:
227 log.debug('Using already configured region: %s', region_namespace)
228 return region_exist
229 cache_dir = region_meta.dogpile_config_defaults['cache_dir']
230 expiration_time = region_obj.expiration_time
231
232 if not os.path.isdir(cache_dir):
233 os.makedirs(cache_dir)
234 new_region = make_region(
235 name=region_uid_name,
236 function_key_generator=backend_key_generator(region_obj.actual_backend)
237 )
238 namespace_filename = os.path.join(
239 cache_dir, "{}.cache.dbm".format(region_namespace))
240 # special type that allows 1db per namespace
241 new_region.configure(
242 backend='dogpile.cache.rc.file_namespace',
243 expiration_time=expiration_time,
244 arguments={"filename": namespace_filename}
245 )
246
247 # create and save in region caches
248 log.debug('configuring new region: %s', region_uid_name)
249 region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
250
251 return region_obj
252
253
254 def clear_cache_namespace(cache_region, cache_namespace_uid, invalidate=False):
255 region = get_or_create_region(cache_region, cache_namespace_uid)
256 cache_keys = region.backend.list_keys(prefix=cache_namespace_uid)
257 num_delete_keys = len(cache_keys)
258 if invalidate:
259 region.invalidate(hard=False)
260 else:
261 if num_delete_keys:
262 region.delete_multi(cache_keys)
263 return num_delete_keys
@@ -201,7 +201,8 b' class SvnRemote(RemoteBase):'
201 def revision_properties(self, wire, revision):
201 def revision_properties(self, wire, revision):
202
202
203 cache_on, context_uid, repo_id = self._cache_on(wire)
203 cache_on, context_uid, repo_id = self._cache_on(wire)
204 @self.region.conditional_cache_on_arguments(condition=cache_on)
204 region = self.region(wire)
205 @region.conditional_cache_on_arguments(condition=cache_on)
205 def _revision_properties(_repo_id, _revision):
206 def _revision_properties(_repo_id, _revision):
206 repo = self._factory.repo(wire)
207 repo = self._factory.repo(wire)
207 fs_ptr = svn.repos.fs(repo)
208 fs_ptr = svn.repos.fs(repo)
@@ -255,7 +256,8 b' class SvnRemote(RemoteBase):'
255 @reraise_safe_exceptions
256 @reraise_safe_exceptions
256 def node_history(self, wire, path, revision, limit):
257 def node_history(self, wire, path, revision, limit):
257 cache_on, context_uid, repo_id = self._cache_on(wire)
258 cache_on, context_uid, repo_id = self._cache_on(wire)
258 @self.region.conditional_cache_on_arguments(condition=cache_on)
259 region = self.region(wire)
260 @region.conditional_cache_on_arguments(condition=cache_on)
259 def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
261 def _assert_correct_path(_context_uid, _repo_id, _path, _revision, _limit):
260 cross_copies = False
262 cross_copies = False
261 repo = self._factory.repo(wire)
263 repo = self._factory.repo(wire)
@@ -276,7 +278,8 b' class SvnRemote(RemoteBase):'
276
278
277 def node_properties(self, wire, path, revision):
279 def node_properties(self, wire, path, revision):
278 cache_on, context_uid, repo_id = self._cache_on(wire)
280 cache_on, context_uid, repo_id = self._cache_on(wire)
279 @self.region.conditional_cache_on_arguments(condition=cache_on)
281 region = self.region(wire)
282 @region.conditional_cache_on_arguments(condition=cache_on)
280 def _node_properties(_repo_id, _path, _revision):
283 def _node_properties(_repo_id, _path, _revision):
281 repo = self._factory.repo(wire)
284 repo = self._factory.repo(wire)
282 fsobj = svn.repos.fs(repo)
285 fsobj = svn.repos.fs(repo)
@@ -314,7 +317,8 b' class SvnRemote(RemoteBase):'
314 def get_node_type(self, wire, path, revision=None):
317 def get_node_type(self, wire, path, revision=None):
315
318
316 cache_on, context_uid, repo_id = self._cache_on(wire)
319 cache_on, context_uid, repo_id = self._cache_on(wire)
317 @self.region.conditional_cache_on_arguments(condition=cache_on)
320 region = self.region(wire)
321 @region.conditional_cache_on_arguments(condition=cache_on)
318 def _get_node_type(_repo_id, _path, _revision):
322 def _get_node_type(_repo_id, _path, _revision):
319 repo = self._factory.repo(wire)
323 repo = self._factory.repo(wire)
320 fs_ptr = svn.repos.fs(repo)
324 fs_ptr = svn.repos.fs(repo)
@@ -328,7 +332,8 b' class SvnRemote(RemoteBase):'
328 def get_nodes(self, wire, path, revision=None):
332 def get_nodes(self, wire, path, revision=None):
329
333
330 cache_on, context_uid, repo_id = self._cache_on(wire)
334 cache_on, context_uid, repo_id = self._cache_on(wire)
331 @self.region.conditional_cache_on_arguments(condition=cache_on)
335 region = self.region(wire)
336 @region.conditional_cache_on_arguments(condition=cache_on)
332 def _get_nodes(_repo_id, _path, _revision):
337 def _get_nodes(_repo_id, _path, _revision):
333 repo = self._factory.repo(wire)
338 repo = self._factory.repo(wire)
334 fsobj = svn.repos.fs(repo)
339 fsobj = svn.repos.fs(repo)
@@ -355,7 +360,8 b' class SvnRemote(RemoteBase):'
355 def get_file_size(self, wire, path, revision=None):
360 def get_file_size(self, wire, path, revision=None):
356
361
357 cache_on, context_uid, repo_id = self._cache_on(wire)
362 cache_on, context_uid, repo_id = self._cache_on(wire)
358 @self.region.conditional_cache_on_arguments(condition=cache_on)
363 region = self.region(wire)
364 @region.conditional_cache_on_arguments(condition=cache_on)
359 def _get_file_size(_repo_id, _path, _revision):
365 def _get_file_size(_repo_id, _path, _revision):
360 repo = self._factory.repo(wire)
366 repo = self._factory.repo(wire)
361 fsobj = svn.repos.fs(repo)
367 fsobj = svn.repos.fs(repo)
@@ -470,7 +476,8 b' class SvnRemote(RemoteBase):'
470 def is_binary(self, wire, rev, path):
476 def is_binary(self, wire, rev, path):
471 cache_on, context_uid, repo_id = self._cache_on(wire)
477 cache_on, context_uid, repo_id = self._cache_on(wire)
472
478
473 @self.region.conditional_cache_on_arguments(condition=cache_on)
479 region = self.region(wire)
480 @region.conditional_cache_on_arguments(condition=cache_on)
474 def _is_binary(_repo_id, _rev, _path):
481 def _is_binary(_repo_id, _rev, _path):
475 raw_bytes = self.get_file_content(wire, path, rev)
482 raw_bytes = self.get_file_content(wire, path, rev)
476 return raw_bytes and '\0' in raw_bytes
483 return raw_bytes and '\0' in raw_bytes
@@ -15,13 +15,15 b''
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 from vcsserver.lib import rc_cache
18
19
19 class RemoteBase(object):
20 class RemoteBase(object):
20 EMPTY_COMMIT = '0' * 40
21 EMPTY_COMMIT = '0' * 40
21
22
22 @property
23 def region(self, wire):
23 def region(self):
24 repo_id = wire.get('repo_id', '')
24 return self._factory._cache_region
25 cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
26 return rc_cache.get_or_create_region('repo_object', cache_namespace_uid)
25
27
26 def _cache_on(self, wire):
28 def _cache_on(self, wire):
27 context = wire.get('context', '')
29 context = wire.get('context', '')
@@ -30,3 +32,14 b' class RemoteBase(object):'
30 cache = wire.get('cache', True)
32 cache = wire.get('cache', True)
31 cache_on = context and cache
33 cache_on = context and cache
32 return cache_on, context_uid, repo_id
34 return cache_on, context_uid, repo_id
35
36 def vcsserver_invalidate_cache(self, wire, delete):
37 from vcsserver.lib import rc_cache
38 repo_id = wire.get('repo_id', '')
39
40 if delete:
41 cache_namespace_uid = 'cache_repo.{}'.format(repo_id)
42 rc_cache.clear_cache_namespace(
43 'repo_object', cache_namespace_uid, invalidate=True)
44
45 return {'invalidated': {'repo_id': repo_id, 'delete': delete}}
General Comments 0
You need to be logged in to leave comments. Login now