archive-cache: synced with ce changes

vcsserver/lib/rc_cache/archive_cache/__init__.py

@@ -19,7 +19,7 b' from .fanout_cache import get_archival_c'
 from .fanout_cache import get_archival_config
 
 from .utils import archive_iterator
-from .utils import ArchiveCacheLock
+from .utils import ArchiveCacheGenerationLock
 
 
 def includeme(config):

vcsserver/lib/rc_cache/archive_cache/fanout_cache.py

@@ -111,6 +111,11 b' class FileSystemCache:'
         self._index = index
         self._directory = directory
 
+    @property
+    def directory(self):
+        """Cache directory."""
+        return self._directory
+
     def _write_file(self, full_path, iterator, mode, encoding=None):
         full_dir, _ = os.path.split(full_path)
 
@@ -168,8 +173,17 b' class FileSystemCache:'
 
         return key, size, MODE_BINARY, filename, _metadata
 
-    def fetch(self, key) -> tuple[typing.BinaryIO, dict]:
+    def fetch(self, key, retry=False, retry_attempts=10) -> tuple[typing.BinaryIO, dict]:
+
+        if retry:
+            for attempt in range(retry_attempts):
+                if key in self:
+                    break
+                # we dind't find the key, wait 1s, and re-check
+                time.sleep(1)
+
         if key not in self:
+            log.exception('requested {key} not found in {self}', key, self)
             raise KeyError(key)
 
         key_file = self._get_keyfile(key)
@@ -179,7 +193,7 b' class FileSystemCache:'
         filename = metadata['filename']
 
         try:
-            return open(os.path.join(self._directory, filename), 'rb'), metadata
+            return open(os.path.join(self.directory, filename), 'rb'), metadata
         finally:
             # update usage stats, count and accessed
             metadata["access_count"] = metadata.get("access_count", 0) + 1
@@ -201,7 +215,7 b' class FileSystemCache:'
         sub_dir = os.path.join(hex_name[:2], hex_name[2:4])
         name = hex_name[4:] + '.archive_cache'
         filename = os.path.join(sub_dir, name)
-        full_path = os.path.join(self._directory, filename)
+        full_path = os.path.join(self.directory, filename)
         return filename, full_path
 
     def hash(self, key):
@@ -224,6 +238,9 b' class FileSystemCache:'
         key_file = self._get_keyfile(key)
         return os.path.exists(key_file)
 
+    def __repr__(self):
+        return f'FileSystemCache(index={self._index}, dir={self.directory})'
+
 
 class FanoutCache:
     """Cache that shards keys and values."""
@@ -261,6 +278,11 b' class FanoutCache:'
         )
         self._hash = self._shards[0].hash
 
+    @property
+    def directory(self):
+        """Cache directory."""
+        return self._directory
+
     def get_lock(self, lock_key):
         return GenerationLock(lock_key, self._locking_url)
 
@@ -273,11 +295,11 b' class FanoutCache:'
         shard = self._get_shard(key)
         return shard.store(key, value_reader, metadata)
 
-    def fetch(self, key):
+    def fetch(self, key, retry=False, retry_attempts=10):
         """Return file handle corresponding to `key` from cache.
         """
         shard = self._get_shard(key)
-        return shard.fetch(key)
+        return shard.fetch(key, retry=retry, retry_attempts=retry_attempts)
 
     def has_key(self, key):
         """Return `True` if `key` matching item is found in cache.
@@ -325,9 +347,9 b' class FanoutCache:'
         data = []
         cnt = 1
         for shard in self._shards:
-            for key_file in os.listdir(shard._directory):
+            for key_file in os.listdir(shard.directory):
                 if key_file.endswith('.key'):
-                    key_file_path = os.path.join(shard._directory, key_file)
+                    key_file_path = os.path.join(shard.directory, key_file)
                     with open(key_file_path, 'rb') as f:
                         metadata = json.loads(f.read())
 

vcsserver/lib/rc_cache/archive_cache/lock.py

@@ -17,7 +17,7 b''
 
 import redis
 from ..._vendor import redis_lock
-from .utils import ArchiveCacheLock
+from .utils import ArchiveCacheGenerationLock
 
 
 class GenerationLock:
@@ -52,7 +52,7 b' class GenerationLock:'
     def __enter__(self):
         acquired = self.lock.acquire(blocking=False)
         if not acquired:
-            raise ArchiveCacheLock('Failed to create a lock')
+            raise ArchiveCacheGenerationLock('Failed to create a lock')
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.lock.release()

vcsserver/lib/rc_cache/archive_cache/utils.py

@@ -18,7 +18,7 b''
 import os
 
 
-class ArchiveCacheLock(Exception):
+class ArchiveCacheGenerationLock(Exception):
     pass