##// END OF EJS Templates
feat(archive-cache): added option to define a configurable top-level bucket for all shards
super-admin -
r5445:fdcdfe77 default
parent child Browse files
Show More
@@ -204,6 +204,7 b' def sanitize_settings_and_apply_defaults'
204 204 settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
205 205 settings_maker.make_setting('archive_cache.objectstore.key', '')
206 206 settings_maker.make_setting('archive_cache.objectstore.secret', '')
207 settings_maker.make_setting('archive_cache.objectstore.bucket_root', 'rhodecode-archive-cache')
207 208 settings_maker.make_setting('archive_cache.objectstore.bucket_shards', 8, parser='int')
208 209
209 210 settings_maker.make_setting('archive_cache.objectstore.cache_size_gb', 10, parser='float')
@@ -115,6 +115,7 b' class FileSystemShard(BaseShard):'
115 115
116 116
117 117 class FileSystemFanoutCache(BaseCache):
118 shard_name = 'shard_%03d'
118 119
119 120 def __init__(self, locking_url, **settings):
120 121 """
@@ -150,7 +151,7 b' class FileSystemFanoutCache(BaseCache):'
150 151 self._shards = tuple(
151 152 FileSystemShard(
152 153 index=num,
153 directory=os.path.join(directory, 'shard_%03d' % num),
154 directory=os.path.join(directory, self.shard_name % num),
154 155 **settings,
155 156 )
156 157 for num in range(self._count)
@@ -41,12 +41,15 b' class S3Shard(BaseShard):'
41 41 key = settings.pop('archive_cache.objectstore.key')
42 42 secret = settings.pop('archive_cache.objectstore.secret')
43 43
44 # TODO: Add it all over the place...
45 self._bucket_root = settings.pop('archive_cache.objectstore.bucket_root')
46
44 47 self.fs = fsspec.filesystem('s3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret)
45 48
46 49 @property
47 50 def bucket(self):
48 51 """Cache bucket."""
49 return self._bucket
52 return os.path.join(self._bucket_root, self._bucket)
50 53
51 54 def _get_keyfile(self, archive_key) -> tuple[str, str]:
52 55 key_file = f'{archive_key}-{self.key_suffix}'
@@ -56,6 +59,10 b' class S3Shard(BaseShard):'
56 59 return self.fs.open(path, 'wb')
57 60
58 61 def _write_file(self, full_path, iterator, mode):
62 if self._bucket_root:
63 if not self.fs.exists(self._bucket_root):
64 self.fs.mkdir(self._bucket_root)
65
59 66 # ensure bucket exists
60 67 destination = self.bucket
61 68 if not self.fs.exists(destination):
@@ -106,6 +113,7 b' class S3Shard(BaseShard):'
106 113
107 114
108 115 class ObjectStoreCache(BaseCache):
116 shard_name = 'shard-bucket-%03d'
109 117
110 118 def __init__(self, locking_url, **settings):
111 119 """
@@ -134,7 +142,7 b' class ObjectStoreCache(BaseCache):'
134 142 self._shards = tuple(
135 143 S3Shard(
136 144 index=num,
137 bucket='rhodecode-archivecache-%03d' % num,
145 bucket=self.shard_name % num,
138 146 **settings,
139 147 )
140 148 for num in range(self._count)
General Comments 0
You need to be logged in to leave comments. Login now