Show More
@@ -204,6 +204,7 b' def sanitize_settings_and_apply_defaults' | |||
|
204 | 204 | settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,) |
|
205 | 205 | settings_maker.make_setting('archive_cache.objectstore.key', '') |
|
206 | 206 | settings_maker.make_setting('archive_cache.objectstore.secret', '') |
|
207 | settings_maker.make_setting('archive_cache.objectstore.bucket_root', 'rhodecode-archive-cache') | |
|
207 | 208 | settings_maker.make_setting('archive_cache.objectstore.bucket_shards', 8, parser='int') |
|
208 | 209 | |
|
209 | 210 | settings_maker.make_setting('archive_cache.objectstore.cache_size_gb', 10, parser='float') |
@@ -115,6 +115,7 b' class FileSystemShard(BaseShard):' | |||
|
115 | 115 | |
|
116 | 116 | |
|
117 | 117 | class FileSystemFanoutCache(BaseCache): |
|
118 | shard_name = 'shard_%03d' | |
|
118 | 119 | |
|
119 | 120 | def __init__(self, locking_url, **settings): |
|
120 | 121 | """ |
@@ -150,7 +151,7 b' class FileSystemFanoutCache(BaseCache):' | |||
|
150 | 151 | self._shards = tuple( |
|
151 | 152 | FileSystemShard( |
|
152 | 153 | index=num, |
|
153 |
directory=os.path.join(directory, |
|
|
154 | directory=os.path.join(directory, self.shard_name % num), | |
|
154 | 155 | **settings, |
|
155 | 156 | ) |
|
156 | 157 | for num in range(self._count) |
@@ -41,12 +41,15 b' class S3Shard(BaseShard):' | |||
|
41 | 41 | key = settings.pop('archive_cache.objectstore.key') |
|
42 | 42 | secret = settings.pop('archive_cache.objectstore.secret') |
|
43 | 43 | |
|
44 | # TODO: Add it all over the place... | |
|
45 | self._bucket_root = settings.pop('archive_cache.objectstore.bucket_root') | |
|
46 | ||
|
44 | 47 | self.fs = fsspec.filesystem('s3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret) |
|
45 | 48 | |
|
46 | 49 | @property |
|
47 | 50 | def bucket(self): |
|
48 | 51 | """Cache bucket.""" |
|
49 | return self._bucket | |
|
52 | return os.path.join(self._bucket_root, self._bucket) | |
|
50 | 53 | |
|
51 | 54 | def _get_keyfile(self, archive_key) -> tuple[str, str]: |
|
52 | 55 | key_file = f'{archive_key}-{self.key_suffix}' |
@@ -56,6 +59,10 b' class S3Shard(BaseShard):' | |||
|
56 | 59 | return self.fs.open(path, 'wb') |
|
57 | 60 | |
|
58 | 61 | def _write_file(self, full_path, iterator, mode): |
|
62 | if self._bucket_root: | |
|
63 | if not self.fs.exists(self._bucket_root): | |
|
64 | self.fs.mkdir(self._bucket_root) | |
|
65 | ||
|
59 | 66 | # ensure bucket exists |
|
60 | 67 | destination = self.bucket |
|
61 | 68 | if not self.fs.exists(destination): |
@@ -106,6 +113,7 b' class S3Shard(BaseShard):' | |||
|
106 | 113 | |
|
107 | 114 | |
|
108 | 115 | class ObjectStoreCache(BaseCache): |
|
116 | shard_name = 'shard-bucket-%03d' | |
|
109 | 117 | |
|
110 | 118 | def __init__(self, locking_url, **settings): |
|
111 | 119 | """ |
@@ -134,7 +142,7 b' class ObjectStoreCache(BaseCache):' | |||
|
134 | 142 | self._shards = tuple( |
|
135 | 143 | S3Shard( |
|
136 | 144 | index=num, |
|
137 |
bucket= |
|
|
145 | bucket=self.shard_name % num, | |
|
138 | 146 | **settings, |
|
139 | 147 | ) |
|
140 | 148 | for num in range(self._count) |
General Comments 0
You need to be logged in to leave comments.
Login now