feat(archive-cache): added option to define a configurable top-level bucket for all shards

rhodecode/config/config_maker.py

@@ -1,221 +1,222 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import tempfile
 import logging
 
 from pyramid.settings import asbool
 
 from rhodecode.config.settings_maker import SettingsMaker
 from rhodecode.config import utils as config_utils
 
 log = logging.getLogger(__name__)
 
 
 def sanitize_settings_and_apply_defaults(global_config, settings):
     """
     Applies settings defaults and does all type conversion.
 
     We would move all settings parsing and preparation into this place, so that
     we have only one place left which deals with this part. The remaining parts
     of the application would start to rely fully on well-prepared settings.
 
     This piece would later be split up per topic to avoid a big fat monster
     function.
     """
     jn = os.path.join
 
     global_settings_maker = SettingsMaker(global_config)
     global_settings_maker.make_setting('debug', default=False, parser='bool')
     debug_enabled = asbool(global_config.get('debug'))
 
     settings_maker = SettingsMaker(settings)
 
     settings_maker.make_setting(
         'logging.autoconfigure',
         default=False,
         parser='bool')
 
     logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini')
     settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
 
     # Default includes, possible to change as a user
     pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
     log.debug(
         "Using the following pyramid.includes: %s",
         pyramid_includes)
 
     settings_maker.make_setting('rhodecode.edition', 'Community Edition')
     settings_maker.make_setting('rhodecode.edition_id', 'CE')
 
     if 'mako.default_filters' not in settings:
         # set custom default filters if we don't have it defined
         settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
         settings['mako.default_filters'] = 'h_filter'
 
     if 'mako.directories' not in settings:
         mako_directories = settings.setdefault('mako.directories', [
             # Base templates of the original application
             'rhodecode:templates',
         ])
         log.debug(
             "Using the following Mako template directories: %s",
             mako_directories)
 
     # NOTE(marcink): fix redis requirement for schema of connection since 3.X
     if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
         raw_url = settings['beaker.session.url']
         if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
             settings['beaker.session.url'] = 'redis://' + raw_url
 
     settings_maker.make_setting('__file__', global_config.get('__file__'))
 
     # TODO: johbo: Re-think this, usually the call to config.include
     # should allow to pass in a prefix.
     settings_maker.make_setting('rhodecode.api.url', '/_admin/api')
 
     # Sanitize generic settings.
     settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
     settings_maker.make_setting('gzip_responses', False, parser='bool')
     settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
 
     # statsd
     settings_maker.make_setting('statsd.enabled', False, parser='bool')
     settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
     settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
     settings_maker.make_setting('statsd.statsd_prefix', '')
     settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
 
     settings_maker.make_setting('vcs.svn.compatible_version', '')
     settings_maker.make_setting('vcs.svn.proxy.enabled', True, parser='bool')
     settings_maker.make_setting('vcs.svn.proxy.host', 'http://svn:8090', parser='string')
     settings_maker.make_setting('vcs.hooks.protocol', 'http')
     settings_maker.make_setting('vcs.hooks.host', '*')
     settings_maker.make_setting('vcs.scm_app_implementation', 'http')
     settings_maker.make_setting('vcs.server', '')
     settings_maker.make_setting('vcs.server.protocol', 'http')
     settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
     settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
     settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
     settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
     settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
 
     settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
 
     # repo_store path
     settings_maker.make_setting('repo_store.path', '/var/opt/rhodecode_repo_store')
     # Support legacy values of vcs.scm_app_implementation. Legacy
     # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
     # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
     scm_app_impl = settings['vcs.scm_app_implementation']
     if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
         settings['vcs.scm_app_implementation'] = 'http'
 
     settings_maker.make_setting('appenlight', False, parser='bool')
 
     temp_store = tempfile.gettempdir()
     tmp_cache_dir = jn(temp_store, 'rc_cache')
 
     # save default, cache dir, and use it for all backends later.
     default_cache_dir = settings_maker.make_setting(
         'cache_dir',
         default=tmp_cache_dir, default_when_empty=True,
         parser='dir:ensured')
 
     # exception store cache
     settings_maker.make_setting(
         'exception_tracker.store_path',
         default=jn(default_cache_dir, 'exc_store'), default_when_empty=True,
         parser='dir:ensured'
     )
 
     settings_maker.make_setting(
         'celerybeat-schedule.path',
         default=jn(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
         parser='file:ensured'
     )
 
     settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
     settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
 
     # sessions, ensure file since no-value is memory
     settings_maker.make_setting('beaker.session.type', 'file')
     settings_maker.make_setting('beaker.session.data_dir',  jn(default_cache_dir, 'session_data'))
 
     # cache_general
     settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
     settings_maker.make_setting('rc_cache.cache_general.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_general.db'))
 
     # cache_perms
     settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
     settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_perms_db'))
 
     # cache_repo
     settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
     settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_repo_db'))
 
     # cache_license
     settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
     settings_maker.make_setting('rc_cache.cache_license.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_license_db'))
 
     # cache_repo_longterm memory, 96H
     settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
     settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
     settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
 
     # sql_cache_short
     settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
     settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
     settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
 
     # archive_cache
     settings_maker.make_setting('archive_cache.locking.url', 'redis://redis:6379/1')
     settings_maker.make_setting('archive_cache.backend.type', 'filesystem')
 
     settings_maker.make_setting('archive_cache.filesystem.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
     settings_maker.make_setting('archive_cache.filesystem.cache_shards', 8, parser='int')
     settings_maker.make_setting('archive_cache.filesystem.cache_size_gb', 10, parser='float')
     settings_maker.make_setting('archive_cache.filesystem.eviction_policy', 'least-recently-stored')
 
     settings_maker.make_setting('archive_cache.filesystem.retry', False, parser='bool')
     settings_maker.make_setting('archive_cache.filesystem.retry_backoff', 1, parser='int')
     settings_maker.make_setting('archive_cache.filesystem.retry_attempts', 10, parser='int')
 
     settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
     settings_maker.make_setting('archive_cache.objectstore.key', '')
     settings_maker.make_setting('archive_cache.objectstore.secret', '')
+    settings_maker.make_setting('archive_cache.objectstore.bucket_root', 'rhodecode-archive-cache')
     settings_maker.make_setting('archive_cache.objectstore.bucket_shards', 8, parser='int')
 
     settings_maker.make_setting('archive_cache.objectstore.cache_size_gb', 10, parser='float')
     settings_maker.make_setting('archive_cache.objectstore.eviction_policy', 'least-recently-stored')
 
     settings_maker.make_setting('archive_cache.objectstore.retry', False, parser='bool')
     settings_maker.make_setting('archive_cache.objectstore.retry_backoff', 1, parser='int')
     settings_maker.make_setting('archive_cache.objectstore.retry_attempts', 10, parser='int')
 
     settings_maker.env_expand()
 
     # configure instance id
     config_utils.set_instance_id(settings)
 
     return settings

rhodecode/lib/archive_cache/backends/fanout_cache.py

@@ -1,166 +1,167 b''
 # Copyright (C) 2015-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import codecs
 import hashlib
 import logging
 import os
 
 import fsspec
 
 from .base import BaseCache, BaseShard
 from ..utils import ShardFileReader, NOT_GIVEN
 from ...type_utils import str2bool
 
 log = logging.getLogger(__name__)
 
 
 class FileSystemShard(BaseShard):
 
     def __init__(self, index, directory, **settings):
         self._index = index
         self._directory = directory
         self.storage_type = 'directory'
         self.fs = fsspec.filesystem('file')
 
     @property
     def directory(self):
         """Cache directory."""
         return self._directory
 
     def _get_keyfile(self, archive_key) -> tuple[str, str]:
         key_file = f'{archive_key}.{self.key_suffix}'
         return key_file, os.path.join(self.directory, key_file)
 
     def _get_writer(self, path, mode):
         for count in range(1, 11):
             try:
                 # Another cache may have deleted the directory before
                 # the file could be opened.
                 return self.fs.open(path, mode)
             except OSError:
                 if count == 10:
                     # Give up after 10 tries to open the file.
                     raise
                 continue
 
     def _write_file(self, full_path, iterator, mode):
         # ensure dir exists
         destination, _ = os.path.split(full_path)
         if not self.fs.exists(destination):
             self.fs.makedirs(destination)
 
         writer = self._get_writer(full_path, mode)
 
         digest = hashlib.sha256()
         with writer:
             size = 0
             for chunk in iterator:
                 size += len(chunk)
                 digest.update(chunk)
                 writer.write(chunk)
             writer.flush()
             # Get the file descriptor
             fd = writer.fileno()
 
             # Sync the file descriptor to disk, helps with NFS cases...
             os.fsync(fd)
             sha256 = digest.hexdigest()
             log.debug('written new archive cache under %s, sha256: %s', full_path, sha256)
             return size, sha256
 
     def store(self, key, value_reader, metadata: dict | None = None):
         return self._store(key, value_reader, metadata, mode='xb')
 
     def fetch(self, key, retry=NOT_GIVEN, retry_attempts=NOT_GIVEN, retry_backoff=1) -> tuple[ShardFileReader, dict]:
         return self._fetch(key, retry, retry_attempts, retry_backoff)
 
     def remove(self, key):
         return self._remove(key)
 
     def random_filename(self):
         """Return filename and full-path tuple for file storage.
 
         Filename will be a randomly generated 28 character hexadecimal string
         with ".archive_cache" suffixed. Two levels of sub-directories will be used to
         reduce the size of directories. On older filesystems, lookups in
         directories with many files may be slow.
         """
 
         hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8')
 
         archive_name = hex_name[4:] + '.archive_cache'
         filename = f"{hex_name[:2]}/{hex_name[2:4]}/{archive_name}"
 
         full_path = os.path.join(self.directory, filename)
         return archive_name, full_path
 
     def __repr__(self):
         return f'{self.__class__.__name__}(index={self._index}, dir={self.directory})'
 
 
 class FileSystemFanoutCache(BaseCache):
+    shard_name = 'shard_%03d'
 
     def __init__(self, locking_url, **settings):
         """
         Initialize file system cache instance.
 
         :param str locking_url: redis url for a lock
         :param settings: settings dict
 
         """
         self._locking_url = locking_url
         self._config = settings
         cache_dir = self.get_conf('archive_cache.filesystem.store_dir')
         directory = str(cache_dir)
         directory = os.path.expanduser(directory)
         directory = os.path.expandvars(directory)
         self._directory = directory
         self._storage_path = directory
 
         # check if it's ok to write, and re-create the archive cache
         if not os.path.isdir(self._directory):
             os.makedirs(self._directory, exist_ok=True)
 
         self._count = int(self.get_conf('archive_cache.filesystem.cache_shards', pop=True))
 
         self._eviction_policy = self.get_conf('archive_cache.filesystem.eviction_policy', pop=True)
         self._cache_size_limit = self.gb_to_bytes(int(self.get_conf('archive_cache.filesystem.cache_size_gb')))
 
         self.retry = str2bool(self.get_conf('archive_cache.filesystem.retry', pop=True))
         self.retry_attempts = int(self.get_conf('archive_cache.filesystem.retry_attempts', pop=True))
         self.retry_backoff = int(self.get_conf('archive_cache.filesystem.retry_backoff', pop=True))
 
         log.debug('Initializing archival cache instance under %s', self._directory)
         self._shards = tuple(
             FileSystemShard(
                 index=num,
-                directory=os.path.join(directory, 'shard_%03d' % num),
+                directory=os.path.join(directory, self.shard_name % num),
                 **settings,
             )
             for num in range(self._count)
         )
         self._hash = self._shards[0].hash
 
     def _get_shard(self, key) -> FileSystemShard:
         index = self._hash(key) % self._count
         shard = self._shards[index]
         return shard
 
     def _get_size(self, shard, archive_path):
         return os.stat(archive_path).st_size

rhodecode/lib/archive_cache/backends/objectstore_cache.py

@@ -1,150 +1,158 b''
 # Copyright (C) 2015-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import codecs
 import hashlib
 import logging
 import os
 
 import fsspec
 
 from .base import BaseCache, BaseShard
 from ..utils import ShardFileReader, NOT_GIVEN
 from ...type_utils import str2bool
 
 log = logging.getLogger(__name__)
 
 
 class S3Shard(BaseShard):
 
     def __init__(self, index, bucket, **settings):
         self._index = index
         self._bucket = bucket
         self.storage_type = 'bucket'
 
         endpoint_url = settings.pop('archive_cache.objectstore.url')
         key = settings.pop('archive_cache.objectstore.key')
         secret = settings.pop('archive_cache.objectstore.secret')
 
+        # TODO: Add it all over the place...
+        self._bucket_root = settings.pop('archive_cache.objectstore.bucket_root')
+
         self.fs = fsspec.filesystem('s3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret)
 
     @property
     def bucket(self):
         """Cache bucket."""
-        return self._bucket
+        return os.path.join(self._bucket_root, self._bucket)
 
     def _get_keyfile(self, archive_key) -> tuple[str, str]:
         key_file = f'{archive_key}-{self.key_suffix}'
         return key_file, os.path.join(self.bucket, key_file)
 
     def _get_writer(self, path, mode):
         return self.fs.open(path, 'wb')
 
     def _write_file(self, full_path, iterator, mode):
+        if self._bucket_root:
+            if not self.fs.exists(self._bucket_root):
+                self.fs.mkdir(self._bucket_root)
+
         # ensure bucket exists
         destination = self.bucket
         if not self.fs.exists(destination):
             self.fs.mkdir(destination, s3_additional_kwargs={})
 
         writer = self._get_writer(full_path, mode)
 
         digest = hashlib.sha256()
         with writer:
             size = 0
             for chunk in iterator:
                 size += len(chunk)
                 digest.update(chunk)
                 writer.write(chunk)
 
             sha256 = digest.hexdigest()
             log.debug('written new archive cache under %s, sha256: %s', full_path, sha256)
             return size, sha256
 
     def store(self, key, value_reader, metadata: dict | None = None):
         return self._store(key, value_reader, metadata, mode='wb')
 
     def fetch(self, key, retry=NOT_GIVEN, retry_attempts=NOT_GIVEN, retry_backoff=1) -> tuple[ShardFileReader, dict]:
         return self._fetch(key, retry, retry_attempts, retry_backoff)
 
     def remove(self, key):
         return self._remove(key)
 
     def random_filename(self):
         """Return filename and full-path tuple for file storage.
 
         Filename will be a randomly generated 28 character hexadecimal string
         with ".archive_cache" suffixed. Two levels of sub-directories will be used to
         reduce the size of directories. On older filesystems, lookups in
         directories with many files may be slow.
         """
 
         hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8')
 
         archive_name = hex_name[4:] + '.archive_cache'
         filename = f"{hex_name[:2]}-{hex_name[2:4]}-{archive_name}"
 
         full_path = os.path.join(self.bucket, filename)
         return archive_name, full_path
 
     def __repr__(self):
         return f'{self.__class__.__name__}(index={self._index}, bucket={self.bucket})'
 
 
 class ObjectStoreCache(BaseCache):
+    shard_name = 'shard-bucket-%03d'
 
     def __init__(self, locking_url, **settings):
         """
         Initialize objectstore cache instance.
 
         :param str locking_url: redis url for a lock
         :param settings: settings dict
 
         """
         self._locking_url = locking_url
         self._config = settings
 
         objectstore_url = self.get_conf('archive_cache.objectstore.url')
         self._storage_path = objectstore_url
 
         self._count = int(self.get_conf('archive_cache.objectstore.bucket_shards', pop=True))
 
         self._eviction_policy = self.get_conf('archive_cache.objectstore.eviction_policy', pop=True)
         self._cache_size_limit = self.gb_to_bytes(int(self.get_conf('archive_cache.objectstore.cache_size_gb')))
 
         self.retry = str2bool(self.get_conf('archive_cache.objectstore.retry', pop=True))
         self.retry_attempts = int(self.get_conf('archive_cache.objectstore.retry_attempts', pop=True))
         self.retry_backoff = int(self.get_conf('archive_cache.objectstore.retry_backoff', pop=True))
 
         log.debug('Initializing archival cache instance under %s', objectstore_url)
         self._shards = tuple(
             S3Shard(
                 index=num,
-                bucket='rhodecode-archivecache-%03d' % num,
+                bucket=self.shard_name % num,
                 **settings,
             )
             for num in range(self._count)
         )
         self._hash = self._shards[0].hash
 
     def _get_shard(self, key) -> S3Shard:
         index = self._hash(key) % self._count
         shard = self._shards[index]
         return shard
 
     def _get_size(self, shard, archive_path):
         return shard.fs.info(archive_path)['size']