# HG changeset patch # User RhodeCode Admin # Date 2024-08-28 21:56:59 # Node ID 3496180b7153f4288d931bfa9581447dd0fc1656 # Parent 4c2ddb505af3f2eaa9f1556dce7d9d29b9b07dce feat(artifacts): new artifact storage engines allowing an s3 based uploads diff --git a/configs/development.ini b/configs/development.ini --- a/configs/development.ini +++ b/configs/development.ini @@ -281,15 +281,56 @@ labs_settings_active = true ; optional prefix to Add to email Subject #exception_tracker.email_prefix = [RHODECODE ERROR] -; File store configuration. This is used to store and serve uploaded files -file_store.enabled = true +; NOTE: this setting IS DEPRECATED: +; file_store backend is always enabled +#file_store.enabled = true +; NOTE: this setting IS DEPRECATED: +; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead ; Storage backend, available options are: local -file_store.backend = local +#file_store.backend = local +; NOTE: this setting IS DEPRECATED: +; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead ; path to store the uploaded binaries and artifacts -file_store.storage_path = /var/opt/rhodecode_data/file_store +#file_store.storage_path = /var/opt/rhodecode_data/file_store + +; Artifacts file-store, is used to store comment attachments and artifacts uploads. +; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options +; filesystem_v1 is backwards compat with pre 5.1 storage changes +; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from +; previous installations to keep the artifacts without a need of migration +#file_store.backend.type = filesystem_v2 + +; filesystem options... +#file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/artifacts_file_store + +; filesystem_v2 options... +#file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/artifacts_file_store +#file_store.filesystem_v2.shards = 8 +; objectstore options... +; url for s3 compatible storage that allows to upload artifacts +; e.g http://minio:9000 +#file_store.backend.type = objectstore +#file_store.objectstore.url = http://s3-minio:9000 + +; a top-level bucket to put all other shards in +; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number +#file_store.objectstore.bucket = rhodecode-file-store + +; number of sharded buckets to create to distribute archives across +; default is 8 shards +#file_store.objectstore.bucket_shards = 8 + +; key for s3 auth +#file_store.objectstore.key = s3admin + +; secret for s3 auth +#file_store.objectstore.secret = s3secret4 + +;region for s3 storage +#file_store.objectstore.region = eu-central-1 ; Redis url to acquire/check generation of archives locks archive_cache.locking.url = redis://redis:6379/1 diff --git a/configs/production.ini b/configs/production.ini --- a/configs/production.ini +++ b/configs/production.ini @@ -249,15 +249,56 @@ labs_settings_active = true ; optional prefix to Add to email Subject #exception_tracker.email_prefix = [RHODECODE ERROR] -; File store configuration. This is used to store and serve uploaded files -file_store.enabled = true +; NOTE: this setting IS DEPRECATED: +; file_store backend is always enabled +#file_store.enabled = true +; NOTE: this setting IS DEPRECATED: +; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead ; Storage backend, available options are: local -file_store.backend = local +#file_store.backend = local +; NOTE: this setting IS DEPRECATED: +; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead ; path to store the uploaded binaries and artifacts -file_store.storage_path = /var/opt/rhodecode_data/file_store +#file_store.storage_path = /var/opt/rhodecode_data/file_store + +; Artifacts file-store, is used to store comment attachments and artifacts uploads. +; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options +; filesystem_v1 is backwards compat with pre 5.1 storage changes +; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from +; previous installations to keep the artifacts without a need of migration +#file_store.backend.type = filesystem_v2 + +; filesystem options... +#file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/artifacts_file_store + +; filesystem_v2 options... +#file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/artifacts_file_store +#file_store.filesystem_v2.shards = 8 +; objectstore options... +; url for s3 compatible storage that allows to upload artifacts +; e.g http://minio:9000 +#file_store.backend.type = objectstore +#file_store.objectstore.url = http://s3-minio:9000 + +; a top-level bucket to put all other shards in +; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number +#file_store.objectstore.bucket = rhodecode-file-store + +; number of sharded buckets to create to distribute archives across +; default is 8 shards +#file_store.objectstore.bucket_shards = 8 + +; key for s3 auth +#file_store.objectstore.key = s3admin + +; secret for s3 auth +#file_store.objectstore.secret = s3secret4 + +;region for s3 storage +#file_store.objectstore.region = eu-central-1 ; Redis url to acquire/check generation of archives locks archive_cache.locking.url = redis://redis:6379/1 diff --git a/rhodecode/api/views/server_api.py b/rhodecode/api/views/server_api.py --- a/rhodecode/api/views/server_api.py +++ b/rhodecode/api/views/server_api.py @@ -33,7 +33,7 @@ from rhodecode.lib.ext_json import json from rhodecode.lib.utils2 import safe_int from rhodecode.model.db import UserIpMap from rhodecode.model.scm import ScmModel -from rhodecode.apps.file_store import utils +from rhodecode.apps.file_store import utils as store_utils from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \ FileOverSizeException diff --git a/rhodecode/apps/admin/views/system_info.py b/rhodecode/apps/admin/views/system_info.py --- a/rhodecode/apps/admin/views/system_info.py +++ b/rhodecode/apps/admin/views/system_info.py @@ -171,11 +171,17 @@ class AdminSystemInfoSettingsView(BaseAp (_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')), ('', '', ''), # spacer - (_('Archive cache storage type'), val('storage_archive')['type'], state('storage_archive')), + (_('Artifacts storage backend'), val('storage_artifacts')['type'], state('storage_artifacts')), + (_('Artifacts storage location'), val('storage_artifacts')['path'], state('storage_artifacts')), + (_('Artifacts info'), val('storage_artifacts')['text'], state('storage_artifacts')), + ('', '', ''), # spacer + + (_('Archive cache storage backend'), val('storage_archive')['type'], state('storage_archive')), (_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')), (_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')), ('', '', ''), # spacer + (_('Temp storage location'), val('storage_temp')['path'], state('storage_temp')), (_('Temp storage info'), val('storage_temp')['text'], state('storage_temp')), ('', '', ''), # spacer diff --git a/rhodecode/apps/file_store/__init__.py b/rhodecode/apps/file_store/__init__.py --- a/rhodecode/apps/file_store/__init__.py +++ b/rhodecode/apps/file_store/__init__.py @@ -16,7 +16,8 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ import os -from rhodecode.apps.file_store import config_keys + + from rhodecode.config.settings_maker import SettingsMaker @@ -24,18 +25,48 @@ def _sanitize_settings_and_apply_default """ Set defaults, convert to python types and validate settings. """ + from rhodecode.apps.file_store import config_keys + + # translate "legacy" params into new config + settings.pop(config_keys.deprecated_enabled, True) + if config_keys.deprecated_backend in settings: + # if legacy backend key is detected we use "legacy" backward compat setting + settings.pop(config_keys.deprecated_backend) + settings[config_keys.backend_type] = config_keys.backend_legacy_filesystem + + if config_keys.deprecated_store_path in settings: + store_path = settings.pop(config_keys.deprecated_store_path) + settings[config_keys.legacy_filesystem_storage_path] = store_path + settings_maker = SettingsMaker(settings) - settings_maker.make_setting(config_keys.enabled, True, parser='bool') - settings_maker.make_setting(config_keys.backend, 'local') + default_cache_dir = settings['cache_dir'] + default_store_dir = os.path.join(default_cache_dir, 'artifacts_filestore') + + # set default backend + settings_maker.make_setting(config_keys.backend_type, config_keys.backend_legacy_filesystem) + + # legacy filesystem defaults + settings_maker.make_setting(config_keys.legacy_filesystem_storage_path, default_store_dir, default_when_empty=True, ) - default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store') - settings_maker.make_setting(config_keys.store_path, default_store) + # filesystem defaults + settings_maker.make_setting(config_keys.filesystem_storage_path, default_store_dir, default_when_empty=True,) + settings_maker.make_setting(config_keys.filesystem_shards, 8, parser='int') + + # objectstore defaults + settings_maker.make_setting(config_keys.objectstore_url, 'http://s3-minio:9000') + settings_maker.make_setting(config_keys.objectstore_bucket, 'rhodecode-artifacts-filestore') + settings_maker.make_setting(config_keys.objectstore_bucket_shards, 8, parser='int') + + settings_maker.make_setting(config_keys.objectstore_region, '') + settings_maker.make_setting(config_keys.objectstore_key, '') + settings_maker.make_setting(config_keys.objectstore_secret, '') settings_maker.env_expand() def includeme(config): + from rhodecode.apps.file_store.views import FileStoreView settings = config.registry.settings diff --git a/rhodecode/apps/file_store/backends/base.py b/rhodecode/apps/file_store/backends/base.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/file_store/backends/base.py @@ -0,0 +1,269 @@ +# Copyright (C) 2016-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import os +import fsspec # noqa +import logging + +from rhodecode.lib.ext_json import json + +from rhodecode.apps.file_store.utils import sha256_safe, ShardFileReader, get_uid_filename +from rhodecode.apps.file_store.extensions import resolve_extensions +from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException # noqa: F401 + +log = logging.getLogger(__name__) + + +class BaseShard: + + metadata_suffix: str = '.metadata' + storage_type: str = '' + fs = None + + @property + def storage_medium(self): + if not self.storage_type: + raise ValueError('No storage type set for this shard storage_type=""') + return getattr(self, self.storage_type) + + def __contains__(self, key): + full_path = self.store_path(key) + return self.fs.exists(full_path) + + def metadata_convert(self, uid_filename, metadata): + return metadata + + def get_metadata_filename(self, uid_filename) -> tuple[str, str]: + metadata_file: str = f'{uid_filename}{self.metadata_suffix}' + return metadata_file, self.store_path(metadata_file) + + def get_metadata(self, uid_filename, ignore_missing=False) -> dict: + _metadata_file, metadata_file_path = self.get_metadata_filename(uid_filename) + if ignore_missing and not self.fs.exists(metadata_file_path): + return {} + + with self.fs.open(metadata_file_path, 'rb') as f: + metadata = json.loads(f.read()) + + metadata = self.metadata_convert(uid_filename, metadata) + return metadata + + def _store(self, key: str, uid_key: str, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs): + raise NotImplementedError + + def store(self, key: str, uid_key: str, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs): + return self._store(key, uid_key, value_reader, max_filesize, metadata, **kwargs) + + def _fetch(self, key, presigned_url_expires: int = 0): + raise NotImplementedError + + def fetch(self, key, **kwargs) -> tuple[ShardFileReader, dict]: + return self._fetch(key) + + def _delete(self, key): + if key not in self: + log.exception(f'requested key={key} not found in {self}') + raise KeyError(key) + + metadata = self.get_metadata(key) + _metadata_file, metadata_file_path = self.get_metadata_filename(key) + artifact_file_path = metadata['filename_uid_path'] + self.fs.rm(artifact_file_path) + self.fs.rm(metadata_file_path) + + return 1 + + def delete(self, key): + raise NotImplementedError + + def store_path(self, uid_filename): + raise NotImplementedError + + +class BaseFileStoreBackend: + _shards = tuple() + _shard_cls = BaseShard + _config: dict | None = None + _storage_path: str = '' + + def __init__(self, settings, extension_groups=None): + self._config = settings + extension_groups = extension_groups or ['any'] + self.extensions = resolve_extensions([], groups=extension_groups) + + def __contains__(self, key): + return self.filename_exists(key) + + def __repr__(self): + return f'<{self.__class__.__name__}(storage={self.storage_path})>' + + @property + def storage_path(self): + return self._storage_path + + @classmethod + def get_shard_index(cls, filename: str, num_shards) -> int: + # Generate a hash value from the filename + hash_value = sha256_safe(filename) + + # Convert the hash value to an integer + hash_int = int(hash_value, 16) + + # Map the hash integer to a shard number between 1 and num_shards + shard_number = (hash_int % num_shards) + + return shard_number + + @classmethod + def apply_counter(cls, counter: int, filename: str) -> str: + """ + Apply a counter to the filename. + + :param counter: The counter value to apply. + :param filename: The original filename. + :return: The modified filename with the counter. + """ + name_counted = f'{counter:d}-{filename}' + return name_counted + + def _get_shard(self, key) -> _shard_cls: + index = self.get_shard_index(key, len(self._shards)) + shard = self._shards[index] + return shard + + def get_conf(self, key, pop=False): + if key not in self._config: + raise ValueError( + f"No configuration key '{key}', please make sure it exists in filestore config") + val = self._config[key] + if pop: + del self._config[key] + return val + + def filename_allowed(self, filename, extensions=None): + """Checks if a filename has an allowed extension + + :param filename: base name of file + :param extensions: iterable of extensions (or self.extensions) + """ + _, ext = os.path.splitext(filename) + return self.extension_allowed(ext, extensions) + + def extension_allowed(self, ext, extensions=None): + """ + Checks if an extension is permitted. Both e.g. ".jpg" and + "jpg" can be passed in. Extension lookup is case-insensitive. + + :param ext: extension to check + :param extensions: iterable of extensions to validate against (or self.extensions) + """ + def normalize_ext(_ext): + if _ext.startswith('.'): + _ext = _ext[1:] + return _ext.lower() + + extensions = extensions or self.extensions + if not extensions: + return True + + ext = normalize_ext(ext) + + return ext in [normalize_ext(x) for x in extensions] + + def filename_exists(self, uid_filename): + shard = self._get_shard(uid_filename) + return uid_filename in shard + + def store_path(self, uid_filename): + """ + Returns absolute file path of the uid_filename + """ + shard = self._get_shard(uid_filename) + return shard.store_path(uid_filename) + + def store_metadata(self, uid_filename): + shard = self._get_shard(uid_filename) + return shard.get_metadata_filename(uid_filename) + + def store(self, filename, value_reader, extensions=None, metadata=None, max_filesize=None, randomized_name=True, **kwargs): + extensions = extensions or self.extensions + + if not self.filename_allowed(filename, extensions): + msg = f'filename {filename} does not allow extensions {extensions}' + raise FileNotAllowedException(msg) + + # # TODO: check why we need this setting ? it looks stupid... + # no_body_seek is used in stream mode importer somehow + # no_body_seek = kwargs.pop('no_body_seek', False) + # if no_body_seek: + # pass + # else: + # value_reader.seek(0) + + uid_filename = kwargs.pop('uid_filename', None) + if uid_filename is None: + uid_filename = get_uid_filename(filename, randomized=randomized_name) + + shard = self._get_shard(uid_filename) + + return shard.store(filename, uid_filename, value_reader, max_filesize, metadata, **kwargs) + + def import_to_store(self, value_reader, org_filename, uid_filename, metadata, **kwargs): + shard = self._get_shard(uid_filename) + max_filesize = None + return shard.store(org_filename, uid_filename, value_reader, max_filesize, metadata, import_mode=True) + + def delete(self, uid_filename): + shard = self._get_shard(uid_filename) + return shard.delete(uid_filename) + + def fetch(self, uid_filename) -> tuple[ShardFileReader, dict]: + shard = self._get_shard(uid_filename) + return shard.fetch(uid_filename) + + def get_metadata(self, uid_filename, ignore_missing=False) -> dict: + shard = self._get_shard(uid_filename) + return shard.get_metadata(uid_filename, ignore_missing=ignore_missing) + + def iter_keys(self): + for shard in self._shards: + if shard.fs.exists(shard.storage_medium): + for path, _dirs, _files in shard.fs.walk(shard.storage_medium): + for key_file_path in _files: + if key_file_path.endswith(shard.metadata_suffix): + yield shard, key_file_path + + def iter_artifacts(self): + for shard, key_file in self.iter_keys(): + json_key = f"{shard.storage_medium}/{key_file}" + with shard.fs.open(json_key, 'rb') as f: + yield shard, json.loads(f.read())['filename_uid'] + + def get_statistics(self): + total_files = 0 + total_size = 0 + meta = {} + + for shard, key_file in self.iter_keys(): + json_key = f"{shard.storage_medium}/{key_file}" + with shard.fs.open(json_key, 'rb') as f: + total_files += 1 + metadata = json.loads(f.read()) + total_size += metadata['size'] + + return total_files, total_size, meta diff --git a/rhodecode/apps/file_store/backends/filesystem.py b/rhodecode/apps/file_store/backends/filesystem.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/file_store/backends/filesystem.py @@ -0,0 +1,183 @@ +# Copyright (C) 2016-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import os +import hashlib +import functools +import time +import logging + +from .. import config_keys +from ..exceptions import FileOverSizeException +from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader + +from ....lib.ext_json import json + +log = logging.getLogger(__name__) + + +class FileSystemShard(BaseShard): + METADATA_VER = 'v2' + BACKEND_TYPE = config_keys.backend_filesystem + storage_type: str = 'directory' + + def __init__(self, index, directory, directory_folder, fs, **settings): + self._index: int = index + self._directory: str = directory + self._directory_folder: str = directory_folder + self.fs = fs + + @property + def directory(self) -> str: + """Cache directory final path.""" + return os.path.join(self._directory, self._directory_folder) + + def _write_file(self, full_path, iterator, max_filesize, mode='wb'): + + # ensure dir exists + destination, _ = os.path.split(full_path) + if not self.fs.exists(destination): + self.fs.makedirs(destination) + + writer = self.fs.open(full_path, mode) + + digest = hashlib.sha256() + oversize_cleanup = False + with writer: + size = 0 + for chunk in iterator: + size += len(chunk) + digest.update(chunk) + writer.write(chunk) + + if max_filesize and size > max_filesize: + oversize_cleanup = True + # free up the copied file, and raise exc + break + + writer.flush() + # Get the file descriptor + fd = writer.fileno() + + # Sync the file descriptor to disk, helps with NFS cases... + os.fsync(fd) + + if oversize_cleanup: + self.fs.rm(full_path) + raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}') + + sha256 = digest.hexdigest() + log.debug('written new artifact under %s, sha256: %s', full_path, sha256) + return size, sha256 + + def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs): + + filename = key + uid_filename = uid_key + full_path = self.store_path(uid_filename) + + # STORE METADATA + _metadata = { + "version": self.METADATA_VER, + "store_type": self.BACKEND_TYPE, + + "filename": filename, + "filename_uid_path": full_path, + "filename_uid": uid_filename, + "sha256": "", # NOTE: filled in by reader iteration + + "store_time": time.time(), + + "size": 0 + } + + if metadata: + if kwargs.pop('import_mode', False): + # in import mode, we don't need to compute metadata, we just take the old version + _metadata["import_mode"] = True + else: + _metadata.update(metadata) + + read_iterator = iter(functools.partial(value_reader.read, 2**22), b'') + size, sha256 = self._write_file(full_path, read_iterator, max_filesize) + _metadata['size'] = size + _metadata['sha256'] = sha256 + + # after storing the artifacts, we write the metadata present + _metadata_file, metadata_file_path = self.get_metadata_filename(uid_key) + + with self.fs.open(metadata_file_path, 'wb') as f: + f.write(json.dumps(_metadata)) + + return uid_filename, _metadata + + def store_path(self, uid_filename): + """ + Returns absolute file path of the uid_filename + """ + return os.path.join(self._directory, self._directory_folder, uid_filename) + + def _fetch(self, key, presigned_url_expires: int = 0): + if key not in self: + log.exception(f'requested key={key} not found in {self}') + raise KeyError(key) + + metadata = self.get_metadata(key) + + file_path = metadata['filename_uid_path'] + if presigned_url_expires and presigned_url_expires > 0: + metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires) + + return ShardFileReader(self.fs.open(file_path, 'rb')), metadata + + def delete(self, key): + return self._delete(key) + + +class FileSystemBackend(BaseFileStoreBackend): + shard_name: str = 'shard_{:03d}' + _shard_cls = FileSystemShard + + def __init__(self, settings): + super().__init__(settings) + + store_dir = self.get_conf(config_keys.filesystem_storage_path) + directory = os.path.expanduser(store_dir) + + self._directory = directory + self._storage_path = directory # common path for all from BaseCache + self._shard_count = int(self.get_conf(config_keys.filesystem_shards, pop=True)) + if self._shard_count < 1: + raise ValueError(f'{config_keys.filesystem_shards} must be 1 or more') + + log.debug('Initializing %s file_store instance', self) + fs = fsspec.filesystem('file') + + if not fs.exists(self._directory): + fs.makedirs(self._directory, exist_ok=True) + + self._shards = tuple( + self._shard_cls( + index=num, + directory=directory, + directory_folder=self.shard_name.format(num), + fs=fs, + **settings, + ) + for num in range(self._shard_count) + ) diff --git a/rhodecode/apps/file_store/backends/filesystem_legacy.py b/rhodecode/apps/file_store/backends/filesystem_legacy.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/file_store/backends/filesystem_legacy.py @@ -0,0 +1,278 @@ +# Copyright (C) 2016-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import errno +import os +import hashlib +import functools +import time +import logging + +from .. import config_keys +from ..exceptions import FileOverSizeException +from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader + +from ....lib.ext_json import json + +log = logging.getLogger(__name__) + + +class LegacyFileSystemShard(BaseShard): + # legacy ver + METADATA_VER = 'v2' + BACKEND_TYPE = config_keys.backend_legacy_filesystem + storage_type: str = 'dir_struct' + + # legacy suffix + metadata_suffix: str = '.meta' + + @classmethod + def _sub_store_from_filename(cls, filename): + return filename[:2] + + @classmethod + def apply_counter(cls, counter, filename): + name_counted = '%d-%s' % (counter, filename) + return name_counted + + @classmethod + def safe_make_dirs(cls, dir_path): + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + return + + @classmethod + def resolve_name(cls, name, directory): + """ + Resolves a unique name and the correct path. If a filename + for that path already exists then a numeric prefix with values > 0 will be + added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix. + + :param name: base name of file + :param directory: absolute directory path + """ + + counter = 0 + while True: + name_counted = cls.apply_counter(counter, name) + + # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file + sub_store: str = cls._sub_store_from_filename(name_counted) + sub_store_path: str = os.path.join(directory, sub_store) + cls.safe_make_dirs(sub_store_path) + + path = os.path.join(sub_store_path, name_counted) + if not os.path.exists(path): + return name_counted, path + counter += 1 + + def __init__(self, index, directory, directory_folder, fs, **settings): + self._index: int = index + self._directory: str = directory + self._directory_folder: str = directory_folder + self.fs = fs + + @property + def dir_struct(self) -> str: + """Cache directory final path.""" + return os.path.join(self._directory, '0-') + + def _write_file(self, full_path, iterator, max_filesize, mode='wb'): + + # ensure dir exists + destination, _ = os.path.split(full_path) + if not self.fs.exists(destination): + self.fs.makedirs(destination) + + writer = self.fs.open(full_path, mode) + + digest = hashlib.sha256() + oversize_cleanup = False + with writer: + size = 0 + for chunk in iterator: + size += len(chunk) + digest.update(chunk) + writer.write(chunk) + + if max_filesize and size > max_filesize: + # free up the copied file, and raise exc + oversize_cleanup = True + break + + writer.flush() + # Get the file descriptor + fd = writer.fileno() + + # Sync the file descriptor to disk, helps with NFS cases... + os.fsync(fd) + + if oversize_cleanup: + self.fs.rm(full_path) + raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}') + + sha256 = digest.hexdigest() + log.debug('written new artifact under %s, sha256: %s', full_path, sha256) + return size, sha256 + + def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs): + + filename = key + uid_filename = uid_key + + # NOTE:, also apply N- Counter... + uid_filename, full_path = self.resolve_name(uid_filename, self._directory) + + # STORE METADATA + # TODO: make it compatible, and backward proof + _metadata = { + "version": self.METADATA_VER, + + "filename": filename, + "filename_uid_path": full_path, + "filename_uid": uid_filename, + "sha256": "", # NOTE: filled in by reader iteration + + "store_time": time.time(), + + "size": 0 + } + if metadata: + _metadata.update(metadata) + + read_iterator = iter(functools.partial(value_reader.read, 2**22), b'') + size, sha256 = self._write_file(full_path, read_iterator, max_filesize) + _metadata['size'] = size + _metadata['sha256'] = sha256 + + # after storing the artifacts, we write the metadata present + _metadata_file, metadata_file_path = self.get_metadata_filename(uid_filename) + + with self.fs.open(metadata_file_path, 'wb') as f: + f.write(json.dumps(_metadata)) + + return uid_filename, _metadata + + def store_path(self, uid_filename): + """ + Returns absolute file path of the uid_filename + """ + prefix_dir = '' + if '/' in uid_filename: + prefix_dir, filename = uid_filename.split('/') + sub_store = self._sub_store_from_filename(filename) + else: + sub_store = self._sub_store_from_filename(uid_filename) + + return os.path.join(self._directory, prefix_dir, sub_store, uid_filename) + + def metadata_convert(self, uid_filename, metadata): + # NOTE: backward compat mode here... this is for file created PRE 5.2 system + if 'meta_ver' in metadata: + full_path = self.store_path(uid_filename) + metadata = { + "_converted": True, + "_org": metadata, + "version": self.METADATA_VER, + "store_type": self.BACKEND_TYPE, + + "filename": metadata['filename'], + "filename_uid_path": full_path, + "filename_uid": uid_filename, + "sha256": metadata['sha256'], + + "store_time": metadata['time'], + + "size": metadata['size'] + } + return metadata + + def _fetch(self, key, presigned_url_expires: int = 0): + if key not in self: + log.exception(f'requested key={key} not found in {self}') + raise KeyError(key) + + metadata = self.get_metadata(key) + + file_path = metadata['filename_uid_path'] + if presigned_url_expires and presigned_url_expires > 0: + metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires) + + return ShardFileReader(self.fs.open(file_path, 'rb')), metadata + + def delete(self, key): + return self._delete(key) + + def _delete(self, key): + if key not in self: + log.exception(f'requested key={key} not found in {self}') + raise KeyError(key) + + metadata = self.get_metadata(key) + metadata_file, metadata_file_path = self.get_metadata_filename(key) + artifact_file_path = metadata['filename_uid_path'] + self.fs.rm(artifact_file_path) + self.fs.rm(metadata_file_path) + + def get_metadata_filename(self, uid_filename) -> tuple[str, str]: + + metadata_file: str = f'{uid_filename}{self.metadata_suffix}' + uid_path_in_store = self.store_path(uid_filename) + + metadata_file_path = f'{uid_path_in_store}{self.metadata_suffix}' + return metadata_file, metadata_file_path + + +class LegacyFileSystemBackend(BaseFileStoreBackend): + _shard_cls = LegacyFileSystemShard + + def __init__(self, settings): + super().__init__(settings) + + store_dir = self.get_conf(config_keys.legacy_filesystem_storage_path) + directory = os.path.expanduser(store_dir) + + self._directory = directory + self._storage_path = directory # common path for all from BaseCache + + log.debug('Initializing %s file_store instance', self) + fs = fsspec.filesystem('file') + + if not fs.exists(self._directory): + fs.makedirs(self._directory, exist_ok=True) + + # legacy system uses single shard + self._shards = tuple( + [ + self._shard_cls( + index=0, + directory=directory, + directory_folder='', + fs=fs, + **settings, + ) + ] + ) + + @classmethod + def get_shard_index(cls, filename: str, num_shards) -> int: + # legacy filesystem doesn't use shards, and always uses single shard + return 0 diff --git a/rhodecode/apps/file_store/backends/local_store.py b/rhodecode/apps/file_store/backends/local_store.py deleted file mode 100755 --- a/rhodecode/apps/file_store/backends/local_store.py +++ /dev/null @@ -1,268 +0,0 @@ -# Copyright (C) 2016-2023 RhodeCode GmbH -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License, version 3 -# (only), as published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -# This program is dual-licensed. If you wish to learn more about the -# RhodeCode Enterprise Edition, including its added features, Support services, -# and proprietary license terms, please see https://rhodecode.com/licenses/ - -import os -import time -import errno -import hashlib - -from rhodecode.lib.ext_json import json -from rhodecode.apps.file_store import utils -from rhodecode.apps.file_store.extensions import resolve_extensions -from rhodecode.apps.file_store.exceptions import ( - FileNotAllowedException, FileOverSizeException) - -METADATA_VER = 'v1' - - -def safe_make_dirs(dir_path): - if not os.path.exists(dir_path): - try: - os.makedirs(dir_path) - except OSError as e: - if e.errno != errno.EEXIST: - raise - return - - -class LocalFileStorage(object): - - @classmethod - def apply_counter(cls, counter, filename): - name_counted = '%d-%s' % (counter, filename) - return name_counted - - @classmethod - def resolve_name(cls, name, directory): - """ - Resolves a unique name and the correct path. If a filename - for that path already exists then a numeric prefix with values > 0 will be - added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix. - - :param name: base name of file - :param directory: absolute directory path - """ - - counter = 0 - while True: - name_counted = cls.apply_counter(counter, name) - - # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file - sub_store = cls._sub_store_from_filename(name_counted) - sub_store_path = os.path.join(directory, sub_store) - safe_make_dirs(sub_store_path) - - path = os.path.join(sub_store_path, name_counted) - if not os.path.exists(path): - return name_counted, path - counter += 1 - - @classmethod - def _sub_store_from_filename(cls, filename): - return filename[:2] - - @classmethod - def calculate_path_hash(cls, file_path): - """ - Efficient calculation of file_path sha256 sum - - :param file_path: - :return: sha256sum - """ - digest = hashlib.sha256() - with open(file_path, 'rb') as f: - for chunk in iter(lambda: f.read(1024 * 100), b""): - digest.update(chunk) - - return digest.hexdigest() - - def __init__(self, base_path, extension_groups=None): - - """ - Local file storage - - :param base_path: the absolute base path where uploads are stored - :param extension_groups: extensions string - """ - - extension_groups = extension_groups or ['any'] - self.base_path = base_path - self.extensions = resolve_extensions([], groups=extension_groups) - - def __repr__(self): - return f'{self.__class__}@{self.base_path}' - - def store_path(self, filename): - """ - Returns absolute file path of the filename, joined to the - base_path. - - :param filename: base name of file - """ - prefix_dir = '' - if '/' in filename: - prefix_dir, filename = filename.split('/') - sub_store = self._sub_store_from_filename(filename) - else: - sub_store = self._sub_store_from_filename(filename) - return os.path.join(self.base_path, prefix_dir, sub_store, filename) - - def delete(self, filename): - """ - Deletes the filename. Filename is resolved with the - absolute path based on base_path. If file does not exist, - returns **False**, otherwise **True** - - :param filename: base name of file - """ - if self.exists(filename): - os.remove(self.store_path(filename)) - return True - return False - - def exists(self, filename): - """ - Checks if file exists. Resolves filename's absolute - path based on base_path. - - :param filename: file_uid name of file, e.g 0-f62b2b2d-9708-4079-a071-ec3f958448d4.svg - """ - return os.path.exists(self.store_path(filename)) - - def filename_allowed(self, filename, extensions=None): - """Checks if a filename has an allowed extension - - :param filename: base name of file - :param extensions: iterable of extensions (or self.extensions) - """ - _, ext = os.path.splitext(filename) - return self.extension_allowed(ext, extensions) - - def extension_allowed(self, ext, extensions=None): - """ - Checks if an extension is permitted. Both e.g. ".jpg" and - "jpg" can be passed in. Extension lookup is case-insensitive. - - :param ext: extension to check - :param extensions: iterable of extensions to validate against (or self.extensions) - """ - def normalize_ext(_ext): - if _ext.startswith('.'): - _ext = _ext[1:] - return _ext.lower() - - extensions = extensions or self.extensions - if not extensions: - return True - - ext = normalize_ext(ext) - - return ext in [normalize_ext(x) for x in extensions] - - def save_file(self, file_obj, filename, directory=None, extensions=None, - extra_metadata=None, max_filesize=None, randomized_name=True, **kwargs): - """ - Saves a file object to the uploads location. - Returns the resolved filename, i.e. the directory + - the (randomized/incremented) base name. - - :param file_obj: **cgi.FieldStorage** object (or similar) - :param filename: original filename - :param directory: relative path of sub-directory - :param extensions: iterable of allowed extensions, if not default - :param max_filesize: maximum size of file that should be allowed - :param randomized_name: generate random generated UID or fixed based on the filename - :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix - - """ - - extensions = extensions or self.extensions - - if not self.filename_allowed(filename, extensions): - raise FileNotAllowedException() - - if directory: - dest_directory = os.path.join(self.base_path, directory) - else: - dest_directory = self.base_path - - safe_make_dirs(dest_directory) - - uid_filename = utils.uid_filename(filename, randomized=randomized_name) - - # resolve also produces special sub-dir for file optimized store - filename, path = self.resolve_name(uid_filename, dest_directory) - stored_file_dir = os.path.dirname(path) - - no_body_seek = kwargs.pop('no_body_seek', False) - if no_body_seek: - pass - else: - file_obj.seek(0) - - with open(path, "wb") as dest: - length = 256 * 1024 - while 1: - buf = file_obj.read(length) - if not buf: - break - dest.write(buf) - - metadata = {} - if extra_metadata: - metadata = extra_metadata - - size = os.stat(path).st_size - - if max_filesize and size > max_filesize: - # free up the copied file, and raise exc - os.remove(path) - raise FileOverSizeException() - - file_hash = self.calculate_path_hash(path) - - metadata.update({ - "filename": filename, - "size": size, - "time": time.time(), - "sha256": file_hash, - "meta_ver": METADATA_VER - }) - - filename_meta = filename + '.meta' - with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta: - dest_meta.write(json.dumps(metadata)) - - if directory: - filename = os.path.join(directory, filename) - - return filename, metadata - - def get_metadata(self, filename, ignore_missing=False): - """ - Reads JSON stored metadata for a file - - :param filename: - :return: - """ - filename = self.store_path(filename) - filename_meta = filename + '.meta' - if ignore_missing and not os.path.isfile(filename_meta): - return {} - with open(filename_meta, "rb") as source_meta: - return json.loads(source_meta.read()) diff --git a/rhodecode/apps/file_store/backends/objectstore.py b/rhodecode/apps/file_store/backends/objectstore.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/file_store/backends/objectstore.py @@ -0,0 +1,184 @@ +# Copyright (C) 2016-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import os +import hashlib +import functools +import time +import logging + +from .. import config_keys +from ..exceptions import FileOverSizeException +from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader + +from ....lib.ext_json import json + +log = logging.getLogger(__name__) + + +class S3Shard(BaseShard): + METADATA_VER = 'v2' + BACKEND_TYPE = config_keys.backend_objectstore + storage_type: str = 'bucket' + + def __init__(self, index, bucket, bucket_folder, fs, **settings): + self._index: int = index + self._bucket_main: str = bucket + self._bucket_folder: str = bucket_folder + + self.fs = fs + + @property + def bucket(self) -> str: + """Cache bucket final path.""" + return os.path.join(self._bucket_main, self._bucket_folder) + + def _write_file(self, full_path, iterator, max_filesize, mode='wb'): + + # ensure dir exists + destination, _ = os.path.split(full_path) + if not self.fs.exists(destination): + self.fs.makedirs(destination) + + writer = self.fs.open(full_path, mode) + + digest = hashlib.sha256() + oversize_cleanup = False + with writer: + size = 0 + for chunk in iterator: + size += len(chunk) + digest.update(chunk) + writer.write(chunk) + + if max_filesize and size > max_filesize: + oversize_cleanup = True + # free up the copied file, and raise exc + break + + if oversize_cleanup: + self.fs.rm(full_path) + raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}') + + sha256 = digest.hexdigest() + log.debug('written new artifact under %s, sha256: %s', full_path, sha256) + return size, sha256 + + def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs): + + filename = key + uid_filename = uid_key + full_path = self.store_path(uid_filename) + + # STORE METADATA + _metadata = { + "version": self.METADATA_VER, + "store_type": self.BACKEND_TYPE, + + "filename": filename, + "filename_uid_path": full_path, + "filename_uid": uid_filename, + "sha256": "", # NOTE: filled in by reader iteration + + "store_time": time.time(), + + "size": 0 + } + + if metadata: + if kwargs.pop('import_mode', False): + # in import mode, we don't need to compute metadata, we just take the old version + _metadata["import_mode"] = True + else: + _metadata.update(metadata) + + read_iterator = iter(functools.partial(value_reader.read, 2**22), b'') + size, sha256 = self._write_file(full_path, read_iterator, max_filesize) + _metadata['size'] = size + _metadata['sha256'] = sha256 + + # after storing the artifacts, we write the metadata present + metadata_file, metadata_file_path = self.get_metadata_filename(uid_key) + + with self.fs.open(metadata_file_path, 'wb') as f: + f.write(json.dumps(_metadata)) + + return uid_filename, _metadata + + def store_path(self, uid_filename): + """ + Returns absolute file path of the uid_filename + """ + return os.path.join(self._bucket_main, self._bucket_folder, uid_filename) + + def _fetch(self, key, presigned_url_expires: int = 0): + if key not in self: + log.exception(f'requested key={key} not found in {self}') + raise KeyError(key) + + metadata_file, metadata_file_path = self.get_metadata_filename(key) + with self.fs.open(metadata_file_path, 'rb') as f: + metadata = json.loads(f.read()) + + file_path = metadata['filename_uid_path'] + if presigned_url_expires and presigned_url_expires > 0: + metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires) + + return ShardFileReader(self.fs.open(file_path, 'rb')), metadata + + def delete(self, key): + return self._delete(key) + + +class ObjectStoreBackend(BaseFileStoreBackend): + shard_name: str = 'shard-{:03d}' + _shard_cls = S3Shard + + def __init__(self, settings): + super().__init__(settings) + + self._shard_count = int(self.get_conf(config_keys.objectstore_bucket_shards, pop=True)) + if self._shard_count < 1: + raise ValueError('cache_shards must be 1 or more') + + self._bucket = settings.pop(config_keys.objectstore_bucket) + if not self._bucket: + raise ValueError(f'{config_keys.objectstore_bucket} needs to have a value') + + objectstore_url = self.get_conf(config_keys.objectstore_url) + key = settings.pop(config_keys.objectstore_key) + secret = settings.pop(config_keys.objectstore_secret) + + self._storage_path = objectstore_url # common path for all from BaseCache + log.debug('Initializing %s file_store instance', self) + fs = fsspec.filesystem('s3', anon=False, endpoint_url=objectstore_url, key=key, secret=secret) + + # init main bucket + if not fs.exists(self._bucket): + fs.mkdir(self._bucket) + + self._shards = tuple( + self._shard_cls( + index=num, + bucket=self._bucket, + bucket_folder=self.shard_name.format(num), + fs=fs, + **settings, + ) + for num in range(self._shard_count) + ) diff --git a/rhodecode/apps/file_store/config_keys.py b/rhodecode/apps/file_store/config_keys.py --- a/rhodecode/apps/file_store/config_keys.py +++ b/rhodecode/apps/file_store/config_keys.py @@ -20,6 +20,38 @@ # Definition of setting keys used to configure this module. Defined here to # avoid repetition of keys throughout the module. -enabled = 'file_store.enabled' -backend = 'file_store.backend' -store_path = 'file_store.storage_path' +# OLD and deprecated keys not used anymore +deprecated_enabled = 'file_store.enabled' +deprecated_backend = 'file_store.backend' +deprecated_store_path = 'file_store.storage_path' + + +backend_type = 'file_store.backend.type' + +backend_legacy_filesystem = 'filesystem_v1' +backend_filesystem = 'filesystem_v2' +backend_objectstore = 'objectstore' + +backend_types = [ + backend_legacy_filesystem, + backend_filesystem, + backend_objectstore, +] + +# filesystem_v1 legacy +legacy_filesystem_storage_path = 'file_store.filesystem_v1.storage_path' + + +# filesystem_v2 new option +filesystem_storage_path = 'file_store.filesystem_v2.storage_path' +filesystem_shards = 'file_store.filesystem_v2.shards' + +# objectstore +objectstore_url = 'file_store.objectstore.url' +objectstore_bucket = 'file_store.objectstore.bucket' +objectstore_bucket_shards = 'file_store.objectstore.bucket_shards' + +objectstore_region = 'file_store.objectstore.region' +objectstore_key = 'file_store.objectstore.key' +objectstore_secret = 'file_store.objectstore.secret' + diff --git a/rhodecode/apps/file_store/tests/__init__.py b/rhodecode/apps/file_store/tests/__init__.py --- a/rhodecode/apps/file_store/tests/__init__.py +++ b/rhodecode/apps/file_store/tests/__init__.py @@ -16,3 +16,42 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ +import os +import random +import tempfile +import string + +import pytest + +from rhodecode.apps.file_store import utils as store_utils + + +@pytest.fixture() +def file_store_instance(ini_settings): + config = ini_settings + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + return f_store + + +@pytest.fixture +def random_binary_file(): + # Generate random binary data + data = bytearray(random.getrandbits(8) for _ in range(1024 * 512)) # 512 KB of random data + + # Create a temporary file + temp_file = tempfile.NamedTemporaryFile(delete=False) + filename = temp_file.name + + try: + # Write the random binary data to the file + temp_file.write(data) + temp_file.seek(0) # Rewind the file pointer to the beginning + yield filename, temp_file + finally: + # Close and delete the temporary file after the test + temp_file.close() + os.remove(filename) + + +def generate_random_filename(length=10): + return ''.join(random.choices(string.ascii_letters + string.digits, k=length)) \ No newline at end of file diff --git a/rhodecode/apps/file_store/tests/test_filestore_backends.py b/rhodecode/apps/file_store/tests/test_filestore_backends.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/file_store/tests/test_filestore_backends.py @@ -0,0 +1,128 @@ +# Copyright (C) 2010-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import pytest + +from rhodecode.apps import file_store +from rhodecode.apps.file_store import config_keys +from rhodecode.apps.file_store.backends.filesystem_legacy import LegacyFileSystemBackend +from rhodecode.apps.file_store.backends.filesystem import FileSystemBackend +from rhodecode.apps.file_store.backends.objectstore import ObjectStoreBackend +from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException + +from rhodecode.apps.file_store import utils as store_utils +from rhodecode.apps.file_store.tests import random_binary_file, file_store_instance + + +class TestFileStoreBackends: + + @pytest.mark.parametrize('backend_type, expected_instance', [ + (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend), + (config_keys.backend_filesystem, FileSystemBackend), + (config_keys.backend_objectstore, ObjectStoreBackend), + ]) + def test_get_backend(self, backend_type, expected_instance, ini_settings): + config = ini_settings + config[config_keys.backend_type] = backend_type + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + assert isinstance(f_store, expected_instance) + + @pytest.mark.parametrize('backend_type, expected_instance', [ + (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend), + (config_keys.backend_filesystem, FileSystemBackend), + (config_keys.backend_objectstore, ObjectStoreBackend), + ]) + def test_store_and_read(self, backend_type, expected_instance, ini_settings, random_binary_file): + filename, temp_file = random_binary_file + config = ini_settings + config[config_keys.backend_type] = backend_type + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + metadata = { + 'user_uploaded': { + 'username': 'user1', + 'user_id': 10, + 'ip': '10.20.30.40' + } + } + store_fid, metadata = f_store.store(filename, temp_file, extra_metadata=metadata) + assert store_fid + assert metadata + + # read-after write + reader, metadata2 = f_store.fetch(store_fid) + assert reader + assert metadata2['filename'] == filename + + @pytest.mark.parametrize('backend_type, expected_instance', [ + (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend), + (config_keys.backend_filesystem, FileSystemBackend), + (config_keys.backend_objectstore, ObjectStoreBackend), + ]) + def test_store_file_not_allowed(self, backend_type, expected_instance, ini_settings, random_binary_file): + filename, temp_file = random_binary_file + config = ini_settings + config[config_keys.backend_type] = backend_type + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + with pytest.raises(FileNotAllowedException): + f_store.store('notallowed.exe', temp_file, extensions=['.txt']) + + @pytest.mark.parametrize('backend_type, expected_instance', [ + (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend), + (config_keys.backend_filesystem, FileSystemBackend), + (config_keys.backend_objectstore, ObjectStoreBackend), + ]) + def test_store_file_over_size(self, backend_type, expected_instance, ini_settings, random_binary_file): + filename, temp_file = random_binary_file + config = ini_settings + config[config_keys.backend_type] = backend_type + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + with pytest.raises(FileOverSizeException): + f_store.store('toobig.exe', temp_file, extensions=['.exe'], max_filesize=124) + + @pytest.mark.parametrize('backend_type, expected_instance, extra_conf', [ + (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend, {}), + (config_keys.backend_filesystem, FileSystemBackend, {config_keys.filesystem_storage_path: '/tmp/test-fs-store'}), + (config_keys.backend_objectstore, ObjectStoreBackend, {config_keys.objectstore_bucket: 'test-bucket'}), + ]) + def test_store_stats_and_keys(self, backend_type, expected_instance, extra_conf, ini_settings, random_binary_file): + config = ini_settings + config[config_keys.backend_type] = backend_type + config.update(extra_conf) + + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + + # purge storage before running + for shard, k in f_store.iter_artifacts(): + f_store.delete(k) + + for i in range(10): + filename, temp_file = random_binary_file + + metadata = { + 'user_uploaded': { + 'username': 'user1', + 'user_id': 10, + 'ip': '10.20.30.40' + } + } + store_fid, metadata = f_store.store(filename, temp_file, extra_metadata=metadata) + assert store_fid + assert metadata + + cnt, size, meta = f_store.get_statistics() + assert cnt == 10 + assert 10 == len(list(f_store.iter_keys())) diff --git a/rhodecode/apps/file_store/tests/test_filestore_filesystem_backend.py b/rhodecode/apps/file_store/tests/test_filestore_filesystem_backend.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/file_store/tests/test_filestore_filesystem_backend.py @@ -0,0 +1,52 @@ +# Copyright (C) 2010-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import pytest + +from rhodecode.apps.file_store import utils as store_utils +from rhodecode.apps.file_store import config_keys +from rhodecode.apps.file_store.tests import generate_random_filename + + +@pytest.fixture() +def file_store_filesystem_instance(ini_settings): + config = ini_settings + config[config_keys.backend_type] = config_keys.backend_filesystem + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + return f_store + + +class TestFileStoreFileSystemBackend: + + @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)]) + def test_get_shard_number(self, filename, file_store_filesystem_instance): + shard_number = file_store_filesystem_instance.get_shard_index(filename, len(file_store_filesystem_instance._shards)) + # Check that the shard number is between 0 and max-shards + assert 0 <= shard_number <= len(file_store_filesystem_instance._shards) + + @pytest.mark.parametrize('filename, expected_shard_num', [ + ('my-name-1', 3), + ('my-name-2', 2), + ('my-name-3', 4), + ('my-name-4', 1), + + ('rhodecode-enterprise-ce', 5), + ('rhodecode-enterprise-ee', 6), + ]) + def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_filesystem_instance): + shard_number = file_store_filesystem_instance.get_shard_index(filename, len(file_store_filesystem_instance._shards)) + assert expected_shard_num == shard_number diff --git a/rhodecode/apps/file_store/tests/test_filestore_legacy_and_v2_compatability.py b/rhodecode/apps/file_store/tests/test_filestore_legacy_and_v2_compatability.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/file_store/tests/test_filestore_legacy_and_v2_compatability.py @@ -0,0 +1,17 @@ +# Copyright (C) 2010-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ \ No newline at end of file diff --git a/rhodecode/apps/file_store/tests/test_filestore_legacy_backend.py b/rhodecode/apps/file_store/tests/test_filestore_legacy_backend.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/file_store/tests/test_filestore_legacy_backend.py @@ -0,0 +1,52 @@ +# Copyright (C) 2010-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import pytest + +from rhodecode.apps.file_store import utils as store_utils +from rhodecode.apps.file_store import config_keys +from rhodecode.apps.file_store.tests import generate_random_filename + + +@pytest.fixture() +def file_store_legacy_instance(ini_settings): + config = ini_settings + config[config_keys.backend_type] = config_keys.backend_legacy_filesystem + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + return f_store + + +class TestFileStoreLegacyBackend: + + @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)]) + def test_get_shard_number(self, filename, file_store_legacy_instance): + shard_number = file_store_legacy_instance.get_shard_index(filename, len(file_store_legacy_instance._shards)) + # Check that the shard number is 0 for legacy filesystem store we don't use shards + assert shard_number == 0 + + @pytest.mark.parametrize('filename, expected_shard_num', [ + ('my-name-1', 0), + ('my-name-2', 0), + ('my-name-3', 0), + ('my-name-4', 0), + + ('rhodecode-enterprise-ce', 0), + ('rhodecode-enterprise-ee', 0), + ]) + def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_legacy_instance): + shard_number = file_store_legacy_instance.get_shard_index(filename, len(file_store_legacy_instance._shards)) + assert expected_shard_num == shard_number diff --git a/rhodecode/apps/file_store/tests/test_filestore_objectstore_backend.py b/rhodecode/apps/file_store/tests/test_filestore_objectstore_backend.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/file_store/tests/test_filestore_objectstore_backend.py @@ -0,0 +1,52 @@ +# Copyright (C) 2010-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import pytest + +from rhodecode.apps.file_store import utils as store_utils +from rhodecode.apps.file_store import config_keys +from rhodecode.apps.file_store.tests import generate_random_filename + + +@pytest.fixture() +def file_store_objectstore_instance(ini_settings): + config = ini_settings + config[config_keys.backend_type] = config_keys.backend_objectstore + f_store = store_utils.get_filestore_backend(config=config, always_init=True) + return f_store + + +class TestFileStoreObjectStoreBackend: + + @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)]) + def test_get_shard_number(self, filename, file_store_objectstore_instance): + shard_number = file_store_objectstore_instance.get_shard_index(filename, len(file_store_objectstore_instance._shards)) + # Check that the shard number is between 0 and shards + assert 0 <= shard_number <= len(file_store_objectstore_instance._shards) + + @pytest.mark.parametrize('filename, expected_shard_num', [ + ('my-name-1', 3), + ('my-name-2', 2), + ('my-name-3', 4), + ('my-name-4', 1), + + ('rhodecode-enterprise-ce', 5), + ('rhodecode-enterprise-ee', 6), + ]) + def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_objectstore_instance): + shard_number = file_store_objectstore_instance.get_shard_index(filename, len(file_store_objectstore_instance._shards)) + assert expected_shard_num == shard_number diff --git a/rhodecode/apps/file_store/tests/test_upload_file.py b/rhodecode/apps/file_store/tests/test_upload_file.py --- a/rhodecode/apps/file_store/tests/test_upload_file.py +++ b/rhodecode/apps/file_store/tests/test_upload_file.py @@ -15,13 +15,16 @@ # This program is dual-licensed. If you wish to learn more about the # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ + import os + import pytest from rhodecode.lib.ext_json import json from rhodecode.model.auth_token import AuthTokenModel from rhodecode.model.db import Session, FileStore, Repository, User -from rhodecode.apps.file_store import utils, config_keys +from rhodecode.apps.file_store import utils as store_utils +from rhodecode.apps.file_store import config_keys from rhodecode.tests import TestController from rhodecode.tests.routes import route_path @@ -29,27 +32,61 @@ from rhodecode.tests.routes import route class TestFileStoreViews(TestController): + @pytest.fixture() + def create_artifact_factory(self, tmpdir, ini_settings): + + def factory(user_id, content, f_name='example.txt'): + + config = ini_settings + config[config_keys.backend_type] = config_keys.backend_legacy_filesystem + + f_store = store_utils.get_filestore_backend(config) + + filesystem_file = os.path.join(str(tmpdir), f_name) + with open(filesystem_file, 'wt') as f: + f.write(content) + + with open(filesystem_file, 'rb') as f: + store_uid, metadata = f_store.store(f_name, f, metadata={'filename': f_name}) + os.remove(filesystem_file) + + entry = FileStore.create( + file_uid=store_uid, filename=metadata["filename"], + file_hash=metadata["sha256"], file_size=metadata["size"], + file_display_name='file_display_name', + file_description='repo artifact `{}`'.format(metadata["filename"]), + check_acl=True, user_id=user_id, + ) + Session().add(entry) + Session().commit() + return entry + return factory + @pytest.mark.parametrize("fid, content, exists", [ ('abcde-0.jpg', "xxxxx", True), ('abcde-0.exe', "1234567", True), ('abcde-0.jpg', "xxxxx", False), ]) - def test_get_files_from_store(self, fid, content, exists, tmpdir, user_util): + def test_get_files_from_store(self, fid, content, exists, tmpdir, user_util, ini_settings): user = self.log_user() user_id = user['user_id'] repo_id = user_util.create_repo().repo_id - store_path = self.app._pyramid_settings[config_keys.store_path] + + config = ini_settings + config[config_keys.backend_type] = config_keys.backend_legacy_filesystem + store_uid = fid if exists: status = 200 - store = utils.get_file_storage({config_keys.store_path: store_path}) + f_store = store_utils.get_filestore_backend(config) filesystem_file = os.path.join(str(tmpdir), fid) with open(filesystem_file, 'wt') as f: f.write(content) with open(filesystem_file, 'rb') as f: - store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid}) + store_uid, metadata = f_store.store(fid, f, metadata={'filename': fid}) + os.remove(filesystem_file) entry = FileStore.create( file_uid=store_uid, filename=metadata["filename"], @@ -69,14 +106,10 @@ class TestFileStoreViews(TestController) if exists: assert response.text == content - file_store_path = os.path.dirname(store.resolve_name(store_uid, store_path)[1]) - metadata_file = os.path.join(file_store_path, store_uid + '.meta') - assert os.path.exists(metadata_file) - with open(metadata_file, 'rb') as f: - json_data = json.loads(f.read()) - assert json_data - assert 'size' in json_data + metadata = f_store.get_metadata(store_uid) + + assert 'size' in metadata def test_upload_files_without_content_to_store(self): self.log_user() @@ -112,32 +145,6 @@ class TestFileStoreViews(TestController) assert response.json['store_fid'] - @pytest.fixture() - def create_artifact_factory(self, tmpdir): - def factory(user_id, content): - store_path = self.app._pyramid_settings[config_keys.store_path] - store = utils.get_file_storage({config_keys.store_path: store_path}) - fid = 'example.txt' - - filesystem_file = os.path.join(str(tmpdir), fid) - with open(filesystem_file, 'wt') as f: - f.write(content) - - with open(filesystem_file, 'rb') as f: - store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid}) - - entry = FileStore.create( - file_uid=store_uid, filename=metadata["filename"], - file_hash=metadata["sha256"], file_size=metadata["size"], - file_display_name='file_display_name', - file_description='repo artifact `{}`'.format(metadata["filename"]), - check_acl=True, user_id=user_id, - ) - Session().add(entry) - Session().commit() - return entry - return factory - def test_download_file_non_scoped(self, user_util, create_artifact_factory): user = self.log_user() user_id = user['user_id'] diff --git a/rhodecode/apps/file_store/utils.py b/rhodecode/apps/file_store/utils.py --- a/rhodecode/apps/file_store/utils.py +++ b/rhodecode/apps/file_store/utils.py @@ -19,21 +19,84 @@ import io import uuid import pathlib +import s3fs + +from rhodecode.lib.hash_utils import sha256_safe +from rhodecode.apps.file_store import config_keys + + +file_store_meta = None + + +def get_filestore_config(config) -> dict: + + final_config = {} + + for k, v in config.items(): + if k.startswith('file_store'): + final_config[k] = v + + return final_config -def get_file_storage(settings): - from rhodecode.apps.file_store.backends.local_store import LocalFileStorage - from rhodecode.apps.file_store import config_keys - store_path = settings.get(config_keys.store_path) - return LocalFileStorage(base_path=store_path) +def get_filestore_backend(config, always_init=False): + """ + + usage:: + from rhodecode.apps.file_store import get_filestore_backend + f_store = get_filestore_backend(config=CONFIG) + + :param config: + :param always_init: + :return: + """ + + global file_store_meta + if file_store_meta is not None and not always_init: + return file_store_meta + + config = get_filestore_config(config) + backend = config[config_keys.backend_type] + + match backend: + case config_keys.backend_legacy_filesystem: + # Legacy backward compatible storage + from rhodecode.apps.file_store.backends.filesystem_legacy import LegacyFileSystemBackend + d_cache = LegacyFileSystemBackend( + settings=config + ) + case config_keys.backend_filesystem: + from rhodecode.apps.file_store.backends.filesystem import FileSystemBackend + d_cache = FileSystemBackend( + settings=config + ) + case config_keys.backend_objectstore: + from rhodecode.apps.file_store.backends.objectstore import ObjectStoreBackend + d_cache = ObjectStoreBackend( + settings=config + ) + case _: + raise ValueError( + f'file_store.backend.type only supports "{config_keys.backend_types}" got {backend}' + ) + + cache_meta = d_cache + return cache_meta def splitext(filename): - ext = ''.join(pathlib.Path(filename).suffixes) + final_ext = [] + for suffix in pathlib.Path(filename).suffixes: + if not suffix.isascii(): + continue + + suffix = " ".join(suffix.split()).replace(" ", "") + final_ext.append(suffix) + ext = ''.join(final_ext) return filename, ext -def uid_filename(filename, randomized=True): +def get_uid_filename(filename, randomized=True): """ Generates a randomized or stable (uuid) filename, preserving the original extension. @@ -46,10 +109,37 @@ def uid_filename(filename, randomized=Tr if randomized: uid = uuid.uuid4() else: - hash_key = '{}.{}'.format(filename, 'store') + store_suffix = "store" + hash_key = f'{filename}.{store_suffix}' uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key) return str(uid) + ext.lower() def bytes_to_file_obj(bytes_data): - return io.StringIO(bytes_data) + return io.BytesIO(bytes_data) + + +class ShardFileReader: + + def __init__(self, file_like_reader): + self._file_like_reader = file_like_reader + + def __getattr__(self, item): + if isinstance(self._file_like_reader, s3fs.core.S3File): + match item: + case 'name': + # S3 FileWrapper doesn't support name attribute, and we use it + return self._file_like_reader.full_name + case _: + return getattr(self._file_like_reader, item) + else: + return getattr(self._file_like_reader, item) + + +def archive_iterator(_reader, block_size: int = 4096 * 512): + # 4096 * 64 = 64KB + while 1: + data = _reader.read(block_size) + if not data: + break + yield data diff --git a/rhodecode/apps/file_store/views.py b/rhodecode/apps/file_store/views.py --- a/rhodecode/apps/file_store/views.py +++ b/rhodecode/apps/file_store/views.py @@ -17,12 +17,11 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ import logging - -from pyramid.response import FileResponse +from pyramid.response import Response from pyramid.httpexceptions import HTTPFound, HTTPNotFound from rhodecode.apps._base import BaseAppView -from rhodecode.apps.file_store import utils +from rhodecode.apps.file_store import utils as store_utils from rhodecode.apps.file_store.exceptions import ( FileNotAllowedException, FileOverSizeException) @@ -31,6 +30,7 @@ from rhodecode.lib import audit_logger from rhodecode.lib.auth import ( CSRFRequired, NotAnonymous, HasRepoPermissionAny, HasRepoGroupPermissionAny, LoginRequired) +from rhodecode.lib.str_utils import header_safe_str from rhodecode.lib.vcs.conf.mtypes import get_mimetypes_db from rhodecode.model.db import Session, FileStore, UserApiKeys @@ -42,7 +42,7 @@ class FileStoreView(BaseAppView): def load_default_context(self): c = self._get_local_tmpl_context() - self.storage = utils.get_file_storage(self.request.registry.settings) + self.f_store = store_utils.get_filestore_backend(self.request.registry.settings) return c def _guess_type(self, file_name): @@ -55,8 +55,8 @@ class FileStoreView(BaseAppView): return _content_type, _encoding def _serve_file(self, file_uid): - if not self.storage.exists(file_uid): - store_path = self.storage.store_path(file_uid) + if not self.f_store.filename_exists(file_uid): + store_path = self.f_store.store_path(file_uid) log.warning('File with FID:%s not found in the store under `%s`', file_uid, store_path) raise HTTPNotFound() @@ -98,28 +98,25 @@ class FileStoreView(BaseAppView): FileStore.bump_access_counter(file_uid) - file_path = self.storage.store_path(file_uid) + file_name = db_obj.file_display_name content_type = 'application/octet-stream' - content_encoding = None - _content_type, _encoding = self._guess_type(file_path) + _content_type, _encoding = self._guess_type(file_name) if _content_type: content_type = _content_type # For file store we don't submit any session data, this logic tells the # Session lib to skip it setattr(self.request, '_file_response', True) - response = FileResponse( - file_path, request=self.request, - content_type=content_type, content_encoding=content_encoding) + reader, _meta = self.f_store.fetch(file_uid) - file_name = db_obj.file_display_name + response = Response(app_iter=store_utils.archive_iterator(reader)) - response.headers["Content-Disposition"] = ( - f'attachment; filename="{str(file_name)}"' - ) + response.content_type = str(content_type) + response.content_disposition = f'attachment; filename="{header_safe_str(file_name)}"' + response.headers["X-RC-Artifact-Id"] = str(db_obj.file_store_id) - response.headers["X-RC-Artifact-Desc"] = str(db_obj.file_description) + response.headers["X-RC-Artifact-Desc"] = header_safe_str(db_obj.file_description) response.headers["X-RC-Artifact-Sha256"] = str(db_obj.file_hash) return response @@ -147,8 +144,8 @@ class FileStoreView(BaseAppView): 'user_id': self._rhodecode_user.user_id, 'ip': self._rhodecode_user.ip_addr}} try: - store_uid, metadata = self.storage.save_file( - file_obj.file, filename, extra_metadata=metadata) + store_uid, metadata = self.f_store.store( + filename, file_obj.file, extra_metadata=metadata) except FileNotAllowedException: return {'store_fid': None, 'access_path': None, @@ -182,7 +179,7 @@ class FileStoreView(BaseAppView): def download_file(self): self.load_default_context() file_uid = self.request.matchdict['fid'] - log.debug('Requesting FID:%s from store %s', file_uid, self.storage) + log.debug('Requesting FID:%s from store %s', file_uid, self.f_store) return self._serve_file(file_uid) # in addition to @LoginRequired ACL is checked by scopes diff --git a/rhodecode/apps/repository/views/repo_commits.py b/rhodecode/apps/repository/views/repo_commits.py --- a/rhodecode/apps/repository/views/repo_commits.py +++ b/rhodecode/apps/repository/views/repo_commits.py @@ -601,26 +601,26 @@ class RepoCommitsView(RepoAppView): max_file_size = 10 * 1024 * 1024 # 10MB, also validated via dropzone.js try: - storage = store_utils.get_file_storage(self.request.registry.settings) - store_uid, metadata = storage.save_file( - file_obj.file, filename, extra_metadata=metadata, + f_store = store_utils.get_filestore_backend(self.request.registry.settings) + store_uid, metadata = f_store.store( + filename, file_obj.file, metadata=metadata, extensions=allowed_extensions, max_filesize=max_file_size) except FileNotAllowedException: self.request.response.status = 400 permitted_extensions = ', '.join(allowed_extensions) - error_msg = 'File `{}` is not allowed. ' \ - 'Only following extensions are permitted: {}'.format( - filename, permitted_extensions) + error_msg = f'File `{filename}` is not allowed. ' \ + f'Only following extensions are permitted: {permitted_extensions}' + return {'store_fid': None, 'access_path': None, 'error': error_msg} except FileOverSizeException: self.request.response.status = 400 limit_mb = h.format_byte_size_binary(max_file_size) + error_msg = f'File {filename} is exceeding allowed limit of {limit_mb}.' return {'store_fid': None, 'access_path': None, - 'error': 'File {} is exceeding allowed limit of {}.'.format( - filename, limit_mb)} + 'error': error_msg} try: entry = FileStore.create( diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py --- a/rhodecode/apps/repository/views/repo_files.py +++ b/rhodecode/apps/repository/views/repo_files.py @@ -48,7 +48,7 @@ from rhodecode.lib.codeblocks import ( filenode_as_lines_tokens, filenode_as_annotated_lines_tokens) from rhodecode.lib.utils2 import convert_line_endings, detect_mode from rhodecode.lib.type_utils import str2bool -from rhodecode.lib.str_utils import safe_str, safe_int +from rhodecode.lib.str_utils import safe_str, safe_int, header_safe_str from rhodecode.lib.auth import ( LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired) from rhodecode.lib.vcs import path as vcspath @@ -820,7 +820,7 @@ class RepoFilesView(RepoAppView): "filename=\"{}\"; " \ "filename*=UTF-8\'\'{}".format(safe_path, encoded_path) - return safe_bytes(headers).decode('latin-1', errors='replace') + return header_safe_str(headers) @LoginRequired() @HasRepoPermissionAnyDecorator( diff --git a/rhodecode/apps/repository/views/repo_settings_advanced.py b/rhodecode/apps/repository/views/repo_settings_advanced.py --- a/rhodecode/apps/repository/views/repo_settings_advanced.py +++ b/rhodecode/apps/repository/views/repo_settings_advanced.py @@ -29,7 +29,7 @@ from rhodecode.lib import audit_logger from rhodecode.lib.auth import ( LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired, HasRepoPermissionAny) -from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError +from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError from rhodecode.lib.utils2 import safe_int from rhodecode.lib.vcs import RepositoryError from rhodecode.model.db import Session, UserFollowing, User, Repository @@ -136,6 +136,9 @@ class RepoSettingsAdvancedView(RepoAppVi elif handle_forks == 'delete_forks': handle_forks = 'delete' + repo_advanced_url = h.route_path( + 'edit_repo_advanced', repo_name=self.db_repo_name, + _anchor='advanced-delete') try: old_data = self.db_repo.get_api_data() RepoModel().delete(self.db_repo, forks=handle_forks) @@ -158,9 +161,6 @@ class RepoSettingsAdvancedView(RepoAppVi category='success') Session().commit() except AttachedForksError: - repo_advanced_url = h.route_path( - 'edit_repo_advanced', repo_name=self.db_repo_name, - _anchor='advanced-delete') delete_anchor = h.link_to(_('detach or delete'), repo_advanced_url) h.flash(_('Cannot delete `{repo}` it still contains attached forks. ' 'Try using {delete_or_detach} option.') @@ -171,9 +171,6 @@ class RepoSettingsAdvancedView(RepoAppVi raise HTTPFound(repo_advanced_url) except AttachedPullRequestsError: - repo_advanced_url = h.route_path( - 'edit_repo_advanced', repo_name=self.db_repo_name, - _anchor='advanced-delete') attached_prs = len(self.db_repo.pull_requests_source + self.db_repo.pull_requests_target) h.flash( @@ -184,6 +181,16 @@ class RepoSettingsAdvancedView(RepoAppVi # redirect to advanced for forks handle action ? raise HTTPFound(repo_advanced_url) + except AttachedArtifactsError: + + attached_artifacts = len(self.db_repo.artifacts) + h.flash( + _('Cannot delete `{repo}` it still contains {num} attached artifacts. ' + 'Consider archiving the repository instead.').format( + repo=self.db_repo_name, num=attached_artifacts), category='warning') + + # redirect to advanced for forks handle action ? + raise HTTPFound(repo_advanced_url) except Exception: log.exception("Exception during deletion of repository") h.flash(_('An error occurred during deletion of `%s`') diff --git a/rhodecode/config/config_maker.py b/rhodecode/config/config_maker.py --- a/rhodecode/config/config_maker.py +++ b/rhodecode/config/config_maker.py @@ -206,7 +206,7 @@ def sanitize_settings_and_apply_defaults settings_maker.make_setting('archive_cache.filesystem.retry_backoff', 1, parser='int') settings_maker.make_setting('archive_cache.filesystem.retry_attempts', 10, parser='int') - settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,) + settings_maker.make_setting('archive_cache.objectstore.url', 'http://s3-minio:9000', default_when_empty=True,) settings_maker.make_setting('archive_cache.objectstore.key', '') settings_maker.make_setting('archive_cache.objectstore.secret', '') settings_maker.make_setting('archive_cache.objectstore.region', 'eu-central-1') diff --git a/rhodecode/config/environment.py b/rhodecode/config/environment.py --- a/rhodecode/config/environment.py +++ b/rhodecode/config/environment.py @@ -16,7 +16,6 @@ # RhodeCode Enterprise Edition, including its added features, Support services, # and proprietary license terms, please see https://rhodecode.com/licenses/ -import os import logging import rhodecode import collections @@ -30,6 +29,21 @@ from rhodecode.lib.vcs import connect_vc log = logging.getLogger(__name__) +def propagate_rhodecode_config(global_config, settings, config): + # Store the settings to make them available to other modules. + settings_merged = global_config.copy() + settings_merged.update(settings) + if config: + settings_merged.update(config) + + rhodecode.PYRAMID_SETTINGS = settings_merged + rhodecode.CONFIG = settings_merged + + if 'default_user_id' not in rhodecode.CONFIG: + rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id() + log.debug('set rhodecode.CONFIG data') + + def load_pyramid_environment(global_config, settings): # Some parts of the code expect a merge of global and app settings. settings_merged = global_config.copy() @@ -75,11 +89,8 @@ def load_pyramid_environment(global_conf utils.configure_vcs(settings) - # Store the settings to make them available to other modules. - - rhodecode.PYRAMID_SETTINGS = settings_merged - rhodecode.CONFIG = settings_merged - rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id() + # first run, to store data... + propagate_rhodecode_config(global_config, settings, {}) if vcs_server_enabled: connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings)) diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py --- a/rhodecode/config/middleware.py +++ b/rhodecode/config/middleware.py @@ -115,6 +115,9 @@ def make_pyramid_app(global_config, **se celery_settings = get_celery_config(settings) config.configure_celery(celery_settings) + # final config set... + propagate_rhodecode_config(global_config, settings, config.registry.settings) + # creating the app uses a connection - return it after we are done meta.Session.remove() diff --git a/rhodecode/lib/exceptions.py b/rhodecode/lib/exceptions.py --- a/rhodecode/lib/exceptions.py +++ b/rhodecode/lib/exceptions.py @@ -80,6 +80,10 @@ class AttachedPullRequestsError(Exceptio pass +class AttachedArtifactsError(Exception): + pass + + class RepoGroupAssignmentError(Exception): pass diff --git a/rhodecode/lib/helpers.py b/rhodecode/lib/helpers.py --- a/rhodecode/lib/helpers.py +++ b/rhodecode/lib/helpers.py @@ -81,7 +81,7 @@ from rhodecode.lib.action_parser import from rhodecode.lib.html_filters import sanitize_html from rhodecode.lib.pagination import Page, RepoPage, SqlPage from rhodecode.lib import ext_json -from rhodecode.lib.ext_json import json +from rhodecode.lib.ext_json import json, formatted_str_json from rhodecode.lib.str_utils import safe_bytes, convert_special_chars, base64_to_str from rhodecode.lib.utils import repo_name_slug, get_custom_lexer from rhodecode.lib.str_utils import safe_str @@ -1416,62 +1416,14 @@ class InitialsGravatar(object): return "data:image/svg+xml;base64,{}".format(img_data) -def initials_gravatar(request, email_address, first_name, last_name, size=30, store_on_disk=False): +def initials_gravatar(request, email_address, first_name, last_name, size=30): svg_type = None if email_address == User.DEFAULT_USER_EMAIL: svg_type = 'default_user' klass = InitialsGravatar(email_address, first_name, last_name, size) - - if store_on_disk: - from rhodecode.apps.file_store import utils as store_utils - from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \ - FileOverSizeException - from rhodecode.model.db import Session - - image_key = md5_safe(email_address.lower() - + first_name.lower() + last_name.lower()) - - storage = store_utils.get_file_storage(request.registry.settings) - filename = '{}.svg'.format(image_key) - subdir = 'gravatars' - # since final name has a counter, we apply the 0 - uid = storage.apply_counter(0, store_utils.uid_filename(filename, randomized=False)) - store_uid = os.path.join(subdir, uid) - - db_entry = FileStore.get_by_store_uid(store_uid) - if db_entry: - return request.route_path('download_file', fid=store_uid) - - img_data = klass.get_img_data(svg_type=svg_type) - img_file = store_utils.bytes_to_file_obj(img_data) - - try: - store_uid, metadata = storage.save_file( - img_file, filename, directory=subdir, - extensions=['.svg'], randomized_name=False) - except (FileNotAllowedException, FileOverSizeException): - raise - - try: - entry = FileStore.create( - file_uid=store_uid, filename=metadata["filename"], - file_hash=metadata["sha256"], file_size=metadata["size"], - file_display_name=filename, - file_description=f'user gravatar `{safe_str(filename)}`', - hidden=True, check_acl=False, user_id=1 - ) - Session().add(entry) - Session().commit() - log.debug('Stored upload in DB as %s', entry) - except Exception: - raise - - return request.route_path('download_file', fid=store_uid) - - else: - return klass.generate_svg(svg_type=svg_type) + return klass.generate_svg(svg_type=svg_type) def gravatar_external(request, gravatar_url_tmpl, email_address, size=30): diff --git a/rhodecode/lib/rc_commands/add_artifact.py b/rhodecode/lib/rc_commands/add_artifact.py --- a/rhodecode/lib/rc_commands/add_artifact.py +++ b/rhodecode/lib/rc_commands/add_artifact.py @@ -91,15 +91,14 @@ def command(ini_path, filename, file_pat auth_user = db_user.AuthUser(ip_addr='127.0.0.1') - storage = store_utils.get_file_storage(request.registry.settings) + f_store = store_utils.get_filestore_backend(request.registry.settings) with open(file_path, 'rb') as f: click.secho(f'Adding new artifact from path: `{file_path}`', fg='green') file_data = _store_file( - storage, auth_user, filename, content=None, check_acl=True, + f_store, auth_user, filename, content=None, check_acl=True, file_obj=f, description=description, scope_repo_id=repo.repo_id) - click.secho(f'File Data: {file_data}', - fg='green') + click.secho(f'File Data: {file_data}', fg='green') diff --git a/rhodecode/lib/rc_commands/migrate_artifact.py b/rhodecode/lib/rc_commands/migrate_artifact.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/rc_commands/migrate_artifact.py @@ -0,0 +1,122 @@ +# Copyright (C) 2016-2023 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import sys +import logging + +import click + +from rhodecode.lib.pyramid_utils import bootstrap +from rhodecode.lib.ext_json import json +from rhodecode.model.db import FileStore +from rhodecode.apps.file_store import utils as store_utils + +log = logging.getLogger(__name__) + + +@click.command() +@click.argument('ini_path', type=click.Path(exists=True)) +@click.argument('file_uid') +@click.option( + '--source-backend-conf', + type=click.Path(exists=True, dir_okay=False, readable=True), + help='Source backend config file path in a json format' +) +@click.option( + '--dest-backend-conf', + type=click.Path(exists=True, dir_okay=False, readable=True), + help='Source backend config file path in a json format' +) +def main(ini_path, file_uid, source_backend_conf, dest_backend_conf): + return command(ini_path, file_uid, source_backend_conf, dest_backend_conf) + + +_source_settings = {} + +_dest_settings = {} + + +def command(ini_path, file_uid, source_backend_conf, dest_backend_conf): + with bootstrap(ini_path, env={'RC_CMD_SETUP_RC': '1'}) as env: + migrate_func(env, file_uid, source_backend_conf, dest_backend_conf) + + +def migrate_func(env, file_uid, source_backend_conf=None, dest_backend_conf=None): + """ + + Example usage:: + + from rhodecode.lib.rc_commands import migrate_artifact + migrate_artifact._source_settings = { + 'file_store.backend.type': 'filesystem_v1', + 'file_store.filesystem_v1.storage_path': '/var/opt/rhodecode_data/file_store', + } + migrate_artifact._dest_settings = { + 'file_store.backend.type': 'objectstore', + 'file_store.objectstore.url': 'http://s3-minio:9000', + 'file_store.objectstore.bucket': 'rhodecode-file-store', + 'file_store.objectstore.key': 's3admin', + 'file_store.objectstore.secret': 's3secret4', + 'file_store.objectstore.region': 'eu-central-1', + } + for db_obj in FileStore.query().all(): + migrate_artifact.migrate_func({}, db_obj.file_uid) + + """ + + try: + from rc_ee.api.views.store_api import _store_file + except ImportError: + click.secho('ERROR: Unable to import store_api. ' + 'store_api is only available in EE edition of RhodeCode', + fg='red') + sys.exit(-1) + + source_settings = _source_settings + if source_backend_conf: + source_settings = json.loads(open(source_backend_conf).read()) + dest_settings = _dest_settings + if dest_backend_conf: + dest_settings = json.loads(open(dest_backend_conf).read()) + + if file_uid.isnumeric(): + file_store_db_obj = FileStore().query() \ + .filter(FileStore.file_store_id == file_uid) \ + .scalar() + else: + file_store_db_obj = FileStore().query() \ + .filter(FileStore.file_uid == file_uid) \ + .scalar() + if not file_store_db_obj: + click.secho(f'ERROR: Unable to fetch artifact from database file_uid={file_uid}', + fg='red') + sys.exit(-1) + + uid_filename = file_store_db_obj.file_uid + org_filename = file_store_db_obj.file_display_name + click.secho(f'Attempting to migrate artifact {uid_filename}, filename: {org_filename}', fg='green') + + # get old version of f_store based on the data. + + origin_f_store = store_utils.get_filestore_backend(source_settings, always_init=True) + reader, metadata = origin_f_store.fetch(uid_filename) + + target_f_store = store_utils.get_filestore_backend(dest_settings, always_init=True) + target_f_store.import_to_store(reader, org_filename, uid_filename, metadata) + + click.secho(f'Migrated artifact {uid_filename}, filename: {org_filename} into {target_f_store} storage', fg='green') diff --git a/rhodecode/lib/str_utils.py b/rhodecode/lib/str_utils.py --- a/rhodecode/lib/str_utils.py +++ b/rhodecode/lib/str_utils.py @@ -181,3 +181,7 @@ def splitnewlines(text: bytes): else: lines[-1] = lines[-1][:-1] return lines + + +def header_safe_str(val): + return safe_bytes(val).decode('latin-1', errors='replace') diff --git a/rhodecode/lib/system_info.py b/rhodecode/lib/system_info.py --- a/rhodecode/lib/system_info.py +++ b/rhodecode/lib/system_info.py @@ -396,17 +396,18 @@ def storage_inodes(): @register_sysinfo -def storage_archives(): +def storage_artifacts(): import rhodecode from rhodecode.lib.helpers import format_byte_size_binary from rhodecode.lib.archive_cache import get_archival_cache_store - storage_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type') + backend_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type') - value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=storage_type) + value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=backend_type) state = STATE_OK_DEFAULT try: d_cache = get_archival_cache_store(config=rhodecode.CONFIG) + backend_type = str(d_cache) total_files, total_size, _directory_stats = d_cache.get_statistics() @@ -415,7 +416,8 @@ def storage_archives(): 'used': total_size, 'total': total_size, 'items': total_files, - 'path': d_cache.storage_path + 'path': d_cache.storage_path, + 'type': backend_type }) except Exception as e: @@ -425,8 +427,44 @@ def storage_archives(): human_value = value.copy() human_value['used'] = format_byte_size_binary(value['used']) human_value['total'] = format_byte_size_binary(value['total']) - human_value['text'] = "{} ({} items)".format( - human_value['used'], value['items']) + human_value['text'] = f"{human_value['used']} ({value['items']} items)" + + return SysInfoRes(value=value, state=state, human_value=human_value) + + +@register_sysinfo +def storage_archives(): + import rhodecode + from rhodecode.lib.helpers import format_byte_size_binary + import rhodecode.apps.file_store.utils as store_utils + from rhodecode import CONFIG + + backend_type = rhodecode.ConfigGet().get_str(store_utils.config_keys.backend_type) + + value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=backend_type) + state = STATE_OK_DEFAULT + try: + f_store = store_utils.get_filestore_backend(config=CONFIG) + backend_type = str(f_store) + total_files, total_size, _directory_stats = f_store.get_statistics() + + value.update({ + 'percent': 100, + 'used': total_size, + 'total': total_size, + 'items': total_files, + 'path': f_store.storage_path, + 'type': backend_type + }) + + except Exception as e: + log.exception('failed to fetch archive cache storage') + state = {'message': str(e), 'type': STATE_ERR} + + human_value = value.copy() + human_value['used'] = format_byte_size_binary(value['used']) + human_value['total'] = format_byte_size_binary(value['total']) + human_value['text'] = f"{human_value['used']} ({value['items']} items)" return SysInfoRes(value=value, state=state, human_value=human_value) @@ -798,6 +836,7 @@ def get_system_info(environ): 'storage': SysInfo(storage)(), 'storage_inodes': SysInfo(storage_inodes)(), 'storage_archive': SysInfo(storage_archives)(), + 'storage_artifacts': SysInfo(storage_artifacts)(), 'storage_gist': SysInfo(storage_gist)(), 'storage_temp': SysInfo(storage_temp)(), diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -5849,8 +5849,7 @@ class FileStore(Base, BaseModel): .filter(FileStoreMetadata.file_store_meta_key == key) \ .scalar() if has_key: - msg = 'key `{}` already defined under section `{}` for this file.'\ - .format(key, section) + msg = f'key `{key}` already defined under section `{section}` for this file.' raise ArtifactMetadataDuplicate(msg, err_section=section, err_key=key) # NOTE(marcink): raises ArtifactMetadataBadValueType @@ -5949,7 +5948,7 @@ class FileStoreMetadata(Base, BaseModel) def valid_value_type(cls, value): if value.split('.')[0] not in cls.SETTINGS_TYPES: raise ArtifactMetadataBadValueType( - 'value_type must be one of %s got %s' % (cls.SETTINGS_TYPES.keys(), value)) + f'value_type must be one of {cls.SETTINGS_TYPES.keys()} got {value}') @hybrid_property def file_store_meta_section(self): diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py --- a/rhodecode/model/repo.py +++ b/rhodecode/model/repo.py @@ -31,7 +31,7 @@ from zope.cachedescriptors.property impo from rhodecode import events from rhodecode.lib.auth import HasUserGroupPermissionAny from rhodecode.lib.caching_query import FromCache -from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError +from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError from rhodecode.lib import hooks_base from rhodecode.lib.user_log_filter import user_log_filter from rhodecode.lib.utils import make_db_config @@ -736,7 +736,7 @@ class RepoModel(BaseModel): log.error(traceback.format_exc()) raise - def delete(self, repo, forks=None, pull_requests=None, fs_remove=True, cur_user=None): + def delete(self, repo, forks=None, pull_requests=None, artifacts=None, fs_remove=True, cur_user=None): """ Delete given repository, forks parameter defines what do do with attached forks. Throws AttachedForksError if deleted repo has attached @@ -745,6 +745,7 @@ class RepoModel(BaseModel): :param repo: :param forks: str 'delete' or 'detach' :param pull_requests: str 'delete' or None + :param artifacts: str 'delete' or None :param fs_remove: remove(archive) repo from filesystem """ if not cur_user: @@ -767,6 +768,13 @@ class RepoModel(BaseModel): if pull_requests != 'delete' and (pr_sources or pr_targets): raise AttachedPullRequestsError() + artifacts_objs = repo.artifacts + if artifacts == 'delete': + for a in artifacts_objs: + self.sa.delete(a) + elif [a for a in artifacts_objs]: + raise AttachedArtifactsError() + old_repo_dict = repo.get_dict() events.trigger(events.RepoPreDeleteEvent(repo)) try: diff --git a/rhodecode/model/user.py b/rhodecode/model/user.py --- a/rhodecode/model/user.py +++ b/rhodecode/model/user.py @@ -557,10 +557,10 @@ class UserModel(BaseModel): elif handle_mode == 'delete': from rhodecode.apps.file_store import utils as store_utils request = get_current_request() - storage = store_utils.get_file_storage(request.registry.settings) + f_store = store_utils.get_filestore_backend(request.registry.settings) for a in artifacts: file_uid = a.file_uid - storage.delete(file_uid) + f_store.delete(file_uid) self.sa.delete(a) left_overs = False diff --git a/rhodecode/templates/admin/repos/repo_edit_advanced.mako b/rhodecode/templates/admin/repos/repo_edit_advanced.mako --- a/rhodecode/templates/admin/repos/repo_edit_advanced.mako +++ b/rhodecode/templates/admin/repos/repo_edit_advanced.mako @@ -215,18 +215,35 @@ %endif + <% attached_prs = len(c.rhodecode_db_repo.pull_requests_source + c.rhodecode_db_repo.pull_requests_target) %> % if c.rhodecode_db_repo.pull_requests_source or c.rhodecode_db_repo.pull_requests_target: ${_ungettext('This repository has %s attached pull request.', 'This repository has %s attached pull requests.', attached_prs) % attached_prs}
- ${_('Consider to archive this repository instead.')} +
+ ${_('Consider to archive this repository instead.')} % endif + + <% attached_artifacts = len(c.rhodecode_db_repo.artifacts) %> + % if attached_artifacts: + + + ${_ungettext('This repository has %s attached artifact.', 'This repository has %s attached artifacts.', attached_artifacts) % attached_artifacts} +
+
+ ${_('Consider to archive this repository instead.')} + + + + + % endif +
diff --git a/rhodecode/tests/fixture.py b/rhodecode/tests/fixture.py --- a/rhodecode/tests/fixture.py +++ b/rhodecode/tests/fixture.py @@ -305,7 +305,7 @@ class Fixture(object): return r def destroy_repo(self, repo_name, **kwargs): - RepoModel().delete(repo_name, pull_requests='delete', **kwargs) + RepoModel().delete(repo_name, pull_requests='delete', artifacts='delete', **kwargs) Session().commit() def destroy_repo_on_filesystem(self, repo_name): diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini --- a/rhodecode/tests/rhodecode.ini +++ b/rhodecode/tests/rhodecode.ini @@ -36,7 +36,7 @@ port = 10020 ; GUNICORN APPLICATION SERVER ; ########################### -; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py +; run with gunicorn --config gunicorn_conf.py --paste rhodecode.ini ; Module to use, this setting shouldn't be changed use = egg:gunicorn#main @@ -249,15 +249,56 @@ labs_settings_active = true ; optional prefix to Add to email Subject #exception_tracker.email_prefix = [RHODECODE ERROR] -; File store configuration. This is used to store and serve uploaded files -file_store.enabled = true +; NOTE: this setting IS DEPRECATED: +; file_store backend is always enabled +#file_store.enabled = true +; NOTE: this setting IS DEPRECATED: +; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead ; Storage backend, available options are: local -file_store.backend = local +#file_store.backend = local +; NOTE: this setting IS DEPRECATED: +; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead ; path to store the uploaded binaries and artifacts -file_store.storage_path = /var/opt/rhodecode_data/file_store +#file_store.storage_path = /var/opt/rhodecode_data/file_store + +; Artifacts file-store, is used to store comment attachments and artifacts uploads. +; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options +; filesystem_v1 is backwards compat with pre 5.1 storage changes +; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from +; previous installations to keep the artifacts without a need of migration +file_store.backend.type = filesystem_v1 + +; filesystem options... +file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/test_artifacts_file_store + +; filesystem_v2 options... +file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/test_artifacts_file_store_2 +file_store.filesystem_v2.shards = 8 +; objectstore options... +; url for s3 compatible storage that allows to upload artifacts +; e.g http://minio:9000 +#file_store.backend.type = objectstore +file_store.objectstore.url = http://s3-minio:9000 + +; a top-level bucket to put all other shards in +; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number +file_store.objectstore.bucket = rhodecode-file-store-tests + +; number of sharded buckets to create to distribute archives across +; default is 8 shards +file_store.objectstore.bucket_shards = 8 + +; key for s3 auth +file_store.objectstore.key = s3admin + +; secret for s3 auth +file_store.objectstore.secret = s3secret4 + +;region for s3 storage +file_store.objectstore.region = eu-central-1 ; Redis url to acquire/check generation of archives locks archive_cache.locking.url = redis://redis:6379/1 @@ -593,6 +634,7 @@ vcs.scm_app_implementation = http ; Push/Pull operations hooks protocol, available options are: ; `http` - use http-rpc backend (default) ; `celery` - use celery based hooks +#DEPRECATED:vcs.hooks.protocol = http vcs.hooks.protocol = http ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be @@ -626,6 +668,10 @@ vcs.methods.cache = false ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible #vcs.svn.compatible_version = 1.8 +; Redis connection settings for svn integrations logic +; This connection string needs to be the same on ce and vcsserver +vcs.svn.redis_conn = redis://redis:6379/0 + ; Enable SVN proxy of requests over HTTP vcs.svn.proxy.enabled = true @@ -681,7 +727,8 @@ ssh.authorized_keys_file_path = %(here)s ; RhodeCode installation directory. ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2 -ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper +#DEPRECATED: ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper +ssh.wrapper_cmd.v2 = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2 ; Allow shell when executing the ssh-wrapper command ssh.wrapper_cmd_allow_shell = false diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -189,6 +189,7 @@ setup( 'rc-upgrade-db=rhodecode.lib.rc_commands.upgrade_db:main', 'rc-ishell=rhodecode.lib.rc_commands.ishell:main', 'rc-add-artifact=rhodecode.lib.rc_commands.add_artifact:main', + 'rc-migrate-artifact=rhodecode.lib.rc_commands.migrate_artifact:main', 'rc-ssh-wrapper=rhodecode.apps.ssh_support.lib.ssh_wrapper_v1:main', 'rc-ssh-wrapper-v2=rhodecode.apps.ssh_support.lib.ssh_wrapper_v2:main', ],