diff --git a/configs/development.ini b/configs/development.ini
--- a/configs/development.ini
+++ b/configs/development.ini
@@ -281,15 +281,56 @@ labs_settings_active = true
; optional prefix to Add to email Subject
#exception_tracker.email_prefix = [RHODECODE ERROR]
-; File store configuration. This is used to store and serve uploaded files
-file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store backend is always enabled
+#file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead
; Storage backend, available options are: local
-file_store.backend = local
+#file_store.backend = local
+; NOTE: this setting IS DEPRECATED:
+; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead
; path to store the uploaded binaries and artifacts
-file_store.storage_path = /var/opt/rhodecode_data/file_store
+#file_store.storage_path = /var/opt/rhodecode_data/file_store
+
+; Artifacts file-store, is used to store comment attachments and artifacts uploads.
+; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options
+; filesystem_v1 is backwards compat with pre 5.1 storage changes
+; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from
+; previous installations to keep the artifacts without a need of migration
+#file_store.backend.type = filesystem_v2
+
+; filesystem options...
+#file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+
+; filesystem_v2 options...
+#file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+#file_store.filesystem_v2.shards = 8
+; objectstore options...
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+#file_store.backend.type = objectstore
+#file_store.objectstore.url = http://s3-minio:9000
+
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number
+#file_store.objectstore.bucket = rhodecode-file-store
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+#file_store.objectstore.bucket_shards = 8
+
+; key for s3 auth
+#file_store.objectstore.key = s3admin
+
+; secret for s3 auth
+#file_store.objectstore.secret = s3secret4
+
+;region for s3 storage
+#file_store.objectstore.region = eu-central-1
; Redis url to acquire/check generation of archives locks
archive_cache.locking.url = redis://redis:6379/1
diff --git a/configs/production.ini b/configs/production.ini
--- a/configs/production.ini
+++ b/configs/production.ini
@@ -249,15 +249,56 @@ labs_settings_active = true
; optional prefix to Add to email Subject
#exception_tracker.email_prefix = [RHODECODE ERROR]
-; File store configuration. This is used to store and serve uploaded files
-file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store backend is always enabled
+#file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead
; Storage backend, available options are: local
-file_store.backend = local
+#file_store.backend = local
+; NOTE: this setting IS DEPRECATED:
+; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead
; path to store the uploaded binaries and artifacts
-file_store.storage_path = /var/opt/rhodecode_data/file_store
+#file_store.storage_path = /var/opt/rhodecode_data/file_store
+
+; Artifacts file-store, is used to store comment attachments and artifacts uploads.
+; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options
+; filesystem_v1 is backwards compat with pre 5.1 storage changes
+; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from
+; previous installations to keep the artifacts without a need of migration
+#file_store.backend.type = filesystem_v2
+
+; filesystem options...
+#file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+
+; filesystem_v2 options...
+#file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+#file_store.filesystem_v2.shards = 8
+; objectstore options...
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+#file_store.backend.type = objectstore
+#file_store.objectstore.url = http://s3-minio:9000
+
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number
+#file_store.objectstore.bucket = rhodecode-file-store
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+#file_store.objectstore.bucket_shards = 8
+
+; key for s3 auth
+#file_store.objectstore.key = s3admin
+
+; secret for s3 auth
+#file_store.objectstore.secret = s3secret4
+
+;region for s3 storage
+#file_store.objectstore.region = eu-central-1
; Redis url to acquire/check generation of archives locks
archive_cache.locking.url = redis://redis:6379/1
diff --git a/rhodecode/api/views/server_api.py b/rhodecode/api/views/server_api.py
--- a/rhodecode/api/views/server_api.py
+++ b/rhodecode/api/views/server_api.py
@@ -33,7 +33,7 @@ from rhodecode.lib.ext_json import json
from rhodecode.lib.utils2 import safe_int
from rhodecode.model.db import UserIpMap
from rhodecode.model.scm import ScmModel
-from rhodecode.apps.file_store import utils
+from rhodecode.apps.file_store import utils as store_utils
from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
FileOverSizeException
diff --git a/rhodecode/apps/admin/views/system_info.py b/rhodecode/apps/admin/views/system_info.py
--- a/rhodecode/apps/admin/views/system_info.py
+++ b/rhodecode/apps/admin/views/system_info.py
@@ -171,11 +171,17 @@ class AdminSystemInfoSettingsView(BaseAp
(_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')),
('', '', ''), # spacer
- (_('Archive cache storage type'), val('storage_archive')['type'], state('storage_archive')),
+ (_('Artifacts storage backend'), val('storage_artifacts')['type'], state('storage_artifacts')),
+ (_('Artifacts storage location'), val('storage_artifacts')['path'], state('storage_artifacts')),
+ (_('Artifacts info'), val('storage_artifacts')['text'], state('storage_artifacts')),
+ ('', '', ''), # spacer
+
+ (_('Archive cache storage backend'), val('storage_archive')['type'], state('storage_archive')),
(_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')),
(_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')),
('', '', ''), # spacer
+
(_('Temp storage location'), val('storage_temp')['path'], state('storage_temp')),
(_('Temp storage info'), val('storage_temp')['text'], state('storage_temp')),
('', '', ''), # spacer
diff --git a/rhodecode/apps/file_store/__init__.py b/rhodecode/apps/file_store/__init__.py
--- a/rhodecode/apps/file_store/__init__.py
+++ b/rhodecode/apps/file_store/__init__.py
@@ -16,7 +16,8 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
import os
-from rhodecode.apps.file_store import config_keys
+
+
from rhodecode.config.settings_maker import SettingsMaker
@@ -24,18 +25,48 @@ def _sanitize_settings_and_apply_default
"""
Set defaults, convert to python types and validate settings.
"""
+ from rhodecode.apps.file_store import config_keys
+
+ # translate "legacy" params into new config
+ settings.pop(config_keys.deprecated_enabled, True)
+ if config_keys.deprecated_backend in settings:
+ # if legacy backend key is detected we use "legacy" backward compat setting
+ settings.pop(config_keys.deprecated_backend)
+ settings[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+
+ if config_keys.deprecated_store_path in settings:
+ store_path = settings.pop(config_keys.deprecated_store_path)
+ settings[config_keys.legacy_filesystem_storage_path] = store_path
+
settings_maker = SettingsMaker(settings)
- settings_maker.make_setting(config_keys.enabled, True, parser='bool')
- settings_maker.make_setting(config_keys.backend, 'local')
+ default_cache_dir = settings['cache_dir']
+ default_store_dir = os.path.join(default_cache_dir, 'artifacts_filestore')
+
+ # set default backend
+ settings_maker.make_setting(config_keys.backend_type, config_keys.backend_legacy_filesystem)
+
+ # legacy filesystem defaults
+ settings_maker.make_setting(config_keys.legacy_filesystem_storage_path, default_store_dir, default_when_empty=True, )
- default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store')
- settings_maker.make_setting(config_keys.store_path, default_store)
+ # filesystem defaults
+ settings_maker.make_setting(config_keys.filesystem_storage_path, default_store_dir, default_when_empty=True,)
+ settings_maker.make_setting(config_keys.filesystem_shards, 8, parser='int')
+
+ # objectstore defaults
+ settings_maker.make_setting(config_keys.objectstore_url, 'http://s3-minio:9000')
+ settings_maker.make_setting(config_keys.objectstore_bucket, 'rhodecode-artifacts-filestore')
+ settings_maker.make_setting(config_keys.objectstore_bucket_shards, 8, parser='int')
+
+ settings_maker.make_setting(config_keys.objectstore_region, '')
+ settings_maker.make_setting(config_keys.objectstore_key, '')
+ settings_maker.make_setting(config_keys.objectstore_secret, '')
settings_maker.env_expand()
def includeme(config):
+
from rhodecode.apps.file_store.views import FileStoreView
settings = config.registry.settings
diff --git a/rhodecode/apps/file_store/backends/base.py b/rhodecode/apps/file_store/backends/base.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/apps/file_store/backends/base.py
@@ -0,0 +1,269 @@
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import fsspec # noqa
+import logging
+
+from rhodecode.lib.ext_json import json
+
+from rhodecode.apps.file_store.utils import sha256_safe, ShardFileReader, get_uid_filename
+from rhodecode.apps.file_store.extensions import resolve_extensions
+from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException # noqa: F401
+
+log = logging.getLogger(__name__)
+
+
+class BaseShard:
+
+ metadata_suffix: str = '.metadata'
+ storage_type: str = ''
+ fs = None
+
+ @property
+ def storage_medium(self):
+ if not self.storage_type:
+ raise ValueError('No storage type set for this shard storage_type=""')
+ return getattr(self, self.storage_type)
+
+ def __contains__(self, key):
+ full_path = self.store_path(key)
+ return self.fs.exists(full_path)
+
+ def metadata_convert(self, uid_filename, metadata):
+ return metadata
+
+ def get_metadata_filename(self, uid_filename) -> tuple[str, str]:
+ metadata_file: str = f'{uid_filename}{self.metadata_suffix}'
+ return metadata_file, self.store_path(metadata_file)
+
+ def get_metadata(self, uid_filename, ignore_missing=False) -> dict:
+ _metadata_file, metadata_file_path = self.get_metadata_filename(uid_filename)
+ if ignore_missing and not self.fs.exists(metadata_file_path):
+ return {}
+
+ with self.fs.open(metadata_file_path, 'rb') as f:
+ metadata = json.loads(f.read())
+
+ metadata = self.metadata_convert(uid_filename, metadata)
+ return metadata
+
+ def _store(self, key: str, uid_key: str, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+ raise NotImplementedError
+
+ def store(self, key: str, uid_key: str, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+ return self._store(key, uid_key, value_reader, max_filesize, metadata, **kwargs)
+
+ def _fetch(self, key, presigned_url_expires: int = 0):
+ raise NotImplementedError
+
+ def fetch(self, key, **kwargs) -> tuple[ShardFileReader, dict]:
+ return self._fetch(key)
+
+ def _delete(self, key):
+ if key not in self:
+ log.exception(f'requested key={key} not found in {self}')
+ raise KeyError(key)
+
+ metadata = self.get_metadata(key)
+ _metadata_file, metadata_file_path = self.get_metadata_filename(key)
+ artifact_file_path = metadata['filename_uid_path']
+ self.fs.rm(artifact_file_path)
+ self.fs.rm(metadata_file_path)
+
+ return 1
+
+ def delete(self, key):
+ raise NotImplementedError
+
+ def store_path(self, uid_filename):
+ raise NotImplementedError
+
+
+class BaseFileStoreBackend:
+ _shards = tuple()
+ _shard_cls = BaseShard
+ _config: dict | None = None
+ _storage_path: str = ''
+
+ def __init__(self, settings, extension_groups=None):
+ self._config = settings
+ extension_groups = extension_groups or ['any']
+ self.extensions = resolve_extensions([], groups=extension_groups)
+
+ def __contains__(self, key):
+ return self.filename_exists(key)
+
+ def __repr__(self):
+ return f'<{self.__class__.__name__}(storage={self.storage_path})>'
+
+ @property
+ def storage_path(self):
+ return self._storage_path
+
+ @classmethod
+ def get_shard_index(cls, filename: str, num_shards) -> int:
+ # Generate a hash value from the filename
+ hash_value = sha256_safe(filename)
+
+ # Convert the hash value to an integer
+ hash_int = int(hash_value, 16)
+
+ # Map the hash integer to a shard number between 1 and num_shards
+ shard_number = (hash_int % num_shards)
+
+ return shard_number
+
+ @classmethod
+ def apply_counter(cls, counter: int, filename: str) -> str:
+ """
+ Apply a counter to the filename.
+
+ :param counter: The counter value to apply.
+ :param filename: The original filename.
+ :return: The modified filename with the counter.
+ """
+ name_counted = f'{counter:d}-{filename}'
+ return name_counted
+
+ def _get_shard(self, key) -> _shard_cls:
+ index = self.get_shard_index(key, len(self._shards))
+ shard = self._shards[index]
+ return shard
+
+ def get_conf(self, key, pop=False):
+ if key not in self._config:
+ raise ValueError(
+ f"No configuration key '{key}', please make sure it exists in filestore config")
+ val = self._config[key]
+ if pop:
+ del self._config[key]
+ return val
+
+ def filename_allowed(self, filename, extensions=None):
+ """Checks if a filename has an allowed extension
+
+ :param filename: base name of file
+ :param extensions: iterable of extensions (or self.extensions)
+ """
+ _, ext = os.path.splitext(filename)
+ return self.extension_allowed(ext, extensions)
+
+ def extension_allowed(self, ext, extensions=None):
+ """
+ Checks if an extension is permitted. Both e.g. ".jpg" and
+ "jpg" can be passed in. Extension lookup is case-insensitive.
+
+ :param ext: extension to check
+ :param extensions: iterable of extensions to validate against (or self.extensions)
+ """
+ def normalize_ext(_ext):
+ if _ext.startswith('.'):
+ _ext = _ext[1:]
+ return _ext.lower()
+
+ extensions = extensions or self.extensions
+ if not extensions:
+ return True
+
+ ext = normalize_ext(ext)
+
+ return ext in [normalize_ext(x) for x in extensions]
+
+ def filename_exists(self, uid_filename):
+ shard = self._get_shard(uid_filename)
+ return uid_filename in shard
+
+ def store_path(self, uid_filename):
+ """
+ Returns absolute file path of the uid_filename
+ """
+ shard = self._get_shard(uid_filename)
+ return shard.store_path(uid_filename)
+
+ def store_metadata(self, uid_filename):
+ shard = self._get_shard(uid_filename)
+ return shard.get_metadata_filename(uid_filename)
+
+ def store(self, filename, value_reader, extensions=None, metadata=None, max_filesize=None, randomized_name=True, **kwargs):
+ extensions = extensions or self.extensions
+
+ if not self.filename_allowed(filename, extensions):
+ msg = f'filename {filename} does not allow extensions {extensions}'
+ raise FileNotAllowedException(msg)
+
+ # # TODO: check why we need this setting ? it looks stupid...
+ # no_body_seek is used in stream mode importer somehow
+ # no_body_seek = kwargs.pop('no_body_seek', False)
+ # if no_body_seek:
+ # pass
+ # else:
+ # value_reader.seek(0)
+
+ uid_filename = kwargs.pop('uid_filename', None)
+ if uid_filename is None:
+ uid_filename = get_uid_filename(filename, randomized=randomized_name)
+
+ shard = self._get_shard(uid_filename)
+
+ return shard.store(filename, uid_filename, value_reader, max_filesize, metadata, **kwargs)
+
+ def import_to_store(self, value_reader, org_filename, uid_filename, metadata, **kwargs):
+ shard = self._get_shard(uid_filename)
+ max_filesize = None
+ return shard.store(org_filename, uid_filename, value_reader, max_filesize, metadata, import_mode=True)
+
+ def delete(self, uid_filename):
+ shard = self._get_shard(uid_filename)
+ return shard.delete(uid_filename)
+
+ def fetch(self, uid_filename) -> tuple[ShardFileReader, dict]:
+ shard = self._get_shard(uid_filename)
+ return shard.fetch(uid_filename)
+
+ def get_metadata(self, uid_filename, ignore_missing=False) -> dict:
+ shard = self._get_shard(uid_filename)
+ return shard.get_metadata(uid_filename, ignore_missing=ignore_missing)
+
+ def iter_keys(self):
+ for shard in self._shards:
+ if shard.fs.exists(shard.storage_medium):
+ for path, _dirs, _files in shard.fs.walk(shard.storage_medium):
+ for key_file_path in _files:
+ if key_file_path.endswith(shard.metadata_suffix):
+ yield shard, key_file_path
+
+ def iter_artifacts(self):
+ for shard, key_file in self.iter_keys():
+ json_key = f"{shard.storage_medium}/{key_file}"
+ with shard.fs.open(json_key, 'rb') as f:
+ yield shard, json.loads(f.read())['filename_uid']
+
+ def get_statistics(self):
+ total_files = 0
+ total_size = 0
+ meta = {}
+
+ for shard, key_file in self.iter_keys():
+ json_key = f"{shard.storage_medium}/{key_file}"
+ with shard.fs.open(json_key, 'rb') as f:
+ total_files += 1
+ metadata = json.loads(f.read())
+ total_size += metadata['size']
+
+ return total_files, total_size, meta
diff --git a/rhodecode/apps/file_store/backends/filesystem.py b/rhodecode/apps/file_store/backends/filesystem.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/apps/file_store/backends/filesystem.py
@@ -0,0 +1,183 @@
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import hashlib
+import functools
+import time
+import logging
+
+from .. import config_keys
+from ..exceptions import FileOverSizeException
+from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
+
+from ....lib.ext_json import json
+
+log = logging.getLogger(__name__)
+
+
+class FileSystemShard(BaseShard):
+ METADATA_VER = 'v2'
+ BACKEND_TYPE = config_keys.backend_filesystem
+ storage_type: str = 'directory'
+
+ def __init__(self, index, directory, directory_folder, fs, **settings):
+ self._index: int = index
+ self._directory: str = directory
+ self._directory_folder: str = directory_folder
+ self.fs = fs
+
+ @property
+ def directory(self) -> str:
+ """Cache directory final path."""
+ return os.path.join(self._directory, self._directory_folder)
+
+ def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
+
+ # ensure dir exists
+ destination, _ = os.path.split(full_path)
+ if not self.fs.exists(destination):
+ self.fs.makedirs(destination)
+
+ writer = self.fs.open(full_path, mode)
+
+ digest = hashlib.sha256()
+ oversize_cleanup = False
+ with writer:
+ size = 0
+ for chunk in iterator:
+ size += len(chunk)
+ digest.update(chunk)
+ writer.write(chunk)
+
+ if max_filesize and size > max_filesize:
+ oversize_cleanup = True
+ # free up the copied file, and raise exc
+ break
+
+ writer.flush()
+ # Get the file descriptor
+ fd = writer.fileno()
+
+ # Sync the file descriptor to disk, helps with NFS cases...
+ os.fsync(fd)
+
+ if oversize_cleanup:
+ self.fs.rm(full_path)
+ raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
+
+ sha256 = digest.hexdigest()
+ log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
+ return size, sha256
+
+ def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+
+ filename = key
+ uid_filename = uid_key
+ full_path = self.store_path(uid_filename)
+
+ # STORE METADATA
+ _metadata = {
+ "version": self.METADATA_VER,
+ "store_type": self.BACKEND_TYPE,
+
+ "filename": filename,
+ "filename_uid_path": full_path,
+ "filename_uid": uid_filename,
+ "sha256": "", # NOTE: filled in by reader iteration
+
+ "store_time": time.time(),
+
+ "size": 0
+ }
+
+ if metadata:
+ if kwargs.pop('import_mode', False):
+ # in import mode, we don't need to compute metadata, we just take the old version
+ _metadata["import_mode"] = True
+ else:
+ _metadata.update(metadata)
+
+ read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
+ size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
+ _metadata['size'] = size
+ _metadata['sha256'] = sha256
+
+ # after storing the artifacts, we write the metadata present
+ _metadata_file, metadata_file_path = self.get_metadata_filename(uid_key)
+
+ with self.fs.open(metadata_file_path, 'wb') as f:
+ f.write(json.dumps(_metadata))
+
+ return uid_filename, _metadata
+
+ def store_path(self, uid_filename):
+ """
+ Returns absolute file path of the uid_filename
+ """
+ return os.path.join(self._directory, self._directory_folder, uid_filename)
+
+ def _fetch(self, key, presigned_url_expires: int = 0):
+ if key not in self:
+ log.exception(f'requested key={key} not found in {self}')
+ raise KeyError(key)
+
+ metadata = self.get_metadata(key)
+
+ file_path = metadata['filename_uid_path']
+ if presigned_url_expires and presigned_url_expires > 0:
+ metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
+
+ return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
+
+ def delete(self, key):
+ return self._delete(key)
+
+
+class FileSystemBackend(BaseFileStoreBackend):
+ shard_name: str = 'shard_{:03d}'
+ _shard_cls = FileSystemShard
+
+ def __init__(self, settings):
+ super().__init__(settings)
+
+ store_dir = self.get_conf(config_keys.filesystem_storage_path)
+ directory = os.path.expanduser(store_dir)
+
+ self._directory = directory
+ self._storage_path = directory # common path for all from BaseCache
+ self._shard_count = int(self.get_conf(config_keys.filesystem_shards, pop=True))
+ if self._shard_count < 1:
+ raise ValueError(f'{config_keys.filesystem_shards} must be 1 or more')
+
+ log.debug('Initializing %s file_store instance', self)
+ fs = fsspec.filesystem('file')
+
+ if not fs.exists(self._directory):
+ fs.makedirs(self._directory, exist_ok=True)
+
+ self._shards = tuple(
+ self._shard_cls(
+ index=num,
+ directory=directory,
+ directory_folder=self.shard_name.format(num),
+ fs=fs,
+ **settings,
+ )
+ for num in range(self._shard_count)
+ )
diff --git a/rhodecode/apps/file_store/backends/filesystem_legacy.py b/rhodecode/apps/file_store/backends/filesystem_legacy.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/apps/file_store/backends/filesystem_legacy.py
@@ -0,0 +1,278 @@
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import errno
+import os
+import hashlib
+import functools
+import time
+import logging
+
+from .. import config_keys
+from ..exceptions import FileOverSizeException
+from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
+
+from ....lib.ext_json import json
+
+log = logging.getLogger(__name__)
+
+
+class LegacyFileSystemShard(BaseShard):
+ # legacy ver
+ METADATA_VER = 'v2'
+ BACKEND_TYPE = config_keys.backend_legacy_filesystem
+ storage_type: str = 'dir_struct'
+
+ # legacy suffix
+ metadata_suffix: str = '.meta'
+
+ @classmethod
+ def _sub_store_from_filename(cls, filename):
+ return filename[:2]
+
+ @classmethod
+ def apply_counter(cls, counter, filename):
+ name_counted = '%d-%s' % (counter, filename)
+ return name_counted
+
+ @classmethod
+ def safe_make_dirs(cls, dir_path):
+ if not os.path.exists(dir_path):
+ try:
+ os.makedirs(dir_path)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ return
+
+ @classmethod
+ def resolve_name(cls, name, directory):
+ """
+ Resolves a unique name and the correct path. If a filename
+ for that path already exists then a numeric prefix with values > 0 will be
+ added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix.
+
+ :param name: base name of file
+ :param directory: absolute directory path
+ """
+
+ counter = 0
+ while True:
+ name_counted = cls.apply_counter(counter, name)
+
+ # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
+ sub_store: str = cls._sub_store_from_filename(name_counted)
+ sub_store_path: str = os.path.join(directory, sub_store)
+ cls.safe_make_dirs(sub_store_path)
+
+ path = os.path.join(sub_store_path, name_counted)
+ if not os.path.exists(path):
+ return name_counted, path
+ counter += 1
+
+ def __init__(self, index, directory, directory_folder, fs, **settings):
+ self._index: int = index
+ self._directory: str = directory
+ self._directory_folder: str = directory_folder
+ self.fs = fs
+
+ @property
+ def dir_struct(self) -> str:
+ """Cache directory final path."""
+ return os.path.join(self._directory, '0-')
+
+ def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
+
+ # ensure dir exists
+ destination, _ = os.path.split(full_path)
+ if not self.fs.exists(destination):
+ self.fs.makedirs(destination)
+
+ writer = self.fs.open(full_path, mode)
+
+ digest = hashlib.sha256()
+ oversize_cleanup = False
+ with writer:
+ size = 0
+ for chunk in iterator:
+ size += len(chunk)
+ digest.update(chunk)
+ writer.write(chunk)
+
+ if max_filesize and size > max_filesize:
+ # free up the copied file, and raise exc
+ oversize_cleanup = True
+ break
+
+ writer.flush()
+ # Get the file descriptor
+ fd = writer.fileno()
+
+ # Sync the file descriptor to disk, helps with NFS cases...
+ os.fsync(fd)
+
+ if oversize_cleanup:
+ self.fs.rm(full_path)
+ raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
+
+ sha256 = digest.hexdigest()
+ log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
+ return size, sha256
+
+ def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+
+ filename = key
+ uid_filename = uid_key
+
+ # NOTE:, also apply N- Counter...
+ uid_filename, full_path = self.resolve_name(uid_filename, self._directory)
+
+ # STORE METADATA
+ # TODO: make it compatible, and backward proof
+ _metadata = {
+ "version": self.METADATA_VER,
+
+ "filename": filename,
+ "filename_uid_path": full_path,
+ "filename_uid": uid_filename,
+ "sha256": "", # NOTE: filled in by reader iteration
+
+ "store_time": time.time(),
+
+ "size": 0
+ }
+ if metadata:
+ _metadata.update(metadata)
+
+ read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
+ size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
+ _metadata['size'] = size
+ _metadata['sha256'] = sha256
+
+ # after storing the artifacts, we write the metadata present
+ _metadata_file, metadata_file_path = self.get_metadata_filename(uid_filename)
+
+ with self.fs.open(metadata_file_path, 'wb') as f:
+ f.write(json.dumps(_metadata))
+
+ return uid_filename, _metadata
+
+ def store_path(self, uid_filename):
+ """
+ Returns absolute file path of the uid_filename
+ """
+ prefix_dir = ''
+ if '/' in uid_filename:
+ prefix_dir, filename = uid_filename.split('/')
+ sub_store = self._sub_store_from_filename(filename)
+ else:
+ sub_store = self._sub_store_from_filename(uid_filename)
+
+ return os.path.join(self._directory, prefix_dir, sub_store, uid_filename)
+
+ def metadata_convert(self, uid_filename, metadata):
+ # NOTE: backward compat mode here... this is for file created PRE 5.2 system
+ if 'meta_ver' in metadata:
+ full_path = self.store_path(uid_filename)
+ metadata = {
+ "_converted": True,
+ "_org": metadata,
+ "version": self.METADATA_VER,
+ "store_type": self.BACKEND_TYPE,
+
+ "filename": metadata['filename'],
+ "filename_uid_path": full_path,
+ "filename_uid": uid_filename,
+ "sha256": metadata['sha256'],
+
+ "store_time": metadata['time'],
+
+ "size": metadata['size']
+ }
+ return metadata
+
+ def _fetch(self, key, presigned_url_expires: int = 0):
+ if key not in self:
+ log.exception(f'requested key={key} not found in {self}')
+ raise KeyError(key)
+
+ metadata = self.get_metadata(key)
+
+ file_path = metadata['filename_uid_path']
+ if presigned_url_expires and presigned_url_expires > 0:
+ metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
+
+ return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
+
+ def delete(self, key):
+ return self._delete(key)
+
+ def _delete(self, key):
+ if key not in self:
+ log.exception(f'requested key={key} not found in {self}')
+ raise KeyError(key)
+
+ metadata = self.get_metadata(key)
+ metadata_file, metadata_file_path = self.get_metadata_filename(key)
+ artifact_file_path = metadata['filename_uid_path']
+ self.fs.rm(artifact_file_path)
+ self.fs.rm(metadata_file_path)
+
+ def get_metadata_filename(self, uid_filename) -> tuple[str, str]:
+
+ metadata_file: str = f'{uid_filename}{self.metadata_suffix}'
+ uid_path_in_store = self.store_path(uid_filename)
+
+ metadata_file_path = f'{uid_path_in_store}{self.metadata_suffix}'
+ return metadata_file, metadata_file_path
+
+
+class LegacyFileSystemBackend(BaseFileStoreBackend):
+ _shard_cls = LegacyFileSystemShard
+
+ def __init__(self, settings):
+ super().__init__(settings)
+
+ store_dir = self.get_conf(config_keys.legacy_filesystem_storage_path)
+ directory = os.path.expanduser(store_dir)
+
+ self._directory = directory
+ self._storage_path = directory # common path for all from BaseCache
+
+ log.debug('Initializing %s file_store instance', self)
+ fs = fsspec.filesystem('file')
+
+ if not fs.exists(self._directory):
+ fs.makedirs(self._directory, exist_ok=True)
+
+ # legacy system uses single shard
+ self._shards = tuple(
+ [
+ self._shard_cls(
+ index=0,
+ directory=directory,
+ directory_folder='',
+ fs=fs,
+ **settings,
+ )
+ ]
+ )
+
+ @classmethod
+ def get_shard_index(cls, filename: str, num_shards) -> int:
+ # legacy filesystem doesn't use shards, and always uses single shard
+ return 0
diff --git a/rhodecode/apps/file_store/backends/local_store.py b/rhodecode/apps/file_store/backends/local_store.py
deleted file mode 100755
--- a/rhodecode/apps/file_store/backends/local_store.py
+++ /dev/null
@@ -1,268 +0,0 @@
-# Copyright (C) 2016-2023 RhodeCode GmbH
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License, version 3
-# (only), as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see .
-#
-# This program is dual-licensed. If you wish to learn more about the
-# RhodeCode Enterprise Edition, including its added features, Support services,
-# and proprietary license terms, please see https://rhodecode.com/licenses/
-
-import os
-import time
-import errno
-import hashlib
-
-from rhodecode.lib.ext_json import json
-from rhodecode.apps.file_store import utils
-from rhodecode.apps.file_store.extensions import resolve_extensions
-from rhodecode.apps.file_store.exceptions import (
- FileNotAllowedException, FileOverSizeException)
-
-METADATA_VER = 'v1'
-
-
-def safe_make_dirs(dir_path):
- if not os.path.exists(dir_path):
- try:
- os.makedirs(dir_path)
- except OSError as e:
- if e.errno != errno.EEXIST:
- raise
- return
-
-
-class LocalFileStorage(object):
-
- @classmethod
- def apply_counter(cls, counter, filename):
- name_counted = '%d-%s' % (counter, filename)
- return name_counted
-
- @classmethod
- def resolve_name(cls, name, directory):
- """
- Resolves a unique name and the correct path. If a filename
- for that path already exists then a numeric prefix with values > 0 will be
- added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix.
-
- :param name: base name of file
- :param directory: absolute directory path
- """
-
- counter = 0
- while True:
- name_counted = cls.apply_counter(counter, name)
-
- # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
- sub_store = cls._sub_store_from_filename(name_counted)
- sub_store_path = os.path.join(directory, sub_store)
- safe_make_dirs(sub_store_path)
-
- path = os.path.join(sub_store_path, name_counted)
- if not os.path.exists(path):
- return name_counted, path
- counter += 1
-
- @classmethod
- def _sub_store_from_filename(cls, filename):
- return filename[:2]
-
- @classmethod
- def calculate_path_hash(cls, file_path):
- """
- Efficient calculation of file_path sha256 sum
-
- :param file_path:
- :return: sha256sum
- """
- digest = hashlib.sha256()
- with open(file_path, 'rb') as f:
- for chunk in iter(lambda: f.read(1024 * 100), b""):
- digest.update(chunk)
-
- return digest.hexdigest()
-
- def __init__(self, base_path, extension_groups=None):
-
- """
- Local file storage
-
- :param base_path: the absolute base path where uploads are stored
- :param extension_groups: extensions string
- """
-
- extension_groups = extension_groups or ['any']
- self.base_path = base_path
- self.extensions = resolve_extensions([], groups=extension_groups)
-
- def __repr__(self):
- return f'{self.__class__}@{self.base_path}'
-
- def store_path(self, filename):
- """
- Returns absolute file path of the filename, joined to the
- base_path.
-
- :param filename: base name of file
- """
- prefix_dir = ''
- if '/' in filename:
- prefix_dir, filename = filename.split('/')
- sub_store = self._sub_store_from_filename(filename)
- else:
- sub_store = self._sub_store_from_filename(filename)
- return os.path.join(self.base_path, prefix_dir, sub_store, filename)
-
- def delete(self, filename):
- """
- Deletes the filename. Filename is resolved with the
- absolute path based on base_path. If file does not exist,
- returns **False**, otherwise **True**
-
- :param filename: base name of file
- """
- if self.exists(filename):
- os.remove(self.store_path(filename))
- return True
- return False
-
- def exists(self, filename):
- """
- Checks if file exists. Resolves filename's absolute
- path based on base_path.
-
- :param filename: file_uid name of file, e.g 0-f62b2b2d-9708-4079-a071-ec3f958448d4.svg
- """
- return os.path.exists(self.store_path(filename))
-
- def filename_allowed(self, filename, extensions=None):
- """Checks if a filename has an allowed extension
-
- :param filename: base name of file
- :param extensions: iterable of extensions (or self.extensions)
- """
- _, ext = os.path.splitext(filename)
- return self.extension_allowed(ext, extensions)
-
- def extension_allowed(self, ext, extensions=None):
- """
- Checks if an extension is permitted. Both e.g. ".jpg" and
- "jpg" can be passed in. Extension lookup is case-insensitive.
-
- :param ext: extension to check
- :param extensions: iterable of extensions to validate against (or self.extensions)
- """
- def normalize_ext(_ext):
- if _ext.startswith('.'):
- _ext = _ext[1:]
- return _ext.lower()
-
- extensions = extensions or self.extensions
- if not extensions:
- return True
-
- ext = normalize_ext(ext)
-
- return ext in [normalize_ext(x) for x in extensions]
-
- def save_file(self, file_obj, filename, directory=None, extensions=None,
- extra_metadata=None, max_filesize=None, randomized_name=True, **kwargs):
- """
- Saves a file object to the uploads location.
- Returns the resolved filename, i.e. the directory +
- the (randomized/incremented) base name.
-
- :param file_obj: **cgi.FieldStorage** object (or similar)
- :param filename: original filename
- :param directory: relative path of sub-directory
- :param extensions: iterable of allowed extensions, if not default
- :param max_filesize: maximum size of file that should be allowed
- :param randomized_name: generate random generated UID or fixed based on the filename
- :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
-
- """
-
- extensions = extensions or self.extensions
-
- if not self.filename_allowed(filename, extensions):
- raise FileNotAllowedException()
-
- if directory:
- dest_directory = os.path.join(self.base_path, directory)
- else:
- dest_directory = self.base_path
-
- safe_make_dirs(dest_directory)
-
- uid_filename = utils.uid_filename(filename, randomized=randomized_name)
-
- # resolve also produces special sub-dir for file optimized store
- filename, path = self.resolve_name(uid_filename, dest_directory)
- stored_file_dir = os.path.dirname(path)
-
- no_body_seek = kwargs.pop('no_body_seek', False)
- if no_body_seek:
- pass
- else:
- file_obj.seek(0)
-
- with open(path, "wb") as dest:
- length = 256 * 1024
- while 1:
- buf = file_obj.read(length)
- if not buf:
- break
- dest.write(buf)
-
- metadata = {}
- if extra_metadata:
- metadata = extra_metadata
-
- size = os.stat(path).st_size
-
- if max_filesize and size > max_filesize:
- # free up the copied file, and raise exc
- os.remove(path)
- raise FileOverSizeException()
-
- file_hash = self.calculate_path_hash(path)
-
- metadata.update({
- "filename": filename,
- "size": size,
- "time": time.time(),
- "sha256": file_hash,
- "meta_ver": METADATA_VER
- })
-
- filename_meta = filename + '.meta'
- with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
- dest_meta.write(json.dumps(metadata))
-
- if directory:
- filename = os.path.join(directory, filename)
-
- return filename, metadata
-
- def get_metadata(self, filename, ignore_missing=False):
- """
- Reads JSON stored metadata for a file
-
- :param filename:
- :return:
- """
- filename = self.store_path(filename)
- filename_meta = filename + '.meta'
- if ignore_missing and not os.path.isfile(filename_meta):
- return {}
- with open(filename_meta, "rb") as source_meta:
- return json.loads(source_meta.read())
diff --git a/rhodecode/apps/file_store/backends/objectstore.py b/rhodecode/apps/file_store/backends/objectstore.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/apps/file_store/backends/objectstore.py
@@ -0,0 +1,184 @@
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import hashlib
+import functools
+import time
+import logging
+
+from .. import config_keys
+from ..exceptions import FileOverSizeException
+from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
+
+from ....lib.ext_json import json
+
+log = logging.getLogger(__name__)
+
+
+class S3Shard(BaseShard):
+ METADATA_VER = 'v2'
+ BACKEND_TYPE = config_keys.backend_objectstore
+ storage_type: str = 'bucket'
+
+ def __init__(self, index, bucket, bucket_folder, fs, **settings):
+ self._index: int = index
+ self._bucket_main: str = bucket
+ self._bucket_folder: str = bucket_folder
+
+ self.fs = fs
+
+ @property
+ def bucket(self) -> str:
+ """Cache bucket final path."""
+ return os.path.join(self._bucket_main, self._bucket_folder)
+
+ def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
+
+ # ensure dir exists
+ destination, _ = os.path.split(full_path)
+ if not self.fs.exists(destination):
+ self.fs.makedirs(destination)
+
+ writer = self.fs.open(full_path, mode)
+
+ digest = hashlib.sha256()
+ oversize_cleanup = False
+ with writer:
+ size = 0
+ for chunk in iterator:
+ size += len(chunk)
+ digest.update(chunk)
+ writer.write(chunk)
+
+ if max_filesize and size > max_filesize:
+ oversize_cleanup = True
+ # free up the copied file, and raise exc
+ break
+
+ if oversize_cleanup:
+ self.fs.rm(full_path)
+ raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
+
+ sha256 = digest.hexdigest()
+ log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
+ return size, sha256
+
+ def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+
+ filename = key
+ uid_filename = uid_key
+ full_path = self.store_path(uid_filename)
+
+ # STORE METADATA
+ _metadata = {
+ "version": self.METADATA_VER,
+ "store_type": self.BACKEND_TYPE,
+
+ "filename": filename,
+ "filename_uid_path": full_path,
+ "filename_uid": uid_filename,
+ "sha256": "", # NOTE: filled in by reader iteration
+
+ "store_time": time.time(),
+
+ "size": 0
+ }
+
+ if metadata:
+ if kwargs.pop('import_mode', False):
+ # in import mode, we don't need to compute metadata, we just take the old version
+ _metadata["import_mode"] = True
+ else:
+ _metadata.update(metadata)
+
+ read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
+ size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
+ _metadata['size'] = size
+ _metadata['sha256'] = sha256
+
+ # after storing the artifacts, we write the metadata present
+ metadata_file, metadata_file_path = self.get_metadata_filename(uid_key)
+
+ with self.fs.open(metadata_file_path, 'wb') as f:
+ f.write(json.dumps(_metadata))
+
+ return uid_filename, _metadata
+
+ def store_path(self, uid_filename):
+ """
+ Returns absolute file path of the uid_filename
+ """
+ return os.path.join(self._bucket_main, self._bucket_folder, uid_filename)
+
+ def _fetch(self, key, presigned_url_expires: int = 0):
+ if key not in self:
+ log.exception(f'requested key={key} not found in {self}')
+ raise KeyError(key)
+
+ metadata_file, metadata_file_path = self.get_metadata_filename(key)
+ with self.fs.open(metadata_file_path, 'rb') as f:
+ metadata = json.loads(f.read())
+
+ file_path = metadata['filename_uid_path']
+ if presigned_url_expires and presigned_url_expires > 0:
+ metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
+
+ return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
+
+ def delete(self, key):
+ return self._delete(key)
+
+
+class ObjectStoreBackend(BaseFileStoreBackend):
+ shard_name: str = 'shard-{:03d}'
+ _shard_cls = S3Shard
+
+ def __init__(self, settings):
+ super().__init__(settings)
+
+ self._shard_count = int(self.get_conf(config_keys.objectstore_bucket_shards, pop=True))
+ if self._shard_count < 1:
+ raise ValueError('cache_shards must be 1 or more')
+
+ self._bucket = settings.pop(config_keys.objectstore_bucket)
+ if not self._bucket:
+ raise ValueError(f'{config_keys.objectstore_bucket} needs to have a value')
+
+ objectstore_url = self.get_conf(config_keys.objectstore_url)
+ key = settings.pop(config_keys.objectstore_key)
+ secret = settings.pop(config_keys.objectstore_secret)
+
+ self._storage_path = objectstore_url # common path for all from BaseCache
+ log.debug('Initializing %s file_store instance', self)
+ fs = fsspec.filesystem('s3', anon=False, endpoint_url=objectstore_url, key=key, secret=secret)
+
+ # init main bucket
+ if not fs.exists(self._bucket):
+ fs.mkdir(self._bucket)
+
+ self._shards = tuple(
+ self._shard_cls(
+ index=num,
+ bucket=self._bucket,
+ bucket_folder=self.shard_name.format(num),
+ fs=fs,
+ **settings,
+ )
+ for num in range(self._shard_count)
+ )
diff --git a/rhodecode/apps/file_store/config_keys.py b/rhodecode/apps/file_store/config_keys.py
--- a/rhodecode/apps/file_store/config_keys.py
+++ b/rhodecode/apps/file_store/config_keys.py
@@ -20,6 +20,38 @@
# Definition of setting keys used to configure this module. Defined here to
# avoid repetition of keys throughout the module.
-enabled = 'file_store.enabled'
-backend = 'file_store.backend'
-store_path = 'file_store.storage_path'
+# OLD and deprecated keys not used anymore
+deprecated_enabled = 'file_store.enabled'
+deprecated_backend = 'file_store.backend'
+deprecated_store_path = 'file_store.storage_path'
+
+
+backend_type = 'file_store.backend.type'
+
+backend_legacy_filesystem = 'filesystem_v1'
+backend_filesystem = 'filesystem_v2'
+backend_objectstore = 'objectstore'
+
+backend_types = [
+ backend_legacy_filesystem,
+ backend_filesystem,
+ backend_objectstore,
+]
+
+# filesystem_v1 legacy
+legacy_filesystem_storage_path = 'file_store.filesystem_v1.storage_path'
+
+
+# filesystem_v2 new option
+filesystem_storage_path = 'file_store.filesystem_v2.storage_path'
+filesystem_shards = 'file_store.filesystem_v2.shards'
+
+# objectstore
+objectstore_url = 'file_store.objectstore.url'
+objectstore_bucket = 'file_store.objectstore.bucket'
+objectstore_bucket_shards = 'file_store.objectstore.bucket_shards'
+
+objectstore_region = 'file_store.objectstore.region'
+objectstore_key = 'file_store.objectstore.key'
+objectstore_secret = 'file_store.objectstore.secret'
+
diff --git a/rhodecode/apps/file_store/tests/__init__.py b/rhodecode/apps/file_store/tests/__init__.py
--- a/rhodecode/apps/file_store/tests/__init__.py
+++ b/rhodecode/apps/file_store/tests/__init__.py
@@ -16,3 +16,42 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
+import os
+import random
+import tempfile
+import string
+
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+
+
+@pytest.fixture()
+def file_store_instance(ini_settings):
+ config = ini_settings
+ f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+ return f_store
+
+
+@pytest.fixture
+def random_binary_file():
+ # Generate random binary data
+ data = bytearray(random.getrandbits(8) for _ in range(1024 * 512)) # 512 KB of random data
+
+ # Create a temporary file
+ temp_file = tempfile.NamedTemporaryFile(delete=False)
+ filename = temp_file.name
+
+ try:
+ # Write the random binary data to the file
+ temp_file.write(data)
+ temp_file.seek(0) # Rewind the file pointer to the beginning
+ yield filename, temp_file
+ finally:
+ # Close and delete the temporary file after the test
+ temp_file.close()
+ os.remove(filename)
+
+
+def generate_random_filename(length=10):
+ return ''.join(random.choices(string.ascii_letters + string.digits, k=length))
\ No newline at end of file
diff --git a/rhodecode/apps/file_store/tests/test_filestore_backends.py b/rhodecode/apps/file_store/tests/test_filestore_backends.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/apps/file_store/tests/test_filestore_backends.py
@@ -0,0 +1,128 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps import file_store
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.backends.filesystem_legacy import LegacyFileSystemBackend
+from rhodecode.apps.file_store.backends.filesystem import FileSystemBackend
+from rhodecode.apps.file_store.backends.objectstore import ObjectStoreBackend
+from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store.tests import random_binary_file, file_store_instance
+
+
+class TestFileStoreBackends:
+
+ @pytest.mark.parametrize('backend_type, expected_instance', [
+ (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+ (config_keys.backend_filesystem, FileSystemBackend),
+ (config_keys.backend_objectstore, ObjectStoreBackend),
+ ])
+ def test_get_backend(self, backend_type, expected_instance, ini_settings):
+ config = ini_settings
+ config[config_keys.backend_type] = backend_type
+ f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+ assert isinstance(f_store, expected_instance)
+
+ @pytest.mark.parametrize('backend_type, expected_instance', [
+ (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+ (config_keys.backend_filesystem, FileSystemBackend),
+ (config_keys.backend_objectstore, ObjectStoreBackend),
+ ])
+ def test_store_and_read(self, backend_type, expected_instance, ini_settings, random_binary_file):
+ filename, temp_file = random_binary_file
+ config = ini_settings
+ config[config_keys.backend_type] = backend_type
+ f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+ metadata = {
+ 'user_uploaded': {
+ 'username': 'user1',
+ 'user_id': 10,
+ 'ip': '10.20.30.40'
+ }
+ }
+ store_fid, metadata = f_store.store(filename, temp_file, extra_metadata=metadata)
+ assert store_fid
+ assert metadata
+
+ # read-after write
+ reader, metadata2 = f_store.fetch(store_fid)
+ assert reader
+ assert metadata2['filename'] == filename
+
+ @pytest.mark.parametrize('backend_type, expected_instance', [
+ (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+ (config_keys.backend_filesystem, FileSystemBackend),
+ (config_keys.backend_objectstore, ObjectStoreBackend),
+ ])
+ def test_store_file_not_allowed(self, backend_type, expected_instance, ini_settings, random_binary_file):
+ filename, temp_file = random_binary_file
+ config = ini_settings
+ config[config_keys.backend_type] = backend_type
+ f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+ with pytest.raises(FileNotAllowedException):
+ f_store.store('notallowed.exe', temp_file, extensions=['.txt'])
+
+ @pytest.mark.parametrize('backend_type, expected_instance', [
+ (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+ (config_keys.backend_filesystem, FileSystemBackend),
+ (config_keys.backend_objectstore, ObjectStoreBackend),
+ ])
+ def test_store_file_over_size(self, backend_type, expected_instance, ini_settings, random_binary_file):
+ filename, temp_file = random_binary_file
+ config = ini_settings
+ config[config_keys.backend_type] = backend_type
+ f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+ with pytest.raises(FileOverSizeException):
+ f_store.store('toobig.exe', temp_file, extensions=['.exe'], max_filesize=124)
+
+ @pytest.mark.parametrize('backend_type, expected_instance, extra_conf', [
+ (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend, {}),
+ (config_keys.backend_filesystem, FileSystemBackend, {config_keys.filesystem_storage_path: '/tmp/test-fs-store'}),
+ (config_keys.backend_objectstore, ObjectStoreBackend, {config_keys.objectstore_bucket: 'test-bucket'}),
+ ])
+ def test_store_stats_and_keys(self, backend_type, expected_instance, extra_conf, ini_settings, random_binary_file):
+ config = ini_settings
+ config[config_keys.backend_type] = backend_type
+ config.update(extra_conf)
+
+ f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+
+ # purge storage before running
+ for shard, k in f_store.iter_artifacts():
+ f_store.delete(k)
+
+ for i in range(10):
+ filename, temp_file = random_binary_file
+
+ metadata = {
+ 'user_uploaded': {
+ 'username': 'user1',
+ 'user_id': 10,
+ 'ip': '10.20.30.40'
+ }
+ }
+ store_fid, metadata = f_store.store(filename, temp_file, extra_metadata=metadata)
+ assert store_fid
+ assert metadata
+
+ cnt, size, meta = f_store.get_statistics()
+ assert cnt == 10
+ assert 10 == len(list(f_store.iter_keys()))
diff --git a/rhodecode/apps/file_store/tests/test_filestore_filesystem_backend.py b/rhodecode/apps/file_store/tests/test_filestore_filesystem_backend.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/apps/file_store/tests/test_filestore_filesystem_backend.py
@@ -0,0 +1,52 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.tests import generate_random_filename
+
+
+@pytest.fixture()
+def file_store_filesystem_instance(ini_settings):
+ config = ini_settings
+ config[config_keys.backend_type] = config_keys.backend_filesystem
+ f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+ return f_store
+
+
+class TestFileStoreFileSystemBackend:
+
+ @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
+ def test_get_shard_number(self, filename, file_store_filesystem_instance):
+ shard_number = file_store_filesystem_instance.get_shard_index(filename, len(file_store_filesystem_instance._shards))
+ # Check that the shard number is between 0 and max-shards
+ assert 0 <= shard_number <= len(file_store_filesystem_instance._shards)
+
+ @pytest.mark.parametrize('filename, expected_shard_num', [
+ ('my-name-1', 3),
+ ('my-name-2', 2),
+ ('my-name-3', 4),
+ ('my-name-4', 1),
+
+ ('rhodecode-enterprise-ce', 5),
+ ('rhodecode-enterprise-ee', 6),
+ ])
+ def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_filesystem_instance):
+ shard_number = file_store_filesystem_instance.get_shard_index(filename, len(file_store_filesystem_instance._shards))
+ assert expected_shard_num == shard_number
diff --git a/rhodecode/apps/file_store/tests/test_filestore_legacy_and_v2_compatability.py b/rhodecode/apps/file_store/tests/test_filestore_legacy_and_v2_compatability.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/apps/file_store/tests/test_filestore_legacy_and_v2_compatability.py
@@ -0,0 +1,17 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
\ No newline at end of file
diff --git a/rhodecode/apps/file_store/tests/test_filestore_legacy_backend.py b/rhodecode/apps/file_store/tests/test_filestore_legacy_backend.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/apps/file_store/tests/test_filestore_legacy_backend.py
@@ -0,0 +1,52 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.tests import generate_random_filename
+
+
+@pytest.fixture()
+def file_store_legacy_instance(ini_settings):
+ config = ini_settings
+ config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+ f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+ return f_store
+
+
+class TestFileStoreLegacyBackend:
+
+ @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
+ def test_get_shard_number(self, filename, file_store_legacy_instance):
+ shard_number = file_store_legacy_instance.get_shard_index(filename, len(file_store_legacy_instance._shards))
+ # Check that the shard number is 0 for legacy filesystem store we don't use shards
+ assert shard_number == 0
+
+ @pytest.mark.parametrize('filename, expected_shard_num', [
+ ('my-name-1', 0),
+ ('my-name-2', 0),
+ ('my-name-3', 0),
+ ('my-name-4', 0),
+
+ ('rhodecode-enterprise-ce', 0),
+ ('rhodecode-enterprise-ee', 0),
+ ])
+ def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_legacy_instance):
+ shard_number = file_store_legacy_instance.get_shard_index(filename, len(file_store_legacy_instance._shards))
+ assert expected_shard_num == shard_number
diff --git a/rhodecode/apps/file_store/tests/test_filestore_objectstore_backend.py b/rhodecode/apps/file_store/tests/test_filestore_objectstore_backend.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/apps/file_store/tests/test_filestore_objectstore_backend.py
@@ -0,0 +1,52 @@
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.tests import generate_random_filename
+
+
+@pytest.fixture()
+def file_store_objectstore_instance(ini_settings):
+ config = ini_settings
+ config[config_keys.backend_type] = config_keys.backend_objectstore
+ f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+ return f_store
+
+
+class TestFileStoreObjectStoreBackend:
+
+ @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
+ def test_get_shard_number(self, filename, file_store_objectstore_instance):
+ shard_number = file_store_objectstore_instance.get_shard_index(filename, len(file_store_objectstore_instance._shards))
+ # Check that the shard number is between 0 and shards
+ assert 0 <= shard_number <= len(file_store_objectstore_instance._shards)
+
+ @pytest.mark.parametrize('filename, expected_shard_num', [
+ ('my-name-1', 3),
+ ('my-name-2', 2),
+ ('my-name-3', 4),
+ ('my-name-4', 1),
+
+ ('rhodecode-enterprise-ce', 5),
+ ('rhodecode-enterprise-ee', 6),
+ ])
+ def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_objectstore_instance):
+ shard_number = file_store_objectstore_instance.get_shard_index(filename, len(file_store_objectstore_instance._shards))
+ assert expected_shard_num == shard_number
diff --git a/rhodecode/apps/file_store/tests/test_upload_file.py b/rhodecode/apps/file_store/tests/test_upload_file.py
--- a/rhodecode/apps/file_store/tests/test_upload_file.py
+++ b/rhodecode/apps/file_store/tests/test_upload_file.py
@@ -15,13 +15,16 @@
# This program is dual-licensed. If you wish to learn more about the
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
+
import os
+
import pytest
from rhodecode.lib.ext_json import json
from rhodecode.model.auth_token import AuthTokenModel
from rhodecode.model.db import Session, FileStore, Repository, User
-from rhodecode.apps.file_store import utils, config_keys
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
from rhodecode.tests import TestController
from rhodecode.tests.routes import route_path
@@ -29,27 +32,61 @@ from rhodecode.tests.routes import route
class TestFileStoreViews(TestController):
+ @pytest.fixture()
+ def create_artifact_factory(self, tmpdir, ini_settings):
+
+ def factory(user_id, content, f_name='example.txt'):
+
+ config = ini_settings
+ config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+
+ f_store = store_utils.get_filestore_backend(config)
+
+ filesystem_file = os.path.join(str(tmpdir), f_name)
+ with open(filesystem_file, 'wt') as f:
+ f.write(content)
+
+ with open(filesystem_file, 'rb') as f:
+ store_uid, metadata = f_store.store(f_name, f, metadata={'filename': f_name})
+ os.remove(filesystem_file)
+
+ entry = FileStore.create(
+ file_uid=store_uid, filename=metadata["filename"],
+ file_hash=metadata["sha256"], file_size=metadata["size"],
+ file_display_name='file_display_name',
+ file_description='repo artifact `{}`'.format(metadata["filename"]),
+ check_acl=True, user_id=user_id,
+ )
+ Session().add(entry)
+ Session().commit()
+ return entry
+ return factory
+
@pytest.mark.parametrize("fid, content, exists", [
('abcde-0.jpg', "xxxxx", True),
('abcde-0.exe', "1234567", True),
('abcde-0.jpg', "xxxxx", False),
])
- def test_get_files_from_store(self, fid, content, exists, tmpdir, user_util):
+ def test_get_files_from_store(self, fid, content, exists, tmpdir, user_util, ini_settings):
user = self.log_user()
user_id = user['user_id']
repo_id = user_util.create_repo().repo_id
- store_path = self.app._pyramid_settings[config_keys.store_path]
+
+ config = ini_settings
+ config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+
store_uid = fid
if exists:
status = 200
- store = utils.get_file_storage({config_keys.store_path: store_path})
+ f_store = store_utils.get_filestore_backend(config)
filesystem_file = os.path.join(str(tmpdir), fid)
with open(filesystem_file, 'wt') as f:
f.write(content)
with open(filesystem_file, 'rb') as f:
- store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid})
+ store_uid, metadata = f_store.store(fid, f, metadata={'filename': fid})
+ os.remove(filesystem_file)
entry = FileStore.create(
file_uid=store_uid, filename=metadata["filename"],
@@ -69,14 +106,10 @@ class TestFileStoreViews(TestController)
if exists:
assert response.text == content
- file_store_path = os.path.dirname(store.resolve_name(store_uid, store_path)[1])
- metadata_file = os.path.join(file_store_path, store_uid + '.meta')
- assert os.path.exists(metadata_file)
- with open(metadata_file, 'rb') as f:
- json_data = json.loads(f.read())
- assert json_data
- assert 'size' in json_data
+ metadata = f_store.get_metadata(store_uid)
+
+ assert 'size' in metadata
def test_upload_files_without_content_to_store(self):
self.log_user()
@@ -112,32 +145,6 @@ class TestFileStoreViews(TestController)
assert response.json['store_fid']
- @pytest.fixture()
- def create_artifact_factory(self, tmpdir):
- def factory(user_id, content):
- store_path = self.app._pyramid_settings[config_keys.store_path]
- store = utils.get_file_storage({config_keys.store_path: store_path})
- fid = 'example.txt'
-
- filesystem_file = os.path.join(str(tmpdir), fid)
- with open(filesystem_file, 'wt') as f:
- f.write(content)
-
- with open(filesystem_file, 'rb') as f:
- store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid})
-
- entry = FileStore.create(
- file_uid=store_uid, filename=metadata["filename"],
- file_hash=metadata["sha256"], file_size=metadata["size"],
- file_display_name='file_display_name',
- file_description='repo artifact `{}`'.format(metadata["filename"]),
- check_acl=True, user_id=user_id,
- )
- Session().add(entry)
- Session().commit()
- return entry
- return factory
-
def test_download_file_non_scoped(self, user_util, create_artifact_factory):
user = self.log_user()
user_id = user['user_id']
diff --git a/rhodecode/apps/file_store/utils.py b/rhodecode/apps/file_store/utils.py
--- a/rhodecode/apps/file_store/utils.py
+++ b/rhodecode/apps/file_store/utils.py
@@ -19,21 +19,84 @@
import io
import uuid
import pathlib
+import s3fs
+
+from rhodecode.lib.hash_utils import sha256_safe
+from rhodecode.apps.file_store import config_keys
+
+
+file_store_meta = None
+
+
+def get_filestore_config(config) -> dict:
+
+ final_config = {}
+
+ for k, v in config.items():
+ if k.startswith('file_store'):
+ final_config[k] = v
+
+ return final_config
-def get_file_storage(settings):
- from rhodecode.apps.file_store.backends.local_store import LocalFileStorage
- from rhodecode.apps.file_store import config_keys
- store_path = settings.get(config_keys.store_path)
- return LocalFileStorage(base_path=store_path)
+def get_filestore_backend(config, always_init=False):
+ """
+
+ usage::
+ from rhodecode.apps.file_store import get_filestore_backend
+ f_store = get_filestore_backend(config=CONFIG)
+
+ :param config:
+ :param always_init:
+ :return:
+ """
+
+ global file_store_meta
+ if file_store_meta is not None and not always_init:
+ return file_store_meta
+
+ config = get_filestore_config(config)
+ backend = config[config_keys.backend_type]
+
+ match backend:
+ case config_keys.backend_legacy_filesystem:
+ # Legacy backward compatible storage
+ from rhodecode.apps.file_store.backends.filesystem_legacy import LegacyFileSystemBackend
+ d_cache = LegacyFileSystemBackend(
+ settings=config
+ )
+ case config_keys.backend_filesystem:
+ from rhodecode.apps.file_store.backends.filesystem import FileSystemBackend
+ d_cache = FileSystemBackend(
+ settings=config
+ )
+ case config_keys.backend_objectstore:
+ from rhodecode.apps.file_store.backends.objectstore import ObjectStoreBackend
+ d_cache = ObjectStoreBackend(
+ settings=config
+ )
+ case _:
+ raise ValueError(
+ f'file_store.backend.type only supports "{config_keys.backend_types}" got {backend}'
+ )
+
+ cache_meta = d_cache
+ return cache_meta
def splitext(filename):
- ext = ''.join(pathlib.Path(filename).suffixes)
+ final_ext = []
+ for suffix in pathlib.Path(filename).suffixes:
+ if not suffix.isascii():
+ continue
+
+ suffix = " ".join(suffix.split()).replace(" ", "")
+ final_ext.append(suffix)
+ ext = ''.join(final_ext)
return filename, ext
-def uid_filename(filename, randomized=True):
+def get_uid_filename(filename, randomized=True):
"""
Generates a randomized or stable (uuid) filename,
preserving the original extension.
@@ -46,10 +109,37 @@ def uid_filename(filename, randomized=Tr
if randomized:
uid = uuid.uuid4()
else:
- hash_key = '{}.{}'.format(filename, 'store')
+ store_suffix = "store"
+ hash_key = f'{filename}.{store_suffix}'
uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
return str(uid) + ext.lower()
def bytes_to_file_obj(bytes_data):
- return io.StringIO(bytes_data)
+ return io.BytesIO(bytes_data)
+
+
+class ShardFileReader:
+
+ def __init__(self, file_like_reader):
+ self._file_like_reader = file_like_reader
+
+ def __getattr__(self, item):
+ if isinstance(self._file_like_reader, s3fs.core.S3File):
+ match item:
+ case 'name':
+ # S3 FileWrapper doesn't support name attribute, and we use it
+ return self._file_like_reader.full_name
+ case _:
+ return getattr(self._file_like_reader, item)
+ else:
+ return getattr(self._file_like_reader, item)
+
+
+def archive_iterator(_reader, block_size: int = 4096 * 512):
+ # 4096 * 64 = 64KB
+ while 1:
+ data = _reader.read(block_size)
+ if not data:
+ break
+ yield data
diff --git a/rhodecode/apps/file_store/views.py b/rhodecode/apps/file_store/views.py
--- a/rhodecode/apps/file_store/views.py
+++ b/rhodecode/apps/file_store/views.py
@@ -17,12 +17,11 @@
# and proprietary license terms, please see https://rhodecode.com/licenses/
import logging
-
-from pyramid.response import FileResponse
+from pyramid.response import Response
from pyramid.httpexceptions import HTTPFound, HTTPNotFound
from rhodecode.apps._base import BaseAppView
-from rhodecode.apps.file_store import utils
+from rhodecode.apps.file_store import utils as store_utils
from rhodecode.apps.file_store.exceptions import (
FileNotAllowedException, FileOverSizeException)
@@ -31,6 +30,7 @@ from rhodecode.lib import audit_logger
from rhodecode.lib.auth import (
CSRFRequired, NotAnonymous, HasRepoPermissionAny, HasRepoGroupPermissionAny,
LoginRequired)
+from rhodecode.lib.str_utils import header_safe_str
from rhodecode.lib.vcs.conf.mtypes import get_mimetypes_db
from rhodecode.model.db import Session, FileStore, UserApiKeys
@@ -42,7 +42,7 @@ class FileStoreView(BaseAppView):
def load_default_context(self):
c = self._get_local_tmpl_context()
- self.storage = utils.get_file_storage(self.request.registry.settings)
+ self.f_store = store_utils.get_filestore_backend(self.request.registry.settings)
return c
def _guess_type(self, file_name):
@@ -55,8 +55,8 @@ class FileStoreView(BaseAppView):
return _content_type, _encoding
def _serve_file(self, file_uid):
- if not self.storage.exists(file_uid):
- store_path = self.storage.store_path(file_uid)
+ if not self.f_store.filename_exists(file_uid):
+ store_path = self.f_store.store_path(file_uid)
log.warning('File with FID:%s not found in the store under `%s`',
file_uid, store_path)
raise HTTPNotFound()
@@ -98,28 +98,25 @@ class FileStoreView(BaseAppView):
FileStore.bump_access_counter(file_uid)
- file_path = self.storage.store_path(file_uid)
+ file_name = db_obj.file_display_name
content_type = 'application/octet-stream'
- content_encoding = None
- _content_type, _encoding = self._guess_type(file_path)
+ _content_type, _encoding = self._guess_type(file_name)
if _content_type:
content_type = _content_type
# For file store we don't submit any session data, this logic tells the
# Session lib to skip it
setattr(self.request, '_file_response', True)
- response = FileResponse(
- file_path, request=self.request,
- content_type=content_type, content_encoding=content_encoding)
+ reader, _meta = self.f_store.fetch(file_uid)
- file_name = db_obj.file_display_name
+ response = Response(app_iter=store_utils.archive_iterator(reader))
- response.headers["Content-Disposition"] = (
- f'attachment; filename="{str(file_name)}"'
- )
+ response.content_type = str(content_type)
+ response.content_disposition = f'attachment; filename="{header_safe_str(file_name)}"'
+
response.headers["X-RC-Artifact-Id"] = str(db_obj.file_store_id)
- response.headers["X-RC-Artifact-Desc"] = str(db_obj.file_description)
+ response.headers["X-RC-Artifact-Desc"] = header_safe_str(db_obj.file_description)
response.headers["X-RC-Artifact-Sha256"] = str(db_obj.file_hash)
return response
@@ -147,8 +144,8 @@ class FileStoreView(BaseAppView):
'user_id': self._rhodecode_user.user_id,
'ip': self._rhodecode_user.ip_addr}}
try:
- store_uid, metadata = self.storage.save_file(
- file_obj.file, filename, extra_metadata=metadata)
+ store_uid, metadata = self.f_store.store(
+ filename, file_obj.file, extra_metadata=metadata)
except FileNotAllowedException:
return {'store_fid': None,
'access_path': None,
@@ -182,7 +179,7 @@ class FileStoreView(BaseAppView):
def download_file(self):
self.load_default_context()
file_uid = self.request.matchdict['fid']
- log.debug('Requesting FID:%s from store %s', file_uid, self.storage)
+ log.debug('Requesting FID:%s from store %s', file_uid, self.f_store)
return self._serve_file(file_uid)
# in addition to @LoginRequired ACL is checked by scopes
diff --git a/rhodecode/apps/repository/views/repo_commits.py b/rhodecode/apps/repository/views/repo_commits.py
--- a/rhodecode/apps/repository/views/repo_commits.py
+++ b/rhodecode/apps/repository/views/repo_commits.py
@@ -601,26 +601,26 @@ class RepoCommitsView(RepoAppView):
max_file_size = 10 * 1024 * 1024 # 10MB, also validated via dropzone.js
try:
- storage = store_utils.get_file_storage(self.request.registry.settings)
- store_uid, metadata = storage.save_file(
- file_obj.file, filename, extra_metadata=metadata,
+ f_store = store_utils.get_filestore_backend(self.request.registry.settings)
+ store_uid, metadata = f_store.store(
+ filename, file_obj.file, metadata=metadata,
extensions=allowed_extensions, max_filesize=max_file_size)
except FileNotAllowedException:
self.request.response.status = 400
permitted_extensions = ', '.join(allowed_extensions)
- error_msg = 'File `{}` is not allowed. ' \
- 'Only following extensions are permitted: {}'.format(
- filename, permitted_extensions)
+ error_msg = f'File `{filename}` is not allowed. ' \
+ f'Only following extensions are permitted: {permitted_extensions}'
+
return {'store_fid': None,
'access_path': None,
'error': error_msg}
except FileOverSizeException:
self.request.response.status = 400
limit_mb = h.format_byte_size_binary(max_file_size)
+ error_msg = f'File {filename} is exceeding allowed limit of {limit_mb}.'
return {'store_fid': None,
'access_path': None,
- 'error': 'File {} is exceeding allowed limit of {}.'.format(
- filename, limit_mb)}
+ 'error': error_msg}
try:
entry = FileStore.create(
diff --git a/rhodecode/apps/repository/views/repo_files.py b/rhodecode/apps/repository/views/repo_files.py
--- a/rhodecode/apps/repository/views/repo_files.py
+++ b/rhodecode/apps/repository/views/repo_files.py
@@ -48,7 +48,7 @@ from rhodecode.lib.codeblocks import (
filenode_as_lines_tokens, filenode_as_annotated_lines_tokens)
from rhodecode.lib.utils2 import convert_line_endings, detect_mode
from rhodecode.lib.type_utils import str2bool
-from rhodecode.lib.str_utils import safe_str, safe_int
+from rhodecode.lib.str_utils import safe_str, safe_int, header_safe_str
from rhodecode.lib.auth import (
LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
from rhodecode.lib.vcs import path as vcspath
@@ -820,7 +820,7 @@ class RepoFilesView(RepoAppView):
"filename=\"{}\"; " \
"filename*=UTF-8\'\'{}".format(safe_path, encoded_path)
- return safe_bytes(headers).decode('latin-1', errors='replace')
+ return header_safe_str(headers)
@LoginRequired()
@HasRepoPermissionAnyDecorator(
diff --git a/rhodecode/apps/repository/views/repo_settings_advanced.py b/rhodecode/apps/repository/views/repo_settings_advanced.py
--- a/rhodecode/apps/repository/views/repo_settings_advanced.py
+++ b/rhodecode/apps/repository/views/repo_settings_advanced.py
@@ -29,7 +29,7 @@ from rhodecode.lib import audit_logger
from rhodecode.lib.auth import (
LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired,
HasRepoPermissionAny)
-from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError
+from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError
from rhodecode.lib.utils2 import safe_int
from rhodecode.lib.vcs import RepositoryError
from rhodecode.model.db import Session, UserFollowing, User, Repository
@@ -136,6 +136,9 @@ class RepoSettingsAdvancedView(RepoAppVi
elif handle_forks == 'delete_forks':
handle_forks = 'delete'
+ repo_advanced_url = h.route_path(
+ 'edit_repo_advanced', repo_name=self.db_repo_name,
+ _anchor='advanced-delete')
try:
old_data = self.db_repo.get_api_data()
RepoModel().delete(self.db_repo, forks=handle_forks)
@@ -158,9 +161,6 @@ class RepoSettingsAdvancedView(RepoAppVi
category='success')
Session().commit()
except AttachedForksError:
- repo_advanced_url = h.route_path(
- 'edit_repo_advanced', repo_name=self.db_repo_name,
- _anchor='advanced-delete')
delete_anchor = h.link_to(_('detach or delete'), repo_advanced_url)
h.flash(_('Cannot delete `{repo}` it still contains attached forks. '
'Try using {delete_or_detach} option.')
@@ -171,9 +171,6 @@ class RepoSettingsAdvancedView(RepoAppVi
raise HTTPFound(repo_advanced_url)
except AttachedPullRequestsError:
- repo_advanced_url = h.route_path(
- 'edit_repo_advanced', repo_name=self.db_repo_name,
- _anchor='advanced-delete')
attached_prs = len(self.db_repo.pull_requests_source +
self.db_repo.pull_requests_target)
h.flash(
@@ -184,6 +181,16 @@ class RepoSettingsAdvancedView(RepoAppVi
# redirect to advanced for forks handle action ?
raise HTTPFound(repo_advanced_url)
+ except AttachedArtifactsError:
+
+ attached_artifacts = len(self.db_repo.artifacts)
+ h.flash(
+ _('Cannot delete `{repo}` it still contains {num} attached artifacts. '
+ 'Consider archiving the repository instead.').format(
+ repo=self.db_repo_name, num=attached_artifacts), category='warning')
+
+ # redirect to advanced for forks handle action ?
+ raise HTTPFound(repo_advanced_url)
except Exception:
log.exception("Exception during deletion of repository")
h.flash(_('An error occurred during deletion of `%s`')
diff --git a/rhodecode/config/config_maker.py b/rhodecode/config/config_maker.py
--- a/rhodecode/config/config_maker.py
+++ b/rhodecode/config/config_maker.py
@@ -206,7 +206,7 @@ def sanitize_settings_and_apply_defaults
settings_maker.make_setting('archive_cache.filesystem.retry_backoff', 1, parser='int')
settings_maker.make_setting('archive_cache.filesystem.retry_attempts', 10, parser='int')
- settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
+ settings_maker.make_setting('archive_cache.objectstore.url', 'http://s3-minio:9000', default_when_empty=True,)
settings_maker.make_setting('archive_cache.objectstore.key', '')
settings_maker.make_setting('archive_cache.objectstore.secret', '')
settings_maker.make_setting('archive_cache.objectstore.region', 'eu-central-1')
diff --git a/rhodecode/config/environment.py b/rhodecode/config/environment.py
--- a/rhodecode/config/environment.py
+++ b/rhodecode/config/environment.py
@@ -16,7 +16,6 @@
# RhodeCode Enterprise Edition, including its added features, Support services,
# and proprietary license terms, please see https://rhodecode.com/licenses/
-import os
import logging
import rhodecode
import collections
@@ -30,6 +29,21 @@ from rhodecode.lib.vcs import connect_vc
log = logging.getLogger(__name__)
+def propagate_rhodecode_config(global_config, settings, config):
+ # Store the settings to make them available to other modules.
+ settings_merged = global_config.copy()
+ settings_merged.update(settings)
+ if config:
+ settings_merged.update(config)
+
+ rhodecode.PYRAMID_SETTINGS = settings_merged
+ rhodecode.CONFIG = settings_merged
+
+ if 'default_user_id' not in rhodecode.CONFIG:
+ rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
+ log.debug('set rhodecode.CONFIG data')
+
+
def load_pyramid_environment(global_config, settings):
# Some parts of the code expect a merge of global and app settings.
settings_merged = global_config.copy()
@@ -75,11 +89,8 @@ def load_pyramid_environment(global_conf
utils.configure_vcs(settings)
- # Store the settings to make them available to other modules.
-
- rhodecode.PYRAMID_SETTINGS = settings_merged
- rhodecode.CONFIG = settings_merged
- rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
+ # first run, to store data...
+ propagate_rhodecode_config(global_config, settings, {})
if vcs_server_enabled:
connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))
diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py
--- a/rhodecode/config/middleware.py
+++ b/rhodecode/config/middleware.py
@@ -115,6 +115,9 @@ def make_pyramid_app(global_config, **se
celery_settings = get_celery_config(settings)
config.configure_celery(celery_settings)
+ # final config set...
+ propagate_rhodecode_config(global_config, settings, config.registry.settings)
+
# creating the app uses a connection - return it after we are done
meta.Session.remove()
diff --git a/rhodecode/lib/exceptions.py b/rhodecode/lib/exceptions.py
--- a/rhodecode/lib/exceptions.py
+++ b/rhodecode/lib/exceptions.py
@@ -80,6 +80,10 @@ class AttachedPullRequestsError(Exceptio
pass
+class AttachedArtifactsError(Exception):
+ pass
+
+
class RepoGroupAssignmentError(Exception):
pass
diff --git a/rhodecode/lib/helpers.py b/rhodecode/lib/helpers.py
--- a/rhodecode/lib/helpers.py
+++ b/rhodecode/lib/helpers.py
@@ -81,7 +81,7 @@ from rhodecode.lib.action_parser import
from rhodecode.lib.html_filters import sanitize_html
from rhodecode.lib.pagination import Page, RepoPage, SqlPage
from rhodecode.lib import ext_json
-from rhodecode.lib.ext_json import json
+from rhodecode.lib.ext_json import json, formatted_str_json
from rhodecode.lib.str_utils import safe_bytes, convert_special_chars, base64_to_str
from rhodecode.lib.utils import repo_name_slug, get_custom_lexer
from rhodecode.lib.str_utils import safe_str
@@ -1416,62 +1416,14 @@ class InitialsGravatar(object):
return "data:image/svg+xml;base64,{}".format(img_data)
-def initials_gravatar(request, email_address, first_name, last_name, size=30, store_on_disk=False):
+def initials_gravatar(request, email_address, first_name, last_name, size=30):
svg_type = None
if email_address == User.DEFAULT_USER_EMAIL:
svg_type = 'default_user'
klass = InitialsGravatar(email_address, first_name, last_name, size)
-
- if store_on_disk:
- from rhodecode.apps.file_store import utils as store_utils
- from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
- FileOverSizeException
- from rhodecode.model.db import Session
-
- image_key = md5_safe(email_address.lower()
- + first_name.lower() + last_name.lower())
-
- storage = store_utils.get_file_storage(request.registry.settings)
- filename = '{}.svg'.format(image_key)
- subdir = 'gravatars'
- # since final name has a counter, we apply the 0
- uid = storage.apply_counter(0, store_utils.uid_filename(filename, randomized=False))
- store_uid = os.path.join(subdir, uid)
-
- db_entry = FileStore.get_by_store_uid(store_uid)
- if db_entry:
- return request.route_path('download_file', fid=store_uid)
-
- img_data = klass.get_img_data(svg_type=svg_type)
- img_file = store_utils.bytes_to_file_obj(img_data)
-
- try:
- store_uid, metadata = storage.save_file(
- img_file, filename, directory=subdir,
- extensions=['.svg'], randomized_name=False)
- except (FileNotAllowedException, FileOverSizeException):
- raise
-
- try:
- entry = FileStore.create(
- file_uid=store_uid, filename=metadata["filename"],
- file_hash=metadata["sha256"], file_size=metadata["size"],
- file_display_name=filename,
- file_description=f'user gravatar `{safe_str(filename)}`',
- hidden=True, check_acl=False, user_id=1
- )
- Session().add(entry)
- Session().commit()
- log.debug('Stored upload in DB as %s', entry)
- except Exception:
- raise
-
- return request.route_path('download_file', fid=store_uid)
-
- else:
- return klass.generate_svg(svg_type=svg_type)
+ return klass.generate_svg(svg_type=svg_type)
def gravatar_external(request, gravatar_url_tmpl, email_address, size=30):
diff --git a/rhodecode/lib/rc_commands/add_artifact.py b/rhodecode/lib/rc_commands/add_artifact.py
--- a/rhodecode/lib/rc_commands/add_artifact.py
+++ b/rhodecode/lib/rc_commands/add_artifact.py
@@ -91,15 +91,14 @@ def command(ini_path, filename, file_pat
auth_user = db_user.AuthUser(ip_addr='127.0.0.1')
- storage = store_utils.get_file_storage(request.registry.settings)
+ f_store = store_utils.get_filestore_backend(request.registry.settings)
with open(file_path, 'rb') as f:
click.secho(f'Adding new artifact from path: `{file_path}`',
fg='green')
file_data = _store_file(
- storage, auth_user, filename, content=None, check_acl=True,
+ f_store, auth_user, filename, content=None, check_acl=True,
file_obj=f, description=description,
scope_repo_id=repo.repo_id)
- click.secho(f'File Data: {file_data}',
- fg='green')
+ click.secho(f'File Data: {file_data}', fg='green')
diff --git a/rhodecode/lib/rc_commands/migrate_artifact.py b/rhodecode/lib/rc_commands/migrate_artifact.py
new file mode 100644
--- /dev/null
+++ b/rhodecode/lib/rc_commands/migrate_artifact.py
@@ -0,0 +1,122 @@
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import sys
+import logging
+
+import click
+
+from rhodecode.lib.pyramid_utils import bootstrap
+from rhodecode.lib.ext_json import json
+from rhodecode.model.db import FileStore
+from rhodecode.apps.file_store import utils as store_utils
+
+log = logging.getLogger(__name__)
+
+
+@click.command()
+@click.argument('ini_path', type=click.Path(exists=True))
+@click.argument('file_uid')
+@click.option(
+ '--source-backend-conf',
+ type=click.Path(exists=True, dir_okay=False, readable=True),
+ help='Source backend config file path in a json format'
+)
+@click.option(
+ '--dest-backend-conf',
+ type=click.Path(exists=True, dir_okay=False, readable=True),
+ help='Source backend config file path in a json format'
+)
+def main(ini_path, file_uid, source_backend_conf, dest_backend_conf):
+ return command(ini_path, file_uid, source_backend_conf, dest_backend_conf)
+
+
+_source_settings = {}
+
+_dest_settings = {}
+
+
+def command(ini_path, file_uid, source_backend_conf, dest_backend_conf):
+ with bootstrap(ini_path, env={'RC_CMD_SETUP_RC': '1'}) as env:
+ migrate_func(env, file_uid, source_backend_conf, dest_backend_conf)
+
+
+def migrate_func(env, file_uid, source_backend_conf=None, dest_backend_conf=None):
+ """
+
+ Example usage::
+
+ from rhodecode.lib.rc_commands import migrate_artifact
+ migrate_artifact._source_settings = {
+ 'file_store.backend.type': 'filesystem_v1',
+ 'file_store.filesystem_v1.storage_path': '/var/opt/rhodecode_data/file_store',
+ }
+ migrate_artifact._dest_settings = {
+ 'file_store.backend.type': 'objectstore',
+ 'file_store.objectstore.url': 'http://s3-minio:9000',
+ 'file_store.objectstore.bucket': 'rhodecode-file-store',
+ 'file_store.objectstore.key': 's3admin',
+ 'file_store.objectstore.secret': 's3secret4',
+ 'file_store.objectstore.region': 'eu-central-1',
+ }
+ for db_obj in FileStore.query().all():
+ migrate_artifact.migrate_func({}, db_obj.file_uid)
+
+ """
+
+ try:
+ from rc_ee.api.views.store_api import _store_file
+ except ImportError:
+ click.secho('ERROR: Unable to import store_api. '
+ 'store_api is only available in EE edition of RhodeCode',
+ fg='red')
+ sys.exit(-1)
+
+ source_settings = _source_settings
+ if source_backend_conf:
+ source_settings = json.loads(open(source_backend_conf).read())
+ dest_settings = _dest_settings
+ if dest_backend_conf:
+ dest_settings = json.loads(open(dest_backend_conf).read())
+
+ if file_uid.isnumeric():
+ file_store_db_obj = FileStore().query() \
+ .filter(FileStore.file_store_id == file_uid) \
+ .scalar()
+ else:
+ file_store_db_obj = FileStore().query() \
+ .filter(FileStore.file_uid == file_uid) \
+ .scalar()
+ if not file_store_db_obj:
+ click.secho(f'ERROR: Unable to fetch artifact from database file_uid={file_uid}',
+ fg='red')
+ sys.exit(-1)
+
+ uid_filename = file_store_db_obj.file_uid
+ org_filename = file_store_db_obj.file_display_name
+ click.secho(f'Attempting to migrate artifact {uid_filename}, filename: {org_filename}', fg='green')
+
+ # get old version of f_store based on the data.
+
+ origin_f_store = store_utils.get_filestore_backend(source_settings, always_init=True)
+ reader, metadata = origin_f_store.fetch(uid_filename)
+
+ target_f_store = store_utils.get_filestore_backend(dest_settings, always_init=True)
+ target_f_store.import_to_store(reader, org_filename, uid_filename, metadata)
+
+ click.secho(f'Migrated artifact {uid_filename}, filename: {org_filename} into {target_f_store} storage', fg='green')
diff --git a/rhodecode/lib/str_utils.py b/rhodecode/lib/str_utils.py
--- a/rhodecode/lib/str_utils.py
+++ b/rhodecode/lib/str_utils.py
@@ -181,3 +181,7 @@ def splitnewlines(text: bytes):
else:
lines[-1] = lines[-1][:-1]
return lines
+
+
+def header_safe_str(val):
+ return safe_bytes(val).decode('latin-1', errors='replace')
diff --git a/rhodecode/lib/system_info.py b/rhodecode/lib/system_info.py
--- a/rhodecode/lib/system_info.py
+++ b/rhodecode/lib/system_info.py
@@ -396,17 +396,18 @@ def storage_inodes():
@register_sysinfo
-def storage_archives():
+def storage_artifacts():
import rhodecode
from rhodecode.lib.helpers import format_byte_size_binary
from rhodecode.lib.archive_cache import get_archival_cache_store
- storage_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type')
+ backend_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type')
- value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=storage_type)
+ value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=backend_type)
state = STATE_OK_DEFAULT
try:
d_cache = get_archival_cache_store(config=rhodecode.CONFIG)
+ backend_type = str(d_cache)
total_files, total_size, _directory_stats = d_cache.get_statistics()
@@ -415,7 +416,8 @@ def storage_archives():
'used': total_size,
'total': total_size,
'items': total_files,
- 'path': d_cache.storage_path
+ 'path': d_cache.storage_path,
+ 'type': backend_type
})
except Exception as e:
@@ -425,8 +427,44 @@ def storage_archives():
human_value = value.copy()
human_value['used'] = format_byte_size_binary(value['used'])
human_value['total'] = format_byte_size_binary(value['total'])
- human_value['text'] = "{} ({} items)".format(
- human_value['used'], value['items'])
+ human_value['text'] = f"{human_value['used']} ({value['items']} items)"
+
+ return SysInfoRes(value=value, state=state, human_value=human_value)
+
+
+@register_sysinfo
+def storage_archives():
+ import rhodecode
+ from rhodecode.lib.helpers import format_byte_size_binary
+ import rhodecode.apps.file_store.utils as store_utils
+ from rhodecode import CONFIG
+
+ backend_type = rhodecode.ConfigGet().get_str(store_utils.config_keys.backend_type)
+
+ value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=backend_type)
+ state = STATE_OK_DEFAULT
+ try:
+ f_store = store_utils.get_filestore_backend(config=CONFIG)
+ backend_type = str(f_store)
+ total_files, total_size, _directory_stats = f_store.get_statistics()
+
+ value.update({
+ 'percent': 100,
+ 'used': total_size,
+ 'total': total_size,
+ 'items': total_files,
+ 'path': f_store.storage_path,
+ 'type': backend_type
+ })
+
+ except Exception as e:
+ log.exception('failed to fetch archive cache storage')
+ state = {'message': str(e), 'type': STATE_ERR}
+
+ human_value = value.copy()
+ human_value['used'] = format_byte_size_binary(value['used'])
+ human_value['total'] = format_byte_size_binary(value['total'])
+ human_value['text'] = f"{human_value['used']} ({value['items']} items)"
return SysInfoRes(value=value, state=state, human_value=human_value)
@@ -798,6 +836,7 @@ def get_system_info(environ):
'storage': SysInfo(storage)(),
'storage_inodes': SysInfo(storage_inodes)(),
'storage_archive': SysInfo(storage_archives)(),
+ 'storage_artifacts': SysInfo(storage_artifacts)(),
'storage_gist': SysInfo(storage_gist)(),
'storage_temp': SysInfo(storage_temp)(),
diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py
--- a/rhodecode/model/db.py
+++ b/rhodecode/model/db.py
@@ -5849,8 +5849,7 @@ class FileStore(Base, BaseModel):
.filter(FileStoreMetadata.file_store_meta_key == key) \
.scalar()
if has_key:
- msg = 'key `{}` already defined under section `{}` for this file.'\
- .format(key, section)
+ msg = f'key `{key}` already defined under section `{section}` for this file.'
raise ArtifactMetadataDuplicate(msg, err_section=section, err_key=key)
# NOTE(marcink): raises ArtifactMetadataBadValueType
@@ -5949,7 +5948,7 @@ class FileStoreMetadata(Base, BaseModel)
def valid_value_type(cls, value):
if value.split('.')[0] not in cls.SETTINGS_TYPES:
raise ArtifactMetadataBadValueType(
- 'value_type must be one of %s got %s' % (cls.SETTINGS_TYPES.keys(), value))
+ f'value_type must be one of {cls.SETTINGS_TYPES.keys()} got {value}')
@hybrid_property
def file_store_meta_section(self):
diff --git a/rhodecode/model/repo.py b/rhodecode/model/repo.py
--- a/rhodecode/model/repo.py
+++ b/rhodecode/model/repo.py
@@ -31,7 +31,7 @@ from zope.cachedescriptors.property impo
from rhodecode import events
from rhodecode.lib.auth import HasUserGroupPermissionAny
from rhodecode.lib.caching_query import FromCache
-from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError
+from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError
from rhodecode.lib import hooks_base
from rhodecode.lib.user_log_filter import user_log_filter
from rhodecode.lib.utils import make_db_config
@@ -736,7 +736,7 @@ class RepoModel(BaseModel):
log.error(traceback.format_exc())
raise
- def delete(self, repo, forks=None, pull_requests=None, fs_remove=True, cur_user=None):
+ def delete(self, repo, forks=None, pull_requests=None, artifacts=None, fs_remove=True, cur_user=None):
"""
Delete given repository, forks parameter defines what do do with
attached forks. Throws AttachedForksError if deleted repo has attached
@@ -745,6 +745,7 @@ class RepoModel(BaseModel):
:param repo:
:param forks: str 'delete' or 'detach'
:param pull_requests: str 'delete' or None
+ :param artifacts: str 'delete' or None
:param fs_remove: remove(archive) repo from filesystem
"""
if not cur_user:
@@ -767,6 +768,13 @@ class RepoModel(BaseModel):
if pull_requests != 'delete' and (pr_sources or pr_targets):
raise AttachedPullRequestsError()
+ artifacts_objs = repo.artifacts
+ if artifacts == 'delete':
+ for a in artifacts_objs:
+ self.sa.delete(a)
+ elif [a for a in artifacts_objs]:
+ raise AttachedArtifactsError()
+
old_repo_dict = repo.get_dict()
events.trigger(events.RepoPreDeleteEvent(repo))
try:
diff --git a/rhodecode/model/user.py b/rhodecode/model/user.py
--- a/rhodecode/model/user.py
+++ b/rhodecode/model/user.py
@@ -557,10 +557,10 @@ class UserModel(BaseModel):
elif handle_mode == 'delete':
from rhodecode.apps.file_store import utils as store_utils
request = get_current_request()
- storage = store_utils.get_file_storage(request.registry.settings)
+ f_store = store_utils.get_filestore_backend(request.registry.settings)
for a in artifacts:
file_uid = a.file_uid
- storage.delete(file_uid)
+ f_store.delete(file_uid)
self.sa.delete(a)
left_overs = False
diff --git a/rhodecode/templates/admin/repos/repo_edit_advanced.mako b/rhodecode/templates/admin/repos/repo_edit_advanced.mako
--- a/rhodecode/templates/admin/repos/repo_edit_advanced.mako
+++ b/rhodecode/templates/admin/repos/repo_edit_advanced.mako
@@ -215,18 +215,35 @@
%endif
+
<% attached_prs = len(c.rhodecode_db_repo.pull_requests_source + c.rhodecode_db_repo.pull_requests_target) %>
% if c.rhodecode_db_repo.pull_requests_source or c.rhodecode_db_repo.pull_requests_target:
${_ungettext('This repository has %s attached pull request.', 'This repository has %s attached pull requests.', attached_prs) % attached_prs}
- ${_('Consider to archive this repository instead.')}
+
+ ${_('Consider to archive this repository instead.')}
|
|
|
% endif
+
+ <% attached_artifacts = len(c.rhodecode_db_repo.artifacts) %>
+ % if attached_artifacts:
+
+
+ ${_ungettext('This repository has %s attached artifact.', 'This repository has %s attached artifacts.', attached_artifacts) % attached_artifacts}
+
+
+ ${_('Consider to archive this repository instead.')}
+ |
+ |
+ |
+
+ % endif
+
diff --git a/rhodecode/tests/fixture.py b/rhodecode/tests/fixture.py
--- a/rhodecode/tests/fixture.py
+++ b/rhodecode/tests/fixture.py
@@ -305,7 +305,7 @@ class Fixture(object):
return r
def destroy_repo(self, repo_name, **kwargs):
- RepoModel().delete(repo_name, pull_requests='delete', **kwargs)
+ RepoModel().delete(repo_name, pull_requests='delete', artifacts='delete', **kwargs)
Session().commit()
def destroy_repo_on_filesystem(self, repo_name):
diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini
--- a/rhodecode/tests/rhodecode.ini
+++ b/rhodecode/tests/rhodecode.ini
@@ -36,7 +36,7 @@ port = 10020
; GUNICORN APPLICATION SERVER
; ###########################
-; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
+; run with gunicorn --config gunicorn_conf.py --paste rhodecode.ini
; Module to use, this setting shouldn't be changed
use = egg:gunicorn#main
@@ -249,15 +249,56 @@ labs_settings_active = true
; optional prefix to Add to email Subject
#exception_tracker.email_prefix = [RHODECODE ERROR]
-; File store configuration. This is used to store and serve uploaded files
-file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store backend is always enabled
+#file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead
; Storage backend, available options are: local
-file_store.backend = local
+#file_store.backend = local
+; NOTE: this setting IS DEPRECATED:
+; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead
; path to store the uploaded binaries and artifacts
-file_store.storage_path = /var/opt/rhodecode_data/file_store
+#file_store.storage_path = /var/opt/rhodecode_data/file_store
+
+; Artifacts file-store, is used to store comment attachments and artifacts uploads.
+; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options
+; filesystem_v1 is backwards compat with pre 5.1 storage changes
+; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from
+; previous installations to keep the artifacts without a need of migration
+file_store.backend.type = filesystem_v1
+
+; filesystem options...
+file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/test_artifacts_file_store
+
+; filesystem_v2 options...
+file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/test_artifacts_file_store_2
+file_store.filesystem_v2.shards = 8
+; objectstore options...
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+#file_store.backend.type = objectstore
+file_store.objectstore.url = http://s3-minio:9000
+
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number
+file_store.objectstore.bucket = rhodecode-file-store-tests
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+file_store.objectstore.bucket_shards = 8
+
+; key for s3 auth
+file_store.objectstore.key = s3admin
+
+; secret for s3 auth
+file_store.objectstore.secret = s3secret4
+
+;region for s3 storage
+file_store.objectstore.region = eu-central-1
; Redis url to acquire/check generation of archives locks
archive_cache.locking.url = redis://redis:6379/1
@@ -593,6 +634,7 @@ vcs.scm_app_implementation = http
; Push/Pull operations hooks protocol, available options are:
; `http` - use http-rpc backend (default)
; `celery` - use celery based hooks
+#DEPRECATED:vcs.hooks.protocol = http
vcs.hooks.protocol = http
; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
@@ -626,6 +668,10 @@ vcs.methods.cache = false
; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
#vcs.svn.compatible_version = 1.8
+; Redis connection settings for svn integrations logic
+; This connection string needs to be the same on ce and vcsserver
+vcs.svn.redis_conn = redis://redis:6379/0
+
; Enable SVN proxy of requests over HTTP
vcs.svn.proxy.enabled = true
@@ -681,7 +727,8 @@ ssh.authorized_keys_file_path = %(here)s
; RhodeCode installation directory.
; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
-ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+#DEPRECATED: ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+ssh.wrapper_cmd.v2 = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
; Allow shell when executing the ssh-wrapper command
ssh.wrapper_cmd_allow_shell = false
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -189,6 +189,7 @@ setup(
'rc-upgrade-db=rhodecode.lib.rc_commands.upgrade_db:main',
'rc-ishell=rhodecode.lib.rc_commands.ishell:main',
'rc-add-artifact=rhodecode.lib.rc_commands.add_artifact:main',
+ 'rc-migrate-artifact=rhodecode.lib.rc_commands.migrate_artifact:main',
'rc-ssh-wrapper=rhodecode.apps.ssh_support.lib.ssh_wrapper_v1:main',
'rc-ssh-wrapper-v2=rhodecode.apps.ssh_support.lib.ssh_wrapper_v2:main',
],