chore: fixed merge conflicts

docs/auth/auth-saml-azure.rst

@@ -0,0 +1,161 b''
+.. _config-saml-azure-ref:
+
+
+SAML 2.0 with Azure Entra ID
+----------------------------
+
+**This plugin is available only in EE Edition.**
+
+|RCE| supports SAML 2.0 Authentication with Azure Entra ID provider. This allows
+users to log-in to RhodeCode via SSO mechanism of external identity provider
+such as Azure AD. The login can be triggered either by the external IDP, or internally
+by clicking specific authentication button on the log-in page.
+
+
+Configuration steps
+^^^^^^^^^^^^^^^^^^^
+
+To configure Duo Security SAML authentication, use the following steps:
+
+1. From the |RCE| interface, select
+   :menuselection:`Admin --> Authentication`
+2. Activate the `Azure Entra ID` plugin and select :guilabel:`Save`
+3. Go to newly available menu option called `Azure Entra ID` on the left side.
+4. Check the `enabled` check box in the plugin configuration section,
+   and fill in the required SAML information and :guilabel:`Save`, for more details,
+   see :ref:`config-saml-azure`
+
+
+.. _config-saml-azure:
+
+
+Example SAML Azure Entra ID configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Example configuration for SAML 2.0 with Azure Entra ID provider
+
+
+Enabled
+    `True`:
+
+    .. note::
+      Enable or disable this authentication plugin.
+
+
+Auth Cache TTL
+    `30`:
+
+    .. note::
+      Amount of seconds to cache the authentication and permissions check response call for this plugin.
+      Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
+
+Debug
+    `True`:
+
+    .. note::
+      Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
+
+
+Auth button name
+    `Azure Entra ID`:
+
+    .. note::
+      Alternative authentication display name. E.g AzureAuth, CorporateID etc.
+
+
+Entity ID
+    `https://sts.windows.net/APP_ID/`:
+
+    .. note::
+      Identity Provider entity/metadata URI. Known as "Microsoft Entra Identifier"
+      E.g. https://sts.windows.net/abcd-c655-dcee-aab7-abcd/
+
+SSO URL
+    `https://login.microsoftonline.com/APP_ID/saml2`:
+
+    .. note::
+      SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login, Known also as Login URL
+      E.g. https://login.microsoftonline.com/abcd-c655-dcee-aab7-abcd/saml2
+
+SLO URL
+    `https://login.microsoftonline.com/APP_ID/saml2`:
+
+    .. note::
+      SLO (SingleLogout) endpoint URL of the IdP. , Known also as Logout URL
+      E.g. https://login.microsoftonline.com/abcd-c655-dcee-aab7-abcd/saml2
+
+x509cert
+    `<CERTIFICATE_STRING>`:
+
+    .. note::
+      Identity provider public x509 certificate. It will be converted to single-line format without headers.
+      Download the raw base64 encoded certificate from the Identity provider and paste it here.
+
+SAML Signature
+    `sha-256`:
+
+    .. note::
+      Type of Algorithm to use for verification of SAML signature on Identity provider side.
+
+SAML Digest
+    `sha-256`:
+
+    .. note::
+      Type of Algorithm to use for verification of SAML digest on Identity provider side.
+
+Service Provider Cert Dir
+    `/etc/rhodecode/conf/saml_ssl/`:
+
+    .. note::
+      Optional directory to store service provider certificate and private keys.
+      Expected certs for the SP should be stored in this folder as:
+
+      * sp.key     Private Key
+      * sp.crt     Public cert
+      * sp_new.crt Future Public cert
+
+      Also you can use other cert to sign the metadata of the SP using the:
+
+      * metadata.key
+      * metadata.crt
+
+Expected NameID Format
+    `nameid-format:emailAddress`:
+
+    .. note::
+      The format that specifies how the NameID is sent to the service provider.
+
+User ID Attribute
+    `user.email`:
+
+    .. note::
+      User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
+      Ensure this is returned from DuoSecurity for example via duo_username.
+
+Username Attribute
+    `user.username`:
+
+    .. note::
+      Username Attribute name. This defines which attribute in SAML response will map to a username.
+
+Email Attribute
+    `user.email`:
+
+    .. note::
+      Email Attribute name. This defines which attribute in SAML response will map to an email address.
+
+
+
+Below is example setup from Azure Administration page that can be used with above config.
+
+.. image:: ../images/saml-azure-service-provider-example.png
+   :alt: Azure SAML setup example
+   :scale: 50 %
+
+
+Below is an example attribute mapping set for IDP provider required by the above config.
+
+
+.. image:: ../images/saml-azure-attributes-example.png
+   :alt: Azure SAML setup example
+   :scale: 50 %
  No newline at end of file

docs/release-notes/release-notes-5.1.1.rst

@@ -0,0 +1,40 b''
+|RCE| 5.1.1 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2024-07-23
+
+
+New Features
+^^^^^^^^^^^^
+
+
+
+General
+^^^^^^^
+
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+
+
+
+Fixes
+^^^^^
+
+- Fixed problems with JS static files build
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.1.1 is unscheduled bugfix release to address some build issues with 5.1 images

docs/release-notes/release-notes-5.1.2.rst

@@ -0,0 +1,41 b''
+|RCE| 5.1.2 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2024-09-12
+
+
+New Features
+^^^^^^^^^^^^
+
+
+
+General
+^^^^^^^
+
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+
+
+
+Fixes
+^^^^^
+
+- Fixed problems with Mercurial authentication after enabling httppostargs.
+  Currently this protocol will be disabled until proper fix is in place
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.1.2 is unscheduled bugfix release to address some build issues with 5.1 images

docs/release-notes/release-notes-5.2.0.rst

@@ -0,0 +1,55 b''
+|RCE| 5.2.0 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2024-10-09
+
+
+New Features
+^^^^^^^^^^^^
+
+- New artifact storage engines allowing an s3 based uploads
+- Enterprise version only: Added security tab to admin interface and possibility to whitelist specific vcs client versions. Some older versions clients have known security vulnerabilities, now you can disallow them.
+- Enterprise version only: Implemented support for Azure SAML authentication
+
+
+General
+^^^^^^^
+- Bumped version of packaging, gunicorn, orjson, zope.interface and some other requirements
+- Few tweaks and changes to saml plugins to allows easier setup
+- Configs: allow json log format for gunicorn
+- Configs: deprecated old ssh wrapper command and make the v2 the default one
+- Make sure commit-caches propagate to parent repo groups
+- Configs: Moved git lfs path and path of hg large files to ini file
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+- description escaper for better performance
+
+Fixes
+^^^^^
+
+- Email notifications not working properly
+- Removed waitress as a default runner
+- Fixed issue with branch permissions
+- Ldap: fixed nested groups extraction logic
+- Fixed possible db corruption in case of filesystem problems
+- Cleanup and improvements to documentation
+- Added Kubernetes deployment section to the documentation
+- Added default value to celery result and broker
+- Fixed broken backends function after python3 migration
+- Explicitly disable mercurial web_push ssl flag to prevent from errors about ssl required
+- VCS: fixed problems with locked repos and with branch permissions reporting
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.2.0 is a planned major release featuring Azure SAML, whitelist for client versions, s3 artifacts backend and more!

rhodecode/apps/admin/views/security.py

@@ -0,0 +1,46 b''
+# Copyright (C) 2010-2024 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+
+from rhodecode.apps._base import BaseAppView
+from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
+
+log = logging.getLogger(__name__)
+
+
+class AdminSecurityView(BaseAppView):
+
+    def load_default_context(self):
+        c = self._get_local_tmpl_context()
+        return c
+
+    @LoginRequired()
+    @HasPermissionAllDecorator('hg.admin')
+    def security(self):
+        c = self.load_default_context()
+        c.active = 'security'
+        return self._get_template_context(c)
+
+
+    @LoginRequired()
+    @HasPermissionAllDecorator('hg.admin')
+    def admin_security_modify_allowed_vcs_client_versions(self):
+        c = self.load_default_context()
+        c.active = 'security'
+        return self._get_template_context(c)

rhodecode/apps/file_store/backends/base.py

@@ -0,0 +1,269 b''
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import fsspec  # noqa
+import logging
+
+from rhodecode.lib.ext_json import json
+
+from rhodecode.apps.file_store.utils import sha256_safe, ShardFileReader, get_uid_filename
+from rhodecode.apps.file_store.extensions import resolve_extensions
+from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException  # noqa: F401
+
+log = logging.getLogger(__name__)
+
+
+class BaseShard:
+
+    metadata_suffix: str = '.metadata'
+    storage_type: str = ''
+    fs = None
+
+    @property
+    def storage_medium(self):
+        if not self.storage_type:
+            raise ValueError('No storage type set for this shard storage_type=""')
+        return getattr(self, self.storage_type)
+
+    def __contains__(self, key):
+        full_path = self.store_path(key)
+        return self.fs.exists(full_path)
+
+    def metadata_convert(self, uid_filename, metadata):
+        return metadata
+
+    def get_metadata_filename(self, uid_filename) -> tuple[str, str]:
+        metadata_file: str = f'{uid_filename}{self.metadata_suffix}'
+        return metadata_file, self.store_path(metadata_file)
+
+    def get_metadata(self, uid_filename, ignore_missing=False) -> dict:
+        _metadata_file, metadata_file_path = self.get_metadata_filename(uid_filename)
+        if ignore_missing and not self.fs.exists(metadata_file_path):
+            return {}
+
+        with self.fs.open(metadata_file_path, 'rb') as f:
+            metadata = json.loads(f.read())
+
+        metadata = self.metadata_convert(uid_filename, metadata)
+        return metadata
+
+    def _store(self, key: str, uid_key: str, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+        raise NotImplementedError
+
+    def store(self, key: str, uid_key: str, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+        return self._store(key, uid_key, value_reader, max_filesize, metadata, **kwargs)
+
+    def _fetch(self, key, presigned_url_expires: int = 0):
+        raise NotImplementedError
+
+    def fetch(self, key, **kwargs) -> tuple[ShardFileReader, dict]:
+        return self._fetch(key)
+
+    def _delete(self, key):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata = self.get_metadata(key)
+        _metadata_file, metadata_file_path = self.get_metadata_filename(key)
+        artifact_file_path = metadata['filename_uid_path']
+        self.fs.rm(artifact_file_path)
+        self.fs.rm(metadata_file_path)
+
+        return 1
+
+    def delete(self, key):
+        raise NotImplementedError
+
+    def store_path(self, uid_filename):
+        raise NotImplementedError
+
+
+class BaseFileStoreBackend:
+    _shards = tuple()
+    _shard_cls = BaseShard
+    _config: dict | None = None
+    _storage_path: str = ''
+
+    def __init__(self, settings, extension_groups=None):
+        self._config = settings
+        extension_groups = extension_groups or ['any']
+        self.extensions = resolve_extensions([], groups=extension_groups)
+
+    def __contains__(self, key):
+        return self.filename_exists(key)
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__}(storage={self.storage_path})>'
+
+    @property
+    def storage_path(self):
+        return self._storage_path
+
+    @classmethod
+    def get_shard_index(cls, filename: str, num_shards) -> int:
+        # Generate a hash value from the filename
+        hash_value = sha256_safe(filename)
+
+        # Convert the hash value to an integer
+        hash_int = int(hash_value, 16)
+
+        # Map the hash integer to a shard number between 1 and num_shards
+        shard_number = (hash_int % num_shards)
+
+        return shard_number
+
+    @classmethod
+    def apply_counter(cls, counter: int, filename: str) -> str:
+        """
+        Apply a counter to the filename.
+
+        :param counter: The counter value to apply.
+        :param filename: The original filename.
+        :return: The modified filename with the counter.
+        """
+        name_counted = f'{counter:d}-{filename}'
+        return name_counted
+
+    def _get_shard(self, key) -> _shard_cls:
+        index = self.get_shard_index(key, len(self._shards))
+        shard = self._shards[index]
+        return shard
+
+    def get_conf(self, key, pop=False):
+        if key not in self._config:
+            raise ValueError(
+                f"No configuration key '{key}', please make sure it exists in filestore config")
+        val = self._config[key]
+        if pop:
+            del self._config[key]
+        return val
+
+    def filename_allowed(self, filename, extensions=None):
+        """Checks if a filename has an allowed extension
+
+        :param filename: base name of file
+        :param extensions: iterable of extensions (or self.extensions)
+        """
+        _, ext = os.path.splitext(filename)
+        return self.extension_allowed(ext, extensions)
+
+    def extension_allowed(self, ext, extensions=None):
+        """
+        Checks if an extension is permitted. Both e.g. ".jpg" and
+        "jpg" can be passed in. Extension lookup is case-insensitive.
+
+        :param ext: extension to check
+        :param extensions: iterable of extensions to validate against (or self.extensions)
+        """
+        def normalize_ext(_ext):
+            if _ext.startswith('.'):
+                _ext = _ext[1:]
+            return _ext.lower()
+
+        extensions = extensions or self.extensions
+        if not extensions:
+            return True
+
+        ext = normalize_ext(ext)
+
+        return ext in [normalize_ext(x) for x in extensions]
+
+    def filename_exists(self, uid_filename):
+        shard = self._get_shard(uid_filename)
+        return uid_filename in shard
+
+    def store_path(self, uid_filename):
+        """
+        Returns absolute file path of the uid_filename
+        """
+        shard = self._get_shard(uid_filename)
+        return shard.store_path(uid_filename)
+
+    def store_metadata(self, uid_filename):
+        shard = self._get_shard(uid_filename)
+        return shard.get_metadata_filename(uid_filename)
+
+    def store(self, filename, value_reader, extensions=None, metadata=None, max_filesize=None, randomized_name=True, **kwargs):
+        extensions = extensions or self.extensions
+
+        if not self.filename_allowed(filename, extensions):
+            msg = f'filename {filename} does not allow extensions {extensions}'
+            raise FileNotAllowedException(msg)
+
+        # # TODO: check why we need this setting ? it looks stupid...
+        # no_body_seek is used in stream mode importer somehow
+        # no_body_seek = kwargs.pop('no_body_seek', False)
+        # if no_body_seek:
+        #     pass
+        # else:
+        #     value_reader.seek(0)
+
+        uid_filename = kwargs.pop('uid_filename', None)
+        if uid_filename is None:
+            uid_filename = get_uid_filename(filename, randomized=randomized_name)
+
+        shard = self._get_shard(uid_filename)
+
+        return shard.store(filename, uid_filename, value_reader, max_filesize, metadata, **kwargs)
+
+    def import_to_store(self, value_reader, org_filename, uid_filename, metadata, **kwargs):
+        shard = self._get_shard(uid_filename)
+        max_filesize = None
+        return shard.store(org_filename, uid_filename, value_reader, max_filesize, metadata, import_mode=True)
+
+    def delete(self, uid_filename):
+        shard = self._get_shard(uid_filename)
+        return shard.delete(uid_filename)
+
+    def fetch(self, uid_filename) -> tuple[ShardFileReader, dict]:
+        shard = self._get_shard(uid_filename)
+        return shard.fetch(uid_filename)
+
+    def get_metadata(self, uid_filename, ignore_missing=False) -> dict:
+        shard = self._get_shard(uid_filename)
+        return shard.get_metadata(uid_filename, ignore_missing=ignore_missing)
+
+    def iter_keys(self):
+        for shard in self._shards:
+            if shard.fs.exists(shard.storage_medium):
+                for path, _dirs, _files in shard.fs.walk(shard.storage_medium):
+                    for key_file_path in _files:
+                        if key_file_path.endswith(shard.metadata_suffix):
+                            yield shard, key_file_path
+
+    def iter_artifacts(self):
+        for shard, key_file in self.iter_keys():
+            json_key = f"{shard.storage_medium}/{key_file}"
+            with shard.fs.open(json_key, 'rb') as f:
+                yield shard, json.loads(f.read())['filename_uid']
+
+    def get_statistics(self):
+        total_files = 0
+        total_size = 0
+        meta = {}
+
+        for shard, key_file in self.iter_keys():
+            json_key = f"{shard.storage_medium}/{key_file}"
+            with shard.fs.open(json_key, 'rb') as f:
+                total_files += 1
+                metadata = json.loads(f.read())
+                total_size += metadata['size']
+
+        return total_files, total_size, meta

rhodecode/apps/file_store/backends/filesystem.py

@@ -0,0 +1,183 b''
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import hashlib
+import functools
+import time
+import logging
+
+from .. import config_keys
+from ..exceptions import FileOverSizeException
+from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
+
+from ....lib.ext_json import json
+
+log = logging.getLogger(__name__)
+
+
+class FileSystemShard(BaseShard):
+    METADATA_VER = 'v2'
+    BACKEND_TYPE = config_keys.backend_filesystem
+    storage_type: str = 'directory'
+
+    def __init__(self, index, directory, directory_folder, fs, **settings):
+        self._index: int = index
+        self._directory: str = directory
+        self._directory_folder: str = directory_folder
+        self.fs = fs
+
+    @property
+    def directory(self) -> str:
+        """Cache directory final path."""
+        return os.path.join(self._directory, self._directory_folder)
+
+    def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
+
+        # ensure dir exists
+        destination, _ = os.path.split(full_path)
+        if not self.fs.exists(destination):
+            self.fs.makedirs(destination)
+
+        writer = self.fs.open(full_path, mode)
+
+        digest = hashlib.sha256()
+        oversize_cleanup = False
+        with writer:
+            size = 0
+            for chunk in iterator:
+                size += len(chunk)
+                digest.update(chunk)
+                writer.write(chunk)
+
+                if max_filesize and size > max_filesize:
+                    oversize_cleanup = True
+                    # free up the copied file, and raise exc
+                    break
+
+            writer.flush()
+            # Get the file descriptor
+            fd = writer.fileno()
+
+            # Sync the file descriptor to disk, helps with NFS cases...
+            os.fsync(fd)
+
+        if oversize_cleanup:
+            self.fs.rm(full_path)
+            raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
+
+        sha256 = digest.hexdigest()
+        log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
+        return size, sha256
+
+    def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+
+        filename = key
+        uid_filename = uid_key
+        full_path = self.store_path(uid_filename)
+
+        # STORE METADATA
+        _metadata = {
+            "version": self.METADATA_VER,
+            "store_type": self.BACKEND_TYPE,
+
+            "filename": filename,
+            "filename_uid_path": full_path,
+            "filename_uid": uid_filename,
+            "sha256": "",  # NOTE: filled in by reader iteration
+
+            "store_time": time.time(),
+
+            "size": 0
+        }
+
+        if metadata:
+            if kwargs.pop('import_mode', False):
+                # in import mode, we don't need to compute metadata, we just take the old version
+                _metadata["import_mode"] = True
+            else:
+                _metadata.update(metadata)
+
+        read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
+        size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
+        _metadata['size'] = size
+        _metadata['sha256'] = sha256
+
+        # after storing the artifacts, we write the metadata present
+        _metadata_file, metadata_file_path = self.get_metadata_filename(uid_key)
+
+        with self.fs.open(metadata_file_path, 'wb') as f:
+            f.write(json.dumps(_metadata))
+
+        return uid_filename, _metadata
+
+    def store_path(self, uid_filename):
+        """
+        Returns absolute file path of the uid_filename
+        """
+        return os.path.join(self._directory, self._directory_folder, uid_filename)
+
+    def _fetch(self, key, presigned_url_expires: int = 0):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata = self.get_metadata(key)
+
+        file_path = metadata['filename_uid_path']
+        if presigned_url_expires and presigned_url_expires > 0:
+            metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
+
+        return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
+
+    def delete(self, key):
+        return self._delete(key)
+
+
+class FileSystemBackend(BaseFileStoreBackend):
+    shard_name: str = 'shard_{:03d}'
+    _shard_cls = FileSystemShard
+
+    def __init__(self, settings):
+        super().__init__(settings)
+
+        store_dir = self.get_conf(config_keys.filesystem_storage_path)
+        directory = os.path.expanduser(store_dir)
+
+        self._directory = directory
+        self._storage_path = directory  # common path for all from BaseCache
+        self._shard_count = int(self.get_conf(config_keys.filesystem_shards, pop=True))
+        if self._shard_count < 1:
+            raise ValueError(f'{config_keys.filesystem_shards} must be 1 or more')
+
+        log.debug('Initializing %s file_store instance', self)
+        fs = fsspec.filesystem('file')
+
+        if not fs.exists(self._directory):
+            fs.makedirs(self._directory, exist_ok=True)
+
+        self._shards = tuple(
+            self._shard_cls(
+                index=num,
+                directory=directory,
+                directory_folder=self.shard_name.format(num),
+                fs=fs,
+                **settings,
+            )
+            for num in range(self._shard_count)
+        )

rhodecode/apps/file_store/backends/filesystem_legacy.py

@@ -0,0 +1,278 b''
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import errno
+import os
+import hashlib
+import functools
+import time
+import logging
+
+from .. import config_keys
+from ..exceptions import FileOverSizeException
+from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
+
+from ....lib.ext_json import json
+
+log = logging.getLogger(__name__)
+
+
+class LegacyFileSystemShard(BaseShard):
+    # legacy ver
+    METADATA_VER = 'v2'
+    BACKEND_TYPE = config_keys.backend_legacy_filesystem
+    storage_type: str = 'dir_struct'
+
+    # legacy suffix
+    metadata_suffix: str = '.meta'
+
+    @classmethod
+    def _sub_store_from_filename(cls, filename):
+        return filename[:2]
+
+    @classmethod
+    def apply_counter(cls, counter, filename):
+        name_counted = '%d-%s' % (counter, filename)
+        return name_counted
+
+    @classmethod
+    def safe_make_dirs(cls, dir_path):
+        if not os.path.exists(dir_path):
+            try:
+                os.makedirs(dir_path)
+            except OSError as e:
+                if e.errno != errno.EEXIST:
+                    raise
+                return
+
+    @classmethod
+    def resolve_name(cls, name, directory):
+        """
+        Resolves a unique name and the correct path. If a filename
+        for that path already exists then a numeric prefix with values > 0 will be
+        added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix.
+
+        :param name: base name of file
+        :param directory: absolute directory path
+        """
+
+        counter = 0
+        while True:
+            name_counted = cls.apply_counter(counter, name)
+
+            # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
+            sub_store: str = cls._sub_store_from_filename(name_counted)
+            sub_store_path: str = os.path.join(directory, sub_store)
+            cls.safe_make_dirs(sub_store_path)
+
+            path = os.path.join(sub_store_path, name_counted)
+            if not os.path.exists(path):
+                return name_counted, path
+            counter += 1
+
+    def __init__(self, index, directory, directory_folder, fs, **settings):
+        self._index: int = index
+        self._directory: str = directory
+        self._directory_folder: str = directory_folder
+        self.fs = fs
+
+    @property
+    def dir_struct(self) -> str:
+        """Cache directory final path."""
+        return os.path.join(self._directory, '0-')
+
+    def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
+
+        # ensure dir exists
+        destination, _ = os.path.split(full_path)
+        if not self.fs.exists(destination):
+            self.fs.makedirs(destination)
+
+        writer = self.fs.open(full_path, mode)
+
+        digest = hashlib.sha256()
+        oversize_cleanup = False
+        with writer:
+            size = 0
+            for chunk in iterator:
+                size += len(chunk)
+                digest.update(chunk)
+                writer.write(chunk)
+
+                if max_filesize and size > max_filesize:
+                    # free up the copied file, and raise exc
+                    oversize_cleanup = True
+                    break
+
+            writer.flush()
+            # Get the file descriptor
+            fd = writer.fileno()
+
+            # Sync the file descriptor to disk, helps with NFS cases...
+            os.fsync(fd)
+
+        if oversize_cleanup:
+            self.fs.rm(full_path)
+            raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
+
+        sha256 = digest.hexdigest()
+        log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
+        return size, sha256
+
+    def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+
+        filename = key
+        uid_filename = uid_key
+
+        # NOTE:, also apply N- Counter...
+        uid_filename, full_path = self.resolve_name(uid_filename, self._directory)
+
+        # STORE METADATA
+        # TODO: make it compatible, and backward proof
+        _metadata = {
+            "version": self.METADATA_VER,
+
+            "filename": filename,
+            "filename_uid_path": full_path,
+            "filename_uid": uid_filename,
+            "sha256": "",  # NOTE: filled in by reader iteration
+
+            "store_time": time.time(),
+
+            "size": 0
+        }
+        if metadata:
+            _metadata.update(metadata)
+
+        read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
+        size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
+        _metadata['size'] = size
+        _metadata['sha256'] = sha256
+
+        # after storing the artifacts, we write the metadata present
+        _metadata_file, metadata_file_path = self.get_metadata_filename(uid_filename)
+
+        with self.fs.open(metadata_file_path, 'wb') as f:
+            f.write(json.dumps(_metadata))
+
+        return uid_filename, _metadata
+
+    def store_path(self, uid_filename):
+        """
+        Returns absolute file path of the uid_filename
+        """
+        prefix_dir = ''
+        if '/' in uid_filename:
+            prefix_dir, filename = uid_filename.split('/')
+            sub_store = self._sub_store_from_filename(filename)
+        else:
+            sub_store = self._sub_store_from_filename(uid_filename)
+
+        return os.path.join(self._directory, prefix_dir, sub_store, uid_filename)
+
+    def metadata_convert(self, uid_filename, metadata):
+        # NOTE: backward compat mode here... this is for file created PRE 5.2 system
+        if 'meta_ver' in metadata:
+            full_path = self.store_path(uid_filename)
+            metadata = {
+                "_converted": True,
+                "_org": metadata,
+                "version": self.METADATA_VER,
+                "store_type": self.BACKEND_TYPE,
+
+                "filename": metadata['filename'],
+                "filename_uid_path": full_path,
+                "filename_uid": uid_filename,
+                "sha256": metadata['sha256'],
+
+                "store_time": metadata['time'],
+
+                "size": metadata['size']
+            }
+        return metadata
+
+    def _fetch(self, key, presigned_url_expires: int = 0):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata = self.get_metadata(key)
+
+        file_path = metadata['filename_uid_path']
+        if presigned_url_expires and presigned_url_expires > 0:
+            metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
+
+        return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
+
+    def delete(self, key):
+        return self._delete(key)
+
+    def _delete(self, key):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata = self.get_metadata(key)
+        metadata_file, metadata_file_path = self.get_metadata_filename(key)
+        artifact_file_path = metadata['filename_uid_path']
+        self.fs.rm(artifact_file_path)
+        self.fs.rm(metadata_file_path)
+
+    def get_metadata_filename(self, uid_filename) -> tuple[str, str]:
+
+        metadata_file: str = f'{uid_filename}{self.metadata_suffix}'
+        uid_path_in_store = self.store_path(uid_filename)
+
+        metadata_file_path = f'{uid_path_in_store}{self.metadata_suffix}'
+        return metadata_file, metadata_file_path
+
+
+class LegacyFileSystemBackend(BaseFileStoreBackend):
+    _shard_cls = LegacyFileSystemShard
+
+    def __init__(self, settings):
+        super().__init__(settings)
+
+        store_dir = self.get_conf(config_keys.legacy_filesystem_storage_path)
+        directory = os.path.expanduser(store_dir)
+
+        self._directory = directory
+        self._storage_path = directory  # common path for all from BaseCache
+
+        log.debug('Initializing %s file_store instance', self)
+        fs = fsspec.filesystem('file')
+
+        if not fs.exists(self._directory):
+            fs.makedirs(self._directory, exist_ok=True)
+
+        # legacy system uses single shard
+        self._shards = tuple(
+            [
+                self._shard_cls(
+                    index=0,
+                    directory=directory,
+                    directory_folder='',
+                    fs=fs,
+                    **settings,
+                )
+            ]
+        )
+
+    @classmethod
+    def get_shard_index(cls, filename: str, num_shards) -> int:
+        # legacy filesystem doesn't use shards, and always uses single shard
+        return 0

rhodecode/apps/file_store/backends/objectstore.py

@@ -0,0 +1,184 b''
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import hashlib
+import functools
+import time
+import logging
+
+from .. import config_keys
+from ..exceptions import FileOverSizeException
+from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
+
+from ....lib.ext_json import json
+
+log = logging.getLogger(__name__)
+
+
+class S3Shard(BaseShard):
+    METADATA_VER = 'v2'
+    BACKEND_TYPE = config_keys.backend_objectstore
+    storage_type: str = 'bucket'
+
+    def __init__(self, index, bucket, bucket_folder, fs, **settings):
+        self._index: int = index
+        self._bucket_main: str = bucket
+        self._bucket_folder: str = bucket_folder
+
+        self.fs = fs
+
+    @property
+    def bucket(self) -> str:
+        """Cache bucket final path."""
+        return os.path.join(self._bucket_main, self._bucket_folder)
+
+    def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
+
+        # ensure dir exists
+        destination, _ = os.path.split(full_path)
+        if not self.fs.exists(destination):
+            self.fs.makedirs(destination)
+
+        writer = self.fs.open(full_path, mode)
+
+        digest = hashlib.sha256()
+        oversize_cleanup = False
+        with writer:
+            size = 0
+            for chunk in iterator:
+                size += len(chunk)
+                digest.update(chunk)
+                writer.write(chunk)
+
+                if max_filesize and size > max_filesize:
+                    oversize_cleanup = True
+                    # free up the copied file, and raise exc
+                    break
+
+        if oversize_cleanup:
+            self.fs.rm(full_path)
+            raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
+
+        sha256 = digest.hexdigest()
+        log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
+        return size, sha256
+
+    def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+
+        filename = key
+        uid_filename = uid_key
+        full_path = self.store_path(uid_filename)
+
+        # STORE METADATA
+        _metadata = {
+            "version": self.METADATA_VER,
+            "store_type": self.BACKEND_TYPE,
+
+            "filename": filename,
+            "filename_uid_path": full_path,
+            "filename_uid": uid_filename,
+            "sha256": "",  # NOTE: filled in by reader iteration
+
+            "store_time": time.time(),
+
+            "size": 0
+        }
+
+        if metadata:
+            if kwargs.pop('import_mode', False):
+                # in import mode, we don't need to compute metadata, we just take the old version
+                _metadata["import_mode"] = True
+            else:
+                _metadata.update(metadata)
+
+        read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
+        size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
+        _metadata['size'] = size
+        _metadata['sha256'] = sha256
+
+        # after storing the artifacts, we write the metadata present
+        metadata_file, metadata_file_path = self.get_metadata_filename(uid_key)
+
+        with self.fs.open(metadata_file_path, 'wb') as f:
+            f.write(json.dumps(_metadata))
+
+        return uid_filename, _metadata
+
+    def store_path(self, uid_filename):
+        """
+        Returns absolute file path of the uid_filename
+        """
+        return os.path.join(self._bucket_main, self._bucket_folder, uid_filename)
+
+    def _fetch(self, key, presigned_url_expires: int = 0):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata_file, metadata_file_path = self.get_metadata_filename(key)
+        with self.fs.open(metadata_file_path, 'rb') as f:
+            metadata = json.loads(f.read())
+
+        file_path = metadata['filename_uid_path']
+        if presigned_url_expires and presigned_url_expires > 0:
+            metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
+
+        return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
+
+    def delete(self, key):
+        return self._delete(key)
+
+
+class ObjectStoreBackend(BaseFileStoreBackend):
+    shard_name: str = 'shard-{:03d}'
+    _shard_cls = S3Shard
+
+    def __init__(self, settings):
+        super().__init__(settings)
+
+        self._shard_count = int(self.get_conf(config_keys.objectstore_bucket_shards, pop=True))
+        if self._shard_count < 1:
+            raise ValueError('cache_shards must be 1 or more')
+
+        self._bucket = settings.pop(config_keys.objectstore_bucket)
+        if not self._bucket:
+            raise ValueError(f'{config_keys.objectstore_bucket} needs to have a value')
+
+        objectstore_url = self.get_conf(config_keys.objectstore_url)
+        key = settings.pop(config_keys.objectstore_key)
+        secret = settings.pop(config_keys.objectstore_secret)
+
+        self._storage_path = objectstore_url  # common path for all from BaseCache
+        log.debug('Initializing %s file_store instance', self)
+        fs = fsspec.filesystem('s3', anon=False, endpoint_url=objectstore_url, key=key, secret=secret)
+
+        # init main bucket
+        if not fs.exists(self._bucket):
+            fs.mkdir(self._bucket)
+
+        self._shards = tuple(
+            self._shard_cls(
+                index=num,
+                bucket=self._bucket,
+                bucket_folder=self.shard_name.format(num),
+                fs=fs,
+                **settings,
+            )
+            for num in range(self._shard_count)
+        )

rhodecode/apps/file_store/tests/test_filestore_backends.py

@@ -0,0 +1,128 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps import file_store
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.backends.filesystem_legacy import LegacyFileSystemBackend
+from rhodecode.apps.file_store.backends.filesystem import FileSystemBackend
+from rhodecode.apps.file_store.backends.objectstore import ObjectStoreBackend
+from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store.tests import random_binary_file, file_store_instance
+
+
+class TestFileStoreBackends:
+
+    @pytest.mark.parametrize('backend_type, expected_instance', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+        (config_keys.backend_filesystem, FileSystemBackend),
+        (config_keys.backend_objectstore, ObjectStoreBackend),
+    ])
+    def test_get_backend(self, backend_type, expected_instance, ini_settings):
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+        assert isinstance(f_store, expected_instance)
+
+    @pytest.mark.parametrize('backend_type, expected_instance', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+        (config_keys.backend_filesystem, FileSystemBackend),
+        (config_keys.backend_objectstore, ObjectStoreBackend),
+    ])
+    def test_store_and_read(self, backend_type, expected_instance, ini_settings, random_binary_file):
+        filename, temp_file = random_binary_file
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+        metadata = {
+            'user_uploaded': {
+                'username': 'user1',
+                'user_id': 10,
+                'ip': '10.20.30.40'
+            }
+        }
+        store_fid, metadata = f_store.store(filename, temp_file, extra_metadata=metadata)
+        assert store_fid
+        assert metadata
+
+        # read-after write
+        reader, metadata2 = f_store.fetch(store_fid)
+        assert reader
+        assert metadata2['filename'] == filename
+
+    @pytest.mark.parametrize('backend_type, expected_instance', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+        (config_keys.backend_filesystem, FileSystemBackend),
+        (config_keys.backend_objectstore, ObjectStoreBackend),
+    ])
+    def test_store_file_not_allowed(self, backend_type, expected_instance, ini_settings, random_binary_file):
+        filename, temp_file = random_binary_file
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+        with pytest.raises(FileNotAllowedException):
+            f_store.store('notallowed.exe', temp_file, extensions=['.txt'])
+
+    @pytest.mark.parametrize('backend_type, expected_instance', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+        (config_keys.backend_filesystem, FileSystemBackend),
+        (config_keys.backend_objectstore, ObjectStoreBackend),
+    ])
+    def test_store_file_over_size(self, backend_type, expected_instance, ini_settings, random_binary_file):
+        filename, temp_file = random_binary_file
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+        with pytest.raises(FileOverSizeException):
+            f_store.store('toobig.exe', temp_file, extensions=['.exe'], max_filesize=124)
+
+    @pytest.mark.parametrize('backend_type, expected_instance, extra_conf', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend, {}),
+        (config_keys.backend_filesystem, FileSystemBackend, {config_keys.filesystem_storage_path: '/tmp/test-fs-store'}),
+        (config_keys.backend_objectstore, ObjectStoreBackend, {config_keys.objectstore_bucket: 'test-bucket'}),
+    ])
+    def test_store_stats_and_keys(self, backend_type, expected_instance, extra_conf, ini_settings, random_binary_file):
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        config.update(extra_conf)
+
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+
+        # purge storage before running
+        for shard, k in f_store.iter_artifacts():
+            f_store.delete(k)
+
+        for i in range(10):
+            filename, temp_file = random_binary_file
+
+            metadata = {
+                'user_uploaded': {
+                    'username': 'user1',
+                    'user_id': 10,
+                    'ip': '10.20.30.40'
+                }
+            }
+            store_fid, metadata = f_store.store(filename, temp_file, extra_metadata=metadata)
+            assert store_fid
+            assert metadata
+
+        cnt, size, meta = f_store.get_statistics()
+        assert cnt == 10
+        assert 10 == len(list(f_store.iter_keys()))

rhodecode/apps/file_store/tests/test_filestore_filesystem_backend.py

@@ -0,0 +1,52 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.tests import generate_random_filename
+
+
[e-mail]()
+def file_store_filesystem_instance(ini_settings):
+    config = ini_settings
+    config[config_keys.backend_type] = config_keys.backend_filesystem
+    f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+    return f_store
+
+
+class TestFileStoreFileSystemBackend:
+
+    @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
+    def test_get_shard_number(self, filename, file_store_filesystem_instance):
+        shard_number = file_store_filesystem_instance.get_shard_index(filename, len(file_store_filesystem_instance._shards))
+        # Check that the shard number is between 0 and max-shards
+        assert 0 <= shard_number <= len(file_store_filesystem_instance._shards)
+
+    @pytest.mark.parametrize('filename, expected_shard_num', [
+        ('my-name-1', 3),
+        ('my-name-2', 2),
+        ('my-name-3', 4),
+        ('my-name-4', 1),
+
+        ('rhodecode-enterprise-ce', 5),
+        ('rhodecode-enterprise-ee', 6),
+    ])
+    def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_filesystem_instance):
+        shard_number = file_store_filesystem_instance.get_shard_index(filename, len(file_store_filesystem_instance._shards))
+        assert expected_shard_num == shard_number

rhodecode/apps/file_store/tests/test_filestore_legacy_and_v2_compatability.py

@@ -0,0 +1,17 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
  No newline at end of file

rhodecode/apps/file_store/tests/test_filestore_legacy_backend.py

@@ -0,0 +1,52 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.tests import generate_random_filename
+
+
[e-mail]()
+def file_store_legacy_instance(ini_settings):
+    config = ini_settings
+    config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+    f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+    return f_store
+
+
+class TestFileStoreLegacyBackend:
+
+    @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
+    def test_get_shard_number(self, filename, file_store_legacy_instance):
+        shard_number = file_store_legacy_instance.get_shard_index(filename, len(file_store_legacy_instance._shards))
+        # Check that the shard number is 0 for legacy filesystem store we don't use shards
+        assert shard_number == 0
+
+    @pytest.mark.parametrize('filename, expected_shard_num', [
+        ('my-name-1', 0),
+        ('my-name-2', 0),
+        ('my-name-3', 0),
+        ('my-name-4', 0),
+
+        ('rhodecode-enterprise-ce', 0),
+        ('rhodecode-enterprise-ee', 0),
+    ])
+    def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_legacy_instance):
+        shard_number = file_store_legacy_instance.get_shard_index(filename, len(file_store_legacy_instance._shards))
+        assert expected_shard_num == shard_number

rhodecode/apps/file_store/tests/test_filestore_objectstore_backend.py

@@ -0,0 +1,52 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.tests import generate_random_filename
+
+
[e-mail]()
+def file_store_objectstore_instance(ini_settings):
+    config = ini_settings
+    config[config_keys.backend_type] = config_keys.backend_objectstore
+    f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+    return f_store
+
+
+class TestFileStoreObjectStoreBackend:
+
+    @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
+    def test_get_shard_number(self, filename, file_store_objectstore_instance):
+        shard_number = file_store_objectstore_instance.get_shard_index(filename, len(file_store_objectstore_instance._shards))
+        # Check that the shard number is between 0 and shards
+        assert 0 <= shard_number <= len(file_store_objectstore_instance._shards)
+
+    @pytest.mark.parametrize('filename, expected_shard_num', [
+        ('my-name-1', 3),
+        ('my-name-2', 2),
+        ('my-name-3', 4),
+        ('my-name-4', 1),
+
+        ('rhodecode-enterprise-ce', 5),
+        ('rhodecode-enterprise-ee', 6),
+    ])
+    def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_objectstore_instance):
+        shard_number = file_store_objectstore_instance.get_shard_index(filename, len(file_store_objectstore_instance._shards))
+        assert expected_shard_num == shard_number

rhodecode/lib/rc_commands/migrate_artifact.py

@@ -0,0 +1,122 b''
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import sys
+import logging
+
+import click
+
+from rhodecode.lib.pyramid_utils import bootstrap
+from rhodecode.lib.ext_json import json
+from rhodecode.model.db import FileStore
+from rhodecode.apps.file_store import utils as store_utils
+
+log = logging.getLogger(__name__)
+
+
[e-mail]()
[e-mail]('ini_path', type=click.Path(exists=True))
[e-mail]('file_uid')
[e-mail](
+    '--source-backend-conf',
+    type=click.Path(exists=True, dir_okay=False, readable=True),
+    help='Source backend config file path in a json format'
+)
[e-mail](
+    '--dest-backend-conf',
+    type=click.Path(exists=True, dir_okay=False, readable=True),
+    help='Source backend config file path in a json format'
+)
+def main(ini_path, file_uid, source_backend_conf, dest_backend_conf):
+    return command(ini_path, file_uid, source_backend_conf, dest_backend_conf)
+
+
+_source_settings = {}
+
+_dest_settings = {}
+
+
+def command(ini_path, file_uid, source_backend_conf, dest_backend_conf):
+    with bootstrap(ini_path, env={'RC_CMD_SETUP_RC': '1'}) as env:
+        migrate_func(env, file_uid, source_backend_conf, dest_backend_conf)
+
+
+def migrate_func(env, file_uid, source_backend_conf=None, dest_backend_conf=None):
+    """
+
+    Example usage::
+
+        from rhodecode.lib.rc_commands import migrate_artifact
+        migrate_artifact._source_settings = {
+            'file_store.backend.type': 'filesystem_v1',
+            'file_store.filesystem_v1.storage_path': '/var/opt/rhodecode_data/file_store',
+        }
+        migrate_artifact._dest_settings = {
+            'file_store.backend.type': 'objectstore',
+            'file_store.objectstore.url': 'http://s3-minio:9000',
+            'file_store.objectstore.bucket': 'rhodecode-file-store',
+            'file_store.objectstore.key': 's3admin',
+            'file_store.objectstore.secret': 's3secret4',
+            'file_store.objectstore.region': 'eu-central-1',
+        }
+        for db_obj in FileStore.query().all():
+            migrate_artifact.migrate_func({}, db_obj.file_uid)
+
+    """
+
+    try:
+        from rc_ee.api.views.store_api import _store_file
+    except ImportError:
+        click.secho('ERROR: Unable to import store_api. '
+                    'store_api is only available in EE edition of RhodeCode',
+                    fg='red')
+        sys.exit(-1)
+
+    source_settings = _source_settings
+    if source_backend_conf:
+        source_settings = json.loads(open(source_backend_conf).read())
+    dest_settings = _dest_settings
+    if dest_backend_conf:
+        dest_settings = json.loads(open(dest_backend_conf).read())
+
+    if file_uid.isnumeric():
+        file_store_db_obj = FileStore().query() \
+            .filter(FileStore.file_store_id == file_uid) \
+            .scalar()
+    else:
+        file_store_db_obj = FileStore().query() \
+            .filter(FileStore.file_uid == file_uid) \
+            .scalar()
+    if not file_store_db_obj:
+        click.secho(f'ERROR: Unable to fetch artifact from database file_uid={file_uid}',
+                    fg='red')
+        sys.exit(-1)
+
+    uid_filename = file_store_db_obj.file_uid
+    org_filename = file_store_db_obj.file_display_name
+    click.secho(f'Attempting to migrate artifact {uid_filename}, filename: {org_filename}', fg='green')
+
+    # get old version of f_store based on the data.
+
+    origin_f_store = store_utils.get_filestore_backend(source_settings, always_init=True)
+    reader, metadata = origin_f_store.fetch(uid_filename)
+
+    target_f_store = store_utils.get_filestore_backend(dest_settings, always_init=True)
+    target_f_store.import_to_store(reader, org_filename, uid_filename, metadata)
+
+    click.secho(f'Migrated artifact {uid_filename}, filename: {org_filename} into {target_f_store} storage', fg='green')

rhodecode/model/license.py

@@ -0,0 +1,17 b''
+
+def apply_license(*args, **kwargs):
+    pass
+
+try:
+    from rc_license.models import apply_license
+except ImportError:
+    pass
+
+
+def apply_license_from_file(*args, **kwargs):
+    pass
+
+try:
+    from rc_license.models import apply_license_from_file
+except ImportError:
+    pass

rhodecode/templates/admin/security/security.mako

@@ -0,0 +1,50 b''
+<%inherit file="/base/base.mako"/>
+
+<%def name="title()">
+    ${_('Security Admin')}
+    %if c.rhodecode_name:
+        &middot; ${h.branding(c.rhodecode_name)}
+    %endif
+</%def>
+
+<%def name="breadcrumbs_links()"></%def>
+
+<%def name="menu_bar_nav()">
+    ${self.menu_items(active='admin')}
+</%def>
+
+<%def name="menu_bar_subnav()">
+    ${self.admin_menu(active='security')}
+</%def>
+
+<%def name="main()">
+
+<div class="box">
+
+<!--    <div class="panel panel-default">-->
+<!--        <div class="panel-heading">-->
+<!--            <h3 class="panel-title">${_('Security Audit')}</h3>-->
+<!--        </div>-->
+<!--        <div class="panel-body">-->
+<!--            <h4>${_('This feature is available in RhodeCode EE edition only. Contact {sales_email} to obtain a trial license.').format(sales_email='<a href="mailto:[e-mail]">[e-mail]</a>')|n}</h4>-->
+<!--            <p>-->
+<!--                ${_('You can scan your repositories for exposed secrets, passwords, etc')}-->
+<!--            </p>-->
+<!--        </div>-->
+<!--    </div>-->
+
+    <div class="panel panel-default">
+        <div class="panel-heading">
+            <h3 class="panel-title">${_('Allowed client versions')}</h3>
+        </div>
+        <div class="panel-body">
+            <h4>${_('This feature is available in RhodeCode EE edition only. Contact {sales_email} to obtain a trial license.').format(sales_email='<a href="mailto:[e-mail]">[e-mail]</a>')|n}</h4>
+            <p>
+                ${_('Some outdated client versions may have security vulnerabilities. This section have rules for whitelisting versions of clients for Git, Mercurial and SVN.')}
+            </p>
+    </div>
+
+
+</div>
+
+</%def>

.bumpversion.cfg

@@ -1,5 +1,5 b''
 [bumpversion]
-current_version = 5.1.2
+current_version = 5.2.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:rhodecode/VERSION]

.hgignore

@@ -54,7 +54,7 b' syntax: regexp'
 ^rhodecode\.log$
 ^rhodecode_dev\.log$
 ^test\.db$
-
+^venv/
 
 # ac-tests
 ^acceptance_tests/\.cache.*$

Makefile

@@ -1,12 +1,49 b''
+.DEFAULT_GOAL := help
+
+# Pretty print values cf. https://misc.flogisoft.com/bash/tip_colors_and_formatting
+RESET := \033[0m # Reset all formatting
+GREEN := \033[0;32m # Resets before setting 16b colour (32 -- green)
+YELLOW := \033[0;33m
+ORANGE := \033[0;38;5;208m # Reset then set 256b colour (208 -- orange)
+PEACH := \033[0;38;5;216m
+
+
+## ---------------------------------------------------------------------------------- ##
+## ------------------------- Help usage builder ------------------------------------- ##
+## ---------------------------------------------------------------------------------- ##
+# use '# >>> Build commands' to create section
+# use '# target: target description' to create help for target
+.PHONY: help
+help:
+	@echo "Usage:"
+	@cat $(MAKEFILE_LIST) | grep -E '^# >>>|^# [A-Za-z0-9_.-]+:' | sed -E 's/^# //' | awk ' \
+		BEGIN { \
+			green="\033[32m"; \
+			yellow="\033[33m"; \
+			reset="\033[0m"; \
+			section=""; \
+		} \
+		/^>>>/ { \
+			section=substr($$0, 5); \
+			printf "\n" green ">>> %s" reset "\n", section; \
+			next; \
+		} \
+		/^([A-Za-z0-9_.-]+):/ { \
+			target=$$1; \
+			gsub(/:$$/, "", target); \
+			description=substr($$0, index($$0, ":") + 2); \
+			if (description == "") { description="-"; } \
+			printf " - " yellow "%-35s" reset " %s\n", target, description; \
+		} \
+	'
+
 # required for pushd to work..
 SHELL = /bin/bash
 
-
-# set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
-OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
+# >>> Tests commands
 
 .PHONY: clean
-## Cleanup compiled and cache py files
+# clean: Cleanup compiled and cache py files
 clean:
 	make test-clean
 	find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
@@ -14,14 +51,14 b' clean:'
 
 
 .PHONY: test
-## run test-clean and tests
+# test: run test-clean and tests
 test:
 	make test-clean
-	make test-only
+	unset RC_SQLALCHEMY_DB1_URL && unset RC_DB_URL && make test-only
 
 
 .PHONY: test-clean
-## run test-clean and tests
+# test-clean: run test-clean and tests
 test-clean:
 	rm -rf coverage.xml htmlcov junit.xml pylint.log result
 	find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
@@ -29,34 +66,36 b' test-clean:'
 
 
 .PHONY: test-only
-## Run tests only without cleanup
+# test-only: Run tests only without cleanup
 test-only:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar \
 	--cov-report=term-missing --cov-report=html \
 	--cov=rhodecode rhodecode
 
+# >>> Docs commands
 
 .PHONY: docs
-## build docs
+# docs: build docs
 docs:
 	(cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make clean html SPHINXOPTS="-W")
 
 
 .PHONY: docs-clean
-## Cleanup docs
+# docs-clean: Cleanup docs
 docs-clean:
 	(cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make clean)
 
 
 .PHONY: docs-cleanup
-## Cleanup docs
+# docs-cleanup: Cleanup docs
 docs-cleanup:
 	(cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make cleanup)
 
+# >>> Dev commands
 
 .PHONY: web-build
-## Build JS packages static/js
+# web-build: Build JS packages static/js
 web-build:
 	rm -rf node_modules
 	docker run -it --rm -v $(PWD):/project --workdir=/project rhodecode/static-files-build:16 -c "npm install && /project/node_modules/.bin/grunt"
@@ -64,25 +103,9 b' web-build:'
 	./rhodecode/tests/scripts/static-file-check.sh rhodecode/public/
 	rm -rf node_modules
 
-.PHONY: ruff-check
-## run a ruff analysis
-ruff-check:
-	ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs  --exclude .dev .
-
-.PHONY: pip-packages
-## Show outdated packages
-pip-packages:
-	python ${OUTDATED_PACKAGES}
-
-
-.PHONY: build
-## Build sdist/egg
-build:
-	python -m build
-
 
 .PHONY: dev-sh
-## make dev-sh
+# dev-sh: make dev-sh
 dev-sh:
 	sudo echo "deb [trusted=yes] https://apt.fury.io/rsteube/ /" | sudo tee -a "/etc/apt/sources.list.d/fury.list"
 	sudo apt-get update
@@ -95,14 +118,14 b' dev-sh:'
 
 
 .PHONY: dev-cleanup
-## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
+# dev-cleanup: Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
 dev-cleanup:
 	pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
 	rm -rf /tmp/*
 
 
 .PHONY: dev-env
-## make dev-env based on the requirements files and install develop of packages
+# dev-env: make dev-env based on the requirements files and install develop of packages
 ## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
 dev-env:
 	sudo -u root chown rhodecode:rhodecode /home/rhodecode/.cache/pip/
@@ -114,7 +137,7 b' dev-env:'
 
 
 .PHONY: sh
-## shortcut for make dev-sh dev-env
+# sh: shortcut for make dev-sh dev-env
 sh:
 	make dev-env
 	make dev-sh
@@ -124,49 +147,12 b' sh:'
 workers?=1
 
 .PHONY: dev-srv
-## run gunicorn web server with reloader, use workers=N to set multiworker mode
+# dev-srv: run gunicorn web server with reloader, use workers=N to set multiworker mode, workers=N allows changes of workers
 dev-srv:
 	gunicorn --paste=.dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py --timeout=120 --reload --workers=$(workers)
 
-
-# Default command on calling make
-.DEFAULT_GOAL := show-help
+.PHONY: ruff-check
+# ruff-check: run a ruff analysis
+ruff-check:
+	ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs  --exclude .dev .
 
-.PHONY: show-help
-show-help:
-	@echo "$$(tput bold)Available rules:$$(tput sgr0)"
-	@echo
-	@sed -n -e "/^## / { \
-		h; \
-		s/.*//; \
-		:doc" \
-		-e "H; \
-		n; \
-		s/^## //; \
-		t doc" \
-		-e "s/:.*//; \
-		G; \
-		s/\\n## /---/; \
-		s/\\n/ /g; \
-		p; \
-	}" ${MAKEFILE_LIST} \
-	| LC_ALL='C' sort --ignore-case \
-	| awk -F '---' \
-		-v ncol=$$(tput cols) \
-		-v indent=19 \
-		-v col_on="$$(tput setaf 6)" \
-		-v col_off="$$(tput sgr0)" \
-	'{ \
-		printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
-		n = split($$2, words, " "); \
-		line_length = ncol - indent; \
-		for (i = 1; i <= n; i++) { \
-			line_length -= length(words[i]) + 1; \
-			if (line_length <= 0) { \
-				line_length = ncol - indent - length(words[i]) - 1; \
-				printf "\n%*s ", -indent, " "; \
-			} \
-			printf "%s ", words[i]; \
-		} \
-		printf "\n"; \
-	}'

configs/development.ini

@@ -257,6 +257,13 b' license_token ='
 ; This flag hides sensitive information on the license page such as token, and license data
 license.hide_license_info = false
 
+; Import EE license from this license path
+#license.import_path = %(here)s/rhodecode_enterprise.license
+
+; import license 'if-missing' or 'force' (always override)
+; if-missing means apply license if it doesn't exist. 'force' option always overrides it
+license.import_path_mode = if-missing
+
 ; supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 
@@ -281,15 +288,56 b' labs_settings_active = true'
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
-; File store configuration. This is used to store and serve uploaded files
-file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store backend is always enabled
+#file_store.enabled = true
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead
 ; Storage backend, available options are: local
-file_store.backend = local
+#file_store.backend = local
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead
 ; path to store the uploaded binaries and artifacts
-file_store.storage_path = /var/opt/rhodecode_data/file_store
+#file_store.storage_path = /var/opt/rhodecode_data/file_store
+
+; Artifacts file-store, is used to store comment attachments and artifacts uploads.
+; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options
+; filesystem_v1 is backwards compat with pre 5.1 storage changes
+; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from
+; previous installations to keep the artifacts without a need of migration
+#file_store.backend.type = filesystem_v2
+
+; filesystem options...
+#file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+
+; filesystem_v2 options...
+#file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+#file_store.filesystem_v2.shards = 8
 
+; objectstore options...
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+#file_store.backend.type = objectstore
+#file_store.objectstore.url = http://s3-minio:9000
+
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number
+#file_store.objectstore.bucket = rhodecode-file-store
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+#file_store.objectstore.bucket_shards = 8
+
+; key for s3 auth
+#file_store.objectstore.key = s3admin
+
+; secret for s3 auth
+#file_store.objectstore.secret = s3secret4
+
+;region for s3 storage
+#file_store.objectstore.region = eu-central-1
 
 ; Redis url to acquire/check generation of archives locks
 archive_cache.locking.url = redis://redis:6379/1
@@ -624,7 +672,8 b' vcs.scm_app_implementation = http'
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
 ; `celery` - use celery based hooks
-vcs.hooks.protocol = http
+#DEPRECATED:vcs.hooks.protocol = http
+vcs.hooks.protocol.v2 = celery
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
 ; accessible via network.
@@ -647,6 +696,12 b' vcs.connection_timeout = 3600'
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = true
 
+; Filesystem location where Git lfs objects should be stored
+vcs.git.lfs.storage_location = /var/opt/rhodecode_repo_store/.cache/git_lfs_store
+
+; Filesystem location where Mercurial largefile objects should be stored
+vcs.hg.largefiles.storage_location = /var/opt/rhodecode_repo_store/.cache/hg_largefiles_store
+
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
 ; Maps RhodeCode repo groups into SVN paths for Apache
@@ -716,7 +771,8 b' ssh.authorized_keys_file_path = /etc/rho'
 ; RhodeCode installation directory.
 ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
-ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+#DEPRECATED: ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+ssh.wrapper_cmd.v2 = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false

configs/gunicorn_config.py

@@ -13,6 +13,7 b' import traceback'
 import random
 import socket
 import dataclasses
+import json
 from gunicorn.glogging import Logger
 
 
@@ -37,17 +38,41 b" accesslog = '-'"
 worker_tmp_dir = None
 tmp_upload_dir = None
 
-# use re-use port logic
-#reuse_port = True
+# use re-use port logic to let linux internals load-balance the requests better.
+reuse_port = True
 
 # Custom log format
 #access_log_format = (
 #    '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
 
 # loki format for easier parsing in grafana
-access_log_format = (
+loki_access_log_format = (
     'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
 
+# JSON format
+json_access_log_format = json.dumps({
+    'time': r'%(t)s',
+    'pid': r'%(p)s',
+    'level': 'INFO',
+    'ip': r'%(h)s',
+    'request_time': r'%(L)s',
+    'remote_address': r'%(h)s',
+    'user_name': r'%(u)s',
+    'status': r'%(s)s',
+    'method': r'%(m)s',
+    'url_path': r'%(U)s',
+    'query_string': r'%(q)s',
+    'protocol': r'%(H)s',
+    'response_length': r'%(B)s',
+    'referer': r'%(f)s',
+    'user_agent': r'%(a)s',
+
+})
+
+access_log_format = loki_access_log_format
+if os.environ.get('RC_LOGGING_FORMATTER') == 'json':
+    access_log_format = json_access_log_format
+
 # self adjust workers based on CPU count, to use maximum of CPU and not overquota the resources
 # workers = get_workers()
 

configs/production.ini

@@ -225,6 +225,13 b' license_token ='
 ; This flag hides sensitive information on the license page such as token, and license data
 license.hide_license_info = false
 
+; Import EE license from this license path
+#license.import_path = %(here)s/rhodecode_enterprise.license
+
+; import license 'if-missing' or 'force' (always override)
+; if-missing means apply license if it doesn't exist. 'force' option always overrides it
+license.import_path_mode = if-missing
+
 ; supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 
@@ -249,15 +256,56 b' labs_settings_active = true'
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
-; File store configuration. This is used to store and serve uploaded files
-file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store backend is always enabled
+#file_store.enabled = true
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead
 ; Storage backend, available options are: local
-file_store.backend = local
+#file_store.backend = local
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead
 ; path to store the uploaded binaries and artifacts
-file_store.storage_path = /var/opt/rhodecode_data/file_store
+#file_store.storage_path = /var/opt/rhodecode_data/file_store
+
+; Artifacts file-store, is used to store comment attachments and artifacts uploads.
+; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options
+; filesystem_v1 is backwards compat with pre 5.1 storage changes
+; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from
+; previous installations to keep the artifacts without a need of migration
+#file_store.backend.type = filesystem_v2
+
+; filesystem options...
+#file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+
+; filesystem_v2 options...
+#file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+#file_store.filesystem_v2.shards = 8
 
+; objectstore options...
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+#file_store.backend.type = objectstore
+#file_store.objectstore.url = http://s3-minio:9000
+
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number
+#file_store.objectstore.bucket = rhodecode-file-store
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+#file_store.objectstore.bucket_shards = 8
+
+; key for s3 auth
+#file_store.objectstore.key = s3admin
+
+; secret for s3 auth
+#file_store.objectstore.secret = s3secret4
+
+;region for s3 storage
+#file_store.objectstore.region = eu-central-1
 
 ; Redis url to acquire/check generation of archives locks
 archive_cache.locking.url = redis://redis:6379/1
@@ -592,7 +640,8 b' vcs.scm_app_implementation = http'
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
 ; `celery` - use celery based hooks
-vcs.hooks.protocol = http
+#DEPRECATED:vcs.hooks.protocol = http
+vcs.hooks.protocol.v2 = celery
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
 ; accessible via network.
@@ -615,6 +664,12 b' vcs.connection_timeout = 3600'
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = true
 
+; Filesystem location where Git lfs objects should be stored
+vcs.git.lfs.storage_location = /var/opt/rhodecode_repo_store/.cache/git_lfs_store
+
+; Filesystem location where Mercurial largefile objects should be stored
+vcs.hg.largefiles.storage_location = /var/opt/rhodecode_repo_store/.cache/hg_largefiles_store
+
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
 ; Maps RhodeCode repo groups into SVN paths for Apache
@@ -684,7 +739,8 b' ssh.authorized_keys_file_path = /etc/rho'
 ; RhodeCode installation directory.
 ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
-ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+#DEPRECATED: ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+ssh.wrapper_cmd.v2 = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false

docs/Dockerfile

@@ -22,6 +22,12 b' RUN apt-get update \\'
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*
 
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip && \
+    ./aws/install && \
+    rm -rf ./aws && \
+    rm awscliv2.zip
+
 RUN \
     python3 -m pip install --no-cache-dir --upgrade pip && \
     python3 -m pip install --no-cache-dir Sphinx Pillow

docs/admin/system-overview.rst

@@ -147,10 +147,6 b' Peer-to-peer Failover Support'
 
 * Yes
 
-Additional Binaries
--------------------
-
-* Yes, see :ref:`rhodecode-nix-ref` for full details.
 
 Remote Connectivity
 -------------------

docs/auth/auth-saml-bulk-enroll-users.rst

@@ -13,7 +13,7 b' This method simply enables SAML authenti'
 From the server RhodeCode Enterprise is running run ishell on the instance which we
 want to apply the SAML migration::
 
-    rccontrol ishell enterprise-1
+    ./rcstack cli ishell
 
 Follow these steps to enable SAML authentication for multiple users.
 
@@ -46,6 +46,8 b' From available options pick only one and'
 
     # for Duo Security
     In [2]: from rc_auth_plugins.auth_duo_security import RhodeCodeAuthPlugin
+    # for Azure Entra
+    In [2]: from rc_auth_plugins.auth_azure import RhodeCodeAuthPlugin
     # for OneLogin
     In [2]: from rc_auth_plugins.auth_onelogin import RhodeCodeAuthPlugin
     # generic SAML plugin
@@ -62,13 +64,13 b' Enter in the ishell prompt'
        ...:     attrs = saml2user.get(user.user_id)
        ...:     provider = RhodeCodeAuthPlugin.uid
        ...:     if existing_identity:
-       ...:         print('Identity for user `{}` already exists, skipping'.format(user.username))
+       ...:         print(f'Identity for user `{user.username}` already exists, skipping')
        ...:         continue
        ...:     if attrs:
        ...:         external_id = attrs['id']
        ...:         new_external_identity = ExternalIdentity()
        ...:         new_external_identity.external_id = external_id
-       ...:         new_external_identity.external_username = '{}-saml-{}'.format(user.username, user.user_id)
+       ...:         new_external_identity.external_username = f'{user.username}-saml-{user.user_id}'
        ...:         new_external_identity.provider_name = provider
        ...:         new_external_identity.local_user_id = user.user_id
        ...:         new_external_identity.access_token = ''
@@ -76,7 +78,7 b' Enter in the ishell prompt'
        ...:         new_external_identity.alt_token = ''
        ...:         Session().add(ex_identity)
        ...:         Session().commit()
-       ...:         print('Set user `{}` external identity bound to ExternalID:{}'.format(user.username, external_id))
+       ...:         print(f'Set user `{user.username}` external identity bound to ExternalID:{external_id}')
 
 .. note::
 

docs/auth/auth-saml-duosecurity.rst

@@ -32,62 +32,118 b' 4. Check the `enabled` check box in the '
 Example SAML Duo Security configuration
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Example configuration for SAML 2.0 with Duo Security provider::
+Example configuration for SAML 2.0 with Duo Security provider
+
+
+Enabled
+    `True`:
 
-    *option*: `enabled` => `True`
-    # Enable or disable this authentication plugin.
+    .. note::
+      Enable or disable this authentication plugin.
+
+
+Auth Cache TTL
+    `30`:
 
-    *option*: `cache_ttl` => `0`
-    # Amount of seconds to cache the authentication and permissions check response call for this plugin.
-    # Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
+    .. note::
+      Amount of seconds to cache the authentication and permissions check response call for this plugin.
+      Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
+
+Debug
+    `True`:
 
-    *option*: `debug` => `True`
-    # Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
+    .. note::
+      Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
+
+
+Auth button name
+    `Azure Entra ID`:
 
-    *option*: `entity_id` => `http://rc-app.com/dag/saml2/idp/metadata.php`
-    # Identity Provider entity/metadata URI.
-    # E.g. https://duo-gateway.com/dag/saml2/idp/metadata.php
+    .. note::
+      Alternative authentication display name. E.g AzureAuth, CorporateID etc.
+
+
+Entity ID
+    `https://my-duo-gateway.com/dag/saml2/idp/metadata.php`:
+
+    .. note::
+      Identity Provider entity/metadata URI.
+      E.g. https://duo-gateway.com/dag/saml2/idp/metadata.php
+
+SSO URL
+    `https://duo-gateway.com/dag/saml2/idp/SSOService.php?spentityid=<metadata_entity_id>`:
 
-    *option*: `sso_service_url` => `http://rc-app.com/dag/saml2/idp/SSOService.php?spentityid=http://rc.local.pl/_admin/auth/duosecurity/saml-metadata`
-    # SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login
-    # E.g. https://duo-gateway.com/dag/saml2/idp/SSOService.php?spentityid=<metadata_entity_id>
+    .. note::
+      SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login, Known also as Login URL
+      E.g. http://rc-app.com/dag/saml2/idp/SSOService.php?spentityid=https://docker-dev/_admin/auth/duosecurity/saml-metadata
+
+SLO URL
+    `https://duo-gateway.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=<return_url>`:
 
-    *option*: `slo_service_url` => `http://rc-app.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=http://rc-app.com/dag/module.php/duosecurity/logout.php`
-    # SLO (SingleLogout) endpoint URL of the IdP.
-    # E.g. https://duo-gateway.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=http://duo-gateway.com/_admin/saml/sign-out-endpoint
+    .. note::
+      SLO (SingleLogout) endpoint URL of the IdP. , Known also as Logout URL
+      E.g. http://rc-app.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=https://docker-dev/_admin/auth/duosecurity/saml-sign-out-endpoint
 
-    *option*: `x509cert` => `<CERTIFICATE_STRING>`
-    # Identity provider public x509 certificate. It will be converted to single-line format without headers
+x509cert
+    `<CERTIFICATE_STRING>`:
 
-    *option*: `name_id_format` => `sha-1`
-    # The format that specifies how the NameID is sent to the service provider.
+    .. note::
+      Identity provider public x509 certificate. It will be converted to single-line format without headers.
+      Download the raw base64 encoded certificate from the Identity provider and paste it here.
+
+SAML Signature
+    `sha-256`:
+
+    .. note::
+      Type of Algorithm to use for verification of SAML signature on Identity provider side.
+
+SAML Digest
+    `sha-256`:
 
-    *option*: `signature_algo` => `sha-256`
-    # Type of Algorithm to use for verification of SAML signature on Identity provider side
+    .. note::
+      Type of Algorithm to use for verification of SAML digest on Identity provider side.
+
+Service Provider Cert Dir
+    `/etc/rhodecode/conf/saml_ssl/`:
 
-    *option*: `digest_algo` => `sha-256`
-    # Type of Algorithm to use for verification of SAML digest on Identity provider side
+    .. note::
+      Optional directory to store service provider certificate and private keys.
+      Expected certs for the SP should be stored in this folder as:
+
+      * sp.key     Private Key
+      * sp.crt     Public cert
+      * sp_new.crt Future Public cert
+
+      Also you can use other cert to sign the metadata of the SP using the:
 
-    *option*: `cert_dir` => `/etc/saml/`
-    # Optional directory to store service provider certificate and private keys.
-    # Expected certs for the SP should be stored in this folder as:
-    #  * sp.key     Private Key
-    #  * sp.crt     Public cert
-    #  * sp_new.crt Future Public cert
-    #
-    # Also you can use other cert to sign the metadata of the SP using the:
-    #  * metadata.key
-    #  * metadata.crt
+      * metadata.key
+      * metadata.crt
+
+Expected NameID Format
+    `nameid-format:emailAddress`:
+
+    .. note::
+      The format that specifies how the NameID is sent to the service provider.
+
+User ID Attribute
+    `PersonImmutableID`:
 
-    *option*: `user_id_attribute` => `PersonImmutableID`
-    # User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
-    # Ensure this is returned from DuoSecurity for example via duo_username
+    .. note::
+      User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
+      Ensure this is returned from DuoSecurity for example via duo_username.
+
+Username Attribute
+    `User.username`:
 
-    *option*: `username_attribute` => `User.username`
-    # Username Attribute name. This defines which attribute in SAML response will map to an username.
+    .. note::
+      Username Attribute name. This defines which attribute in SAML response will map to a username.
 
-    *option*: `email_attribute` => `User.email`
-    # Email Attribute name. This defines which attribute in SAML response will map to an email address.
+Email Attribute
+    `User.email`:
+
+    .. note::
+      Email Attribute name. This defines which attribute in SAML response will map to an email address.
+
 
 
 Below is example setup from DUO Administration page that can be used with above config.

docs/auth/auth-saml-generic.rst

@@ -15,5 +15,6 b' Please check for reference two example p'
 
    auth-saml-duosecurity
    auth-saml-onelogin
+   auth-saml-azure
    auth-saml-bulk-enroll-users
 

docs/auth/auth-saml-onelogin.rst

@@ -32,62 +32,117 b' 4. Check the `enabled` check box in the '
 Example SAML OneLogin configuration
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Example configuration for SAML 2.0 with OneLogin provider::
+Example configuration for SAML 2.0 with OneLogin provider
+
+
+Enabled
+    `True`:
 
-    *option*: `enabled` => `True`
-    # Enable or disable this authentication plugin.
+    .. note::
+      Enable or disable this authentication plugin.
+
+
+Auth Cache TTL
+    `30`:
 
-    *option*: `cache_ttl` => `0`
-    # Amount of seconds to cache the authentication and permissions check response call for this plugin.
-    # Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
+    .. note::
+      Amount of seconds to cache the authentication and permissions check response call for this plugin.
+      Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
+
+Debug
+    `True`:
 
-    *option*: `debug` => `True`
-    # Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
+    .. note::
+      Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
+
+
+Auth button name
+    `Azure Entra ID`:
 
-    *option*: `entity_id` => `https://app.onelogin.com/saml/metadata/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`
-    # Identity Provider entity/metadata URI.
-    # E.g. https://app.onelogin.com/saml/metadata/<onelogin_connector_id>
+    .. note::
+      Alternative authentication display name. E.g AzureAuth, CorporateID etc.
+
+
+Entity ID
+    `https://app.onelogin.com/saml/metadata/<onelogin_connector_id>`:
+
+    .. note::
+      Identity Provider entity/metadata URI.
+      E.g. https://app.onelogin.com/saml/metadata/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+
+SSO URL
+    `https://app.onelogin.com/trust/saml2/http-post/sso/<onelogin_connector_id>`:
 
-    *option*: `sso_service_url` => `https://customer-domain.onelogin.com/trust/saml2/http-post/sso/xxxxxx`
-    # SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login
-    # E.g. https://app.onelogin.com/trust/saml2/http-post/sso/<onelogin_connector_id>
+    .. note::
+      SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login, Known also as Login URL
+      E.g.  https://app.onelogin.com/trust/saml2/http-post/sso/<onelogin_connector_id>
+
+SLO URL
+    `https://app.onelogin.com/trust/saml2/http-redirect/slo/<onelogin_connector_id>`:
 
-    *option*: `slo_service_url` => `https://customer-domain.onelogin.com/trust/saml2/http-redirect/slo/xxxxxx`
-    # SLO (SingleLogout) endpoint URL of the IdP.
-    # E.g. https://app.onelogin.com/trust/saml2/http-redirect/slo/<onelogin_connector_id>
+    .. note::
+      SLO (SingleLogout) endpoint URL of the IdP. , Known also as Logout URL
+      E.g. https://app.onelogin.com/trust/saml2/http-redirect/slo/<onelogin_connector_id>
 
-    *option*: `x509cert` => `<CERTIFICATE_STRING>`
-    # Identity provider public x509 certificate. It will be converted to single-line format without headers
+x509cert
+    `<CERTIFICATE_STRING>`:
 
-    *option*: `name_id_format` => `sha-1`
-    # The format that specifies how the NameID is sent to the service provider.
+    .. note::
+      Identity provider public x509 certificate. It will be converted to single-line format without headers.
+      Download the raw base64 encoded certificate from the Identity provider and paste it here.
+
+SAML Signature
+    `sha-256`:
+
+    .. note::
+      Type of Algorithm to use for verification of SAML signature on Identity provider side.
+
+SAML Digest
+    `sha-256`:
 
-    *option*: `signature_algo` => `sha-256`
-    # Type of Algorithm to use for verification of SAML signature on Identity provider side
+    .. note::
+      Type of Algorithm to use for verification of SAML digest on Identity provider side.
+
+Service Provider Cert Dir
+    `/etc/rhodecode/conf/saml_ssl/`:
 
-    *option*: `digest_algo` => `sha-256`
-    # Type of Algorithm to use for verification of SAML digest on Identity provider side
+    .. note::
+      Optional directory to store service provider certificate and private keys.
+      Expected certs for the SP should be stored in this folder as:
+
+      * sp.key     Private Key
+      * sp.crt     Public cert
+      * sp_new.crt Future Public cert
 
-    *option*: `cert_dir` => `/etc/saml/`
-    # Optional directory to store service provider certificate and private keys.
-    # Expected certs for the SP should be stored in this folder as:
-    #  * sp.key     Private Key
-    #  * sp.crt     Public cert
-    #  * sp_new.crt Future Public cert
-    #
-    # Also you can use other cert to sign the metadata of the SP using the:
-    #  * metadata.key
-    #  * metadata.crt
+      Also you can use other cert to sign the metadata of the SP using the:
+
+      * metadata.key
+      * metadata.crt
+
+Expected NameID Format
+    `nameid-format:emailAddress`:
+
+    .. note::
+      The format that specifies how the NameID is sent to the service provider.
+
+User ID Attribute
+    `PersonImmutableID`:
 
-    *option*: `user_id_attribute` => `PersonImmutableID`
-    # User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
-    # Ensure this is returned from OneLogin for example via Internal ID
+    .. note::
+      User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
+      Ensure this is returned from DuoSecurity for example via duo_username.
+
+Username Attribute
+    `User.username`:
 
-    *option*: `username_attribute` => `User.username`
-    # Username Attribute name. This defines which attribute in SAML response will map to an username.
+    .. note::
+      Username Attribute name. This defines which attribute in SAML response will map to a username.
 
-    *option*: `email_attribute` => `User.email`
-    # Email Attribute name. This defines which attribute in SAML response will map to an email address.
+Email Attribute
+    `User.email`:
+
+    .. note::
+      Email Attribute name. This defines which attribute in SAML response will map to an email address.
 
 
 

docs/auth/auth.rst

@@ -29,6 +29,7 b' administrator greater control over how u'
     auth-saml-generic
     auth-saml-onelogin
     auth-saml-duosecurity
+    auth-saml-azure
     auth-crowd
     auth-pam
     ssh-connection

docs/contributing/dev-setup.rst

@@ -4,237 +4,8 b''
  Development setup
 ===================
 
-
-RhodeCode Enterprise runs inside a Nix managed environment. This ensures build
-environment dependencies are correctly declared and installed during setup.
-It also enables atomic upgrades, rollbacks, and multiple instances of RhodeCode
-Enterprise running with isolation.
-
-To set up RhodeCode Enterprise inside the Nix environment, use the following steps:
-
-
-
-Setup Nix Package Manager
--------------------------
-
-To install the Nix Package Manager, please run::
-
-   $ curl https://releases.nixos.org/nix/nix-2.3.4/install | sh
-
-or go to https://nixos.org/nix/ and follow the installation instructions.
-Once this is correctly set up on your system, you should be able to use the
-following commands:
-
-* `nix-env`
-
-* `nix-shell`
-
-
-.. tip::
-
-   Update your channels frequently by running ``nix-channel --update``.
-
-.. note::
-
-   To uninstall nix run the following:
-
-    remove the . "$HOME/.nix-profile/etc/profile.d/nix.sh" line in your ~/.profile or ~/.bash_profile
-    rm -rf $HOME/{.nix-channels,.nix-defexpr,.nix-profile,.config/nixpkgs}
-    sudo rm -rf /nix
-
-Switch nix to the latest STABLE channel
----------------------------------------
-
-run::
-
-   nix-channel --add https://nixos.org/channels/nixos-20.03 nixpkgs
-
-Followed by::
-
-   nix-channel --update
-   nix-env -i nix-2.3.4
-
-
-Install required binaries
--------------------------
-
-We need some handy tools first.
-
-run::
-
-    nix-env -i nix-prefetch-hg
-    nix-env -i nix-prefetch-git
-
-
-Speed up JS build by installing PhantomJS
------------------------------------------
-
-PhantomJS will be downloaded each time nix-shell is invoked. To speed this by
-setting already downloaded version do this::
-
-    nix-env -i phantomjs-2.1.1
-
-    # and set nix bin path
-    export PATH=$PATH:~/.nix-profile/bin
-
-
-Clone the required repositories
--------------------------------
-
-After Nix is set up, clone the RhodeCode Enterprise Community Edition and
-RhodeCode VCSServer repositories into the same directory.
-RhodeCode currently is using Mercurial Version Control System, please make sure
-you have it installed before continuing.
-
-To obtain the required sources, use the following commands::
-
-    mkdir rhodecode-develop && cd rhodecode-develop
-    hg clone -u default https://code.rhodecode.com/rhodecode-enterprise-ce
-    hg clone -u default https://code.rhodecode.com/rhodecode-vcsserver
-
-.. note::
-
-   If you cannot clone the repository, please contact us via [e-mail]
-
-
-Install some required libraries
--------------------------------
-
-There are some required drivers and dev libraries that we need to install to
-test RhodeCode under different types of databases. For example in Ubuntu we
-need to install the following.
-
-required libraries::
-
-    # svn related
-    sudo apt-get install libapr1-dev libaprutil1-dev
-    sudo apt-get install libsvn-dev
-    # libcurl required too
-    sudo apt-get install libcurl4-openssl-dev
-    # mysql/pg server for development, optional
-    sudo apt-get install mysql-server libmysqlclient-dev
-    sudo apt-get install postgresql postgresql-contrib libpq-dev
-
-
-
-Enter the Development Shell
----------------------------
-
-The final step is to start the development shells. To do this, run the
-following command from inside the cloned repository::
-
-   # first, the vcsserver
-   cd ~/rhodecode-vcsserver
-   nix-shell
-
-   # then enterprise sources
-   cd ~/rhodecode-enterprise-ce
-   nix-shell
-
-.. note::
-
-   On the first run, this will take a while to download and optionally compile
-   a few things. The following runs will be faster. The development shell works
-   fine on both MacOS and Linux platforms.
-
-
-Create config.nix for development
----------------------------------
-
-In order to run proper tests and setup linking across projects, a config.nix
-file needs to be setup::
-
-    # create config
-    mkdir -p ~/.nixpkgs
-    touch ~/.nixpkgs/config.nix
-
-    # put the below content into the ~/.nixpkgs/config.nix file
-    # adjusts, the path to where you cloned your repositories.
-
-    {
-      rc = {
-       sources = {
-        rhodecode-vcsserver = "/home/dev/rhodecode-vcsserver";
-        rhodecode-enterprise-ce = "/home/dev/rhodecode-enterprise-ce";
-        rhodecode-enterprise-ee = "/home/dev/rhodecode-enterprise-ee";
-       };
-      };
-    }
-
-
-
-Creating a Development Configuration
-------------------------------------
-
-To create a development environment for RhodeCode Enterprise,
-use the following steps:
-
-1. Create a copy of vcsserver config:
-    `cp ~/rhodecode-vcsserver/configs/development.ini ~/rhodecode-vcsserver/configs/dev.ini`
-2. Create a copy of rhodocode config:
-    `cp ~/rhodecode-enterprise-ce/configs/development.ini ~/rhodecode-enterprise-ce/configs/dev.ini`
-3. Adjust the configuration settings to your needs if needed.
-
-.. note::
-
-  It is recommended to use the name `dev.ini` since it's included in .hgignore file.
-
-
-Setup the Development Database
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-To create a development database, use the following example. This is a one
-time operation executed from the nix-shell of rhodecode-enterprise-ce sources ::
-
-    rc-setup-app dev.ini \
-        --user=admin --password=secret \
-        --email=[e-mail] \
-        --repos=~/my_dev_repos
-
-
-Compile CSS and JavaScript
-^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-To use the application's frontend and prepare it for production deployment,
-you will need to compile the CSS and JavaScript with Grunt.
-This is easily done from within the nix-shell using the following command::
-
-    make web-build
-
-When developing new features you will need to recompile following any
-changes made to the CSS or JavaScript files when developing the code::
-
-    grunt watch
-
-This prepares the development (with comments/whitespace) versions of files.
-
-Start the Development Servers
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-From the rhodecode-vcsserver directory, start the development server in another
-nix-shell, using the following command::
-
-      pserve configs/dev.ini
-
-In the adjacent nix-shell which you created for your development server, you may
-now start CE with the following command::
-
-
-      pserve --reload configs/dev.ini
-
-.. note::
-
-  `--reload` flag will automatically reload the server when source file changes.
-
-
-Run the Environment Tests
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Please make sure that the tests are passing to verify that your environment is
-set up correctly. RhodeCode uses py.test to run tests.
-While your instance is running, start a new nix-shell and simply run
-``make test`` to run the basic test suite.
-
+Please refer to RCstack installed documentation for instructions on setting up dev environment:
+https://docs.rhodecode.com/rcstack/dev/dev-setup.html
 
 Need Help?
 ^^^^^^^^^^

docs/index.rst

@@ -37,6 +37,18 b' and commit files and |repos| while manag'
 
 .. toctree::
    :maxdepth: 1
+   :caption: Documentation directory
+
+   Back to documentation directory <https://docs.rhodecode.com/>
+
+.. toctree::
+   :maxdepth: 1
+   :caption: RhodeCode RCstack Documentation
+
+   RhodeCode RCstack Installer <https://docs.rhodecode.com/rcstack/>
+
+.. toctree::
+   :maxdepth: 1
    :caption: Admin Documentation
 
    install/quick-start

docs/install/quick-start.rst

@@ -66,6 +66,7 b' Output should look similar to this:'
     fb77fb6496c6   channelstream/channelstream:0.7.1       Up 2 hours (healthy)   rc_cluster_services-channelstream-1   8000/tcp
     cb6c5c022f5b   postgres:14.6                           Up 2 hours (healthy)   rc_cluster_services-database-1        5432/tcp
 
+
 At this point you should be able to access:
 
 - RhodeCode instance at your domain entered, e.g http://rhodecode.local, the default access
@@ -76,6 +77,7 b' At this point you should be able to acce'
     RHODECODE_USER_PASS=super-secret-password
 
 
+
 .. note::
 
    Recommended post quick start install instructions:
@@ -85,7 +87,6 b' At this point you should be able to acce'
    * Set up :ref:`indexing-ref`
    * Familiarise yourself with the :ref:`rhodecode-admin-ref` section.
 
-.. _rhodecode.com/download/: https://rhodecode.com/download/
 .. _rhodecode.com: https://rhodecode.com/
 .. _rhodecode.com/register: https://rhodecode.com/register/
 .. _rhodecode.com/download: https://rhodecode.com/download/

docs/install/using-sqllite.rst

@@ -1,23 +1,12 b''
 .. _install-sqlite-database:
 
-SQLite
-------
+SQLite (Deprecated)
+-------------------
 
 .. important::
 
-    We do not recommend using SQLite in a large development environment
-    as it has an internal locking mechanism which can become a performance
-    bottleneck when there are more than 5 concurrent users.
+    As of 5.x, SQLite is no longer supported, we advise to migrate to MySQL or PostgreSQL.
 
-|RCE| installs SQLite as the default database if you do not specify another
-during installation. SQLite is suitable for small teams,
-projects with a low load, and evaluation purposes since it is built into
-|RCE| and does not require any additional database server.
-
-Using MySQL or PostgreSQL in an large setup gives you much greater
-performance, and while migration tools exist to move from one database type
-to another, it is better to get it right first time and to immediately use
-MySQL or PostgreSQL when you deploy |RCE| in a production environment.
 
 Migrating From SQLite to PostgreSQL
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

docs/known-issues/known-issues.rst

@@ -42,6 +42,7 b' Newer Operating system locales'
 the local-archive format, which is now incompatible with our used glibc 2.26.
 
 Mostly affected are:
+
 - Fedora 23+
 - Ubuntu 18.04
 - CentOS / RHEL 8
@@ -93,3 +94,24 b' example to pass the correct locale infor'
     [Install]
     WantedBy=multi-user.target
 
+
+Merge stucks in "merging" status
+--------------------------------
+
+Similar issues:
+
+- Pull Request duplicated and/or stucks in "creating" status.
+
+Mostly affected are:
+
+- Kubernetes AWS EKS setup with NFS as shared storage
+- AWS EFS as shared storage
+
+Workaround:
+
+1. Manually clear the repo cache via UI: 
+:menuselection:`Repository Settings --> Caches --> Invalidate repository cache`
+
+1. Open problematic PR and reset status to "created"
+
+Now you can merge PR normally

docs/release-notes/release-notes-5.1.0.rst

@@ -10,20 +10,20 b' Release Date'
 New Features
 ^^^^^^^^^^^^
 
-- We've introduced 2FA for users. Now alongside the external auth 2fa support RhodeCode allows to enable 2FA for users
+- We've introduced 2FA for users. Now alongside the external auth 2FA support RhodeCode allows to enable 2FA for users.
   2FA options will be available for each user individually, or enforced via authentication plugins like ldap, or internal.
 - Email based log-in. RhodeCode now allows to log-in using email as well as username for main authentication type.
 - Ability to replace a file using web UI. Now one can replace an existing file from the web-ui.
 - GIT LFS Sync automation. Remote push/pull commands now can also sync GIT LFS objects.
-- Added ability to remove or close branches from the web ui
-- Added ability to delete a branch automatically after merging PR for git repositories
-- Added support for S3 based archive_cache based that allows storing cached archives in S3 compatible object store.
+- Added ability to remove or close branches from the web ui.
+- Added ability to delete a branch automatically after merging PR for git repositories.
+- Added support for S3 based archive_cache that allows storing cached archives in S3 compatible object store.
 
 
 General
 ^^^^^^^
 
-- Upgraded all dependency libraries to their latest available versions
+- Upgraded all dependency libraries to their latest available versions.
 - Repository storage is no longer controlled via DB settings, but .ini file. This allows easier automated deployments.
 - Bumped mercurial to 6.7.4
 - Mercurial: enable httppostarguments for better support of large repositories with lots of heads.
@@ -39,21 +39,20 b' Performance'
 ^^^^^^^^^^^
 
 - Introduced a full rewrite of ssh backend for performance. The result is 2-5x speed improvement for operation with ssh.
-  enable new ssh wrapper by setting: `ssh.wrapper_cmd = /home/rhodecode/venv/bin/rc-ssh-wrapper-v2`
-- Introduced a new hooks subsystem that is more scalable and faster, enable it by settings: `vcs.hooks.protocol = celery`
+  Enable new ssh wrapper by setting: `ssh.wrapper_cmd = /home/rhodecode/venv/bin/rc-ssh-wrapper-v2`
+- Introduced a new hooks subsystem that is more scalable and faster, enable it by setting: `vcs.hooks.protocol = celery`
 
 
 Fixes
 ^^^^^
 
-- Archives: Zip archive download breaks when a gitmodules file is present
-- Branch permissions: fixed bug preventing to specify own rules from 4.X install
-- SVN: refactored svn events, thus fixing support for it in dockerized env
-- Fixed empty server url in PR link after push from cli
+- Archives: Zip archive download breaks when a gitmodules file is present.
+- Branch permissions: fixed bug preventing to specify own rules from 4.X install.
+- SVN: refactored svn events, thus fixing support for it in dockerized environment.
+- Fixed empty server url in PR link after push from cli.
 
 
 Upgrade notes
 ^^^^^^^^^^^^^
 
-- RhodeCode 5.1.0 is a mayor feature release after big 5.0.0 python3 migration. Happy to ship a first time feature
-  rich release
+- RhodeCode 5.1.0 is a major feature release after big 5.0.0 python3 migration. Happy to ship a first time feature-rich release.

docs/release-notes/release-notes.rst

@@ -9,7 +9,9 b' Release Notes'
 .. toctree::
    :maxdepth: 1
 
-
+   release-notes-5.2.0.rst
+   release-notes-5.1.2.rst
+   release-notes-5.1.1.rst
    release-notes-5.1.0.rst
    release-notes-5.0.3.rst
    release-notes-5.0.2.rst

docs/requirements_docs.txt

@@ -4,7 +4,7 b' furo==2023.9.10'
 sphinx-press-theme==0.8.0
 sphinx-rtd-theme==1.3.0
 
-pygments==2.16.1
+pygments==2.18.0
 
 docutils<0.19
 markupsafe==2.1.3

requirements.txt

@@ -5,7 +5,7 b' alembic==1.13.1'
     markupsafe==2.1.2
   sqlalchemy==1.4.52
     greenlet==3.0.3
-  typing_extensions==4.9.0
+  typing_extensions==4.12.2
 async-timeout==4.0.3
 babel==2.12.1
 beaker==1.12.1
@@ -18,8 +18,8 b' celery==5.3.6'
     click==8.1.3
   click-repl==0.2.0
     click==8.1.3
-    prompt-toolkit==3.0.38
-      wcwidth==0.2.6
+    prompt_toolkit==3.0.47
+      wcwidth==0.2.13
     six==1.16.0
   kombu==5.3.5
     amqp==5.2.0
@@ -33,7 +33,7 b' channelstream==0.7.1'
   gevent==24.2.1
     greenlet==3.0.3
     zope.event==5.0.0
-    zope.interface==6.3.0
+    zope.interface==7.0.3
   itsdangerous==1.1.0
   marshmallow==2.18.0
   pyramid==2.0.2
@@ -46,7 +46,7 b' channelstream==0.7.1'
     venusian==3.0.0
     webob==1.8.7
     zope.deprecation==5.0.0
-    zope.interface==6.3.0
+    zope.interface==7.0.3
   pyramid-jinja2==2.10
     jinja2==3.1.2
       markupsafe==2.1.2
@@ -61,7 +61,7 b' channelstream==0.7.1'
       venusian==3.0.0
       webob==1.8.7
       zope.deprecation==5.0.0
-      zope.interface==6.3.0
+      zope.interface==7.0.3
     zope.deprecation==5.0.0
   python-dateutil==2.8.2
     six==1.16.0
@@ -87,32 +87,31 b' dogpile.cache==1.3.3'
     pbr==5.11.1
 formencode==2.1.0
   six==1.16.0
-fsspec==2024.6.0
-gunicorn==21.2.0
-  packaging==24.0
+fsspec==2024.9.0
+gunicorn==23.0.0
+  packaging==24.1
 gevent==24.2.1
   greenlet==3.0.3
   zope.event==5.0.0
-  zope.interface==6.3.0
-ipython==8.14.0
-  backcall==0.2.0
+  zope.interface==7.0.3
+ipython==8.26.0
   decorator==5.1.1
-  jedi==0.19.0
-    parso==0.8.3
-  matplotlib-inline==0.1.6
-    traitlets==5.9.0
-  pexpect==4.8.0
+  jedi==0.19.1
+    parso==0.8.4
+  matplotlib-inline==0.1.7
+    traitlets==5.14.3
+  pexpect==4.9.0
     ptyprocess==0.7.0
-  pickleshare==0.7.5
-  prompt-toolkit==3.0.38
-    wcwidth==0.2.6
-  pygments==2.15.1
-  stack-data==0.6.2
-    asttokens==2.2.1
+  prompt_toolkit==3.0.47
+    wcwidth==0.2.13
+  pygments==2.18.0
+  stack-data==0.6.3
+    asttokens==2.4.1
       six==1.16.0
-    executing==1.2.0
-    pure-eval==0.2.2
-  traitlets==5.9.0
+    executing==2.0.1
+    pure_eval==0.2.3
+  traitlets==5.14.3
+  typing_extensions==4.12.2
 markdown==3.4.3
 msgpack==1.0.8
 mysqlclient==2.1.1
@@ -127,7 +126,7 b' nbconvert==7.7.3'
     markupsafe==2.1.2
   jupyter_core==5.3.1
     platformdirs==3.10.0
-    traitlets==5.9.0
+    traitlets==5.14.3
   jupyterlab-pygments==0.2.2
   markupsafe==2.1.2
   mistune==2.0.5
@@ -135,15 +134,15 b' nbconvert==7.7.3'
     jupyter_client==8.3.0
       jupyter_core==5.3.1
         platformdirs==3.10.0
-        traitlets==5.9.0
+        traitlets==5.14.3
       python-dateutil==2.8.2
         six==1.16.0
       pyzmq==25.0.0
       tornado==6.2
-      traitlets==5.9.0
+      traitlets==5.14.3
     jupyter_core==5.3.1
       platformdirs==3.10.0
-      traitlets==5.9.0
+      traitlets==5.14.3
     nbformat==5.9.2
       fastjsonschema==2.18.0
       jsonschema==4.18.6
@@ -151,9 +150,9 b' nbconvert==7.7.3'
         pyrsistent==0.19.3
       jupyter_core==5.3.1
         platformdirs==3.10.0
-        traitlets==5.9.0
-      traitlets==5.9.0
-    traitlets==5.9.0
+        traitlets==5.14.3
+      traitlets==5.14.3
+    traitlets==5.14.3
   nbformat==5.9.2
     fastjsonschema==2.18.0
     jsonschema==4.18.6
@@ -161,20 +160,20 b' nbconvert==7.7.3'
       pyrsistent==0.19.3
     jupyter_core==5.3.1
       platformdirs==3.10.0
-      traitlets==5.9.0
-    traitlets==5.9.0
+      traitlets==5.14.3
+    traitlets==5.14.3
   pandocfilters==1.5.0
-  pygments==2.15.1
+  pygments==2.18.0
   tinycss2==1.2.1
     webencodings==0.5.1
-  traitlets==5.9.0
-orjson==3.10.3
+  traitlets==5.14.3
+orjson==3.10.7
 paste==3.10.1
 premailer==3.10.0
   cachetools==5.3.3
   cssselect==1.2.0
   cssutils==2.6.0
-  lxml==4.9.3
+  lxml==5.3.0
   requests==2.28.2
     certifi==2022.12.7
     charset-normalizer==3.1.0
@@ -191,33 +190,6 b' pycurl==7.45.3'
 pymysql==1.0.3
 pyotp==2.8.0
 pyparsing==3.1.1
-pyramid-debugtoolbar==4.12.1
-  pygments==2.15.1
-  pyramid==2.0.2
-    hupper==1.12
-    plaster==1.1.2
-    plaster-pastedeploy==1.0.1
-      pastedeploy==3.1.0
-      plaster==1.1.2
-    translationstring==1.4
-    venusian==3.0.0
-    webob==1.8.7
-    zope.deprecation==5.0.0
-    zope.interface==6.3.0
-  pyramid-mako==1.1.0
-    mako==1.2.4
-      markupsafe==2.1.2
-    pyramid==2.0.2
-      hupper==1.12
-      plaster==1.1.2
-      plaster-pastedeploy==1.0.1
-        pastedeploy==3.1.0
-        plaster==1.1.2
-      translationstring==1.4
-      venusian==3.0.0
-      webob==1.8.7
-      zope.deprecation==5.0.0
-      zope.interface==6.3.0
 pyramid-mailer==0.15.1
   pyramid==2.0.2
     hupper==1.12
@@ -229,13 +201,27 b' pyramid-mailer==0.15.1'
     venusian==3.0.0
     webob==1.8.7
     zope.deprecation==5.0.0
-    zope.interface==6.3.0
+    zope.interface==7.0.3
   repoze.sendmail==4.4.1
-    transaction==3.1.0
-      zope.interface==6.3.0
-    zope.interface==6.3.0
-  transaction==3.1.0
-    zope.interface==6.3.0
+    transaction==5.0.0
+      zope.interface==7.0.3
+    zope.interface==7.0.3
+  transaction==5.0.0
+    zope.interface==7.0.3
+pyramid-mako==1.1.0
+  mako==1.2.4
+    markupsafe==2.1.2
+  pyramid==2.0.2
+    hupper==1.12
+    plaster==1.1.2
+    plaster-pastedeploy==1.0.1
+      pastedeploy==3.1.0
+      plaster==1.1.2
+    translationstring==1.4
+    venusian==3.0.0
+    webob==1.8.7
+    zope.deprecation==5.0.0
+    zope.interface==7.0.3
 python-ldap==3.4.3
   pyasn1==0.4.8
   pyasn1-modules==0.2.8
@@ -243,20 +229,20 b' python-ldap==3.4.3'
 python-memcached==1.59
   six==1.16.0
 python-pam==2.0.2
-python3-saml==1.15.0
+python3-saml==1.16.0
   isodate==0.6.1
     six==1.16.0
-  lxml==4.9.3
-  xmlsec==1.3.13
-    lxml==4.9.3
+  lxml==5.3.0
+  xmlsec==1.3.14
+    lxml==5.3.0
 pyyaml==6.0.1
-redis==5.0.4
+redis==5.1.0
   async-timeout==4.0.3
 regex==2022.10.31
 routes==2.5.1
   repoze.lru==0.7
   six==1.16.0
-s3fs==2024.6.0
+s3fs==2024.9.0
   aiobotocore==2.13.0
     aiohttp==3.9.5
       aiosignal==1.3.1
@@ -283,7 +269,7 b' s3fs==2024.6.0'
     yarl==1.9.4
       idna==3.4
       multidict==6.0.5
-  fsspec==2024.6.0
+  fsspec==2024.9.0
 simplejson==3.19.2
 sshpubkeys==3.3.1
   cryptography==40.0.2
@@ -293,7 +279,7 b' sshpubkeys==3.3.1'
     six==1.16.0
 sqlalchemy==1.4.52
   greenlet==3.0.3
-  typing_extensions==4.9.0
+  typing_extensions==4.12.2
 supervisor==4.2.5
 tzlocal==4.3
   pytz-deprecation-shim==0.1.0.post0

requirements_debug.txt

@@ -9,6 +9,7 b' pympler'
 ipdb
 ipython
 rich
+pyramid-debugtoolbar
 
 # format
 flake8

requirements_test.txt

@@ -4,38 +4,38 b' pytest-cov==4.1.0'
   coverage==7.4.3
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
 pytest-env==1.1.3
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
 pytest-profiling==1.7.0
   gprof2dot==2022.7.29
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
   six==1.16.0
 pytest-rerunfailures==13.0
-  packaging==24.0
+  packaging==24.1
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
 pytest-runner==6.0.1
 pytest-sugar==1.0.0
-  packaging==24.0
+  packaging==24.1
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
   termcolor==2.4.0
 pytest-timeout==2.3.1
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
 webtest==3.0.0
   beautifulsoup4==4.12.3

rhodecode/VERSION

@@ -1,1 +1,1 b''
-5.1.2
  No newline at end of file
+5.2.0

rhodecode/api/__init__.py

@@ -40,6 +40,7 b' from rhodecode.lib import ext_json'
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.plugins.utils import get_plugin_settings
 from rhodecode.model.db import User, UserApiKeys
+from rhodecode.config.patches import inspect_getargspec
 
 log = logging.getLogger(__name__)
 
@@ -186,7 +187,6 b' def request_view(request):'
     exposed method
     """
     # cython compatible inspect
-    from rhodecode.config.patches import inspect_getargspec
     inspect = inspect_getargspec()
 
     # check if we can find this session using api_key, get_by_auth_token

rhodecode/api/views/server_api.py

@@ -33,7 +33,7 b' from rhodecode.lib.ext_json import json'
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.model.db import UserIpMap
 from rhodecode.model.scm import ScmModel
-from rhodecode.apps.file_store import utils
+from rhodecode.apps.file_store import utils as store_utils
 from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
     FileOverSizeException
 
@@ -328,8 +328,8 b' def get_method(request, apiuser, pattern'
       ]
       error :  null
     """
-    from rhodecode.config.patches import inspect_getargspec
-    inspect = inspect_getargspec()
+    from rhodecode.config import patches
+    inspect = patches.inspect_getargspec()
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()

rhodecode/apps/admin/__init__.py

@@ -43,7 +43,38 b' def admin_routes(config):'
     from rhodecode.apps.admin.views.system_info import AdminSystemInfoSettingsView
     from rhodecode.apps.admin.views.user_groups import AdminUserGroupsView
     from rhodecode.apps.admin.views.users import AdminUsersView, UsersView
-    
+    from rhodecode.apps.admin.views.security import AdminSecurityView
+
+    # Security EE feature
+
+    config.add_route(
+        'admin_security',
+        pattern='/security')
+    config.add_view(
+        AdminSecurityView,
+        attr='security',
+        route_name='admin_security', request_method='GET',
+        renderer='rhodecode:templates/admin/security/security.mako')
+
+    config.add_route(
+        name='admin_security_update',
+        pattern='/security/update')
+    config.add_view(
+        AdminSecurityView,
+        attr='security_update',
+        route_name='admin_security_update', request_method='POST',
+        renderer='rhodecode:templates/admin/security/security.mako')
+
+    config.add_route(
+        name='admin_security_modify_allowed_vcs_client_versions',
+        pattern=ADMIN_PREFIX + '/security/modify/allowed_vcs_client_versions')
+    config.add_view(
+        AdminSecurityView,
+        attr='vcs_whitelisted_client_versions_edit',
+        route_name='admin_security_modify_allowed_vcs_client_versions', request_method=('GET', 'POST'),
+        renderer='rhodecode:templates/admin/security/edit_allowed_vcs_client_versions.mako')
+
+
     config.add_route(
         name='admin_audit_logs',
         pattern='/audit_logs')

rhodecode/apps/admin/views/settings.py

@@ -75,14 +75,21 b' class AdminSettingsView(BaseAppView):'
 
         if not ret:
             raise Exception('Could not get application ui settings !')
-        settings = {}
+        settings = {
+            # legacy param that needs to be kept
+            'web_push_ssl': False
+        }
         for each in ret:
             k = each.ui_key
             v = each.ui_value
+            # skip some options if they are defined
+            if k in ['push_ssl']:
+                continue
+
             if k == '/':
                 k = 'root_path'
 
-            if k in ['push_ssl', 'publish', 'enabled']:
+            if k in ['publish', 'enabled']:
                 v = str2bool(v)
 
             if k.find('.') != -1:
@@ -92,6 +99,7 b' class AdminSettingsView(BaseAppView):'
                 v = each.ui_active
 
             settings[each.ui_section + '_' + k] = v
+
         return settings
 
     @classmethod
@@ -164,7 +172,6 b' class AdminSettingsView(BaseAppView):'
             return Response(html)
 
         try:
-            model.update_global_ssl_setting(form_result['web_push_ssl'])
             model.update_global_hook_settings(form_result)
 
             model.create_or_update_global_svn_settings(form_result)

rhodecode/apps/admin/views/system_info.py

@@ -171,11 +171,17 b' class AdminSystemInfoSettingsView(BaseAp'
             (_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')),
             ('', '', ''),  # spacer
 
-            (_('Archive cache storage type'), val('storage_archive')['type'], state('storage_archive')),
+            (_('Artifacts storage backend'), val('storage_artifacts')['type'], state('storage_artifacts')),
+            (_('Artifacts storage location'), val('storage_artifacts')['path'], state('storage_artifacts')),
+            (_('Artifacts info'), val('storage_artifacts')['text'], state('storage_artifacts')),
+            ('', '', ''),  # spacer
+
+            (_('Archive cache storage backend'), val('storage_archive')['type'], state('storage_archive')),
             (_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')),
             (_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')),
             ('', '', ''),  # spacer
 
+
             (_('Temp storage location'), val('storage_temp')['path'], state('storage_temp')),
             (_('Temp storage info'), val('storage_temp')['text'], state('storage_temp')),
             ('', '', ''),  # spacer

rhodecode/apps/file_store/__init__.py

@@ -16,7 +16,8 b''
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import os
-from rhodecode.apps.file_store import config_keys
+
+
 from rhodecode.config.settings_maker import SettingsMaker
 
 
@@ -24,18 +25,48 b' def _sanitize_settings_and_apply_default'
     """
     Set defaults, convert to python types and validate settings.
     """
+    from rhodecode.apps.file_store import config_keys
+
+    # translate "legacy" params into new config
+    settings.pop(config_keys.deprecated_enabled, True)
+    if config_keys.deprecated_backend in settings:
+        # if legacy backend key is detected we use "legacy" backward compat setting
+        settings.pop(config_keys.deprecated_backend)
+        settings[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+
+    if config_keys.deprecated_store_path in settings:
+        store_path = settings.pop(config_keys.deprecated_store_path)
+        settings[config_keys.legacy_filesystem_storage_path] = store_path
+
     settings_maker = SettingsMaker(settings)
 
-    settings_maker.make_setting(config_keys.enabled, True, parser='bool')
-    settings_maker.make_setting(config_keys.backend, 'local')
+    default_cache_dir = settings['cache_dir']
+    default_store_dir = os.path.join(default_cache_dir, 'artifacts_filestore')
+
+    # set default backend
+    settings_maker.make_setting(config_keys.backend_type, config_keys.backend_legacy_filesystem)
+
+    # legacy filesystem defaults
+    settings_maker.make_setting(config_keys.legacy_filesystem_storage_path, default_store_dir, default_when_empty=True, )
 
-    default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store')
-    settings_maker.make_setting(config_keys.store_path, default_store)
+    # filesystem defaults
+    settings_maker.make_setting(config_keys.filesystem_storage_path, default_store_dir, default_when_empty=True,)
+    settings_maker.make_setting(config_keys.filesystem_shards, 8, parser='int')
+
+    # objectstore defaults
+    settings_maker.make_setting(config_keys.objectstore_url, 'http://s3-minio:9000')
+    settings_maker.make_setting(config_keys.objectstore_bucket, 'rhodecode-artifacts-filestore')
+    settings_maker.make_setting(config_keys.objectstore_bucket_shards, 8, parser='int')
+
+    settings_maker.make_setting(config_keys.objectstore_region, '')
+    settings_maker.make_setting(config_keys.objectstore_key, '')
+    settings_maker.make_setting(config_keys.objectstore_secret, '')
 
     settings_maker.env_expand()
 
 
 def includeme(config):
+
     from rhodecode.apps.file_store.views import FileStoreView
 
     settings = config.registry.settings

rhodecode/apps/file_store/config_keys.py

@@ -20,6 +20,38 b''
 # Definition of setting keys used to configure this module. Defined here to
 # avoid repetition of keys throughout the module.
 
-enabled = 'file_store.enabled'
-backend = 'file_store.backend'
-store_path = 'file_store.storage_path'
+# OLD and deprecated keys not used anymore
+deprecated_enabled = 'file_store.enabled'
+deprecated_backend = 'file_store.backend'
+deprecated_store_path = 'file_store.storage_path'
+
+
+backend_type = 'file_store.backend.type'
+
+backend_legacy_filesystem = 'filesystem_v1'
+backend_filesystem = 'filesystem_v2'
+backend_objectstore = 'objectstore'
+
+backend_types = [
+    backend_legacy_filesystem,
+    backend_filesystem,
+    backend_objectstore,
+]
+
+# filesystem_v1 legacy
+legacy_filesystem_storage_path = 'file_store.filesystem_v1.storage_path'
+
+
+# filesystem_v2 new option
+filesystem_storage_path = 'file_store.filesystem_v2.storage_path'
+filesystem_shards = 'file_store.filesystem_v2.shards'
+
+# objectstore
+objectstore_url = 'file_store.objectstore.url'
+objectstore_bucket = 'file_store.objectstore.bucket'
+objectstore_bucket_shards = 'file_store.objectstore.bucket_shards'
+
+objectstore_region = 'file_store.objectstore.region'
+objectstore_key = 'file_store.objectstore.key'
+objectstore_secret = 'file_store.objectstore.secret'
+

rhodecode/apps/file_store/tests/__init__.py

@@ -16,3 +16,42 b''
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
+import os
+import random
+import tempfile
+import string
+
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+
+
[e-mail]()
+def file_store_instance(ini_settings):
+    config = ini_settings
+    f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+    return f_store
+
+
[e-mail]
+def random_binary_file():
+    # Generate random binary data
+    data = bytearray(random.getrandbits(8) for _ in range(1024 * 512))  # 512 KB of random data
+
+    # Create a temporary file
+    temp_file = tempfile.NamedTemporaryFile(delete=False)
+    filename = temp_file.name
+
+    try:
+        # Write the random binary data to the file
+        temp_file.write(data)
+        temp_file.seek(0)  # Rewind the file pointer to the beginning
+        yield filename, temp_file
+    finally:
+        # Close and delete the temporary file after the test
+        temp_file.close()
+        os.remove(filename)
+
+
+def generate_random_filename(length=10):
+    return ''.join(random.choices(string.ascii_letters + string.digits, k=length))
  No newline at end of file

rhodecode/apps/file_store/tests/test_upload_file.py

@@ -15,13 +15,16 b''
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
+
 import os
+
 import pytest
 
 from rhodecode.lib.ext_json import json
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.db import Session, FileStore, Repository, User
-from rhodecode.apps.file_store import utils, config_keys
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
 
 from rhodecode.tests import TestController
 from rhodecode.tests.routes import route_path
@@ -29,27 +32,61 b' from rhodecode.tests.routes import route'
 
 class TestFileStoreViews(TestController):
 
+    @pytest.fixture()
+    def create_artifact_factory(self, tmpdir, ini_settings):
+
+        def factory(user_id, content, f_name='example.txt'):
+
+            config = ini_settings
+            config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+
+            f_store = store_utils.get_filestore_backend(config)
+
+            filesystem_file = os.path.join(str(tmpdir), f_name)
+            with open(filesystem_file, 'wt') as f:
+                f.write(content)
+
+            with open(filesystem_file, 'rb') as f:
+                store_uid, metadata = f_store.store(f_name, f, metadata={'filename': f_name})
+            os.remove(filesystem_file)
+
+            entry = FileStore.create(
+                file_uid=store_uid, filename=metadata["filename"],
+                file_hash=metadata["sha256"], file_size=metadata["size"],
+                file_display_name='file_display_name',
+                file_description='repo artifact `{}`'.format(metadata["filename"]),
+                check_acl=True, user_id=user_id,
+            )
+            Session().add(entry)
+            Session().commit()
+            return entry
+        return factory
+
     @pytest.mark.parametrize("fid, content, exists", [
         ('abcde-0.jpg', "xxxxx", True),
         ('abcde-0.exe', "1234567", True),
         ('abcde-0.jpg', "xxxxx", False),
     ])
-    def test_get_files_from_store(self, fid, content, exists, tmpdir, user_util):
+    def test_get_files_from_store(self, fid, content, exists, tmpdir, user_util, ini_settings):
         user = self.log_user()
         user_id = user['user_id']
         repo_id = user_util.create_repo().repo_id
-        store_path = self.app._pyramid_settings[config_keys.store_path]
+
+        config = ini_settings
+        config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+
         store_uid = fid
 
         if exists:
             status = 200
-            store = utils.get_file_storage({config_keys.store_path: store_path})
+            f_store = store_utils.get_filestore_backend(config)
             filesystem_file = os.path.join(str(tmpdir), fid)
             with open(filesystem_file, 'wt') as f:
                 f.write(content)
 
             with open(filesystem_file, 'rb') as f:
-                store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid})
+                store_uid, metadata = f_store.store(fid, f, metadata={'filename': fid})
+            os.remove(filesystem_file)
 
             entry = FileStore.create(
                 file_uid=store_uid, filename=metadata["filename"],
@@ -69,14 +106,10 b' class TestFileStoreViews(TestController)'
 
         if exists:
             assert response.text == content
-            file_store_path = os.path.dirname(store.resolve_name(store_uid, store_path)[1])
-            metadata_file = os.path.join(file_store_path, store_uid + '.meta')
-            assert os.path.exists(metadata_file)
-            with open(metadata_file, 'rb') as f:
-                json_data = json.loads(f.read())
 
-            assert json_data
-            assert 'size' in json_data
+            metadata = f_store.get_metadata(store_uid)
+
+            assert 'size' in metadata
 
     def test_upload_files_without_content_to_store(self):
         self.log_user()
@@ -112,32 +145,6 b' class TestFileStoreViews(TestController)'
 
         assert response.json['store_fid']
 
-    @pytest.fixture()
-    def create_artifact_factory(self, tmpdir):
-        def factory(user_id, content):
-            store_path = self.app._pyramid_settings[config_keys.store_path]
-            store = utils.get_file_storage({config_keys.store_path: store_path})
-            fid = 'example.txt'
-
-            filesystem_file = os.path.join(str(tmpdir), fid)
-            with open(filesystem_file, 'wt') as f:
-                f.write(content)
-
-            with open(filesystem_file, 'rb') as f:
-                store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid})
-
-            entry = FileStore.create(
-                file_uid=store_uid, filename=metadata["filename"],
-                file_hash=metadata["sha256"], file_size=metadata["size"],
-                file_display_name='file_display_name',
-                file_description='repo artifact `{}`'.format(metadata["filename"]),
-                check_acl=True, user_id=user_id,
-            )
-            Session().add(entry)
-            Session().commit()
-            return entry
-        return factory
-
     def test_download_file_non_scoped(self, user_util, create_artifact_factory):
         user = self.log_user()
         user_id = user['user_id']

rhodecode/apps/file_store/utils.py

@@ -19,21 +19,84 b''
 import io
 import uuid
 import pathlib
+import s3fs
+
+from rhodecode.lib.hash_utils import sha256_safe
+from rhodecode.apps.file_store import config_keys
+
+
+file_store_meta = None
+
+
+def get_filestore_config(config) -> dict:
+
+    final_config = {}
+
+    for k, v in config.items():
+        if k.startswith('file_store'):
+            final_config[k] = v
+
+    return final_config
 
 
-def get_file_storage(settings):
-    from rhodecode.apps.file_store.backends.local_store import LocalFileStorage
-    from rhodecode.apps.file_store import config_keys
-    store_path = settings.get(config_keys.store_path)
-    return LocalFileStorage(base_path=store_path)
+def get_filestore_backend(config, always_init=False):
+    """
+
+    usage::
+        from rhodecode.apps.file_store import get_filestore_backend
+        f_store = get_filestore_backend(config=CONFIG)
+
+    :param config:
+    :param always_init:
+    :return:
+    """
+
+    global file_store_meta
+    if file_store_meta is not None and not always_init:
+        return file_store_meta
+
+    config = get_filestore_config(config)
+    backend = config[config_keys.backend_type]
+
+    match backend:
+        case config_keys.backend_legacy_filesystem:
+            # Legacy backward compatible storage
+            from rhodecode.apps.file_store.backends.filesystem_legacy import LegacyFileSystemBackend
+            d_cache = LegacyFileSystemBackend(
+                settings=config
+            )
+        case config_keys.backend_filesystem:
+            from rhodecode.apps.file_store.backends.filesystem import FileSystemBackend
+            d_cache = FileSystemBackend(
+                settings=config
+            )
+        case config_keys.backend_objectstore:
+            from rhodecode.apps.file_store.backends.objectstore import ObjectStoreBackend
+            d_cache = ObjectStoreBackend(
+                settings=config
+            )
+        case _:
+            raise ValueError(
+                f'file_store.backend.type only supports "{config_keys.backend_types}" got {backend}'
+            )
+
+    cache_meta = d_cache
+    return cache_meta
 
 
 def splitext(filename):
-    ext = ''.join(pathlib.Path(filename).suffixes)
+    final_ext = []
+    for suffix in pathlib.Path(filename).suffixes:
+        if not suffix.isascii():
+            continue
+
+        suffix = " ".join(suffix.split()).replace(" ", "")
+        final_ext.append(suffix)
+    ext = ''.join(final_ext)
     return filename, ext
 
 
-def uid_filename(filename, randomized=True):
+def get_uid_filename(filename, randomized=True):
     """
     Generates a randomized or stable (uuid) filename,
     preserving the original extension.
@@ -46,10 +109,37 b' def uid_filename(filename, randomized=Tr'
     if randomized:
         uid = uuid.uuid4()
     else:
-        hash_key = '{}.{}'.format(filename, 'store')
+        store_suffix = "store"
+        hash_key = f'{filename}.{store_suffix}'
         uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
     return str(uid) + ext.lower()
 
 
 def bytes_to_file_obj(bytes_data):
-    return io.StringIO(bytes_data)
+    return io.BytesIO(bytes_data)
+
+
+class ShardFileReader:
+
+    def __init__(self, file_like_reader):
+        self._file_like_reader = file_like_reader
+
+    def __getattr__(self, item):
+        if isinstance(self._file_like_reader, s3fs.core.S3File):
+            match item:
+                case 'name':
+                    # S3 FileWrapper doesn't support name attribute, and we use it
+                    return self._file_like_reader.full_name
+                case _:
+                    return getattr(self._file_like_reader, item)
+        else:
+            return getattr(self._file_like_reader, item)
+
+
+def archive_iterator(_reader, block_size: int = 4096 * 512):
+    # 4096 * 64 = 64KB
+    while 1:
+        data = _reader.read(block_size)
+        if not data:
+            break
+        yield data

rhodecode/apps/file_store/views.py

@@ -17,12 +17,11 b''
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import logging
 
-
-from pyramid.response import FileResponse
+from pyramid.response import Response
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 
 from rhodecode.apps._base import BaseAppView
-from rhodecode.apps.file_store import utils
+from rhodecode.apps.file_store import utils as store_utils
 from rhodecode.apps.file_store.exceptions import (
     FileNotAllowedException, FileOverSizeException)
 
@@ -31,6 +30,7 b' from rhodecode.lib import audit_logger'
 from rhodecode.lib.auth import (
     CSRFRequired, NotAnonymous, HasRepoPermissionAny, HasRepoGroupPermissionAny,
     LoginRequired)
+from rhodecode.lib.str_utils import header_safe_str
 from rhodecode.lib.vcs.conf.mtypes import get_mimetypes_db
 from rhodecode.model.db import Session, FileStore, UserApiKeys
 
@@ -42,7 +42,7 b' class FileStoreView(BaseAppView):'
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
-        self.storage = utils.get_file_storage(self.request.registry.settings)
+        self.f_store = store_utils.get_filestore_backend(self.request.registry.settings)
         return c
 
     def _guess_type(self, file_name):
@@ -55,10 +55,10 b' class FileStoreView(BaseAppView):'
         return _content_type, _encoding
 
     def _serve_file(self, file_uid):
-        if not self.storage.exists(file_uid):
-            store_path = self.storage.store_path(file_uid)
-            log.debug('File with FID:%s not found in the store under `%s`',
-                      file_uid, store_path)
+        if not self.f_store.filename_exists(file_uid):
+            store_path = self.f_store.store_path(file_uid)
+            log.warning('File with FID:%s not found in the store under `%s`',
+                        file_uid, store_path)
             raise HTTPNotFound()
 
         db_obj = FileStore.get_by_store_uid(file_uid, safe=True)
@@ -98,28 +98,25 b' class FileStoreView(BaseAppView):'
 
         FileStore.bump_access_counter(file_uid)
 
-        file_path = self.storage.store_path(file_uid)
+        file_name = db_obj.file_display_name
         content_type = 'application/octet-stream'
-        content_encoding = None
 
-        _content_type, _encoding = self._guess_type(file_path)
+        _content_type, _encoding = self._guess_type(file_name)
         if _content_type:
             content_type = _content_type
 
         # For file store we don't submit any session data, this logic tells the
         # Session lib to skip it
         setattr(self.request, '_file_response', True)
-        response = FileResponse(
-            file_path, request=self.request,
-            content_type=content_type, content_encoding=content_encoding)
+        reader, _meta = self.f_store.fetch(file_uid)
 
-        file_name = db_obj.file_display_name
+        response = Response(app_iter=store_utils.archive_iterator(reader))
 
-        response.headers["Content-Disposition"] = (
-            f'attachment; filename="{str(file_name)}"'
-        )
+        response.content_type = str(content_type)
+        response.content_disposition = f'attachment; filename="{header_safe_str(file_name)}"'
+
         response.headers["X-RC-Artifact-Id"] = str(db_obj.file_store_id)
-        response.headers["X-RC-Artifact-Desc"] = str(db_obj.file_description)
+        response.headers["X-RC-Artifact-Desc"] = header_safe_str(db_obj.file_description)
         response.headers["X-RC-Artifact-Sha256"] = str(db_obj.file_hash)
         return response
 
@@ -147,8 +144,8 b' class FileStoreView(BaseAppView):'
                               'user_id': self._rhodecode_user.user_id,
                               'ip': self._rhodecode_user.ip_addr}}
         try:
-            store_uid, metadata = self.storage.save_file(
-                file_obj.file, filename, extra_metadata=metadata)
+            store_uid, metadata = self.f_store.store(
+                filename, file_obj.file, extra_metadata=metadata)
         except FileNotAllowedException:
             return {'store_fid': None,
                     'access_path': None,
@@ -182,7 +179,7 b' class FileStoreView(BaseAppView):'
     def download_file(self):
         self.load_default_context()
         file_uid = self.request.matchdict['fid']
-        log.debug('Requesting FID:%s from store %s', file_uid, self.storage)
+        log.debug('Requesting FID:%s from store %s', file_uid, self.f_store)
         return self._serve_file(file_uid)
 
     # in addition to @LoginRequired ACL is checked by scopes

rhodecode/apps/repository/views/repo_commits.py

@@ -601,26 +601,26 b' class RepoCommitsView(RepoAppView):'
         max_file_size = 10 * 1024 * 1024  # 10MB, also validated via dropzone.js
 
         try:
-            storage = store_utils.get_file_storage(self.request.registry.settings)
-            store_uid, metadata = storage.save_file(
-                file_obj.file, filename, extra_metadata=metadata,
+            f_store = store_utils.get_filestore_backend(self.request.registry.settings)
+            store_uid, metadata = f_store.store(
+                filename, file_obj.file, metadata=metadata,
                 extensions=allowed_extensions, max_filesize=max_file_size)
         except FileNotAllowedException:
             self.request.response.status = 400
             permitted_extensions = ', '.join(allowed_extensions)
-            error_msg = 'File `{}` is not allowed. ' \
-                        'Only following extensions are permitted: {}'.format(
-                            filename, permitted_extensions)
+            error_msg = f'File `{filename}` is not allowed. ' \
+                        f'Only following extensions are permitted: {permitted_extensions}'
+
             return {'store_fid': None,
                     'access_path': None,
                     'error': error_msg}
         except FileOverSizeException:
             self.request.response.status = 400
             limit_mb = h.format_byte_size_binary(max_file_size)
+            error_msg = f'File {filename} is exceeding allowed limit of {limit_mb}.'
             return {'store_fid': None,
                     'access_path': None,
-                    'error': 'File {} is exceeding allowed limit of {}.'.format(
-                        filename, limit_mb)}
+                    'error': error_msg}
 
         try:
             entry = FileStore.create(

rhodecode/apps/repository/views/repo_files.py

@@ -48,7 +48,7 b' from rhodecode.lib.codeblocks import ('
     filenode_as_lines_tokens, filenode_as_annotated_lines_tokens)
 from rhodecode.lib.utils2 import convert_line_endings, detect_mode
 from rhodecode.lib.type_utils import str2bool
-from rhodecode.lib.str_utils import safe_str, safe_int
+from rhodecode.lib.str_utils import safe_str, safe_int, header_safe_str
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
 from rhodecode.lib.vcs import path as vcspath
@@ -820,7 +820,7 b' class RepoFilesView(RepoAppView):'
                   "filename=\"{}\"; " \
                   "filename*=UTF-8\'\'{}".format(safe_path, encoded_path)
 
-        return safe_bytes(headers).decode('latin-1', errors='replace')
+        return header_safe_str(headers)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(

rhodecode/apps/repository/views/repo_settings_advanced.py

@@ -29,7 +29,7 b' from rhodecode.lib import audit_logger'
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired,
     HasRepoPermissionAny)
-from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError
+from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.lib.vcs import RepositoryError
 from rhodecode.model.db import Session, UserFollowing, User, Repository
@@ -136,6 +136,9 b' class RepoSettingsAdvancedView(RepoAppVi'
         elif handle_forks == 'delete_forks':
             handle_forks = 'delete'
 
+        repo_advanced_url = h.route_path(
+            'edit_repo_advanced', repo_name=self.db_repo_name,
+            _anchor='advanced-delete')
         try:
             old_data = self.db_repo.get_api_data()
             RepoModel().delete(self.db_repo, forks=handle_forks)
@@ -158,9 +161,6 b' class RepoSettingsAdvancedView(RepoAppVi'
                 category='success')
             Session().commit()
         except AttachedForksError:
-            repo_advanced_url = h.route_path(
-                'edit_repo_advanced', repo_name=self.db_repo_name,
-                _anchor='advanced-delete')
             delete_anchor = h.link_to(_('detach or delete'), repo_advanced_url)
             h.flash(_('Cannot delete `{repo}` it still contains attached forks. '
                       'Try using {delete_or_detach} option.')
@@ -171,9 +171,6 b' class RepoSettingsAdvancedView(RepoAppVi'
             raise HTTPFound(repo_advanced_url)
 
         except AttachedPullRequestsError:
-            repo_advanced_url = h.route_path(
-                'edit_repo_advanced', repo_name=self.db_repo_name,
-                _anchor='advanced-delete')
             attached_prs = len(self.db_repo.pull_requests_source +
                                self.db_repo.pull_requests_target)
             h.flash(
@@ -184,6 +181,16 b' class RepoSettingsAdvancedView(RepoAppVi'
             # redirect to advanced for forks handle action ?
             raise HTTPFound(repo_advanced_url)
 
+        except AttachedArtifactsError:
+
+            attached_artifacts = len(self.db_repo.artifacts)
+            h.flash(
+                _('Cannot delete `{repo}` it still contains {num} attached artifacts. '
+                  'Consider archiving the repository instead.').format(
+                    repo=self.db_repo_name, num=attached_artifacts), category='warning')
+
+            # redirect to advanced for forks handle action ?
+            raise HTTPFound(repo_advanced_url)
         except Exception:
             log.exception("Exception during deletion of repository")
             h.flash(_('An error occurred during deletion of `%s`')

rhodecode/apps/ssh_support/__init__.py

@@ -37,7 +37,7 b' def _sanitize_settings_and_apply_default'
     settings_maker.make_setting(config_keys.ssh_key_generator_enabled, True, parser='bool')
 
     settings_maker.make_setting(config_keys.authorized_keys_file_path, '~/.ssh/authorized_keys_rhodecode')
-    settings_maker.make_setting(config_keys.wrapper_cmd, '')
+    settings_maker.make_setting(config_keys.wrapper_cmd, '/usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2')
     settings_maker.make_setting(config_keys.authorized_keys_line_ssh_opts, '')
 
     settings_maker.make_setting(config_keys.ssh_hg_bin, '/usr/local/bin/rhodecode_bin/vcs_bin/hg')

rhodecode/apps/ssh_support/config_keys.py

@@ -23,7 +23,7 b" generate_authorized_keyfile = 'ssh.gener"
 authorized_keys_file_path = 'ssh.authorized_keys_file_path'
 authorized_keys_line_ssh_opts = 'ssh.authorized_keys_ssh_opts'
 ssh_key_generator_enabled = 'ssh.enable_ui_key_generator'
-wrapper_cmd = 'ssh.wrapper_cmd'
+wrapper_cmd = 'ssh.wrapper_cmd.v2'
 wrapper_allow_shell = 'ssh.wrapper_cmd_allow_shell'
 enable_debug_logging = 'ssh.enable_debug_logging'
 

rhodecode/apps/ssh_support/lib/backends/base.py

@@ -157,7 +157,7 b' class SshVcsServer(object):'
         return exit_code, action == "push"
 
     def run(self, tunnel_extras=None):
-        self.hooks_protocol = self.settings['vcs.hooks.protocol']
+        self.hooks_protocol = self.settings['vcs.hooks.protocol.v2']
         tunnel_extras = tunnel_extras or {}
         extras = {}
         extras.update(tunnel_extras)

rhodecode/apps/ssh_support/tests/test_server_git.py

@@ -32,7 +32,7 b' class GitServerCreator(object):'
     config_data = {
         'app:main': {
             'ssh.executable.git': git_path,
-            'vcs.hooks.protocol': 'http',
+            'vcs.hooks.protocol.v2': 'celery',
         }
     }
     repo_name = 'test_git'

rhodecode/apps/ssh_support/tests/test_server_hg.py

@@ -31,7 +31,7 b' class MercurialServerCreator(object):'
     config_data = {
         'app:main': {
             'ssh.executable.hg': hg_path,
-            'vcs.hooks.protocol': 'http',
+            'vcs.hooks.protocol.v2': 'celery',
         }
     }
     repo_name = 'test_hg'

rhodecode/apps/ssh_support/tests/test_server_svn.py

@@ -29,7 +29,7 b' class SubversionServerCreator(object):'
     config_data = {
         'app:main': {
             'ssh.executable.svn': svn_path,
-            'vcs.hooks.protocol': 'http',
+            'vcs.hooks.protocol.v2': 'celery',
         }
     }
     repo_name = 'test-svn'

rhodecode/authentication/routes.py

@@ -52,6 +52,7 b' class AuthnRootResource(AuthnResourceBas'
     """
     This is the root traversal resource object for the authentication settings.
     """
+    is_root = True
 
     def __init__(self):
         self._store = collections.OrderedDict()

rhodecode/config/config_maker.py

@@ -52,7 +52,8 b' def sanitize_settings_and_apply_defaults'
         default=False,
         parser='bool')
 
-    logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini')
+    ini_loc = os.path.dirname(global_config.get('__file__'))
+    logging_conf = jn(ini_loc, 'logging.ini')
     settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
 
     # Default includes, possible to change as a user
@@ -95,6 +96,11 b' def sanitize_settings_and_apply_defaults'
     settings_maker.make_setting('gzip_responses', False, parser='bool')
     settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
 
+    # License settings.
+    settings_maker.make_setting('license.hide_license_info', False, parser='bool')
+    settings_maker.make_setting('license.import_path', '')
+    settings_maker.make_setting('license.import_path_mode', 'if-missing')
+
     # statsd
     settings_maker.make_setting('statsd.enabled', False, parser='bool')
     settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
@@ -106,7 +112,7 b' def sanitize_settings_and_apply_defaults'
     settings_maker.make_setting('vcs.svn.redis_conn', 'redis://redis:6379/0')
     settings_maker.make_setting('vcs.svn.proxy.enabled', True, parser='bool')
     settings_maker.make_setting('vcs.svn.proxy.host', 'http://svn:8090', parser='string')
-    settings_maker.make_setting('vcs.hooks.protocol', 'http')
+    settings_maker.make_setting('vcs.hooks.protocol.v2', 'celery')
     settings_maker.make_setting('vcs.hooks.host', '*')
     settings_maker.make_setting('vcs.scm_app_implementation', 'http')
     settings_maker.make_setting('vcs.server', '')
@@ -116,6 +122,9 b' def sanitize_settings_and_apply_defaults'
     settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
     settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
     settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
+    settings_maker.make_setting('vcs.git.lfs.storage_location', '/var/opt/rhodecode_repo_store/.cache/git_lfs_store')
+    settings_maker.make_setting('vcs.hg.largefiles.storage_location',
+                                '/var/opt/rhodecode_repo_store/.cache/hg_largefiles_store')
 
     settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
 
@@ -152,6 +161,10 b' def sanitize_settings_and_apply_defaults'
         parser='file:ensured'
     )
 
+    # celery
+    broker_url = settings_maker.make_setting('celery.broker_url', 'redis://redis:6379/8')
+    settings_maker.make_setting('celery.result_backend', broker_url)
+
     settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
     settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
 
@@ -202,7 +215,7 b' def sanitize_settings_and_apply_defaults'
     settings_maker.make_setting('archive_cache.filesystem.retry_backoff', 1, parser='int')
     settings_maker.make_setting('archive_cache.filesystem.retry_attempts', 10, parser='int')
 
-    settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
+    settings_maker.make_setting('archive_cache.objectstore.url', 'http://s3-minio:9000', default_when_empty=True,)
     settings_maker.make_setting('archive_cache.objectstore.key', '')
     settings_maker.make_setting('archive_cache.objectstore.secret', '')
     settings_maker.make_setting('archive_cache.objectstore.region', 'eu-central-1')

rhodecode/config/environment.py

@@ -16,8 +16,8 b''
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-import os
 import logging
+
 import rhodecode
 import collections
 
@@ -30,6 +30,21 b' from rhodecode.lib.vcs import connect_vc'
 log = logging.getLogger(__name__)
 
 
+def propagate_rhodecode_config(global_config, settings, config):
+    # Store the settings to make them available to other modules.
+    settings_merged = global_config.copy()
+    settings_merged.update(settings)
+    if config:
+        settings_merged.update(config)
+
+    rhodecode.PYRAMID_SETTINGS = settings_merged
+    rhodecode.CONFIG = settings_merged
+
+    if 'default_user_id' not in rhodecode.CONFIG:
+        rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
+    log.debug('set rhodecode.CONFIG data')
+
+
 def load_pyramid_environment(global_config, settings):
     # Some parts of the code expect a merge of global and app settings.
     settings_merged = global_config.copy()
@@ -75,11 +90,8 b' def load_pyramid_environment(global_conf'
 
     utils.configure_vcs(settings)
 
-    # Store the settings to make them available to other modules.
-
-    rhodecode.PYRAMID_SETTINGS = settings_merged
-    rhodecode.CONFIG = settings_merged
-    rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
+    # first run, to store data...
+    propagate_rhodecode_config(global_config, settings, {})
 
     if vcs_server_enabled:
         connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))

rhodecode/config/middleware.py

@@ -35,7 +35,7 b' from pyramid.renderers import render_to_'
 from rhodecode.model import meta
 from rhodecode.config import patches
 
-from rhodecode.config.environment import load_pyramid_environment
+from rhodecode.config.environment import load_pyramid_environment, propagate_rhodecode_config
 
 import rhodecode.events
 from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
@@ -50,7 +50,7 b' from rhodecode.lib.utils2 import Attribu'
 from rhodecode.lib.exc_tracking import store_exception, format_exc
 from rhodecode.subscribers import (
     scan_repositories_if_enabled, write_js_routes_if_enabled,
-    write_metadata_if_needed, write_usage_data)
+    write_metadata_if_needed, write_usage_data, import_license_if_present)
 from rhodecode.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
@@ -99,6 +99,7 b' def make_pyramid_app(global_config, **se'
 
     # Apply compatibility patches
     patches.inspect_getargspec()
+    patches.repoze_sendmail_lf_fix()
 
     load_pyramid_environment(global_config, settings)
 
@@ -114,6 +115,9 b' def make_pyramid_app(global_config, **se'
     celery_settings = get_celery_config(settings)
     config.configure_celery(celery_settings)
 
+    # final config set...
+    propagate_rhodecode_config(global_config, settings, config.registry.settings)
+
     # creating the app uses a connection - return it after we are done
     meta.Session.remove()
 
@@ -396,7 +400,8 b' def includeme(config, auth_resources=Non'
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_js_routes_if_enabled,
                               pyramid.events.ApplicationCreated)
-
+        config.add_subscriber(import_license_if_present,
+                              pyramid.events.ApplicationCreated)
 
     # Set the default renderer for HTML templates to mako.
     config.add_mako_renderer('.html')

rhodecode/config/patches.py

@@ -158,3 +158,10 b' def inspect_getargspec():'
     inspect.getargspec = inspect.getfullargspec
 
     return inspect
+
+
+def repoze_sendmail_lf_fix():
+    from repoze.sendmail import encoding
+    from email.policy import SMTP
+
+    encoding.encode_message = lambda message, *args, **kwargs: message.as_bytes(policy=SMTP)

rhodecode/config/utils.py

@@ -35,7 +35,7 b' def configure_vcs(config):'
         'svn': 'rhodecode.lib.vcs.backends.svn.SubversionRepository',
     }
 
-    conf.settings.HOOKS_PROTOCOL = config['vcs.hooks.protocol']
+    conf.settings.HOOKS_PROTOCOL = config['vcs.hooks.protocol.v2']
     conf.settings.HOOKS_HOST = config['vcs.hooks.host']
     conf.settings.DEFAULT_ENCODINGS = config['default_encoding']
     conf.settings.ALIASES[:] = config['vcs.backends']

rhodecode/lib/archive_cache/__init__.py

@@ -31,9 +31,11 b' cache_meta = None'
 
 
 def includeme(config):
+    return  # don't init cache currently for faster startup time
+
     # init our cache at start
-    settings = config.get_settings()
-    get_archival_cache_store(settings)
+    # settings = config.get_settings()
+    # get_archival_cache_store(settings)
 
 
 def get_archival_config(config):

rhodecode/lib/archive_cache/backends/objectstore_cache.py

@@ -58,7 +58,7 b' class S3Shard(BaseShard):'
         # ensure folder in bucket exists
         destination = self.bucket
         if not self.fs.exists(destination):
-            self.fs.mkdir(destination, s3_additional_kwargs={})
+            self.fs.mkdir(destination)
 
         writer = self._get_writer(full_path, mode)
 

rhodecode/lib/celerylib/loader.py

@@ -27,10 +27,11 b' Celery loader, run with::'
         --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler \
         --loglevel DEBUG --ini=.dev/dev.ini
 """
-from rhodecode.config.patches import inspect_getargspec, inspect_formatargspec
-inspect_getargspec()
-inspect_formatargspec()
+from rhodecode.config import patches
+patches.inspect_getargspec()
+patches.inspect_formatargspec()
 # python3.11 inspect patches for backward compat on `paste` code
+patches.repoze_sendmail_lf_fix()
 
 import sys
 import logging

rhodecode/lib/celerylib/tasks.py

@@ -1,4 +1,4 b''
-# Copyright (C) 2012-2023 RhodeCode GmbH
+# Copyright (C) 2012-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
@@ -64,8 +64,9 b' def send_email(recipients, subject, body'
                   "Make sure that `smtp_server` variable is configured "
                   "inside the .ini file")
         return False
-
-    subject = "%s %s" % (email_config.get('email_prefix', ''), subject)
+    conf_prefix = email_config.get('email_prefix', None)
+    prefix = f'{conf_prefix} ' if conf_prefix else ''
+    subject = f"{prefix}{subject}"
 
     if recipients:
         if isinstance(recipients, str):
@@ -86,8 +87,8 b' def send_email(recipients, subject, body'
     email_conf = dict(
         host=mail_server,
         port=email_config.get('smtp_port', 25),
-        username=email_config.get('smtp_username'),
-        password=email_config.get('smtp_password'),
+        username=email_config.get('smtp_username', None),
+        password=email_config.get('smtp_password', None),
 
         tls=str2bool(email_config.get('smtp_use_tls')),
         ssl=str2bool(email_config.get('smtp_use_ssl')),
@@ -207,7 +208,7 b' def create_repo(form_data, cur_user):'
         hooks_base.create_repository(created_by=owner.username, **repo.get_dict())
 
         # update repo commit caches initially
-        repo.update_commit_cache()
+        repo.update_commit_cache(recursive=False)
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
@@ -298,7 +299,7 b' def create_repo_fork(form_data, cur_user'
         # update repo commit caches initially
         config = repo._config
         config.set('extensions', 'largefiles', '')
-        repo.update_commit_cache(config=config)
+        repo.update_commit_cache(config=config, recursive=False)
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
@@ -390,7 +391,7 b' def sync_last_update_for_objects(*args, '
             .order_by(Repository.group_id.asc())
 
         for repo in repos:
-            repo.update_commit_cache()
+            repo.update_commit_cache(recursive=False)
 
     skip_groups = kwargs.get('skip_groups')
     if not skip_groups:

rhodecode/lib/db_manage.py

@@ -570,7 +570,6 b' class DbManage(object):'
         self.create_ui_settings(path)
 
         ui_config = [
-            ('web', 'push_ssl', 'False'),
             ('web', 'allow_archive', 'gz zip bz2'),
             ('web', 'allow_push', '*'),
             ('web', 'baseurl', '/'),

rhodecode/lib/dbmigrate/versions/115_version_5_1_0.py

@@ -35,16 +35,19 b' def downgrade(migrate_engine):'
 
 
 def fixups(models, _SESSION):
+
     for db_repo in _SESSION.query(models.Repository).all():
 
-        config = db_repo._config
-        config.set('extensions', 'largefiles', '')
+        try:
+            config = db_repo._config
+            config.set('extensions', 'largefiles', '')
 
-        try:
-            scm = db_repo.scm_instance(cache=False, config=config)
+            scm = db_repo.scm_instance(cache=False, config=config, vcs_full_cache=False)
             if scm:
                 print(f'installing hook for repo: {db_repo}')
                 scm.install_hooks(force=True)
+            del scm  # force GC
+            del config
         except Exception as e:
             print(e)
             print('continue...')

rhodecode/lib/exceptions.py

@@ -80,6 +80,10 b' class AttachedPullRequestsError(Exceptio'
     pass
 
 
+class AttachedArtifactsError(Exception):
+    pass
+
+
 class RepoGroupAssignmentError(Exception):
     pass
 
@@ -98,6 +102,11 b' class HTTPRequirementError(HTTPClientErr'
         self.args = (message, )
 
 
+class ClientNotSupportedError(HTTPRequirementError):
+    title = explanation = 'Client Not Supported'
+    reason = None
+
+
 class HTTPLockedRC(HTTPClientError):
     """
     Special Exception For locked Repos in RhodeCode, the return code can

rhodecode/lib/helpers.py

@@ -81,7 +81,7 b' from rhodecode.lib.action_parser import '
 from rhodecode.lib.html_filters import sanitize_html
 from rhodecode.lib.pagination import Page, RepoPage, SqlPage
 from rhodecode.lib import ext_json
-from rhodecode.lib.ext_json import json
+from rhodecode.lib.ext_json import json, formatted_str_json
 from rhodecode.lib.str_utils import safe_bytes, convert_special_chars, base64_to_str
 from rhodecode.lib.utils import repo_name_slug, get_custom_lexer
 from rhodecode.lib.str_utils import safe_str
@@ -1416,62 +1416,14 b' class InitialsGravatar(object):'
         return "data:image/svg+xml;base64,{}".format(img_data)
 
 
-def initials_gravatar(request, email_address, first_name, last_name, size=30, store_on_disk=False):
+def initials_gravatar(request, email_address, first_name, last_name, size=30):
 
     svg_type = None
     if email_address == User.DEFAULT_USER_EMAIL:
         svg_type = 'default_user'
 
     klass = InitialsGravatar(email_address, first_name, last_name, size)
-
-    if store_on_disk:
-        from rhodecode.apps.file_store import utils as store_utils
-        from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
-            FileOverSizeException
-        from rhodecode.model.db import Session
-
-        image_key = md5_safe(email_address.lower()
-                             + first_name.lower() + last_name.lower())
-
-        storage = store_utils.get_file_storage(request.registry.settings)
-        filename = '{}.svg'.format(image_key)
-        subdir = 'gravatars'
-        # since final name has a counter, we apply the 0
-        uid = storage.apply_counter(0, store_utils.uid_filename(filename, randomized=False))
-        store_uid = os.path.join(subdir, uid)
-
-        db_entry = FileStore.get_by_store_uid(store_uid)
-        if db_entry:
-            return request.route_path('download_file', fid=store_uid)
-
-        img_data = klass.get_img_data(svg_type=svg_type)
-        img_file = store_utils.bytes_to_file_obj(img_data)
-
-        try:
-            store_uid, metadata = storage.save_file(
-                img_file, filename, directory=subdir,
-                extensions=['.svg'], randomized_name=False)
-        except (FileNotAllowedException, FileOverSizeException):
-            raise
-
-        try:
-            entry = FileStore.create(
-                file_uid=store_uid, filename=metadata["filename"],
-                file_hash=metadata["sha256"], file_size=metadata["size"],
-                file_display_name=filename,
-                file_description=f'user gravatar `{safe_str(filename)}`',
-                hidden=True, check_acl=False, user_id=1
-            )
-            Session().add(entry)
-            Session().commit()
-            log.debug('Stored upload in DB as %s', entry)
-        except Exception:
-            raise
-
-        return request.route_path('download_file', fid=store_uid)
-
-    else:
-        return klass.generate_svg(svg_type=svg_type)
+    return klass.generate_svg(svg_type=svg_type)
 
 
 def gravatar_external(request, gravatar_url_tmpl, email_address, size=30):

rhodecode/lib/hook_daemon/hook_module.py

@@ -66,12 +66,12 b' class Hooks(object):'
             result = hook(extras)
             if result is None:
                 raise Exception(f'Failed to obtain hook result from func: {hook}')
-        except HTTPBranchProtected as handled_error:
+        except HTTPBranchProtected as error:
             # Those special cases don't need error reporting. It's a case of
             # locked repo or protected branch
             result = AttributeDict({
-                'status': handled_error.code,
-                'output': handled_error.explanation
+                'status': error.code,
+                'output': error.explanation
             })
         except (HTTPLockedRC, Exception) as error:
             # locked needs different handling since we need to also

rhodecode/lib/hooks_base.py

@@ -30,7 +30,7 b' from rhodecode.lib import helpers as h'
 from rhodecode.lib import audit_logger
 from rhodecode.lib.utils2 import safe_str, user_agent_normalizer
 from rhodecode.lib.exceptions import (
-    HTTPLockedRC, HTTPBranchProtected, UserCreationError)
+    HTTPLockedRC, HTTPBranchProtected, UserCreationError, ClientNotSupportedError)
 from rhodecode.model.db import Repository, User
 from rhodecode.lib.statsd_client import StatsdClient
 
@@ -64,6 +64,18 b' def is_shadow_repo(extras):'
     return extras['is_shadow_repo']
 
 
+def check_vcs_client(extras):
+    """
+    Checks if vcs client is allowed (Only works in enterprise edition)
+    """
+    try:
+        from rc_ee.lib.security.utils import is_vcs_client_whitelisted
+    except ModuleNotFoundError:
+        is_vcs_client_whitelisted = lambda *x: True
+    backend = extras.get('scm')
+    if not is_vcs_client_whitelisted(extras.get('user_agent'), backend):
+        raise ClientNotSupportedError(f"Your {backend} client is forbidden")
+
 def _get_scm_size(alias, root_path):
 
     if not alias.startswith('.'):
@@ -108,6 +120,7 b' def pre_push(extras):'
     It bans pushing when the repository is locked.
     """
 
+    check_vcs_client(extras)
     user = User.get_by_username(extras.username)
     output = ''
     if extras.locked_by[0] and user.user_id != int(extras.locked_by[0]):
@@ -129,6 +142,8 b' def pre_push(extras):'
         if extras.commit_ids and extras.check_branch_perms:
             auth_user = user.AuthUser()
             repo = Repository.get_by_repo_name(extras.repository)
+            if not repo:
+                raise ValueError(f'Repo for {extras.repository} not found')
             affected_branches = []
             if repo.repo_type == 'hg':
                 for entry in extras.commit_ids:
@@ -180,6 +195,7 b' def pre_pull(extras):'
     It bans pulling when the repository is locked.
     """
 
+    check_vcs_client(extras)
     output = ''
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username

rhodecode/lib/middleware/request_wrapper.py

@@ -46,7 +46,7 b' class RequestWrapperTween(object):'
 
     def __call__(self, request):
         start = time.time()
-        log.debug('Starting request time measurement')
+        log.debug('Starting request processing')
         response = None
         request.req_wrapper_start = start
 
@@ -63,7 +63,7 b' class RequestWrapperTween(object):'
 
             total = time.time() - start
             log.info(
-                'Req[%4s] %s %s Request to %s time: %.4fs [%s], RhodeCode %s',
+                'Finished request processing: req[%4s] %s %s Request to %s time: %.4fs [%s], RhodeCode %s',
                 count, _auth_user, request.environ.get('REQUEST_METHOD'),
                 _path, total, get_user_agent(request. environ), _ver_,
                 extra={"time": total, "ver": _ver_, "ip": ip,

rhodecode/lib/middleware/simplehg.py

@@ -53,25 +53,31 b' class SimpleHg(simplevcs.SimpleVCS):'
         return repo_name.rstrip('/')
 
     _ACTION_MAPPING = {
+        'between': 'pull',
+        'branches': 'pull',
+        'branchmap': 'pull',
+        'capabilities': 'pull',
         'changegroup': 'pull',
         'changegroupsubset': 'pull',
+        'changesetdata': 'pull',
+        'clonebundles': 'pull',
+        'clonebundles_manifest': 'pull',
+        'debugwireargs': 'pull',
+        'filedata': 'pull',
         'getbundle': 'pull',
-        'stream_out': 'pull',
-        'listkeys': 'pull',
-        'between': 'pull',
-        'branchmap': 'pull',
-        'branches': 'pull',
-        'clonebundles': 'pull',
-        'capabilities': 'pull',
-        'debugwireargs': 'pull',
         'heads': 'pull',
-        'lookup': 'pull',
         'hello': 'pull',
         'known': 'pull',
+        'listkeys': 'pull',
+        'lookup': 'pull',
+        'manifestdata': 'pull',
+        'narrow_widen': 'pull',
+        'protocaps': 'pull',
+        'stream_out': 'pull',
 
         # largefiles
+        'getlfile': 'pull',
         'putlfile': 'push',
-        'getlfile': 'pull',
         'statlfile': 'pull',
         'lheads': 'pull',
 

rhodecode/lib/middleware/simplevcs.py

@@ -293,7 +293,7 b' class SimpleVCS(object):'
         def compute_perm_vcs(
                 cache_name, plugin_id, action, user_id, repo_name, ip_addr):
 
-            log.debug('auth: calculating permission access now...')
+            log.debug('auth: calculating permission access now for vcs operation: %s', action)
             # check IP
             inherit = user.inherit_default_permissions
             ip_allowed = AuthUser.check_ip_allowed(
@@ -339,21 +339,6 b' class SimpleVCS(object):'
             log.exception('Failed to read http scheme')
             return 'http'
 
-    def _check_ssl(self, environ, start_response):
-        """
-        Checks the SSL check flag and returns False if SSL is not present
-        and required True otherwise
-        """
-        org_proto = environ['wsgi._org_proto']
-        # check if we have SSL required  ! if not it's a bad request !
-        require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
-        if require_ssl and org_proto == 'http':
-            log.debug(
-                'Bad request: detected protocol is `%s` and '
-                'SSL/HTTPS is required.', org_proto)
-            return False
-        return True
-
     def _get_default_cache_ttl(self):
         # take AUTH_CACHE_TTL from the `rhodecode` auth plugin
         plugin = loadplugin('egg:rhodecode-enterprise-ce#rhodecode')
@@ -373,12 +358,6 b' class SimpleVCS(object):'
             meta.Session.remove()
 
     def _handle_request(self, environ, start_response):
-        if not self._check_ssl(environ, start_response):
-            reason = ('SSL required, while RhodeCode was unable '
-                      'to detect this as SSL request')
-            log.debug('User not allowed to proceed, %s', reason)
-            return HTTPNotAcceptable(reason)(environ, start_response)
-
         if not self.url_repo_name:
             log.warning('Repository name is empty: %s', self.url_repo_name)
             # failed to get repo name, we fail now

rhodecode/lib/middleware/vcs.py

@@ -159,11 +159,18 b' def detect_vcs_request(environ, backends'
         # favicon often requested by browsers
         'favicon.ico',
 
+        # static files no detection
+        '_static++',
+
+        # debug-toolbar
+        '_debug_toolbar++',
+
         # e.g /_file_store/download
         '_file_store++',
 
         # login
-        "_admin/login",
+        f"{ADMIN_PREFIX}/login",
+        f"{ADMIN_PREFIX}/logout",
 
         # 2fa
         f"{ADMIN_PREFIX}/check_2fa",
@@ -178,12 +185,6 b' def detect_vcs_request(environ, backends'
         # _admin/my_account is safe too
         f'{ADMIN_PREFIX}/my_account++',
 
-        # static files no detection
-        '_static++',
-
-        # debug-toolbar
-        '_debug_toolbar++',
-
         # skip ops ping, status
         f'{ADMIN_PREFIX}/ops/ping',
         f'{ADMIN_PREFIX}/ops/status',
@@ -193,11 +194,14 b' def detect_vcs_request(environ, backends'
 
         '++/repo_creating_check'
     ]
+
     path_info = get_path_info(environ)
     path_url = path_info.lstrip('/')
     req_method = environ.get('REQUEST_METHOD')
 
     for item in white_list:
+        item = item.lstrip('/')
+
         if item.endswith('++') and path_url.startswith(item[:-2]):
             log.debug('path `%s` in whitelist (match:%s), skipping...', path_url, item)
             return handler

rhodecode/lib/rc_cache/backends.py

@@ -38,9 +38,9 b' from dogpile.cache.backends import redis'
 from dogpile.cache.backends.file import FileLock
 from dogpile.cache.util import memoized_property
 
-from rhodecode.lib.memory_lru_dict import LRUDict, LRUDictDebug
-from rhodecode.lib.str_utils import safe_bytes, safe_str
-from rhodecode.lib.type_utils import str2bool
+from ...lib.memory_lru_dict import LRUDict, LRUDictDebug
+from ...lib.str_utils import safe_bytes, safe_str
+from ...lib.type_utils import str2bool
 
 _default_max_size = 1024
 
@@ -198,6 +198,13 b' class FileNamespaceBackend(PickleSeriali'
     def get_store(self):
         return self.filename
 
+    def cleanup_store(self):
+        for ext in ("db", "dat", "pag", "dir"):
+            final_filename = self.filename + os.extsep + ext
+            if os.path.exists(final_filename):
+                os.remove(final_filename)
+                log.warning('Removed dbm file %s', final_filename)
+
 
 class BaseRedisBackend(redis_backend.RedisBackend):
     key_prefix = ''
@@ -289,7 +296,7 b' class RedisMsgPackBackend(MsgPackSeriali'
 
 
 def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
-    from rhodecode.lib._vendor import redis_lock
+    from ...lib._vendor import redis_lock
 
     class _RedisLockWrapper:
         """LockWrapper for redis_lock"""

rhodecode/lib/rc_cache/utils.py

@@ -26,9 +26,9 b' import decorator'
 from dogpile.cache import CacheRegion
 
 import rhodecode
-from rhodecode.lib.hash_utils import sha1
-from rhodecode.lib.str_utils import safe_bytes
-from rhodecode.lib.type_utils import str2bool # noqa :required by imports from .utils
+from ...lib.hash_utils import sha1
+from ...lib.str_utils import safe_bytes
+from ...lib.type_utils import str2bool # noqa :required by imports from .utils
 
 from . import region_meta
 

rhodecode/lib/rc_commands/add_artifact.py

@@ -91,15 +91,14 b' def command(ini_path, filename, file_pat'
 
         auth_user = db_user.AuthUser(ip_addr='127.0.0.1')
 
-        storage = store_utils.get_file_storage(request.registry.settings)
+        f_store = store_utils.get_filestore_backend(request.registry.settings)
 
         with open(file_path, 'rb') as f:
             click.secho(f'Adding new artifact from path: `{file_path}`',
                         fg='green')
 
             file_data = _store_file(
-                storage, auth_user, filename, content=None, check_acl=True,
+                f_store, auth_user, filename, content=None, check_acl=True,
                 file_obj=f, description=description,
                 scope_repo_id=repo.repo_id)
-            click.secho(f'File Data: {file_data}',
-                        fg='green')
+            click.secho(f'File Data: {file_data}', fg='green')

rhodecode/lib/rc_commands/setup_rc.py

@@ -108,11 +108,10 b' def command(ini_path, force_yes, user, e'
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     if apply_license_key:
-        try:
-            from rc_license.models import apply_trial_license_if_missing
-            apply_trial_license_if_missing(force=True)
-        except ImportError:
-            pass
+        from rhodecode.model.license import apply_license_from_file
+        license_file_path = config.get('license.import_path')
+        if license_file_path:
+            apply_license_from_file(license_file_path, force=True)
 
     Session().commit()
 

rhodecode/lib/str_utils.py

@@ -181,3 +181,7 b' def splitnewlines(text: bytes):'
         else:
             lines[-1] = lines[-1][:-1]
     return lines
+
+
+def header_safe_str(val):
+    return safe_bytes(val).decode('latin-1', errors='replace')

rhodecode/lib/system_info.py

@@ -396,17 +396,18 b' def storage_inodes():'
 
 
 @register_sysinfo
-def storage_archives():
+def storage_artifacts():
     import rhodecode
     from rhodecode.lib.helpers import format_byte_size_binary
     from rhodecode.lib.archive_cache import get_archival_cache_store
 
-    storage_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type')
+    backend_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type')
 
-    value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=storage_type)
+    value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=backend_type)
     state = STATE_OK_DEFAULT
     try:
         d_cache = get_archival_cache_store(config=rhodecode.CONFIG)
+        backend_type = str(d_cache)
 
         total_files, total_size, _directory_stats = d_cache.get_statistics()
 
@@ -415,7 +416,8 b' def storage_archives():'
             'used': total_size,
             'total': total_size,
             'items': total_files,
-            'path': d_cache.storage_path
+            'path': d_cache.storage_path,
+            'type': backend_type
         })
 
     except Exception as e:
@@ -425,8 +427,44 b' def storage_archives():'
     human_value = value.copy()
     human_value['used'] = format_byte_size_binary(value['used'])
     human_value['total'] = format_byte_size_binary(value['total'])
-    human_value['text'] = "{} ({} items)".format(
-        human_value['used'], value['items'])
+    human_value['text'] = f"{human_value['used']} ({value['items']} items)"
+
+    return SysInfoRes(value=value, state=state, human_value=human_value)
+
+
+@register_sysinfo
+def storage_archives():
+    import rhodecode
+    from rhodecode.lib.helpers import format_byte_size_binary
+    import rhodecode.apps.file_store.utils as store_utils
+    from rhodecode import CONFIG
+
+    backend_type = rhodecode.ConfigGet().get_str(store_utils.config_keys.backend_type)
+
+    value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=backend_type)
+    state = STATE_OK_DEFAULT
+    try:
+        f_store = store_utils.get_filestore_backend(config=CONFIG)
+        backend_type = str(f_store)
+        total_files, total_size, _directory_stats = f_store.get_statistics()
+
+        value.update({
+            'percent': 100,
+            'used': total_size,
+            'total': total_size,
+            'items': total_files,
+            'path': f_store.storage_path,
+            'type': backend_type
+        })
+
+    except Exception as e:
+        log.exception('failed to fetch archive cache storage')
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    human_value = value.copy()
+    human_value['used'] = format_byte_size_binary(value['used'])
+    human_value['total'] = format_byte_size_binary(value['total'])
+    human_value['text'] = f"{human_value['used']} ({value['items']} items)"
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
@@ -798,6 +836,7 b' def get_system_info(environ):'
         'storage': SysInfo(storage)(),
         'storage_inodes': SysInfo(storage_inodes)(),
         'storage_archive': SysInfo(storage_archives)(),
+        'storage_artifacts': SysInfo(storage_artifacts)(),
         'storage_gist': SysInfo(storage_gist)(),
         'storage_temp': SysInfo(storage_temp)(),
 

rhodecode/lib/utils.py

@@ -42,6 +42,8 b' from webhelpers2.text import collapse, s'
 
 from mako import exceptions
 
+from rhodecode import ConfigGet
+from rhodecode.lib.exceptions import HTTPBranchProtected, HTTPLockedRC
 from rhodecode.lib.hash_utils import sha256_safe, md5, sha1
 from rhodecode.lib.type_utils import AttributeDict
 from rhodecode.lib.str_utils import safe_bytes, safe_str
@@ -84,8 +86,39 b' def adopt_for_celery(func):'
     @wraps(func)
     def wrapper(extras):
         extras = AttributeDict(extras)
-        # HooksResponse implements to_json method which must be used there.
-        return func(extras).to_json()
+        try:
+            # HooksResponse implements to_json method which must be used there.
+            return func(extras).to_json()
+        except HTTPBranchProtected as error:
+            # Those special cases don't need error reporting. It's a case of
+            # locked repo or protected branch
+            error_args = error.args
+            return {
+                'status': error.code,
+                'output': error.explanation,
+                'exception': type(error).__name__,
+                'exception_args': error_args,
+                'exception_traceback': '',
+            }
+        except HTTPLockedRC as error:
+            # Those special cases don't need error reporting. It's a case of
+            # locked repo or protected branch
+            error_args = error.args
+            return {
+                'status': error.code,
+                'output': error.explanation,
+                'exception': type(error).__name__,
+                'exception_args': error_args,
+                'exception_traceback': '',
+            }
+        except Exception as e:
+            return {
+                'status': 128,
+                'output': '',
+                'exception': type(e).__name__,
+                'exception_args': e.args,
+                'exception_traceback': '',
+            }
     return wrapper
 
 
@@ -361,32 +394,39 b' ui_sections = ['
     'ui', 'web', ]
 
 
-def config_data_from_db(clear_session=True, repo=None):
+def prepare_config_data(clear_session=True, repo=None):
     """
-    Read the configuration data from the database and return configuration
+    Read the configuration data from the database, *.ini files and return configuration
     tuples.
     """
     from rhodecode.model.settings import VcsSettingsModel
 
-    config = []
-
     sa = meta.Session()
     settings_model = VcsSettingsModel(repo=repo, sa=sa)
 
     ui_settings = settings_model.get_ui_settings()
 
     ui_data = []
+    config = [
+        ('web', 'push_ssl', 'false'),
+    ]
     for setting in ui_settings:
+        # Todo: remove this section once transition to *.ini files will be completed
+        if setting.section in ('largefiles', 'vcs_git_lfs'):
+            if setting.key != 'enabled':
+                continue
         if setting.active:
             ui_data.append((setting.section, setting.key, setting.value))
             config.append((
                 safe_str(setting.section), safe_str(setting.key),
                 safe_str(setting.value)))
         if setting.key == 'push_ssl':
-            # force set push_ssl requirement to False, rhodecode
-            # handles that
+            # force set push_ssl requirement to False this is deprecated, and we must force it to False
             config.append((
                 safe_str(setting.section), safe_str(setting.key), False))
+    config_getter = ConfigGet()
+    config.append(('vcs_git_lfs', 'store_location', config_getter.get_str('vcs.git.lfs.storage_location')))
+    config.append(('largefiles', 'usercache', config_getter.get_str('vcs.hg.largefiles.storage_location')))
     log.debug(
         'settings ui from db@repo[%s]: %s',
         repo,
@@ -415,7 +455,7 b' def make_db_config(clear_session=True, r'
     Create a :class:`Config` instance based on the values in the database.
     """
     config = Config()
-    config_data = config_data_from_db(clear_session=clear_session, repo=repo)
+    config_data = prepare_config_data(clear_session=clear_session, repo=repo)
     for section, option, value in config_data:
         config.set(section, option, value)
     return config
@@ -582,7 +622,7 b' def repo2db_mapper(initial_repo_list, re'
                 log.debug('Running update server info')
                 git_repo._update_server_info(force=True)
 
-            db_repo.update_commit_cache()
+            db_repo.update_commit_cache(recursive=False)
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')

rhodecode/model/db.py

@@ -2568,10 +2568,10 b' class Repository(Base, BaseModel):'
         return commit
 
     def flush_commit_cache(self):
-        self.update_commit_cache(cs_cache={'raw_id':'0'})
+        self.update_commit_cache(cs_cache={'raw_id': '0'})
         self.update_commit_cache()
 
-    def update_commit_cache(self, cs_cache=None, config=None):
+    def update_commit_cache(self, cs_cache=None, config=None, recursive=True):
         """
         Update cache of last commit for repository
         cache_keys should be::
@@ -2610,6 +2610,14 b' class Repository(Base, BaseModel):'
         if isinstance(cs_cache, BaseCommit):
             cs_cache = cs_cache.__json__()
 
+        def maybe_update_recursive(instance, _config, _recursive, _cs_cache, _last_change):
+            if _recursive:
+                repo_id = instance.repo_id
+                _cs_cache['source_repo_id'] = repo_id
+                for gr in instance.groups_with_parents:
+                    gr.changeset_cache = _cs_cache
+                    gr.updated_on = _last_change
+
         def is_outdated(new_cs_cache):
             if (new_cs_cache['raw_id'] != self.changeset_cache['raw_id'] or
                 new_cs_cache['revision'] != self.changeset_cache['revision']):
@@ -2636,6 +2644,7 b' class Repository(Base, BaseModel):'
             self.changeset_cache = cs_cache
             self.updated_on = last_change
             Session().add(self)
+            maybe_update_recursive(self, config, recursive, cs_cache, last_change)
             Session().commit()
 
         else:
@@ -2650,6 +2659,7 b' class Repository(Base, BaseModel):'
             self.changeset_cache = cs_cache
             self.updated_on = _date_latest
             Session().add(self)
+            maybe_update_recursive(self, config, recursive, cs_cache, _date_latest)
             Session().commit()
 
         log.debug('updated repo `%s` with new commit cache %s, and last update_date: %s',
@@ -5839,8 +5849,7 b' class FileStore(Base, BaseModel):'
                 .filter(FileStoreMetadata.file_store_meta_key == key) \
                 .scalar()
             if has_key:
-                msg = 'key `{}` already defined under section `{}` for this file.'\
-                    .format(key, section)
+                msg = f'key `{key}` already defined under section `{section}` for this file.'
                 raise ArtifactMetadataDuplicate(msg, err_section=section, err_key=key)
 
             # NOTE(marcink): raises ArtifactMetadataBadValueType
@@ -5939,7 +5948,7 b' class FileStoreMetadata(Base, BaseModel)'
     def valid_value_type(cls, value):
         if value.split('.')[0] not in cls.SETTINGS_TYPES:
             raise ArtifactMetadataBadValueType(
-                'value_type must be one of %s got %s' % (cls.SETTINGS_TYPES.keys(), value))
+                f'value_type must be one of {cls.SETTINGS_TYPES.keys()} got {value}')
 
     @hybrid_property
     def file_store_meta_section(self):

rhodecode/model/forms.py

@@ -129,6 +129,20 b' def TOTPForm(localizer, user, allow_reco'
     return _TOTPForm
 
 
+def WhitelistedVcsClientsForm(localizer):
+    _ = localizer
+
+    class _WhitelistedVcsClientsForm(formencode.Schema):
+        regexp = r'^(?:\s*[<>=~^!]*\s*\d{1,2}\.\d{1,2}(?:\.\d{1,2})?\s*|\*)\s*(?:,\s*[<>=~^!]*\s*\d{1,2}\.\d{1,2}(?:\.\d{1,2})?\s*|\s*\*\s*)*$'
+        allow_extra_fields = True
+        filter_extra_fields = True
+        git = v.Regex(regexp)
+        hg = v.Regex(regexp)
+        svn = v.Regex(regexp)
+
+    return _WhitelistedVcsClientsForm
+
+
 def UserForm(localizer, edit=False, available_languages=None, old_data=None):
     old_data = old_data or {}
     available_languages = available_languages or []
@@ -454,13 +468,6 b' def ApplicationUiSettingsForm(localizer)'
     _ = localizer
 
     class _ApplicationUiSettingsForm(_BaseVcsSettingsForm):
-        web_push_ssl = v.StringBoolean(if_missing=False)
-        largefiles_usercache = All(
-            v.ValidPath(localizer),
-            v.UnicodeString(strip=True, min=2, not_empty=True))
-        vcs_git_lfs_store_location = All(
-            v.ValidPath(localizer),
-            v.UnicodeString(strip=True, min=2, not_empty=True))
         extensions_hggit = v.StringBoolean(if_missing=False)
         new_svn_branch = v.ValidSvnPattern(localizer, section='vcs_svn_branch')
         new_svn_tag = v.ValidSvnPattern(localizer, section='vcs_svn_tag')

rhodecode/model/notification.py

@@ -117,14 +117,16 b' class NotificationModel(BaseModel):'
         # add mentioned users into recipients
         final_recipients = set(recipients_objs).union(mention_recipients)
 
-        (subject, email_body, email_body_plaintext) = \
-            EmailNotificationModel().render_email(notification_type, **email_kwargs)
+        # No need to render email if we are sending just notification
+        if with_email:
+            (subject, email_body, email_body_plaintext) = \
+                EmailNotificationModel().render_email(notification_type, **email_kwargs)
 
-        if not notification_subject:
-            notification_subject = subject
+            if not notification_subject:
+                notification_subject = subject
 
-        if not notification_body:
-            notification_body = email_body_plaintext
+            if not notification_body:
+                notification_body = email_body_plaintext
 
         notification = Notification.create(
             created_by=created_by_obj, subject=notification_subject,

rhodecode/model/repo.py

@@ -31,7 +31,7 b' from zope.cachedescriptors.property impo'
 from rhodecode import events
 from rhodecode.lib.auth import HasUserGroupPermissionAny
 from rhodecode.lib.caching_query import FromCache
-from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError
+from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError
 from rhodecode.lib import hooks_base
 from rhodecode.lib.user_log_filter import user_log_filter
 from rhodecode.lib.utils import make_db_config
@@ -736,7 +736,7 b' class RepoModel(BaseModel):'
                 log.error(traceback.format_exc())
                 raise
 
-    def delete(self, repo, forks=None, pull_requests=None, fs_remove=True, cur_user=None):
+    def delete(self, repo, forks=None, pull_requests=None, artifacts=None, fs_remove=True, cur_user=None):
         """
         Delete given repository, forks parameter defines what do do with
         attached forks. Throws AttachedForksError if deleted repo has attached
@@ -745,6 +745,7 b' class RepoModel(BaseModel):'
         :param repo:
         :param forks: str 'delete' or 'detach'
         :param pull_requests: str 'delete' or None
+        :param artifacts: str 'delete' or None
         :param fs_remove: remove(archive) repo from filesystem
         """
         if not cur_user:
@@ -767,6 +768,13 b' class RepoModel(BaseModel):'
             if pull_requests != 'delete' and (pr_sources or pr_targets):
                 raise AttachedPullRequestsError()
 
+            artifacts_objs = repo.artifacts
+            if artifacts == 'delete':
+                for a in artifacts_objs:
+                    self.sa.delete(a)
+            elif [a for a in artifacts_objs]:
+                raise AttachedArtifactsError()
+
             old_repo_dict = repo.get_dict()
             events.trigger(events.RepoPreDeleteEvent(repo))
             try:

rhodecode/model/settings.py

@@ -486,7 +486,6 b' class VcsSettingsModel(object):'
     )
     GLOBAL_HG_SETTINGS = (
         ('extensions', 'largefiles'),
-        ('largefiles', 'usercache'),
         ('phases', 'publish'),
         ('extensions', 'evolve'),
         ('extensions', 'topic'),
@@ -496,12 +495,10 b' class VcsSettingsModel(object):'
 
     GLOBAL_GIT_SETTINGS = (
         ('vcs_git_lfs', 'enabled'),
-        ('vcs_git_lfs', 'store_location')
     )
 
     SVN_BRANCH_SECTION = 'vcs_svn_branch'
     SVN_TAG_SECTION = 'vcs_svn_tag'
-    SSL_SETTING = ('web', 'push_ssl')
     PATH_SETTING = ('paths', '/')
 
     def __init__(self, sa=None, repo=None):
@@ -666,18 +663,16 b' class VcsSettingsModel(object):'
             self.repo_settings, *phases, value=safe_str(data[phases_key]))
 
     def create_or_update_global_hg_settings(self, data):
-        opts_len = 4
-        largefiles, largefiles_store, phases, evolve \
+        opts_len = 3
+        largefiles, phases, evolve \
             = self.GLOBAL_HG_SETTINGS[:opts_len]
-        largefiles_key, largefiles_store_key, phases_key, evolve_key \
+        largefiles_key, phases_key, evolve_key \
             = self._get_settings_keys(self.GLOBAL_HG_SETTINGS[:opts_len], data)
 
         self._create_or_update_ui(
             self.global_settings, *largefiles, value='',
             active=data[largefiles_key])
         self._create_or_update_ui(
-            self.global_settings, *largefiles_store, value=data[largefiles_store_key])
-        self._create_or_update_ui(
             self.global_settings, *phases, value=safe_str(data[phases_key]))
         self._create_or_update_ui(
             self.global_settings, *evolve, value='',
@@ -697,26 +692,17 b' class VcsSettingsModel(object):'
             active=data[lfs_enabled_key])
 
     def create_or_update_global_git_settings(self, data):
-        lfs_enabled, lfs_store_location \
-            = self.GLOBAL_GIT_SETTINGS
-        lfs_enabled_key, lfs_store_location_key \
-            = self._get_settings_keys(self.GLOBAL_GIT_SETTINGS, data)
+        lfs_enabled = self.GLOBAL_GIT_SETTINGS[0]
+        lfs_enabled_key = self._get_settings_keys(self.GLOBAL_GIT_SETTINGS, data)[0]
 
         self._create_or_update_ui(
             self.global_settings, *lfs_enabled, value=data[lfs_enabled_key],
             active=data[lfs_enabled_key])
-        self._create_or_update_ui(
-            self.global_settings, *lfs_store_location,
-            value=data[lfs_store_location_key])
 
     def create_or_update_global_svn_settings(self, data):
         # branch/tags patterns
         self._create_svn_settings(self.global_settings, data)
 
-    def update_global_ssl_setting(self, value):
-        self._create_or_update_ui(
-            self.global_settings, *self.SSL_SETTING, value=value)
-
     @assert_repo_settings
     def delete_repo_svn_pattern(self, id_):
         ui = self.repo_settings.UiDbModel.get(id_)

rhodecode/model/user.py

@@ -557,10 +557,10 b' class UserModel(BaseModel):'
         elif handle_mode == 'delete':
             from rhodecode.apps.file_store import utils as store_utils
             request = get_current_request()
-            storage = store_utils.get_file_storage(request.registry.settings)
+            f_store = store_utils.get_filestore_backend(request.registry.settings)
             for a in artifacts:
                 file_uid = a.file_uid
-                storage.delete(file_uid)
+                f_store.delete(file_uid)
                 self.sa.delete(a)
 
             left_overs = False

rhodecode/public/js/rhodecode/routes.js

@@ -86,6 +86,7 b' function registerRCRoutes() {'
     pyroutes.register('admin_settings_vcs_update', '/_admin/settings/vcs/update', []);
     pyroutes.register('admin_settings_visual', '/_admin/settings/visual', []);
     pyroutes.register('admin_settings_visual_update', '/_admin/settings/visual/update', []);
+    pyroutes.register('admin_security_modify_allowed_vcs_client_versions', '/_admin/security/modify/allowed_vcs_client_versions', []);
     pyroutes.register('apiv2', '/_admin/api', []);
     pyroutes.register('atom_feed_home', '/%(repo_name)s/feed-atom', ['repo_name']);
     pyroutes.register('atom_feed_home_old', '/%(repo_name)s/feed/atom', ['repo_name']);

rhodecode/subscribers.py

@@ -111,9 +111,11 b' def scan_repositories_if_enabled(event):'
     This is subscribed to the `pyramid.events.ApplicationCreated` event. It
     does a repository scan if enabled in the settings.
     """
+
     settings = event.app.registry.settings
     vcs_server_enabled = settings['vcs.server.enable']
     import_on_startup = settings['startup.import_repos']
+
     if vcs_server_enabled and import_on_startup:
         from rhodecode.model.scm import ScmModel
         from rhodecode.lib.utils import repo2db_mapper
@@ -205,7 +207,7 b' def write_usage_data(event):'
         return
 
     def get_update_age(dest_file):
-        now = datetime.datetime.utcnow()
+        now = datetime.datetime.now(datetime.UTC)
 
         with open(dest_file, 'rb') as f:
             data = ext_json.json.loads(f.read())
@@ -216,10 +218,9 b' def write_usage_data(event):'
 
         return 0
 
-    utc_date = datetime.datetime.utcnow()
+    utc_date = datetime.datetime.now(datetime.UTC)
     hour_quarter = int(math.ceil((utc_date.hour + utc_date.minute/60.0) / 6.))
-    fname = '.rc_usage_{date.year}{date.month:02d}{date.day:02d}_{hour}.json'.format(
-        date=utc_date, hour=hour_quarter)
+    fname = f'.rc_usage_{utc_date.year}{utc_date.month:02d}{utc_date.day:02d}_{hour_quarter}.json'
     ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
 
     usage_dir = os.path.join(ini_loc, '.rcusage')
@@ -314,6 +315,28 b' def write_js_routes_if_enabled(event):'
             log.exception('Failed to write routes.js into %s', jsroutes_file_path)
 
 
+def import_license_if_present(event):
+    """
+    This is subscribed to the `pyramid.events.ApplicationCreated` event. It
+    does a import license key based on a presence of the file.
+    """
+    settings = event.app.registry.settings
+
+    rhodecode_edition_id = settings.get('rhodecode.edition_id')
+    license_file_path = settings.get('license.import_path')
+    force = settings.get('license.import_path_mode') == 'force'
+
+    if license_file_path and rhodecode_edition_id == 'EE':
+        log.debug('license.import_path= is set importing license from %s', license_file_path)
+        from rhodecode.model.meta import Session
+        from rhodecode.model.license import apply_license_from_file
+        try:
+            apply_license_from_file(license_file_path, force=force)
+            Session().commit()
+        except OSError:
+            log.exception('Failed to import license from %s, make sure this file exists', license_file_path)
+
+
 class Subscriber(object):
     """
     Base class for subscribers to the pyramid event system.

rhodecode/templates/admin/auth/auth_settings.mako

@@ -26,8 +26,13 b''
     <div class="sidebar">
         <ul class="nav nav-pills nav-stacked">
           % for item in resource.get_root().get_nav_list():
+
             <li ${('class=active' if item == resource else '')}>
-              <a href="${request.resource_path(item, route_name='auth_home')}">${item.display_name}</a>
+              % if getattr(item, 'is_root', False):
+                <a href="${request.resource_path(item, route_name='auth_home')}">${item.display_name}</a>
+              % else:
+                <a style="padding-left: 10px" href="${request.resource_path(item, route_name='auth_home')}">${item.display_name}</a>
+              % endif
             </li>
           % endfor
         </ul>

rhodecode/templates/admin/auth/plugin_settings.mako

@@ -50,6 +50,13 b''
                 ${h.secure_form(request.resource_path(resource, route_name='auth_home'), request=request)}
                 <div class="form">
 
+                  ## Allow derived templates to add something above the form
+                  ## input fields
+                  %if hasattr(next, 'above_form_fields'):
+                    ${next.above_form_fields()}
+                  %endif
+
+                  <h4>${_('Plugin Configuration')}</h4>
                   %for node in plugin.get_settings_schema():
                     <%
                       label_to_type = {'label-checkbox': 'bool', 'label-textarea': 'textarea'}

rhodecode/templates/admin/repos/repo_edit_advanced.mako

@@ -215,18 +215,35 b''
                     %endif
                 </td>
             </tr>
+
             <% attached_prs = len(c.rhodecode_db_repo.pull_requests_source + c.rhodecode_db_repo.pull_requests_target) %>
             % if c.rhodecode_db_repo.pull_requests_source or c.rhodecode_db_repo.pull_requests_target:
             <tr>
                 <td>
                     ${_ungettext('This repository has %s attached pull request.', 'This repository has %s attached pull requests.', attached_prs) % attached_prs}
                     <br/>
-                    ${_('Consider to archive this repository instead.')}
+                    <br/>
+                    <strong>${_('Consider to archive this repository instead.')}</strong>
                 </td>
                 <td></td>
                 <td></td>
             </tr>
             % endif
+
+            <% attached_artifacts = len(c.rhodecode_db_repo.artifacts) %>
+            % if attached_artifacts:
+            <tr>
+                <td>
+                    ${_ungettext('This repository has %s attached artifact.', 'This repository has %s attached artifacts.', attached_artifacts) % attached_artifacts}
+                    <br/>
+                    <br/>
+                    <strong>${_('Consider to archive this repository instead.')}</strong>
+                </td>
+                <td></td>
+                <td></td>
+            </tr>
+            % endif
+
         </table>
         <div style="margin: 0 0 20px 0" class="fake-space"></div>
 

rhodecode/templates/base/base.mako

@@ -114,6 +114,7 b''
           <li class="${h.is_active('repository_groups', active)}"><a href="${h.route_path('repo_groups')}">${_('Repository groups')}</a></li>
           <li class="${h.is_active('users', active)}"><a href="${h.route_path('users')}">${_('Users')}</a></li>
           <li class="${h.is_active('user_groups', active)}"><a href="${h.route_path('user_groups')}">${_('User groups')}</a></li>
+          <li class="${h.is_active('security', active)}"><a href="${h.route_path('admin_security')}">${_('Security')}</a></li>
           <li class="${h.is_active('artifacts', active)}"><a href="${h.route_path('admin_artifacts')}">${_('Artifacts')}</a></li>
           <li class="${h.is_active('automation', active)}"><a href="${h.route_path('admin_automation')}">${_('Automation')}</a></li>
           <li class="${h.is_active('scheduler', active)}"><a href="${h.route_path('admin_scheduler')}">${_('Scheduler')}</a></li>

rhodecode/templates/base/vcs_settings.mako

@@ -5,22 +5,7 b''
 
 <%def name="vcs_settings_fields(suffix='', svn_branch_patterns=None, svn_tag_patterns=None, repo_type=None, display_globals=False, **kwargs)">
     % if display_globals:
-        <div class="panel panel-default">
-            <div class="panel-heading" id="general">
-                <h3 class="panel-title">${_('General')}<a class="permalink" href="#general"> ¶</a></h3>
-            </div>
-            <div class="panel-body">
-                <div class="field">
-                    <div class="checkbox">
-                        ${h.checkbox('web_push_ssl' + suffix, 'True')}
-                        <label for="web_push_ssl${suffix}">${_('Require SSL for vcs operations')}</label>
-                    </div>
-                    <div class="label">
-                        <span class="help-block">${_('Activate to set RhodeCode to require SSL for pushing or pulling. If SSL certificate is missing it will return a HTTP Error 406: Not Acceptable.')}</span>
-                    </div>
-                 </div>
-            </div>
-        </div>
+
     % endif
 
     % if display_globals or repo_type in ['git', 'hg']:
@@ -75,17 +60,6 b''
                     % endif
                 </div>
 
-                % if display_globals:
-                <div class="field">
-                    <div class="input">
-                        ${h.text('largefiles_usercache' + suffix, size=59)}
-                    </div>
-                </div>
-                <div class="label">
-                    <span class="help-block">${_('Filesystem location where Mercurial largefile objects should be stored.')}</span>
-                </div>
-                % endif
-
                 <div class="checkbox">
                     ${h.checkbox('phases_publish' + suffix, 'True', **kwargs)}
                     <label for="phases_publish${suffix}">${_('Set repositories as publishing') if display_globals else _('Set repository as publishing')}</label>
@@ -127,17 +101,6 b''
                         <span class="help-block">${_('Enable lfs extensions for this repository.')}</span>
                     % endif
                 </div>
-
-                % if display_globals:
-                <div class="field">
-                    <div class="input">
-                        ${h.text('vcs_git_lfs_store_location' + suffix, size=59)}
-                    </div>
-                </div>
-                <div class="label">
-                    <span class="help-block">${_('Filesystem location where Git lfs objects should be stored.')}</span>
-                </div>
-                % endif
             </div>
         </div>
     % endif

rhodecode/tests/config/test_sanitize_settings.py

@@ -117,7 +117,7 b' class TestSanitizeVcsSettings(object):'
 
     _string_funcs = [
         ('vcs.svn.compatible_version', ''),
-        ('vcs.hooks.protocol', 'http'),
+        ('vcs.hooks.protocol.v2', 'celery'),
         ('vcs.hooks.host', '*'),
         ('vcs.scm_app_implementation', 'http'),
         ('vcs.server', ''),

rhodecode/tests/fixture.py

@@ -305,7 +305,7 b' class Fixture(object):'
         return r
 
     def destroy_repo(self, repo_name, **kwargs):
-        RepoModel().delete(repo_name, pull_requests='delete', **kwargs)
+        RepoModel().delete(repo_name, pull_requests='delete', artifacts='delete', **kwargs)
         Session().commit()
 
     def destroy_repo_on_filesystem(self, repo_name):

rhodecode/tests/fixture_mods/fixture_pyramid.py

@@ -110,7 +110,7 b' def ini_config(request, tmpdir_factory, '
             'vcs.server.protocol': 'http',
             'vcs.scm_app_implementation': 'http',
             'vcs.svn.proxy.enabled': 'true',
-            'vcs.hooks.protocol': 'http',
+            'vcs.hooks.protocol.v2': 'celery',
             'vcs.hooks.host': '*',
             'repo_store.path': TESTS_TMP_PATH,
             'app.service_api.token': 'service_secret_token',

rhodecode/tests/lib/middleware/test_simplehg.py

@@ -120,7 +120,6 b' def test_get_config(user_util, baseapp, '
 
     expected_config = [
         ('vcs_svn_tag', 'ff89f8c714d135d865f44b90e5413b88de19a55f', '/tags/*'),
-        ('web', 'push_ssl', 'False'),
         ('web', 'allow_push', '*'),
         ('web', 'allow_archive', 'gz zip bz2'),
         ('web', 'baseurl', '/'),

rhodecode/tests/lib/middleware/test_simplevcs.py

@@ -239,7 +239,6 b' class TestShadowRepoExposure(object):'
         """
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
-        controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'pull'
         controller._is_shadow_repo_dir = True
@@ -267,7 +266,6 b' class TestShadowRepoExposure(object):'
         """
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
-        controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'pull'
         controller._is_shadow_repo_dir = False
@@ -291,7 +289,6 b' class TestShadowRepoExposure(object):'
         """
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
-        controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'push'
         controller.stub_response_body = (b'dummy body value',)
@@ -399,7 +396,7 b' class TestGenerateVcsResponse(object):'
     def call_controller_with_response_body(self, response_body):
         settings = {
             'base_path': 'fake_base_path',
-            'vcs.hooks.protocol': 'http',
+            'vcs.hooks.protocol.v2': 'celery',
             'vcs.hooks.direct_calls': False,
         }
         registry = AttributeDict()

rhodecode/tests/lib/test_utils.py

@@ -371,7 +371,7 b' class TestMakeDbConfig(object):'
             ('section2', 'option2', 'value2'),
             ('section3', 'option3', 'value3'),
         ]
-        with mock.patch.object(utils, 'config_data_from_db') as config_mock:
+        with mock.patch.object(utils, 'prepare_config_data') as config_mock:
             config_mock.return_value = test_data
             kwargs = {'clear_session': False, 'repo': 'test_repo'}
             result = utils.make_db_config(**kwargs)
@@ -381,8 +381,8 b' class TestMakeDbConfig(object):'
             assert value == expected_value
 
 
-class TestConfigDataFromDb(object):
-    def test_config_data_from_db_returns_active_settings(self):
+class TestPrepareConfigData(object):
+    def test_prepare_config_data_returns_active_settings(self):
         test_data = [
             UiSetting('section1', 'option1', 'value1', True),
             UiSetting('section2', 'option2', 'value2', True),
@@ -398,7 +398,7 b' class TestConfigDataFromDb(object):'
             instance_mock = mock.Mock()
             model_mock.return_value = instance_mock
             instance_mock.get_ui_settings.return_value = test_data
-            result = utils.config_data_from_db(
+            result = utils.prepare_config_data(
                 clear_session=False, repo=repo_name)
 
         self._assert_repo_name_passed(model_mock, repo_name)
@@ -407,7 +407,8 b' class TestConfigDataFromDb(object):'
             ('section1', 'option1', 'value1'),
             ('section2', 'option2', 'value2'),
         ]
-        assert result == expected_result
+        # We have extra config items returned, so we're ignoring two last items
+        assert result[:2] == expected_result
 
     def _assert_repo_name_passed(self, model_mock, repo_name):
         assert model_mock.call_count == 1

rhodecode/tests/models/settings/test_vcs_settings.py

@@ -578,21 +578,9 b' class TestCreateOrUpdateRepoHgSettings(o'
         assert str(exc_info.value) == 'Repository is not specified'
 
 
-class TestUpdateGlobalSslSetting(object):
-    def test_updates_global_hg_settings(self):
-        model = VcsSettingsModel()
-        with mock.patch.object(model, '_create_or_update_ui') as create_mock:
-            model.update_global_ssl_setting('False')
-            Session().commit()
-
-        create_mock.assert_called_once_with(
-            model.global_settings, 'web', 'push_ssl', value='False')
-
-
 class TestCreateOrUpdateGlobalHgSettings(object):
     FORM_DATA = {
         'extensions_largefiles': False,
-        'largefiles_usercache': '/example/largefiles-store',
         'phases_publish': False,
         'extensions_evolve': False
     }
@@ -605,7 +593,6 b' class TestCreateOrUpdateGlobalHgSettings'
 
         expected_calls = [
             mock.call(model.global_settings, 'extensions', 'largefiles', active=False, value=''),
-            mock.call(model.global_settings, 'largefiles', 'usercache', value='/example/largefiles-store'),
             mock.call(model.global_settings, 'phases', 'publish', value='False'),
             mock.call(model.global_settings, 'extensions', 'evolve', active=False, value=''),
             mock.call(model.global_settings, 'experimental', 'evolution', active=False, value=''),
@@ -632,7 +619,6 b' class TestCreateOrUpdateGlobalHgSettings'
 class TestCreateOrUpdateGlobalGitSettings(object):
     FORM_DATA = {
         'vcs_git_lfs_enabled': False,
-        'vcs_git_lfs_store_location': '/example/lfs-store',
     }
 
     def test_creates_repo_hg_settings_when_data_is_correct(self):
@@ -643,7 +629,6 b' class TestCreateOrUpdateGlobalGitSetting'
 
         expected_calls = [
             mock.call(model.global_settings, 'vcs_git_lfs', 'enabled', active=False, value=False),
-            mock.call(model.global_settings, 'vcs_git_lfs', 'store_location', value='/example/lfs-store'),
         ]
         assert expected_calls == create_mock.call_args_list
 
@@ -1001,9 +986,7 b' class TestCreateOrUpdateRepoSettings(obj'
         'hooks_outgoing_pull_logger': False,
         'extensions_largefiles': False,
         'extensions_evolve': False,
-        'largefiles_usercache': '/example/largefiles-store',
         'vcs_git_lfs_enabled': False,
-        'vcs_git_lfs_store_location': '/',
         'phases_publish': 'False',
         'rhodecode_pr_merge_enabled': False,
         'rhodecode_use_outdated_comments': False,

rhodecode/tests/models/test_pullrequest.py

@@ -449,7 +449,7 b' class TestPullRequestModel(object):'
 @pytest.mark.usefixtures('config_stub')
 class TestIntegrationMerge(object):
     @pytest.mark.parametrize('extra_config', (
-        {'vcs.hooks.protocol': 'http', 'vcs.hooks.direct_calls': False},
+        {'vcs.hooks.protocol.v2': 'celery', 'vcs.hooks.direct_calls': False},
     ))
     def test_merge_triggers_push_hooks(
             self, pr_util, user_admin, capture_rcextensions, merge_extras,

rhodecode/tests/rhodecode.ini

@@ -36,7 +36,7 b' port = 10020'
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
-; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
+; run with gunicorn  --config gunicorn_conf.py --paste rhodecode.ini
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
@@ -249,15 +249,56 b' labs_settings_active = true'
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
-; File store configuration. This is used to store and serve uploaded files
-file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store backend is always enabled
+#file_store.enabled = true
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead
 ; Storage backend, available options are: local
-file_store.backend = local
+#file_store.backend = local
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead
 ; path to store the uploaded binaries and artifacts
-file_store.storage_path = /var/opt/rhodecode_data/file_store
+#file_store.storage_path = /var/opt/rhodecode_data/file_store
+
+; Artifacts file-store, is used to store comment attachments and artifacts uploads.
+; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options
+; filesystem_v1 is backwards compat with pre 5.1 storage changes
+; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from
+; previous installations to keep the artifacts without a need of migration
+file_store.backend.type = filesystem_v1
+
+; filesystem options...
+file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/test_artifacts_file_store
+
+; filesystem_v2 options...
+file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/test_artifacts_file_store_2
+file_store.filesystem_v2.shards = 8
 
+; objectstore options...
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+#file_store.backend.type = objectstore
+file_store.objectstore.url = http://s3-minio:9000
+
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number
+file_store.objectstore.bucket = rhodecode-file-store-tests
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+file_store.objectstore.bucket_shards = 8
+
+; key for s3 auth
+file_store.objectstore.key = s3admin
+
+; secret for s3 auth
+file_store.objectstore.secret = s3secret4
+
+;region for s3 storage
+file_store.objectstore.region = eu-central-1
 
 ; Redis url to acquire/check generation of archives locks
 archive_cache.locking.url = redis://redis:6379/1
@@ -593,6 +634,7 b' vcs.scm_app_implementation = http'
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
 ; `celery` - use celery based hooks
+#DEPRECATED:vcs.hooks.protocol = http
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
@@ -626,6 +668,10 b' vcs.methods.cache = false'
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
+; Redis connection settings for svn integrations logic
+; This connection string needs to be the same on ce and vcsserver
+vcs.svn.redis_conn = redis://redis:6379/0
+
 ; Enable SVN proxy of requests over HTTP
 vcs.svn.proxy.enabled = true
 
@@ -681,7 +727,8 b' ssh.authorized_keys_file_path = %(here)s'
 ; RhodeCode installation directory.
 ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
-ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+#DEPRECATED: ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+ssh.wrapper_cmd.v2 = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false

setup.py

@@ -189,6 +189,7 b' setup('
             'rc-upgrade-db=rhodecode.lib.rc_commands.upgrade_db:main',
             'rc-ishell=rhodecode.lib.rc_commands.ishell:main',
             'rc-add-artifact=rhodecode.lib.rc_commands.add_artifact:main',
+            'rc-migrate-artifact=rhodecode.lib.rc_commands.migrate_artifact:main',
             'rc-ssh-wrapper=rhodecode.apps.ssh_support.lib.ssh_wrapper_v1:main',
             'rc-ssh-wrapper-v2=rhodecode.apps.ssh_support.lib.ssh_wrapper_v2:main',
         ],

configs/init.d/.readme.txt

@@ -1,1 +0,0 b''
-Example init scripts.
  No newline at end of file

configs/init.d/supervisord.conf

@@ -1,61 +0,0 b''
-; Sample supervisor RhodeCode config file.
-;
-; For more information on the config file, please see:
-; http://supervisord.org/configuration.html
-;
-; Note: shell expansion ("~" or "$HOME") is not supported.  Environment
-; variables can be expanded using this syntax: "%(ENV_HOME)s".
-
-[unix_http_server]
-file=/tmp/supervisor.sock   ; (the path to the socket file)
-;chmod=0700                 ; socket file mode (default 0700)
-;chown=nobody:nogroup       ; socket file uid:gid owner
-;username=user              ; (default is no username (open server))
-;password=123               ; (default is no password (open server))
-
-[supervisord]
-logfile=/home/ubuntu/rhodecode/supervisord.log ; (main log file;default $CWD/supervisord.log)
-logfile_maxbytes=50MB        ; (max main logfile bytes b4 rotation;default 50MB)
-logfile_backups=10           ; (num of main logfile rotation backups;default 10)
-loglevel=info                ; (log level;default info; others: debug,warn,trace)
-pidfile=/home/ubuntu/rhodecode/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
-nodaemon=true               ; (start in foreground if true;default false)
-minfds=1024                  ; (min. avail startup file descriptors;default 1024)
-minprocs=200                 ; (min. avail process descriptors;default 200)
-;umask=022                   ; (process file creation umask;default 022)
-user=ubuntu                 ; (default is current user, required if root)
-;identifier=supervisor       ; (supervisord identifier, default is 'supervisor')
-;directory=/tmp              ; (default is not to cd during start)
-;nocleanup=true              ; (don't clean up tempfiles at start;default false)
-;childlogdir=/tmp            ; ('AUTO' child log dir, default $TEMP)
-environment=HOME=/home/ubuntu,LANG=en_US.UTF-8     ; (key value pairs to add to environment)
-;strip_ansi=false            ; (strip ansi escape codes in logs; def. false)
-
-; the below section must remain in the config file for RPC
-; (supervisorctl/web interface) to work, additional interfaces may be
-; added by defining them in separate rpcinterface: sections
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[supervisorctl]
-serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL  for a unix socket
-;username=chris              ; should be same as http_username if set
-;password=123                ; should be same as http_password if set
-
-
-; restart with supervisorctl restart rhodecode:*
-[program:rhodecode]
-numprocs = 1
-numprocs_start = 5000
-directory=/home/ubuntu/rhodecode/source
-command = /home/ubuntu/rhodecode/venv/bin/paster serve /home/ubuntu/rhodecode/source/prod.ini
-process_name = %(program_name)s_%(process_num)04d
-redirect_stderr=true
-stdout_logfile=/home/ubuntu/rhodecode/rhodecode.log
-
-[program:rhodecode_workers]
-numproces = 1
-directory = /home/ubuntu/rhodecode/source
-command = /home/ubuntu/rhodecode/venv/bin/paster celeryd /home/ubuntu/rhodecode/source/prod.ini --autoscale=10,2
-redirect_stderr=true
-stdout_logfile=/%(here)s/rhodecode_workers.log

rhodecode/apps/file_store/backends/local_store.py

@@ -1,268 +0,0 b''
-# Copyright (C) 2016-2023 RhodeCode GmbH
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License, version 3
-# (only), as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This program is dual-licensed. If you wish to learn more about the
-# RhodeCode Enterprise Edition, including its added features, Support services,
-# and proprietary license terms, please see https://rhodecode.com/licenses/
-
-import os
-import time
-import errno
-import hashlib
-
-from rhodecode.lib.ext_json import json
-from rhodecode.apps.file_store import utils
-from rhodecode.apps.file_store.extensions import resolve_extensions
-from rhodecode.apps.file_store.exceptions import (
-    FileNotAllowedException, FileOverSizeException)
-
-METADATA_VER = 'v1'
-
-
-def safe_make_dirs(dir_path):
-    if not os.path.exists(dir_path):
-        try:
-            os.makedirs(dir_path)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
-            return
-
-
-class LocalFileStorage(object):
-
-    @classmethod
-    def apply_counter(cls, counter, filename):
-        name_counted = '%d-%s' % (counter, filename)
-        return name_counted
-
-    @classmethod
-    def resolve_name(cls, name, directory):
-        """
-        Resolves a unique name and the correct path. If a filename
-        for that path already exists then a numeric prefix with values > 0 will be
-        added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix.
-
-        :param name: base name of file
-        :param directory: absolute directory path
-        """
-
-        counter = 0
-        while True:
-            name_counted = cls.apply_counter(counter, name)
-
-            # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
-            sub_store = cls._sub_store_from_filename(name_counted)
-            sub_store_path = os.path.join(directory, sub_store)
-            safe_make_dirs(sub_store_path)
-
-            path = os.path.join(sub_store_path, name_counted)
-            if not os.path.exists(path):
-                return name_counted, path
-            counter += 1
-
-    @classmethod
-    def _sub_store_from_filename(cls, filename):
-        return filename[:2]
-
-    @classmethod
-    def calculate_path_hash(cls, file_path):
-        """
-        Efficient calculation of file_path sha256 sum
-
-        :param file_path:
-        :return: sha256sum
-        """
-        digest = hashlib.sha256()
-        with open(file_path, 'rb') as f:
-            for chunk in iter(lambda: f.read(1024 * 100), b""):
-                digest.update(chunk)
-
-        return digest.hexdigest()
-
-    def __init__(self, base_path, extension_groups=None):
-
-        """
-        Local file storage
-
-        :param base_path: the absolute base path where uploads are stored
-        :param extension_groups: extensions string
-        """
-
-        extension_groups = extension_groups or ['any']
-        self.base_path = base_path
-        self.extensions = resolve_extensions([], groups=extension_groups)
-
-    def __repr__(self):
-        return f'{self.__class__}@{self.base_path}'
-
-    def store_path(self, filename):
-        """
-        Returns absolute file path of the filename, joined to the
-        base_path.
-
-        :param filename: base name of file
-        """
-        prefix_dir = ''
-        if '/' in filename:
-            prefix_dir, filename = filename.split('/')
-            sub_store = self._sub_store_from_filename(filename)
-        else:
-            sub_store = self._sub_store_from_filename(filename)
-        return os.path.join(self.base_path, prefix_dir, sub_store, filename)
-
-    def delete(self, filename):
-        """
-        Deletes the filename. Filename is resolved with the
-        absolute path based on base_path. If file does not exist,
-        returns **False**, otherwise **True**
-
-        :param filename: base name of file
-        """
-        if self.exists(filename):
-            os.remove(self.store_path(filename))
-            return True
-        return False
-
-    def exists(self, filename):
-        """
-        Checks if file exists. Resolves filename's absolute
-        path based on base_path.
-
-        :param filename: file_uid name of file, e.g 0-f62b2b2d-9708-4079-a071-ec3f958448d4.svg
-        """
-        return os.path.exists(self.store_path(filename))
-
-    def filename_allowed(self, filename, extensions=None):
-        """Checks if a filename has an allowed extension
-
-        :param filename: base name of file
-        :param extensions: iterable of extensions (or self.extensions)
-        """
-        _, ext = os.path.splitext(filename)
-        return self.extension_allowed(ext, extensions)
-
-    def extension_allowed(self, ext, extensions=None):
-        """
-        Checks if an extension is permitted. Both e.g. ".jpg" and
-        "jpg" can be passed in. Extension lookup is case-insensitive.
-
-        :param ext: extension to check
-        :param extensions: iterable of extensions to validate against (or self.extensions)
-        """
-        def normalize_ext(_ext):
-            if _ext.startswith('.'):
-                _ext = _ext[1:]
-            return _ext.lower()
-
-        extensions = extensions or self.extensions
-        if not extensions:
-            return True
-
-        ext = normalize_ext(ext)
-
-        return ext in [normalize_ext(x) for x in extensions]
-
-    def save_file(self, file_obj, filename, directory=None, extensions=None,
-                  extra_metadata=None, max_filesize=None, randomized_name=True, **kwargs):
-        """
-        Saves a file object to the uploads location.
-        Returns the resolved filename, i.e. the directory +
-        the (randomized/incremented) base name.
-
-        :param file_obj: **cgi.FieldStorage** object (or similar)
-        :param filename: original filename
-        :param directory: relative path of sub-directory
-        :param extensions: iterable of allowed extensions, if not default
-        :param max_filesize: maximum size of file that should be allowed
-        :param randomized_name: generate random generated UID or fixed based on the filename
-        :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
-
-        """
-
-        extensions = extensions or self.extensions
-
-        if not self.filename_allowed(filename, extensions):
-            raise FileNotAllowedException()
-
-        if directory:
-            dest_directory = os.path.join(self.base_path, directory)
-        else:
-            dest_directory = self.base_path
-
-        safe_make_dirs(dest_directory)
-
-        uid_filename = utils.uid_filename(filename, randomized=randomized_name)
-
-        # resolve also produces special sub-dir for file optimized store
-        filename, path = self.resolve_name(uid_filename, dest_directory)
-        stored_file_dir = os.path.dirname(path)
-
-        no_body_seek = kwargs.pop('no_body_seek', False)
-        if no_body_seek:
-            pass
-        else:
-            file_obj.seek(0)
-
-        with open(path, "wb") as dest:
-            length = 256 * 1024
-            while 1:
-                buf = file_obj.read(length)
-                if not buf:
-                    break
-                dest.write(buf)
-
-        metadata = {}
-        if extra_metadata:
-            metadata = extra_metadata
-
-        size = os.stat(path).st_size
-
-        if max_filesize and size > max_filesize:
-            # free up the copied file, and raise exc
-            os.remove(path)
-            raise FileOverSizeException()
-
-        file_hash = self.calculate_path_hash(path)
-
-        metadata.update({
-            "filename": filename,
-             "size": size,
-             "time": time.time(),
-             "sha256": file_hash,
-             "meta_ver": METADATA_VER
-        })
-
-        filename_meta = filename + '.meta'
-        with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
-            dest_meta.write(json.dumps(metadata))
-
-        if directory:
-            filename = os.path.join(directory, filename)
-
-        return filename, metadata
-
-    def get_metadata(self, filename, ignore_missing=False):
-        """
-        Reads JSON stored metadata for a file
-
-        :param filename:
-        :return:
-        """
-        filename = self.store_path(filename)
-        filename_meta = filename + '.meta'
-        if ignore_missing and not os.path.isfile(filename_meta):
-            return {}
-        with open(filename_meta, "rb") as source_meta:
-            return json.loads(source_meta.read())