chore: fixed merge conflicts

docs/auth/auth-saml-azure.rst

@@ -0,0 +1,161 b''
+.. _config-saml-azure-ref:
+
+
+SAML 2.0 with Azure Entra ID
+----------------------------
+
+**This plugin is available only in EE Edition.**
+
+|RCE| supports SAML 2.0 Authentication with Azure Entra ID provider. This allows
+users to log-in to RhodeCode via SSO mechanism of external identity provider
+such as Azure AD. The login can be triggered either by the external IDP, or internally
+by clicking specific authentication button on the log-in page.
+
+
+Configuration steps
+^^^^^^^^^^^^^^^^^^^
+
+To configure Duo Security SAML authentication, use the following steps:
+
+1. From the |RCE| interface, select
+   :menuselection:`Admin --> Authentication`
+2. Activate the `Azure Entra ID` plugin and select :guilabel:`Save`
+3. Go to newly available menu option called `Azure Entra ID` on the left side.
+4. Check the `enabled` check box in the plugin configuration section,
+   and fill in the required SAML information and :guilabel:`Save`, for more details,
+   see :ref:`config-saml-azure`
+
+
+.. _config-saml-azure:
+
+
+Example SAML Azure Entra ID configuration
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Example configuration for SAML 2.0 with Azure Entra ID provider
+
+
+Enabled
+    `True`:
+
+    .. note::
+      Enable or disable this authentication plugin.
+
+
+Auth Cache TTL
+    `30`:
+
+    .. note::
+      Amount of seconds to cache the authentication and permissions check response call for this plugin.
+      Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
+
+Debug
+    `True`:
+
+    .. note::
+      Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
+
+
+Auth button name
+    `Azure Entra ID`:
+
+    .. note::
+      Alternative authentication display name. E.g AzureAuth, CorporateID etc.
+
+
+Entity ID
+    `https://sts.windows.net/APP_ID/`:
+
+    .. note::
+      Identity Provider entity/metadata URI. Known as "Microsoft Entra Identifier"
+      E.g. https://sts.windows.net/abcd-c655-dcee-aab7-abcd/
+
+SSO URL
+    `https://login.microsoftonline.com/APP_ID/saml2`:
+
+    .. note::
+      SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login, Known also as Login URL
+      E.g. https://login.microsoftonline.com/abcd-c655-dcee-aab7-abcd/saml2
+
+SLO URL
+    `https://login.microsoftonline.com/APP_ID/saml2`:
+
+    .. note::
+      SLO (SingleLogout) endpoint URL of the IdP. , Known also as Logout URL
+      E.g. https://login.microsoftonline.com/abcd-c655-dcee-aab7-abcd/saml2
+
+x509cert
+    `<CERTIFICATE_STRING>`:
+
+    .. note::
+      Identity provider public x509 certificate. It will be converted to single-line format without headers.
+      Download the raw base64 encoded certificate from the Identity provider and paste it here.
+
+SAML Signature
+    `sha-256`:
+
+    .. note::
+      Type of Algorithm to use for verification of SAML signature on Identity provider side.
+
+SAML Digest
+    `sha-256`:
+
+    .. note::
+      Type of Algorithm to use for verification of SAML digest on Identity provider side.
+
+Service Provider Cert Dir
+    `/etc/rhodecode/conf/saml_ssl/`:
+
+    .. note::
+      Optional directory to store service provider certificate and private keys.
+      Expected certs for the SP should be stored in this folder as:
+
+      * sp.key     Private Key
+      * sp.crt     Public cert
+      * sp_new.crt Future Public cert
+
+      Also you can use other cert to sign the metadata of the SP using the:
+
+      * metadata.key
+      * metadata.crt
+
+Expected NameID Format
+    `nameid-format:emailAddress`:
+
+    .. note::
+      The format that specifies how the NameID is sent to the service provider.
+
+User ID Attribute
+    `user.email`:
+
+    .. note::
+      User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
+      Ensure this is returned from DuoSecurity for example via duo_username.
+
+Username Attribute
+    `user.username`:
+
+    .. note::
+      Username Attribute name. This defines which attribute in SAML response will map to a username.
+
+Email Attribute
+    `user.email`:
+
+    .. note::
+      Email Attribute name. This defines which attribute in SAML response will map to an email address.
+
+
+
+Below is example setup from Azure Administration page that can be used with above config.
+
+.. image:: ../images/saml-azure-service-provider-example.png
+   :alt: Azure SAML setup example
+   :scale: 50 %
+
+
+Below is an example attribute mapping set for IDP provider required by the above config.
+
+
+.. image:: ../images/saml-azure-attributes-example.png
+   :alt: Azure SAML setup example
+   :scale: 50 %
  No newline at end of file

docs/release-notes/release-notes-5.1.1.rst

@@ -0,0 +1,40 b''
+|RCE| 5.1.1 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2024-07-23
+
+
+New Features
+^^^^^^^^^^^^
+
+
+
+General
+^^^^^^^
+
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+
+
+
+Fixes
+^^^^^
+
+- Fixed problems with JS static files build
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.1.1 is unscheduled bugfix release to address some build issues with 5.1 images

docs/release-notes/release-notes-5.1.2.rst

@@ -0,0 +1,41 b''
+|RCE| 5.1.2 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2024-09-12
+
+
+New Features
+^^^^^^^^^^^^
+
+
+
+General
+^^^^^^^
+
+
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+
+
+
+Fixes
+^^^^^
+
+- Fixed problems with Mercurial authentication after enabling httppostargs.
+  Currently this protocol will be disabled until proper fix is in place
+
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.1.2 is unscheduled bugfix release to address some build issues with 5.1 images

docs/release-notes/release-notes-5.2.0.rst

@@ -0,0 +1,55 b''
+|RCE| 5.2.0 |RNS|
+-----------------
+
+Release Date
+^^^^^^^^^^^^
+
+- 2024-10-09
+
+
+New Features
+^^^^^^^^^^^^
+
+- New artifact storage engines allowing an s3 based uploads
+- Enterprise version only: Added security tab to admin interface and possibility to whitelist specific vcs client versions. Some older versions clients have known security vulnerabilities, now you can disallow them.
+- Enterprise version only: Implemented support for Azure SAML authentication
+
+
+General
+^^^^^^^
+- Bumped version of packaging, gunicorn, orjson, zope.interface and some other requirements
+- Few tweaks and changes to saml plugins to allows easier setup
+- Configs: allow json log format for gunicorn
+- Configs: deprecated old ssh wrapper command and make the v2 the default one
+- Make sure commit-caches propagate to parent repo groups
+- Configs: Moved git lfs path and path of hg large files to ini file
+
+Security
+^^^^^^^^
+
+
+
+Performance
+^^^^^^^^^^^
+
+- description escaper for better performance
+
+Fixes
+^^^^^
+
+- Email notifications not working properly
+- Removed waitress as a default runner
+- Fixed issue with branch permissions
+- Ldap: fixed nested groups extraction logic
+- Fixed possible db corruption in case of filesystem problems
+- Cleanup and improvements to documentation
+- Added Kubernetes deployment section to the documentation
+- Added default value to celery result and broker
+- Fixed broken backends function after python3 migration
+- Explicitly disable mercurial web_push ssl flag to prevent from errors about ssl required
+- VCS: fixed problems with locked repos and with branch permissions reporting
+
+Upgrade notes
+^^^^^^^^^^^^^
+
+- RhodeCode 5.2.0 is a planned major release featuring Azure SAML, whitelist for client versions, s3 artifacts backend and more!

rhodecode/apps/admin/views/security.py

@@ -0,0 +1,46 b''
+# Copyright (C) 2010-2024 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+
+from rhodecode.apps._base import BaseAppView
+from rhodecode.lib.auth import LoginRequired, HasPermissionAllDecorator
+
+log = logging.getLogger(__name__)
+
+
+class AdminSecurityView(BaseAppView):
+
+    def load_default_context(self):
+        c = self._get_local_tmpl_context()
+        return c
+
+    @LoginRequired()
+    @HasPermissionAllDecorator('hg.admin')
+    def security(self):
+        c = self.load_default_context()
+        c.active = 'security'
+        return self._get_template_context(c)
+
+
+    @LoginRequired()
+    @HasPermissionAllDecorator('hg.admin')
+    def admin_security_modify_allowed_vcs_client_versions(self):
+        c = self.load_default_context()
+        c.active = 'security'
+        return self._get_template_context(c)

rhodecode/apps/file_store/backends/base.py

@@ -0,0 +1,269 b''
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import fsspec  # noqa
+import logging
+
+from rhodecode.lib.ext_json import json
+
+from rhodecode.apps.file_store.utils import sha256_safe, ShardFileReader, get_uid_filename
+from rhodecode.apps.file_store.extensions import resolve_extensions
+from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException  # noqa: F401
+
+log = logging.getLogger(__name__)
+
+
+class BaseShard:
+
+    metadata_suffix: str = '.metadata'
+    storage_type: str = ''
+    fs = None
+
+    @property
+    def storage_medium(self):
+        if not self.storage_type:
+            raise ValueError('No storage type set for this shard storage_type=""')
+        return getattr(self, self.storage_type)
+
+    def __contains__(self, key):
+        full_path = self.store_path(key)
+        return self.fs.exists(full_path)
+
+    def metadata_convert(self, uid_filename, metadata):
+        return metadata
+
+    def get_metadata_filename(self, uid_filename) -> tuple[str, str]:
+        metadata_file: str = f'{uid_filename}{self.metadata_suffix}'
+        return metadata_file, self.store_path(metadata_file)
+
+    def get_metadata(self, uid_filename, ignore_missing=False) -> dict:
+        _metadata_file, metadata_file_path = self.get_metadata_filename(uid_filename)
+        if ignore_missing and not self.fs.exists(metadata_file_path):
+            return {}
+
+        with self.fs.open(metadata_file_path, 'rb') as f:
+            metadata = json.loads(f.read())
+
+        metadata = self.metadata_convert(uid_filename, metadata)
+        return metadata
+
+    def _store(self, key: str, uid_key: str, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+        raise NotImplementedError
+
+    def store(self, key: str, uid_key: str, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+        return self._store(key, uid_key, value_reader, max_filesize, metadata, **kwargs)
+
+    def _fetch(self, key, presigned_url_expires: int = 0):
+        raise NotImplementedError
+
+    def fetch(self, key, **kwargs) -> tuple[ShardFileReader, dict]:
+        return self._fetch(key)
+
+    def _delete(self, key):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata = self.get_metadata(key)
+        _metadata_file, metadata_file_path = self.get_metadata_filename(key)
+        artifact_file_path = metadata['filename_uid_path']
+        self.fs.rm(artifact_file_path)
+        self.fs.rm(metadata_file_path)
+
+        return 1
+
+    def delete(self, key):
+        raise NotImplementedError
+
+    def store_path(self, uid_filename):
+        raise NotImplementedError
+
+
+class BaseFileStoreBackend:
+    _shards = tuple()
+    _shard_cls = BaseShard
+    _config: dict | None = None
+    _storage_path: str = ''
+
+    def __init__(self, settings, extension_groups=None):
+        self._config = settings
+        extension_groups = extension_groups or ['any']
+        self.extensions = resolve_extensions([], groups=extension_groups)
+
+    def __contains__(self, key):
+        return self.filename_exists(key)
+
+    def __repr__(self):
+        return f'<{self.__class__.__name__}(storage={self.storage_path})>'
+
+    @property
+    def storage_path(self):
+        return self._storage_path
+
+    @classmethod
+    def get_shard_index(cls, filename: str, num_shards) -> int:
+        # Generate a hash value from the filename
+        hash_value = sha256_safe(filename)
+
+        # Convert the hash value to an integer
+        hash_int = int(hash_value, 16)
+
+        # Map the hash integer to a shard number between 1 and num_shards
+        shard_number = (hash_int % num_shards)
+
+        return shard_number
+
+    @classmethod
+    def apply_counter(cls, counter: int, filename: str) -> str:
+        """
+        Apply a counter to the filename.
+
+        :param counter: The counter value to apply.
+        :param filename: The original filename.
+        :return: The modified filename with the counter.
+        """
+        name_counted = f'{counter:d}-{filename}'
+        return name_counted
+
+    def _get_shard(self, key) -> _shard_cls:
+        index = self.get_shard_index(key, len(self._shards))
+        shard = self._shards[index]
+        return shard
+
+    def get_conf(self, key, pop=False):
+        if key not in self._config:
+            raise ValueError(
+                f"No configuration key '{key}', please make sure it exists in filestore config")
+        val = self._config[key]
+        if pop:
+            del self._config[key]
+        return val
+
+    def filename_allowed(self, filename, extensions=None):
+        """Checks if a filename has an allowed extension
+
+        :param filename: base name of file
+        :param extensions: iterable of extensions (or self.extensions)
+        """
+        _, ext = os.path.splitext(filename)
+        return self.extension_allowed(ext, extensions)
+
+    def extension_allowed(self, ext, extensions=None):
+        """
+        Checks if an extension is permitted. Both e.g. ".jpg" and
+        "jpg" can be passed in. Extension lookup is case-insensitive.
+
+        :param ext: extension to check
+        :param extensions: iterable of extensions to validate against (or self.extensions)
+        """
+        def normalize_ext(_ext):
+            if _ext.startswith('.'):
+                _ext = _ext[1:]
+            return _ext.lower()
+
+        extensions = extensions or self.extensions
+        if not extensions:
+            return True
+
+        ext = normalize_ext(ext)
+
+        return ext in [normalize_ext(x) for x in extensions]
+
+    def filename_exists(self, uid_filename):
+        shard = self._get_shard(uid_filename)
+        return uid_filename in shard
+
+    def store_path(self, uid_filename):
+        """
+        Returns absolute file path of the uid_filename
+        """
+        shard = self._get_shard(uid_filename)
+        return shard.store_path(uid_filename)
+
+    def store_metadata(self, uid_filename):
+        shard = self._get_shard(uid_filename)
+        return shard.get_metadata_filename(uid_filename)
+
+    def store(self, filename, value_reader, extensions=None, metadata=None, max_filesize=None, randomized_name=True, **kwargs):
+        extensions = extensions or self.extensions
+
+        if not self.filename_allowed(filename, extensions):
+            msg = f'filename {filename} does not allow extensions {extensions}'
+            raise FileNotAllowedException(msg)
+
+        # # TODO: check why we need this setting ? it looks stupid...
+        # no_body_seek is used in stream mode importer somehow
+        # no_body_seek = kwargs.pop('no_body_seek', False)
+        # if no_body_seek:
+        #     pass
+        # else:
+        #     value_reader.seek(0)
+
+        uid_filename = kwargs.pop('uid_filename', None)
+        if uid_filename is None:
+            uid_filename = get_uid_filename(filename, randomized=randomized_name)
+
+        shard = self._get_shard(uid_filename)
+
+        return shard.store(filename, uid_filename, value_reader, max_filesize, metadata, **kwargs)
+
+    def import_to_store(self, value_reader, org_filename, uid_filename, metadata, **kwargs):
+        shard = self._get_shard(uid_filename)
+        max_filesize = None
+        return shard.store(org_filename, uid_filename, value_reader, max_filesize, metadata, import_mode=True)
+
+    def delete(self, uid_filename):
+        shard = self._get_shard(uid_filename)
+        return shard.delete(uid_filename)
+
+    def fetch(self, uid_filename) -> tuple[ShardFileReader, dict]:
+        shard = self._get_shard(uid_filename)
+        return shard.fetch(uid_filename)
+
+    def get_metadata(self, uid_filename, ignore_missing=False) -> dict:
+        shard = self._get_shard(uid_filename)
+        return shard.get_metadata(uid_filename, ignore_missing=ignore_missing)
+
+    def iter_keys(self):
+        for shard in self._shards:
+            if shard.fs.exists(shard.storage_medium):
+                for path, _dirs, _files in shard.fs.walk(shard.storage_medium):
+                    for key_file_path in _files:
+                        if key_file_path.endswith(shard.metadata_suffix):
+                            yield shard, key_file_path
+
+    def iter_artifacts(self):
+        for shard, key_file in self.iter_keys():
+            json_key = f"{shard.storage_medium}/{key_file}"
+            with shard.fs.open(json_key, 'rb') as f:
+                yield shard, json.loads(f.read())['filename_uid']
+
+    def get_statistics(self):
+        total_files = 0
+        total_size = 0
+        meta = {}
+
+        for shard, key_file in self.iter_keys():
+            json_key = f"{shard.storage_medium}/{key_file}"
+            with shard.fs.open(json_key, 'rb') as f:
+                total_files += 1
+                metadata = json.loads(f.read())
+                total_size += metadata['size']
+
+        return total_files, total_size, meta

rhodecode/apps/file_store/backends/filesystem.py

@@ -0,0 +1,183 b''
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import hashlib
+import functools
+import time
+import logging
+
+from .. import config_keys
+from ..exceptions import FileOverSizeException
+from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
+
+from ....lib.ext_json import json
+
+log = logging.getLogger(__name__)
+
+
+class FileSystemShard(BaseShard):
+    METADATA_VER = 'v2'
+    BACKEND_TYPE = config_keys.backend_filesystem
+    storage_type: str = 'directory'
+
+    def __init__(self, index, directory, directory_folder, fs, **settings):
+        self._index: int = index
+        self._directory: str = directory
+        self._directory_folder: str = directory_folder
+        self.fs = fs
+
+    @property
+    def directory(self) -> str:
+        """Cache directory final path."""
+        return os.path.join(self._directory, self._directory_folder)
+
+    def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
+
+        # ensure dir exists
+        destination, _ = os.path.split(full_path)
+        if not self.fs.exists(destination):
+            self.fs.makedirs(destination)
+
+        writer = self.fs.open(full_path, mode)
+
+        digest = hashlib.sha256()
+        oversize_cleanup = False
+        with writer:
+            size = 0
+            for chunk in iterator:
+                size += len(chunk)
+                digest.update(chunk)
+                writer.write(chunk)
+
+                if max_filesize and size > max_filesize:
+                    oversize_cleanup = True
+                    # free up the copied file, and raise exc
+                    break
+
+            writer.flush()
+            # Get the file descriptor
+            fd = writer.fileno()
+
+            # Sync the file descriptor to disk, helps with NFS cases...
+            os.fsync(fd)
+
+        if oversize_cleanup:
+            self.fs.rm(full_path)
+            raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
+
+        sha256 = digest.hexdigest()
+        log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
+        return size, sha256
+
+    def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+
+        filename = key
+        uid_filename = uid_key
+        full_path = self.store_path(uid_filename)
+
+        # STORE METADATA
+        _metadata = {
+            "version": self.METADATA_VER,
+            "store_type": self.BACKEND_TYPE,
+
+            "filename": filename,
+            "filename_uid_path": full_path,
+            "filename_uid": uid_filename,
+            "sha256": "",  # NOTE: filled in by reader iteration
+
+            "store_time": time.time(),
+
+            "size": 0
+        }
+
+        if metadata:
+            if kwargs.pop('import_mode', False):
+                # in import mode, we don't need to compute metadata, we just take the old version
+                _metadata["import_mode"] = True
+            else:
+                _metadata.update(metadata)
+
+        read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
+        size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
+        _metadata['size'] = size
+        _metadata['sha256'] = sha256
+
+        # after storing the artifacts, we write the metadata present
+        _metadata_file, metadata_file_path = self.get_metadata_filename(uid_key)
+
+        with self.fs.open(metadata_file_path, 'wb') as f:
+            f.write(json.dumps(_metadata))
+
+        return uid_filename, _metadata
+
+    def store_path(self, uid_filename):
+        """
+        Returns absolute file path of the uid_filename
+        """
+        return os.path.join(self._directory, self._directory_folder, uid_filename)
+
+    def _fetch(self, key, presigned_url_expires: int = 0):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata = self.get_metadata(key)
+
+        file_path = metadata['filename_uid_path']
+        if presigned_url_expires and presigned_url_expires > 0:
+            metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
+
+        return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
+
+    def delete(self, key):
+        return self._delete(key)
+
+
+class FileSystemBackend(BaseFileStoreBackend):
+    shard_name: str = 'shard_{:03d}'
+    _shard_cls = FileSystemShard
+
+    def __init__(self, settings):
+        super().__init__(settings)
+
+        store_dir = self.get_conf(config_keys.filesystem_storage_path)
+        directory = os.path.expanduser(store_dir)
+
+        self._directory = directory
+        self._storage_path = directory  # common path for all from BaseCache
+        self._shard_count = int(self.get_conf(config_keys.filesystem_shards, pop=True))
+        if self._shard_count < 1:
+            raise ValueError(f'{config_keys.filesystem_shards} must be 1 or more')
+
+        log.debug('Initializing %s file_store instance', self)
+        fs = fsspec.filesystem('file')
+
+        if not fs.exists(self._directory):
+            fs.makedirs(self._directory, exist_ok=True)
+
+        self._shards = tuple(
+            self._shard_cls(
+                index=num,
+                directory=directory,
+                directory_folder=self.shard_name.format(num),
+                fs=fs,
+                **settings,
+            )
+            for num in range(self._shard_count)
+        )

rhodecode/apps/file_store/backends/filesystem_legacy.py

@@ -0,0 +1,278 b''
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import errno
+import os
+import hashlib
+import functools
+import time
+import logging
+
+from .. import config_keys
+from ..exceptions import FileOverSizeException
+from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
+
+from ....lib.ext_json import json
+
+log = logging.getLogger(__name__)
+
+
+class LegacyFileSystemShard(BaseShard):
+    # legacy ver
+    METADATA_VER = 'v2'
+    BACKEND_TYPE = config_keys.backend_legacy_filesystem
+    storage_type: str = 'dir_struct'
+
+    # legacy suffix
+    metadata_suffix: str = '.meta'
+
+    @classmethod
+    def _sub_store_from_filename(cls, filename):
+        return filename[:2]
+
+    @classmethod
+    def apply_counter(cls, counter, filename):
+        name_counted = '%d-%s' % (counter, filename)
+        return name_counted
+
+    @classmethod
+    def safe_make_dirs(cls, dir_path):
+        if not os.path.exists(dir_path):
+            try:
+                os.makedirs(dir_path)
+            except OSError as e:
+                if e.errno != errno.EEXIST:
+                    raise
+                return
+
+    @classmethod
+    def resolve_name(cls, name, directory):
+        """
+        Resolves a unique name and the correct path. If a filename
+        for that path already exists then a numeric prefix with values > 0 will be
+        added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix.
+
+        :param name: base name of file
+        :param directory: absolute directory path
+        """
+
+        counter = 0
+        while True:
+            name_counted = cls.apply_counter(counter, name)
+
+            # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
+            sub_store: str = cls._sub_store_from_filename(name_counted)
+            sub_store_path: str = os.path.join(directory, sub_store)
+            cls.safe_make_dirs(sub_store_path)
+
+            path = os.path.join(sub_store_path, name_counted)
+            if not os.path.exists(path):
+                return name_counted, path
+            counter += 1
+
+    def __init__(self, index, directory, directory_folder, fs, **settings):
+        self._index: int = index
+        self._directory: str = directory
+        self._directory_folder: str = directory_folder
+        self.fs = fs
+
+    @property
+    def dir_struct(self) -> str:
+        """Cache directory final path."""
+        return os.path.join(self._directory, '0-')
+
+    def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
+
+        # ensure dir exists
+        destination, _ = os.path.split(full_path)
+        if not self.fs.exists(destination):
+            self.fs.makedirs(destination)
+
+        writer = self.fs.open(full_path, mode)
+
+        digest = hashlib.sha256()
+        oversize_cleanup = False
+        with writer:
+            size = 0
+            for chunk in iterator:
+                size += len(chunk)
+                digest.update(chunk)
+                writer.write(chunk)
+
+                if max_filesize and size > max_filesize:
+                    # free up the copied file, and raise exc
+                    oversize_cleanup = True
+                    break
+
+            writer.flush()
+            # Get the file descriptor
+            fd = writer.fileno()
+
+            # Sync the file descriptor to disk, helps with NFS cases...
+            os.fsync(fd)
+
+        if oversize_cleanup:
+            self.fs.rm(full_path)
+            raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
+
+        sha256 = digest.hexdigest()
+        log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
+        return size, sha256
+
+    def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+
+        filename = key
+        uid_filename = uid_key
+
+        # NOTE:, also apply N- Counter...
+        uid_filename, full_path = self.resolve_name(uid_filename, self._directory)
+
+        # STORE METADATA
+        # TODO: make it compatible, and backward proof
+        _metadata = {
+            "version": self.METADATA_VER,
+
+            "filename": filename,
+            "filename_uid_path": full_path,
+            "filename_uid": uid_filename,
+            "sha256": "",  # NOTE: filled in by reader iteration
+
+            "store_time": time.time(),
+
+            "size": 0
+        }
+        if metadata:
+            _metadata.update(metadata)
+
+        read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
+        size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
+        _metadata['size'] = size
+        _metadata['sha256'] = sha256
+
+        # after storing the artifacts, we write the metadata present
+        _metadata_file, metadata_file_path = self.get_metadata_filename(uid_filename)
+
+        with self.fs.open(metadata_file_path, 'wb') as f:
+            f.write(json.dumps(_metadata))
+
+        return uid_filename, _metadata
+
+    def store_path(self, uid_filename):
+        """
+        Returns absolute file path of the uid_filename
+        """
+        prefix_dir = ''
+        if '/' in uid_filename:
+            prefix_dir, filename = uid_filename.split('/')
+            sub_store = self._sub_store_from_filename(filename)
+        else:
+            sub_store = self._sub_store_from_filename(uid_filename)
+
+        return os.path.join(self._directory, prefix_dir, sub_store, uid_filename)
+
+    def metadata_convert(self, uid_filename, metadata):
+        # NOTE: backward compat mode here... this is for file created PRE 5.2 system
+        if 'meta_ver' in metadata:
+            full_path = self.store_path(uid_filename)
+            metadata = {
+                "_converted": True,
+                "_org": metadata,
+                "version": self.METADATA_VER,
+                "store_type": self.BACKEND_TYPE,
+
+                "filename": metadata['filename'],
+                "filename_uid_path": full_path,
+                "filename_uid": uid_filename,
+                "sha256": metadata['sha256'],
+
+                "store_time": metadata['time'],
+
+                "size": metadata['size']
+            }
+        return metadata
+
+    def _fetch(self, key, presigned_url_expires: int = 0):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata = self.get_metadata(key)
+
+        file_path = metadata['filename_uid_path']
+        if presigned_url_expires and presigned_url_expires > 0:
+            metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
+
+        return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
+
+    def delete(self, key):
+        return self._delete(key)
+
+    def _delete(self, key):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata = self.get_metadata(key)
+        metadata_file, metadata_file_path = self.get_metadata_filename(key)
+        artifact_file_path = metadata['filename_uid_path']
+        self.fs.rm(artifact_file_path)
+        self.fs.rm(metadata_file_path)
+
+    def get_metadata_filename(self, uid_filename) -> tuple[str, str]:
+
+        metadata_file: str = f'{uid_filename}{self.metadata_suffix}'
+        uid_path_in_store = self.store_path(uid_filename)
+
+        metadata_file_path = f'{uid_path_in_store}{self.metadata_suffix}'
+        return metadata_file, metadata_file_path
+
+
+class LegacyFileSystemBackend(BaseFileStoreBackend):
+    _shard_cls = LegacyFileSystemShard
+
+    def __init__(self, settings):
+        super().__init__(settings)
+
+        store_dir = self.get_conf(config_keys.legacy_filesystem_storage_path)
+        directory = os.path.expanduser(store_dir)
+
+        self._directory = directory
+        self._storage_path = directory  # common path for all from BaseCache
+
+        log.debug('Initializing %s file_store instance', self)
+        fs = fsspec.filesystem('file')
+
+        if not fs.exists(self._directory):
+            fs.makedirs(self._directory, exist_ok=True)
+
+        # legacy system uses single shard
+        self._shards = tuple(
+            [
+                self._shard_cls(
+                    index=0,
+                    directory=directory,
+                    directory_folder='',
+                    fs=fs,
+                    **settings,
+                )
+            ]
+        )
+
+    @classmethod
+    def get_shard_index(cls, filename: str, num_shards) -> int:
+        # legacy filesystem doesn't use shards, and always uses single shard
+        return 0

rhodecode/apps/file_store/backends/objectstore.py

@@ -0,0 +1,184 b''
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import hashlib
+import functools
+import time
+import logging
+
+from .. import config_keys
+from ..exceptions import FileOverSizeException
+from ..backends.base import BaseFileStoreBackend, fsspec, BaseShard, ShardFileReader
+
+from ....lib.ext_json import json
+
+log = logging.getLogger(__name__)
+
+
+class S3Shard(BaseShard):
+    METADATA_VER = 'v2'
+    BACKEND_TYPE = config_keys.backend_objectstore
+    storage_type: str = 'bucket'
+
+    def __init__(self, index, bucket, bucket_folder, fs, **settings):
+        self._index: int = index
+        self._bucket_main: str = bucket
+        self._bucket_folder: str = bucket_folder
+
+        self.fs = fs
+
+    @property
+    def bucket(self) -> str:
+        """Cache bucket final path."""
+        return os.path.join(self._bucket_main, self._bucket_folder)
+
+    def _write_file(self, full_path, iterator, max_filesize, mode='wb'):
+
+        # ensure dir exists
+        destination, _ = os.path.split(full_path)
+        if not self.fs.exists(destination):
+            self.fs.makedirs(destination)
+
+        writer = self.fs.open(full_path, mode)
+
+        digest = hashlib.sha256()
+        oversize_cleanup = False
+        with writer:
+            size = 0
+            for chunk in iterator:
+                size += len(chunk)
+                digest.update(chunk)
+                writer.write(chunk)
+
+                if max_filesize and size > max_filesize:
+                    oversize_cleanup = True
+                    # free up the copied file, and raise exc
+                    break
+
+        if oversize_cleanup:
+            self.fs.rm(full_path)
+            raise FileOverSizeException(f'given file is over size limit ({max_filesize}): {full_path}')
+
+        sha256 = digest.hexdigest()
+        log.debug('written new artifact under %s, sha256: %s', full_path, sha256)
+        return size, sha256
+
+    def _store(self, key: str, uid_key, value_reader, max_filesize: int | None = None, metadata: dict | None = None, **kwargs):
+
+        filename = key
+        uid_filename = uid_key
+        full_path = self.store_path(uid_filename)
+
+        # STORE METADATA
+        _metadata = {
+            "version": self.METADATA_VER,
+            "store_type": self.BACKEND_TYPE,
+
+            "filename": filename,
+            "filename_uid_path": full_path,
+            "filename_uid": uid_filename,
+            "sha256": "",  # NOTE: filled in by reader iteration
+
+            "store_time": time.time(),
+
+            "size": 0
+        }
+
+        if metadata:
+            if kwargs.pop('import_mode', False):
+                # in import mode, we don't need to compute metadata, we just take the old version
+                _metadata["import_mode"] = True
+            else:
+                _metadata.update(metadata)
+
+        read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
+        size, sha256 = self._write_file(full_path, read_iterator, max_filesize)
+        _metadata['size'] = size
+        _metadata['sha256'] = sha256
+
+        # after storing the artifacts, we write the metadata present
+        metadata_file, metadata_file_path = self.get_metadata_filename(uid_key)
+
+        with self.fs.open(metadata_file_path, 'wb') as f:
+            f.write(json.dumps(_metadata))
+
+        return uid_filename, _metadata
+
+    def store_path(self, uid_filename):
+        """
+        Returns absolute file path of the uid_filename
+        """
+        return os.path.join(self._bucket_main, self._bucket_folder, uid_filename)
+
+    def _fetch(self, key, presigned_url_expires: int = 0):
+        if key not in self:
+            log.exception(f'requested key={key} not found in {self}')
+            raise KeyError(key)
+
+        metadata_file, metadata_file_path = self.get_metadata_filename(key)
+        with self.fs.open(metadata_file_path, 'rb') as f:
+            metadata = json.loads(f.read())
+
+        file_path = metadata['filename_uid_path']
+        if presigned_url_expires and presigned_url_expires > 0:
+            metadata['url'] = self.fs.url(file_path, expires=presigned_url_expires)
+
+        return ShardFileReader(self.fs.open(file_path, 'rb')), metadata
+
+    def delete(self, key):
+        return self._delete(key)
+
+
+class ObjectStoreBackend(BaseFileStoreBackend):
+    shard_name: str = 'shard-{:03d}'
+    _shard_cls = S3Shard
+
+    def __init__(self, settings):
+        super().__init__(settings)
+
+        self._shard_count = int(self.get_conf(config_keys.objectstore_bucket_shards, pop=True))
+        if self._shard_count < 1:
+            raise ValueError('cache_shards must be 1 or more')
+
+        self._bucket = settings.pop(config_keys.objectstore_bucket)
+        if not self._bucket:
+            raise ValueError(f'{config_keys.objectstore_bucket} needs to have a value')
+
+        objectstore_url = self.get_conf(config_keys.objectstore_url)
+        key = settings.pop(config_keys.objectstore_key)
+        secret = settings.pop(config_keys.objectstore_secret)
+
+        self._storage_path = objectstore_url  # common path for all from BaseCache
+        log.debug('Initializing %s file_store instance', self)
+        fs = fsspec.filesystem('s3', anon=False, endpoint_url=objectstore_url, key=key, secret=secret)
+
+        # init main bucket
+        if not fs.exists(self._bucket):
+            fs.mkdir(self._bucket)
+
+        self._shards = tuple(
+            self._shard_cls(
+                index=num,
+                bucket=self._bucket,
+                bucket_folder=self.shard_name.format(num),
+                fs=fs,
+                **settings,
+            )
+            for num in range(self._shard_count)
+        )

rhodecode/apps/file_store/tests/test_filestore_backends.py

@@ -0,0 +1,128 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps import file_store
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.backends.filesystem_legacy import LegacyFileSystemBackend
+from rhodecode.apps.file_store.backends.filesystem import FileSystemBackend
+from rhodecode.apps.file_store.backends.objectstore import ObjectStoreBackend
+from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store.tests import random_binary_file, file_store_instance
+
+
+class TestFileStoreBackends:
+
+    @pytest.mark.parametrize('backend_type, expected_instance', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+        (config_keys.backend_filesystem, FileSystemBackend),
+        (config_keys.backend_objectstore, ObjectStoreBackend),
+    ])
+    def test_get_backend(self, backend_type, expected_instance, ini_settings):
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+        assert isinstance(f_store, expected_instance)
+
+    @pytest.mark.parametrize('backend_type, expected_instance', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+        (config_keys.backend_filesystem, FileSystemBackend),
+        (config_keys.backend_objectstore, ObjectStoreBackend),
+    ])
+    def test_store_and_read(self, backend_type, expected_instance, ini_settings, random_binary_file):
+        filename, temp_file = random_binary_file
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+        metadata = {
+            'user_uploaded': {
+                'username': 'user1',
+                'user_id': 10,
+                'ip': '10.20.30.40'
+            }
+        }
+        store_fid, metadata = f_store.store(filename, temp_file, extra_metadata=metadata)
+        assert store_fid
+        assert metadata
+
+        # read-after write
+        reader, metadata2 = f_store.fetch(store_fid)
+        assert reader
+        assert metadata2['filename'] == filename
+
+    @pytest.mark.parametrize('backend_type, expected_instance', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+        (config_keys.backend_filesystem, FileSystemBackend),
+        (config_keys.backend_objectstore, ObjectStoreBackend),
+    ])
+    def test_store_file_not_allowed(self, backend_type, expected_instance, ini_settings, random_binary_file):
+        filename, temp_file = random_binary_file
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+        with pytest.raises(FileNotAllowedException):
+            f_store.store('notallowed.exe', temp_file, extensions=['.txt'])
+
+    @pytest.mark.parametrize('backend_type, expected_instance', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend),
+        (config_keys.backend_filesystem, FileSystemBackend),
+        (config_keys.backend_objectstore, ObjectStoreBackend),
+    ])
+    def test_store_file_over_size(self, backend_type, expected_instance, ini_settings, random_binary_file):
+        filename, temp_file = random_binary_file
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+        with pytest.raises(FileOverSizeException):
+            f_store.store('toobig.exe', temp_file, extensions=['.exe'], max_filesize=124)
+
+    @pytest.mark.parametrize('backend_type, expected_instance, extra_conf', [
+        (config_keys.backend_legacy_filesystem, LegacyFileSystemBackend, {}),
+        (config_keys.backend_filesystem, FileSystemBackend, {config_keys.filesystem_storage_path: '/tmp/test-fs-store'}),
+        (config_keys.backend_objectstore, ObjectStoreBackend, {config_keys.objectstore_bucket: 'test-bucket'}),
+    ])
+    def test_store_stats_and_keys(self, backend_type, expected_instance, extra_conf, ini_settings, random_binary_file):
+        config = ini_settings
+        config[config_keys.backend_type] = backend_type
+        config.update(extra_conf)
+
+        f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+
+        # purge storage before running
+        for shard, k in f_store.iter_artifacts():
+            f_store.delete(k)
+
+        for i in range(10):
+            filename, temp_file = random_binary_file
+
+            metadata = {
+                'user_uploaded': {
+                    'username': 'user1',
+                    'user_id': 10,
+                    'ip': '10.20.30.40'
+                }
+            }
+            store_fid, metadata = f_store.store(filename, temp_file, extra_metadata=metadata)
+            assert store_fid
+            assert metadata
+
+        cnt, size, meta = f_store.get_statistics()
+        assert cnt == 10
+        assert 10 == len(list(f_store.iter_keys()))

rhodecode/apps/file_store/tests/test_filestore_filesystem_backend.py

@@ -0,0 +1,52 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.tests import generate_random_filename
+
+
[e-mail]()
+def file_store_filesystem_instance(ini_settings):
+    config = ini_settings
+    config[config_keys.backend_type] = config_keys.backend_filesystem
+    f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+    return f_store
+
+
+class TestFileStoreFileSystemBackend:
+
+    @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
+    def test_get_shard_number(self, filename, file_store_filesystem_instance):
+        shard_number = file_store_filesystem_instance.get_shard_index(filename, len(file_store_filesystem_instance._shards))
+        # Check that the shard number is between 0 and max-shards
+        assert 0 <= shard_number <= len(file_store_filesystem_instance._shards)
+
+    @pytest.mark.parametrize('filename, expected_shard_num', [
+        ('my-name-1', 3),
+        ('my-name-2', 2),
+        ('my-name-3', 4),
+        ('my-name-4', 1),
+
+        ('rhodecode-enterprise-ce', 5),
+        ('rhodecode-enterprise-ee', 6),
+    ])
+    def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_filesystem_instance):
+        shard_number = file_store_filesystem_instance.get_shard_index(filename, len(file_store_filesystem_instance._shards))
+        assert expected_shard_num == shard_number

rhodecode/apps/file_store/tests/test_filestore_legacy_and_v2_compatability.py

@@ -0,0 +1,17 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
  No newline at end of file

rhodecode/apps/file_store/tests/test_filestore_legacy_backend.py

@@ -0,0 +1,52 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.tests import generate_random_filename
+
+
[e-mail]()
+def file_store_legacy_instance(ini_settings):
+    config = ini_settings
+    config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+    f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+    return f_store
+
+
+class TestFileStoreLegacyBackend:
+
+    @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
+    def test_get_shard_number(self, filename, file_store_legacy_instance):
+        shard_number = file_store_legacy_instance.get_shard_index(filename, len(file_store_legacy_instance._shards))
+        # Check that the shard number is 0 for legacy filesystem store we don't use shards
+        assert shard_number == 0
+
+    @pytest.mark.parametrize('filename, expected_shard_num', [
+        ('my-name-1', 0),
+        ('my-name-2', 0),
+        ('my-name-3', 0),
+        ('my-name-4', 0),
+
+        ('rhodecode-enterprise-ce', 0),
+        ('rhodecode-enterprise-ee', 0),
+    ])
+    def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_legacy_instance):
+        shard_number = file_store_legacy_instance.get_shard_index(filename, len(file_store_legacy_instance._shards))
+        assert expected_shard_num == shard_number

rhodecode/apps/file_store/tests/test_filestore_objectstore_backend.py

@@ -0,0 +1,52 b''
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
+from rhodecode.apps.file_store.tests import generate_random_filename
+
+
[e-mail]()
+def file_store_objectstore_instance(ini_settings):
+    config = ini_settings
+    config[config_keys.backend_type] = config_keys.backend_objectstore
+    f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+    return f_store
+
+
+class TestFileStoreObjectStoreBackend:
+
+    @pytest.mark.parametrize('filename', [generate_random_filename() for _ in range(10)])
+    def test_get_shard_number(self, filename, file_store_objectstore_instance):
+        shard_number = file_store_objectstore_instance.get_shard_index(filename, len(file_store_objectstore_instance._shards))
+        # Check that the shard number is between 0 and shards
+        assert 0 <= shard_number <= len(file_store_objectstore_instance._shards)
+
+    @pytest.mark.parametrize('filename, expected_shard_num', [
+        ('my-name-1', 3),
+        ('my-name-2', 2),
+        ('my-name-3', 4),
+        ('my-name-4', 1),
+
+        ('rhodecode-enterprise-ce', 5),
+        ('rhodecode-enterprise-ee', 6),
+    ])
+    def test_get_shard_number_consistency(self, filename, expected_shard_num, file_store_objectstore_instance):
+        shard_number = file_store_objectstore_instance.get_shard_index(filename, len(file_store_objectstore_instance._shards))
+        assert expected_shard_num == shard_number

rhodecode/lib/rc_commands/migrate_artifact.py

@@ -0,0 +1,122 b''
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import sys
+import logging
+
+import click
+
+from rhodecode.lib.pyramid_utils import bootstrap
+from rhodecode.lib.ext_json import json
+from rhodecode.model.db import FileStore
+from rhodecode.apps.file_store import utils as store_utils
+
+log = logging.getLogger(__name__)
+
+
[e-mail]()
[e-mail]('ini_path', type=click.Path(exists=True))
[e-mail]('file_uid')
[e-mail](
+    '--source-backend-conf',
+    type=click.Path(exists=True, dir_okay=False, readable=True),
+    help='Source backend config file path in a json format'
+)
[e-mail](
+    '--dest-backend-conf',
+    type=click.Path(exists=True, dir_okay=False, readable=True),
+    help='Source backend config file path in a json format'
+)
+def main(ini_path, file_uid, source_backend_conf, dest_backend_conf):
+    return command(ini_path, file_uid, source_backend_conf, dest_backend_conf)
+
+
+_source_settings = {}
+
+_dest_settings = {}
+
+
+def command(ini_path, file_uid, source_backend_conf, dest_backend_conf):
+    with bootstrap(ini_path, env={'RC_CMD_SETUP_RC': '1'}) as env:
+        migrate_func(env, file_uid, source_backend_conf, dest_backend_conf)
+
+
+def migrate_func(env, file_uid, source_backend_conf=None, dest_backend_conf=None):
+    """
+
+    Example usage::
+
+        from rhodecode.lib.rc_commands import migrate_artifact
+        migrate_artifact._source_settings = {
+            'file_store.backend.type': 'filesystem_v1',
+            'file_store.filesystem_v1.storage_path': '/var/opt/rhodecode_data/file_store',
+        }
+        migrate_artifact._dest_settings = {
+            'file_store.backend.type': 'objectstore',
+            'file_store.objectstore.url': 'http://s3-minio:9000',
+            'file_store.objectstore.bucket': 'rhodecode-file-store',
+            'file_store.objectstore.key': 's3admin',
+            'file_store.objectstore.secret': 's3secret4',
+            'file_store.objectstore.region': 'eu-central-1',
+        }
+        for db_obj in FileStore.query().all():
+            migrate_artifact.migrate_func({}, db_obj.file_uid)
+
+    """
+
+    try:
+        from rc_ee.api.views.store_api import _store_file
+    except ImportError:
+        click.secho('ERROR: Unable to import store_api. '
+                    'store_api is only available in EE edition of RhodeCode',
+                    fg='red')
+        sys.exit(-1)
+
+    source_settings = _source_settings
+    if source_backend_conf:
+        source_settings = json.loads(open(source_backend_conf).read())
+    dest_settings = _dest_settings
+    if dest_backend_conf:
+        dest_settings = json.loads(open(dest_backend_conf).read())
+
+    if file_uid.isnumeric():
+        file_store_db_obj = FileStore().query() \
+            .filter(FileStore.file_store_id == file_uid) \
+            .scalar()
+    else:
+        file_store_db_obj = FileStore().query() \
+            .filter(FileStore.file_uid == file_uid) \
+            .scalar()
+    if not file_store_db_obj:
+        click.secho(f'ERROR: Unable to fetch artifact from database file_uid={file_uid}',
+                    fg='red')
+        sys.exit(-1)
+
+    uid_filename = file_store_db_obj.file_uid
+    org_filename = file_store_db_obj.file_display_name
+    click.secho(f'Attempting to migrate artifact {uid_filename}, filename: {org_filename}', fg='green')
+
+    # get old version of f_store based on the data.
+
+    origin_f_store = store_utils.get_filestore_backend(source_settings, always_init=True)
+    reader, metadata = origin_f_store.fetch(uid_filename)
+
+    target_f_store = store_utils.get_filestore_backend(dest_settings, always_init=True)
+    target_f_store.import_to_store(reader, org_filename, uid_filename, metadata)
+
+    click.secho(f'Migrated artifact {uid_filename}, filename: {org_filename} into {target_f_store} storage', fg='green')

rhodecode/model/license.py

@@ -0,0 +1,17 b''
+
+def apply_license(*args, **kwargs):
+    pass
+
+try:
+    from rc_license.models import apply_license
+except ImportError:
+    pass
+
+
+def apply_license_from_file(*args, **kwargs):
+    pass
+
+try:
+    from rc_license.models import apply_license_from_file
+except ImportError:
+    pass

rhodecode/templates/admin/security/security.mako

@@ -0,0 +1,50 b''
+<%inherit file="/base/base.mako"/>
+
+<%def name="title()">
+    ${_('Security Admin')}
+    %if c.rhodecode_name:
+        &middot; ${h.branding(c.rhodecode_name)}
+    %endif
+</%def>
+
+<%def name="breadcrumbs_links()"></%def>
+
+<%def name="menu_bar_nav()">
+    ${self.menu_items(active='admin')}
+</%def>
+
+<%def name="menu_bar_subnav()">
+    ${self.admin_menu(active='security')}
+</%def>
+
+<%def name="main()">
+
+<div class="box">
+
+<!--    <div class="panel panel-default">-->
+<!--        <div class="panel-heading">-->
+<!--            <h3 class="panel-title">${_('Security Audit')}</h3>-->
+<!--        </div>-->
+<!--        <div class="panel-body">-->
+<!--            <h4>${_('This feature is available in RhodeCode EE edition only. Contact {sales_email} to obtain a trial license.').format(sales_email='<a href="mailto:[e-mail]">[e-mail]</a>')|n}</h4>-->
+<!--            <p>-->
+<!--                ${_('You can scan your repositories for exposed secrets, passwords, etc')}-->
+<!--            </p>-->
+<!--        </div>-->
+<!--    </div>-->
+
+    <div class="panel panel-default">
+        <div class="panel-heading">
+            <h3 class="panel-title">${_('Allowed client versions')}</h3>
+        </div>
+        <div class="panel-body">
+            <h4>${_('This feature is available in RhodeCode EE edition only. Contact {sales_email} to obtain a trial license.').format(sales_email='<a href="mailto:[e-mail]">[e-mail]</a>')|n}</h4>
+            <p>
+                ${_('Some outdated client versions may have security vulnerabilities. This section have rules for whitelisting versions of clients for Git, Mercurial and SVN.')}
+            </p>
+    </div>
+
+
+</div>
+
+</%def>

.bumpversion.cfg

@@ -1,5 +1,5 b''
 [bumpversion]
-current_version = 5.1.2
+current_version = 5.2.0
 message = release: Bump version {current_version} to {new_version}
 
 [bumpversion:file:rhodecode/VERSION]

.hgignore

@@ -1,71 +1,71 b''
 syntax: glob
 
 *.egg
 *.egg-info
 *.idea
 *.orig
 *.pyc
 *.sqlite-journal
 *.swp
 *.tox
 *.DS_Store*
 rhodecode/public/js/src/components/**/*.css
 
 syntax: regexp
 
 #.filename
 ^\.settings$
 ^\.project$
 ^\.pydevproject$
 ^\.coverage$
 ^\.cache.*$
 ^\.ruff_cache.*$
 ^\.rhodecode$
 
 ^rcextensions
 ^.dev
 ^._dev
 ^build/
 ^coverage\.xml$
 ^data$
 ^\.eggs/
 ^configs/data$
 ^dev.ini$
 ^acceptance_tests/dev.*\.ini$
 ^dist/
 ^fabfile.py
 ^htmlcov
 ^junit\.xml$
 ^node_modules/
 ^node_binaries/
 ^pylint.log$
 ^rcextensions/
 ^result$
 ^rhodecode/public/css/style.css$
 ^rhodecode/public/css/style-polymer.css$
 ^rhodecode/public/css/style-ipython.css$
 ^rhodecode/public/js/rhodecode-components.html$
 ^rhodecode/public/js/rhodecode-components.js$
 ^rhodecode/public/js/scripts.js$
 ^rhodecode/public/js/scripts.min.js$
 ^rhodecode/public/js/src/components/root-styles.gen.html$
 ^rhodecode/public/js/vendors/webcomponentsjs/
 ^rhodecode\.db$
 ^rhodecode\.log$
 ^rhodecode_dev\.log$
 ^test\.db$
-
+^venv/
 
 # ac-tests
 ^acceptance_tests/\.cache.*$
 ^acceptance_tests/externals
 ^acceptance_tests/ghostdriver.log$
 ^acceptance_tests/local(_.+)?\.ini$
 
 # docs
 ^docs/_build$
 ^docs/result$
 ^docs-internal/_build$
 
 # Cythonized things
 ^rhodecode/.*\.(c|so)$

Makefile

@@ -1,172 +1,158 b''
+.DEFAULT_GOAL := help
+
+# Pretty print values cf. https://misc.flogisoft.com/bash/tip_colors_and_formatting
+RESET := \033[0m # Reset all formatting
+GREEN := \033[0;32m # Resets before setting 16b colour (32 -- green)
+YELLOW := \033[0;33m
+ORANGE := \033[0;38;5;208m # Reset then set 256b colour (208 -- orange)
+PEACH := \033[0;38;5;216m
+
+
+## ---------------------------------------------------------------------------------- ##
+## ------------------------- Help usage builder ------------------------------------- ##
+## ---------------------------------------------------------------------------------- ##
+# use '# >>> Build commands' to create section
+# use '# target: target description' to create help for target
+.PHONY: help
+help:
+	@echo "Usage:"
+	@cat $(MAKEFILE_LIST) | grep -E '^# >>>|^# [A-Za-z0-9_.-]+:' | sed -E 's/^# //' | awk ' \
+		BEGIN { \
+			green="\033[32m"; \
+			yellow="\033[33m"; \
+			reset="\033[0m"; \
+			section=""; \
+		} \
+		/^>>>/ { \
+			section=substr($$0, 5); \
+			printf "\n" green ">>> %s" reset "\n", section; \
+			next; \
+		} \
+		/^([A-Za-z0-9_.-]+):/ { \
+			target=$$1; \
+			gsub(/:$$/, "", target); \
+			description=substr($$0, index($$0, ":") + 2); \
+			if (description == "") { description="-"; } \
+			printf " - " yellow "%-35s" reset " %s\n", target, description; \
+		} \
+	'
+
 # required for pushd to work..
 SHELL = /bin/bash
 
-
-# set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
-OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
+# >>> Tests commands
 
 .PHONY: clean
-## Cleanup compiled and cache py files
+# clean: Cleanup compiled and cache py files
 clean:
 	make test-clean
 	find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
 	find . -type d -name "build" -prune -exec rm -rf '{}' ';'
 
 
 .PHONY: test
-## run test-clean and tests
+# test: run test-clean and tests
 test:
 	make test-clean
-	make test-only
+	unset RC_SQLALCHEMY_DB1_URL && unset RC_DB_URL && make test-only
 
 
 .PHONY: test-clean
-## run test-clean and tests
+# test-clean: run test-clean and tests
 test-clean:
 	rm -rf coverage.xml htmlcov junit.xml pylint.log result
 	find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
 	find . -type f \( -iname '.coverage.*' \) -exec rm '{}' ';'
 
 
 .PHONY: test-only
-## Run tests only without cleanup
+# test-only: Run tests only without cleanup
 test-only:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar \
 	--cov-report=term-missing --cov-report=html \
 	--cov=rhodecode rhodecode
 
+# >>> Docs commands
 
 .PHONY: docs
-## build docs
+# docs: build docs
 docs:
 	(cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make clean html SPHINXOPTS="-W")
 
 
 .PHONY: docs-clean
-## Cleanup docs
+# docs-clean: Cleanup docs
 docs-clean:
 	(cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make clean)
 
 
 .PHONY: docs-cleanup
-## Cleanup docs
+# docs-cleanup: Cleanup docs
 docs-cleanup:
 	(cd docs; docker run --rm -v $(PWD):/project --workdir=/project/docs sphinx-doc-build-rc make cleanup)
 
+# >>> Dev commands
 
 .PHONY: web-build
-## Build JS packages static/js
+# web-build: Build JS packages static/js
 web-build:
 	rm -rf node_modules
 	docker run -it --rm -v $(PWD):/project --workdir=/project rhodecode/static-files-build:16 -c "npm install && /project/node_modules/.bin/grunt"
 	# run static file check
 	./rhodecode/tests/scripts/static-file-check.sh rhodecode/public/
 	rm -rf node_modules
 
-.PHONY: ruff-check
-## run a ruff analysis
-ruff-check:
-	ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs  --exclude .dev .
-
-.PHONY: pip-packages
-## Show outdated packages
-pip-packages:
-	python ${OUTDATED_PACKAGES}
-
-
-.PHONY: build
-## Build sdist/egg
-build:
-	python -m build
-
 
 .PHONY: dev-sh
-## make dev-sh
+# dev-sh: make dev-sh
 dev-sh:
 	sudo echo "deb [trusted=yes] https://apt.fury.io/rsteube/ /" | sudo tee -a "/etc/apt/sources.list.d/fury.list"
 	sudo apt-get update
 	sudo apt-get install -y zsh carapace-bin
 	rm -rf /home/rhodecode/.oh-my-zsh
 	curl https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh | sh
 	@echo "source <(carapace _carapace)" > /home/rhodecode/.zsrc
 	@echo "${RC_DEV_CMD_HELP}"
 	@PROMPT='%(?.%F{green}√.%F{red}?%?)%f %B%F{240}%1~%f%b %# ' zsh
 
 
 .PHONY: dev-cleanup
-## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
+# dev-cleanup: Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
 dev-cleanup:
 	pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
 	rm -rf /tmp/*
 
 
 .PHONY: dev-env
-## make dev-env based on the requirements files and install develop of packages
+# dev-env: make dev-env based on the requirements files and install develop of packages
 ## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
 dev-env:
 	sudo -u root chown rhodecode:rhodecode /home/rhodecode/.cache/pip/
 	pip install build virtualenv
 	pushd ../rhodecode-vcsserver/ && make dev-env && popd
 	pip wheel --wheel-dir=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_rc_tools.txt -r requirements_test.txt -r requirements_debug.txt
 	pip install --no-index --find-links=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_rc_tools.txt -r requirements_test.txt -r requirements_debug.txt
 	pip install -e .
 
 
 .PHONY: sh
-## shortcut for make dev-sh dev-env
+# sh: shortcut for make dev-sh dev-env
 sh:
 	make dev-env
 	make dev-sh
 
 
 ## Allows changes of workers e.g make dev-srv-g workers=2
 workers?=1
 
 .PHONY: dev-srv
-## run gunicorn web server with reloader, use workers=N to set multiworker mode
+# dev-srv: run gunicorn web server with reloader, use workers=N to set multiworker mode, workers=N allows changes of workers
 dev-srv:
 	gunicorn --paste=.dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py --timeout=120 --reload --workers=$(workers)
 
-
-# Default command on calling make
-.DEFAULT_GOAL := show-help
+.PHONY: ruff-check
+# ruff-check: run a ruff analysis
+ruff-check:
+	ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs  --exclude .dev .
 
-.PHONY: show-help
-show-help:
-	@echo "$$(tput bold)Available rules:$$(tput sgr0)"
-	@echo
-	@sed -n -e "/^## / { \
-		h; \
-		s/.*//; \
-		:doc" \
-		-e "H; \
-		n; \
-		s/^## //; \
-		t doc" \
-		-e "s/:.*//; \
-		G; \
-		s/\\n## /---/; \
-		s/\\n/ /g; \
-		p; \
-	}" ${MAKEFILE_LIST} \
-	| LC_ALL='C' sort --ignore-case \
-	| awk -F '---' \
-		-v ncol=$$(tput cols) \
-		-v indent=19 \
-		-v col_on="$$(tput setaf 6)" \
-		-v col_off="$$(tput sgr0)" \
-	'{ \
-		printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
-		n = split($$2, words, " "); \
-		line_length = ncol - indent; \
-		for (i = 1; i <= n; i++) { \
-			line_length -= length(words[i]) + 1; \
-			if (line_length <= 0) { \
-				line_length = ncol - indent - length(words[i]) - 1; \
-				printf "\n%*s ", -indent, " "; \
-			} \
-			printf "%s ", words[i]; \
-		} \
-		printf "\n"; \
-	}'

configs/development.ini

@@ -1,856 +1,912 b''
 
 ; #########################################
 ; RHODECODE COMMUNITY EDITION CONFIGURATION
 ; #########################################
 
 [DEFAULT]
 ; Debug flag sets all loggers to debug, and enables request tracking
 debug = true
 
 ; ########################################################################
 ; EMAIL CONFIGURATION
 ; These settings will be used by the RhodeCode mailing system
 ; ########################################################################
 
 ; prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
 ; email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 
 [server:main]
 ; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
 ; Host port for gunicorn are controlled by gunicorn_conf.py
 host = 127.0.0.1
 port = 10020
 
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
 ; run with gunicorn  --config gunicorn_conf.py --paste rhodecode.ini
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
 
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.
 ; allows to set RhodeCode under a prefix in server.
 ; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ; And set your prefix like: `prefix = /custom_prefix`
 ; be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ; to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
 ; Each option in the app:main can be override by an environmental variable
 ;
 ;To override an option:
 ;
 ;RC_<KeyName>
 ;Everything should be uppercase, . and - should be replaced by _.
 ;For example, if you have these configuration settings:
 ;rc_cache.repo_object.backend = foo
 ;can be overridden by
 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
 
 use = egg:rhodecode-enterprise-ce
 
 ; enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 ; #############
 ; DEBUG OPTIONS
 ; #############
 
 pyramid.reload_templates = true
 
 # During development the we want to have the debug toolbar enabled
 pyramid.includes =
     pyramid_debugtoolbar
 
 debugtoolbar.hosts = 0.0.0.0/0
 debugtoolbar.exclude_prefixes =
     /css
     /fonts
     /images
     /js
 
 ## RHODECODE PLUGINS ##
 rhodecode.includes =
     rhodecode.api
 
 
 # api prefix url
 rhodecode.api.url = /_admin/api
 
 ; enable debug style page
 debug_style = true
 
 ; #################
 ; END DEBUG OPTIONS
 ; #################
 
 ; encryption key used to encrypt social plugin tokens,
 ; remote_urls with credentials etc, if not set it defaults to
 ; `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
 ; decryption strict mode (enabled by default). It controls if decryption raises
 ; `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
 ; Pick algorithm for encryption. Either fernet (more secure) or aes (default)
 ; fernet is safer, and we strongly recommend switching to it.
 ; Due to backward compatibility aes is used as default.
 #rhodecode.encrypted_values.algorithm = fernet
 
 ; Return gzipped responses from RhodeCode (static files/application)
 gzip_responses = false
 
 ; Auto-generate javascript routes file on startup
 generate_js_files = false
 
 ; System global default language.
 ; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ; Perform a full repository scan and import on each server start.
 ; Settings this to true could lead to very long startup time.
 startup.import_repos = false
 
 ; URL at which the application is running. This is used for Bootstrapping
 ; requests in context when no web request is available. Used in ishell, or
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
 ; Host at which the Service API is running.
 app.service_api.host = http://rhodecode.local:10020
 
 ; Secret for Service API authentication.
 app.service_api.token =
 
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
 ; Cut off limit for large diffs (size in bytes). If overall diff size on
 ; commit, or pull request exceeds this limit this diff will be displayed
 ; partially. E.g 512000 == 512Kb
 cut_off_limit_diff = 512000
 
 ; Cut off limit for large files inside diffs (size in bytes). Each individual
 ; file inside diff which exceeds this limit will be displayed partially.
 ;  E.g 128000 == 128Kb
 cut_off_limit_file = 128000
 
 ; Use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = true
 
 ; Force https in RhodeCode, fixes https redirects, assumes it's always https.
 ; Normally this is controlled by proper flags sent from http server such as Nginx or Apache
 force_https = false
 
 ; use Strict-Transport-Security headers
 use_htsts = false
 
 ; Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ; RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ; gist URL alias, used to create nicer urls for gist. This should be an
 ; url that does rewrites to _admin/gists/{gistid}.
 ; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ; List of views (using glob pattern syntax) that AUTH TOKENS could be
 ; used for access.
 ; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ; came from the the logged in user who own this authentication token.
 ; Additionally @TOKEN syntax can be used to bound the view to specific
 ; authentication token. Such view would be only accessible when used together
 ; with this authentication token
 ; list of all views can be found under `/_admin/permissions/auth_token_access`
 ; The list should be "," separated and on a single line.
 ; Most common views to enable:
 
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
 ; Default encoding used to convert from and to unicode
 ; can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ; instance-id prefix
 ; a prefix key for this instance used for cache invalidation when running
 ; multiple instances of RhodeCode, make sure it's globally unique for
 ; all running RhodeCode instances. Leave empty if you don't use it
 instance_id =
 
 ; Fallback authentication plugin. Set this to a plugin ID to force the usage
 ; of an authentication plugin also if it is disabled by it's settings.
 ; This could be useful if you are unable to log in to the system due to broken
 ; authentication settings. Then you can enable e.g. the internal RhodeCode auth
 ; module to log in again and fix the settings.
 ; Available builtin plugin IDs (hash is part of the ID):
 ; egg:rhodecode-enterprise-ce#rhodecode
 ; egg:rhodecode-enterprise-ce#pam
 ; egg:rhodecode-enterprise-ce#ldap
 ; egg:rhodecode-enterprise-ce#jasig_cas
 ; egg:rhodecode-enterprise-ce#headers
 ; egg:rhodecode-enterprise-ce#crowd
 
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
 ; Flag to control loading of legacy plugins in py:/path format
 auth_plugin.import_legacy_plugins = true
 
 ; alternative return HTTP header for failed authentication. Default HTTP
 ; response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ; handling that causing a series of failed authentication calls.
 ; Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ; This will be served instead of default 401 on bad authentication
 auth_ret_code =
 
 ; use special detection method when serving auth_ret_code, instead of serving
 ; ret_code directly, use 401 initially (Which triggers credentials prompt)
 ; and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ; locking return code. When repository is locked return this HTTP code. 2XX
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ; Filesystem location were repositories should be stored
 repo_store.path = /var/opt/rhodecode_repo_store
 
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ; Generated license token required for EE edition license.
 ; New generated token value can be found in Admin > settings > license page.
 license_token =
 
 ; This flag hides sensitive information on the license page such as token, and license data
 license.hide_license_info = false
 
+; Import EE license from this license path
+#license.import_path = %(here)s/rhodecode_enterprise.license
+
+; import license 'if-missing' or 'force' (always override)
+; if-missing means apply license if it doesn't exist. 'force' option always overrides it
+license.import_path_mode = if-missing
+
 ; supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 
 ; supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = dev
 
 ; Display extended labs settings
 labs_settings_active = true
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ; Send email with exception details when it happens
 #exception_tracker.send_email = false
 
 ; Comma separated list of recipients for exception emails,
 ; e.g [e-mail],[e-mail]
 ; Can be left empty, then emails will be sent to ALL super-admins
 #exception_tracker.send_email_recipients =
 
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
-; File store configuration. This is used to store and serve uploaded files
-file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store backend is always enabled
+#file_store.enabled = true
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead
 ; Storage backend, available options are: local
-file_store.backend = local
+#file_store.backend = local
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead
 ; path to store the uploaded binaries and artifacts
-file_store.storage_path = /var/opt/rhodecode_data/file_store
+#file_store.storage_path = /var/opt/rhodecode_data/file_store
+
+; Artifacts file-store, is used to store comment attachments and artifacts uploads.
+; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options
+; filesystem_v1 is backwards compat with pre 5.1 storage changes
+; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from
+; previous installations to keep the artifacts without a need of migration
+#file_store.backend.type = filesystem_v2
+
+; filesystem options...
+#file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+
+; filesystem_v2 options...
+#file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+#file_store.filesystem_v2.shards = 8
 
+; objectstore options...
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+#file_store.backend.type = objectstore
+#file_store.objectstore.url = http://s3-minio:9000
+
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number
+#file_store.objectstore.bucket = rhodecode-file-store
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+#file_store.objectstore.bucket_shards = 8
+
+; key for s3 auth
+#file_store.objectstore.key = s3admin
+
+; secret for s3 auth
+#file_store.objectstore.secret = s3secret4
+
+;region for s3 storage
+#file_store.objectstore.region = eu-central-1
 
 ; Redis url to acquire/check generation of archives locks
 archive_cache.locking.url = redis://redis:6379/1
 
 ; Storage backend, only 'filesystem' and 'objectstore' are available now
 archive_cache.backend.type = filesystem
 
 ; url for s3 compatible storage that allows to upload artifacts
 ; e.g http://minio:9000
 archive_cache.objectstore.url = http://s3-minio:9000
 
 ; key for s3 auth
 archive_cache.objectstore.key = key
 
 ; secret for s3 auth
 archive_cache.objectstore.secret = secret
 
 ;region for s3 storage
 archive_cache.objectstore.region = eu-central-1
 
 ; number of sharded buckets to create to distribute archives across
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8
 
 ; a top-level bucket to put all other shards in
 ; objects will be stored in rhodecode-archive-cache/shard-N based on the bucket_shards number
 archive_cache.objectstore.bucket = rhodecode-archive-cache
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.objectstore.retry = false
 
 ; number of seconds to wait for next try using retry
 archive_cache.objectstore.retry_backoff = 1
 
 ; how many tries do do a retry fetch from this backend
 archive_cache.objectstore.retry_attempts = 10
 
 ; Default is $cache_dir/archive_cache if not set
 ; Generated repo archives will be cached at this location
 ; and served from the cache during subsequent requests for the same archive of
 ; the repository. This path is important to be shared across filesystems and with
 ; RhodeCode and vcsserver
 archive_cache.filesystem.store_dir = /var/opt/rhodecode_data/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
 archive_cache.filesystem.cache_size_gb = 1
 
 ; Eviction policy used to clear out after cache_size_gb limit is reached
 archive_cache.filesystem.eviction_policy = least-recently-stored
 
 ; By default cache uses sharding technique, this specifies how many shards are there
 ; default is 8 shards
 archive_cache.filesystem.cache_shards = 8
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.filesystem.retry = false
 
 ; number of seconds to wait for next try using retry
 archive_cache.filesystem.retry_backoff = 1
 
 ; how many tries do do a retry fetch from this backend
 archive_cache.filesystem.retry_attempts = 10
 
 
 ; #############
 ; CELERY CONFIG
 ; #############
 
 ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
 
 use_celery = true
 
 ; path to store schedule database
 #celerybeat-schedule.path =
 
 ; connection url to the message broker (default redis)
 celery.broker_url = redis://redis:6379/8
 
 ; results backend to get results for (default redis)
 celery.result_backend = redis://redis:6379/8
 
 ; rabbitmq example
 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
 ; maximum tasks to execute before worker restart
 celery.max_tasks_per_child = 20
 
 ; tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = false
 
 ; #############
 ; DOGPILE CACHE
 ; #############
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
 cache_dir = /var/opt/rhodecode_data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
 ; Only supported backend is `memory_lru`
 ; *********************************************
 rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
 rc_cache.sql_cache_short.expiration_time = 30
 
 
 ; *****************************************************
 ; `cache_repo_longterm` cache for repo object instances
 ; Only supported backend is `memory_lru`
 ; *****************************************************
 rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
 ; by default we use 30 Days, cache is still invalidated on push
 rc_cache.cache_repo_longterm.expiration_time = 2592000
 ; max items in LRU cache, set to smaller number to save memory, and expire last used caches
 rc_cache.cache_repo_longterm.max_size = 10000
 
 
 ; *********************************************
 ; `cache_general` cache for general purpose use
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *********************************************
 rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_general.expiration_time = 43200
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_general.arguments.filename = /tmp/cache_general_db
 
 ; alternative `cache_general` redis backend with distributed lock
 #rc_cache.cache_general.backend = dogpile.cache.rc.redis
 #rc_cache.cache_general.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_general.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_general.arguments.host = localhost
 #rc_cache.cache_general.arguments.port = 6379
 #rc_cache.cache_general.arguments.db = 0
 #rc_cache.cache_general.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_general.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_general.arguments.lock_auto_renewal = true
 
 ; *************************************************
 ; `cache_perms` cache for permission tree, auth TTL
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *************************************************
 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_perms.expiration_time = 3600
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_perms.arguments.filename = /tmp/cache_perms_db
 
 ; alternative `cache_perms` redis backend with distributed lock
 #rc_cache.cache_perms.backend = dogpile.cache.rc.redis
 #rc_cache.cache_perms.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_perms.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_perms.arguments.host = localhost
 #rc_cache.cache_perms.arguments.port = 6379
 #rc_cache.cache_perms.arguments.db = 0
 #rc_cache.cache_perms.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_perms.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_perms.arguments.lock_auto_renewal = true
 
 ; ***************************************************
 ; `cache_repo` cache for file tree, Readme, RSS FEEDS
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; ***************************************************
 rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_repo.expiration_time = 2592000
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_repo.arguments.filename = /tmp/cache_repo_db
 
 ; alternative `cache_repo` redis backend with distributed lock
 #rc_cache.cache_repo.backend = dogpile.cache.rc.redis
 #rc_cache.cache_repo.expiration_time = 2592000
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
 
 #rc_cache.cache_repo.arguments.host = localhost
 #rc_cache.cache_repo.arguments.port = 6379
 #rc_cache.cache_repo.arguments.db = 1
 #rc_cache.cache_repo.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_repo.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_repo.arguments.lock_auto_renewal = true
 
 ; ##############
 ; BEAKER SESSION
 ; ##############
 
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
 ; types are file, ext:redis, ext:database, ext:memcached
 ; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session
 #beaker.session.type = file
 #beaker.session.data_dir = %(here)s/data/sessions
 
 ; Redis based sessions
 beaker.session.type = ext:redis
 beaker.session.url = redis://redis:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = develop-rc-uytcxaz
 beaker.session.lock_dir = /data_ramdisk/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
 ; Sets session as invalid (also logging out user) if it haven not been
 ; accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 
 ; Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
 ; Set https secure cookie
 beaker.session.secure = false
 
 ; default cookie expiration time in seconds, set to `true` to set expire
 ; at browser close
 #beaker.session.cookie_expires = 3600
 
 ; #############################
 ; SEARCH INDEXING CONFIGURATION
 ; #############################
 
 ; Full text search indexer is available in rhodecode-tools under
 ; `rhodecode-tools index` command
 
 ; WHOOSH Backend, doesn't require additional services to run
 ; it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
 ; ####################
 ; CHANNELSTREAM CONFIG
 ; ####################
 
 ; channelstream enables persistent connections and live notification
 ; in the system. It's also used by the chat system
 
 channelstream.enabled = true
 
 ; server address for channelstream server on the backend
 channelstream.server = channelstream:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = ENV_GENERATED
 channelstream.history.location = /var/opt/rhodecode_data/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
 ; ##############################
 ; MAIN RHODECODE DATABASE CONFIG
 ; ##############################
 
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8
 ; pymysql is an alternative driver for MySQL, use in case of problems with default one
 #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
 sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 
 ; see sqlalchemy docs for other advanced settings
 ; print the sql statements to output
 sqlalchemy.db1.echo = false
 
 ; recycle the connections after this amount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 
 ; the number of connections to keep open inside the connection pool.
 ; 0 indicates no limit
 ; the general calculus with gevent is:
 ; if your system allows 500 concurrent greenlets (max_connections) that all do database access,
 ; then increase pool size + max overflow so that they add up to 500.
 #sqlalchemy.db1.pool_size = 5
 
 ; The number of connections to allow in connection pool "overflow", that is
 ; connections that can be opened above and beyond the pool_size setting,
 ; which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 ; Connection check ping, used to detect broken database connections
 ; could be enabled to better handle cases if MySQL has gone away errors
 #sqlalchemy.db1.ping_connection = true
 
 ; ##########
 ; VCS CONFIG
 ; ##########
 vcs.server.enable = true
 vcs.server = vcsserver:10010
 
 ; Web server connectivity protocol, responsible for web based VCS operations
 ; Available protocols are:
 ; `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
 ; Push/Pull operations protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
 ; `celery` - use celery based hooks
-vcs.hooks.protocol = http
+#DEPRECATED:vcs.hooks.protocol = http
+vcs.hooks.protocol.v2 = celery
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
 ; accessible via network.
 ; Use vcs.hooks.host = "*" to bind to current hostname (for Docker)
 vcs.hooks.host = *
 
 ; Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
 ; List of enabled VCS backends, available options are:
 ; `hg`  - mercurial
 ; `git` - git
 ; `svn` - subversion
 vcs.backends = hg, git, svn
 
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
 ; Cache flag to cache vcsserver remote calls locally
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = true
 
+; Filesystem location where Git lfs objects should be stored
+vcs.git.lfs.storage_location = /var/opt/rhodecode_repo_store/.cache/git_lfs_store
+
+; Filesystem location where Mercurial largefile objects should be stored
+vcs.hg.largefiles.storage_location = /var/opt/rhodecode_repo_store/.cache/hg_largefiles_store
+
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
 ; Maps RhodeCode repo groups into SVN paths for Apache
 ; ####################################################
 
 ; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ; Set a numeric version for your current SVN e.g 1.8, or 1.12
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
 ; Redis connection settings for svn integrations logic
 ; This connection string needs to be the same on ce and vcsserver
 vcs.svn.redis_conn = redis://redis:6379/0
 
 ; Enable SVN proxy of requests over HTTP
 vcs.svn.proxy.enabled = true
 
 ; host to connect to running SVN subsystem
 vcs.svn.proxy.host = http://svn:8090
 
 ; Enable or disable the config file generation.
 svn.proxy.generate_config = true
 
 ; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
 svn.proxy.config_file_path = /etc/rhodecode/conf/svn/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
 ; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako
 #svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako
 
 ; Used as a prefix to the `Location` block in the generated config file.
 ; In most cases it should be set to `/`.
 svn.proxy.location_root = /
 
 ; Command to reload the mod dav svn configuration on change.
 ; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh
 ; Make sure user who runs RhodeCode process is allowed to reload Apache
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
 
 ; If the timeout expires before the reload command finishes, the command will
 ; be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
 ; ####################
 ; SSH Support Settings
 ; ####################
 
 ; Defines if a custom authorized_keys file should be created and written on
 ; any change user ssh keys. Setting this to false also disables possibility
 ; of adding SSH keys by users from web interface. Super admins can still
 ; manage SSH Keys.
 ssh.generate_authorized_keyfile = true
 
 ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
 ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
-ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+#DEPRECATED: ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+ssh.wrapper_cmd.v2 = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
 ; Enables logging, and detailed output send back to the client during SSH
 ; operations. Useful for debugging, shouldn't be used in production.
 ssh.enable_debug_logging = true
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
 ssh.executable.hg = /usr/local/bin/rhodecode_bin/vcs_bin/hg
 ssh.executable.git = /usr/local/bin/rhodecode_bin/vcs_bin/git
 ssh.executable.svn = /usr/local/bin/rhodecode_bin/vcs_bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.
 ssh.enable_ui_key_generator = true
 
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Prometheus
 #statsd.enabled = false
 #statsd.statsd_host = 0.0.0.0
 #statsd.statsd_port = 8125
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
 ; configure logging automatically at server startup set to false
 ; to use the below custom logging config.
 ; RC_LOGGING_FORMATTER
 ; RC_LOGGING_LEVEL
 ; env variables can control the settings for logging in case of autoconfigure
 
 #logging.autoconfigure = true
 
 ; specify your own custom logging config file to configure logging
 #logging.logging_conf_file = /path/to/custom_logging.ini
 
 ; Dummy marker to add new entries after.
 ; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1
 
 
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
 
 [loggers]
 keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, json, color_formatter, color_formatter_sql
 
 ; #######
 ; LOGGERS
 ; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
 [logger_celery]
 level = DEBUG
 handlers =
 qualname = celery
 
 
 ; ########
 ; HANDLERS
 ; ########
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr, )
 level = DEBUG
 ; To enable JSON formatted logs replace 'generic/color_formatter' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = color_formatter
 
 [handler_console_sql]
 ; "level = DEBUG" logs SQL queries and results.
 ; "level = INFO" logs SQL queries.
 ; "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
 args = (sys.stderr, )
 level = WARN
 ; To enable JSON formatted logs replace 'generic/color_formatter_sql' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = color_formatter_sql
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_json]
 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
 class = rhodecode.lib._vendor.jsonlogger.JsonFormatter

configs/gunicorn_config.py

@@ -1,520 +1,545 b''
 """
 Gunicorn config extension and hooks. This config file adds some extra settings and memory management.
 Gunicorn configuration should be managed by .ini files entries of RhodeCode or VCSServer
 """
 
 import gc
 import os
 import sys
 import math
 import time
 import threading
 import traceback
 import random
 import socket
 import dataclasses
+import json
 from gunicorn.glogging import Logger
 
 
 def get_workers():
     import multiprocessing
     return multiprocessing.cpu_count() * 2 + 1
 
 
 bind = "127.0.0.1:10020"
 
 
 # Error logging output for gunicorn (-) is stdout
 errorlog = '-'
 
 # Access logging output for gunicorn (-) is stdout
 accesslog = '-'
 
 
 # SERVER MECHANICS
 # None == system temp dir
 # worker_tmp_dir is recommended to be set to some tmpfs
 worker_tmp_dir = None
 tmp_upload_dir = None
 
-# use re-use port logic
-#reuse_port = True
+# use re-use port logic to let linux internals load-balance the requests better.
+reuse_port = True
 
 # Custom log format
 #access_log_format = (
 #    '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
 
 # loki format for easier parsing in grafana
-access_log_format = (
+loki_access_log_format = (
     'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
 
+# JSON format
+json_access_log_format = json.dumps({
+    'time': r'%(t)s',
+    'pid': r'%(p)s',
+    'level': 'INFO',
+    'ip': r'%(h)s',
+    'request_time': r'%(L)s',
+    'remote_address': r'%(h)s',
+    'user_name': r'%(u)s',
+    'status': r'%(s)s',
+    'method': r'%(m)s',
+    'url_path': r'%(U)s',
+    'query_string': r'%(q)s',
+    'protocol': r'%(H)s',
+    'response_length': r'%(B)s',
+    'referer': r'%(f)s',
+    'user_agent': r'%(a)s',
+
+})
+
+access_log_format = loki_access_log_format
+if os.environ.get('RC_LOGGING_FORMATTER') == 'json':
+    access_log_format = json_access_log_format
+
 # self adjust workers based on CPU count, to use maximum of CPU and not overquota the resources
 # workers = get_workers()
 
 # Gunicorn access log level
 loglevel = 'info'
 
 # Process name visible in a process list
 proc_name = 'rhodecode_enterprise'
 
 # Type of worker class, one of `sync`, `gevent` or `gthread`
 # currently `sync` is the only option allowed for vcsserver and for rhodecode all of 3 are allowed
 # gevent:
 # In this case, the maximum number of concurrent requests is (N workers * X worker_connections)
 # e.g. workers =3 worker_connections=10 = 3*10, 30 concurrent requests can be handled
 # gthread:
 # In this case, the maximum number of concurrent requests is (N workers * X threads)
 # e.g. workers = 3 threads=3 = 3*3, 9 concurrent requests can be handled
 worker_class = 'gthread'
 
 # Sets the number of process workers. More workers means more concurrent connections
 # RhodeCode can handle at the same time. Each additional worker also it increases
 # memory usage as each has its own set of caches.
 # The Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
 # than 8-10 unless for huge deployments .e.g 700-1000 users.
 # `instance_id = *` must be set in the [app:main] section below (which is the default)
 # when using more than 1 worker.
 workers = 2
 
 # Threads numbers for worker class gthread
 threads = 1
 
 # The maximum number of simultaneous clients. Valid only for gevent
 # In this case, the maximum number of concurrent requests is (N workers * X worker_connections)
 # e.g workers =3 worker_connections=10 = 3*10, 30 concurrent requests can be handled
 worker_connections = 10
 
 # Max number of requests that worker will handle before being gracefully restarted.
 # Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
 max_requests = 2000
 max_requests_jitter = int(max_requests * 0.2)  # 20% of max_requests
 
 # The maximum number of pending connections.
 # Exceeding this number results in the client getting an error when attempting to connect.
 backlog = 64
 
 # The Amount of time a worker can spend with handling a request before it
 # gets killed and restarted. By default, set to 21600 (6hrs)
 # Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
 timeout = 21600
 
 # The maximum size of HTTP request line in bytes.
 # 0 for unlimited
 limit_request_line = 0
 
 # Limit the number of HTTP headers fields in a request.
 # By default this value is 100 and can't be larger than 32768.
 limit_request_fields = 32768
 
 # Limit the allowed size of an HTTP request header field.
 # Value is a positive number or 0.
 # Setting it to 0 will allow unlimited header field sizes.
 limit_request_field_size = 0
 
 # Timeout for graceful workers restart.
 # After receiving a restart signal, workers have this much time to finish
 # serving requests. Workers still alive after the timeout (starting from the
 # receipt of the restart signal) are force killed.
 # Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
 graceful_timeout = 21600
 
 # The number of seconds to wait for requests on a Keep-Alive connection.
 # Generally set in the 1-5 seconds range.
 keepalive = 2
 
 # Maximum memory usage that each worker can use before it will receive a
 # graceful restart signal 0 = memory monitoring is disabled
 # Examples: 268435456 (256MB), 536870912 (512MB)
 # 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
 # Dynamic formula 1024 * 1024 * 256 == 256MBs
 memory_max_usage = 0
 
 # How often in seconds to check for memory usage for each gunicorn worker
 memory_usage_check_interval = 60
 
 # Threshold value for which we don't recycle worker if GarbageCollection
 # frees up enough resources. Before each restart, we try to run GC on worker
 # in case we get enough free memory after that; restart will not happen.
 memory_usage_recovery_threshold = 0.8
 
 
 @dataclasses.dataclass
 class MemoryCheckConfig:
     max_usage: int
     check_interval: int
     recovery_threshold: float
 
 
 def _get_process_rss(pid=None):
     try:
         import psutil
         if pid:
             proc = psutil.Process(pid)
         else:
             proc = psutil.Process()
         return proc.memory_info().rss
     except Exception:
         return None
 
 
 def _get_config(ini_path):
     import configparser
 
     try:
         config = configparser.RawConfigParser()
         config.read(ini_path)
         return config
     except Exception:
         return None
 
 
 def get_memory_usage_params(config=None):
     # memory spec defaults
     _memory_max_usage = memory_max_usage
     _memory_usage_check_interval = memory_usage_check_interval
     _memory_usage_recovery_threshold = memory_usage_recovery_threshold
 
     if config:
         ini_path = os.path.abspath(config)
         conf = _get_config(ini_path)
 
         section = 'server:main'
         if conf and conf.has_section(section):
 
             if conf.has_option(section, 'memory_max_usage'):
                 _memory_max_usage = conf.getint(section, 'memory_max_usage')
 
             if conf.has_option(section, 'memory_usage_check_interval'):
                 _memory_usage_check_interval = conf.getint(section, 'memory_usage_check_interval')
 
             if conf.has_option(section, 'memory_usage_recovery_threshold'):
                 _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
 
     _memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
                             or _memory_max_usage)
     _memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
                                        or _memory_usage_check_interval)
     _memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
                                              or _memory_usage_recovery_threshold)
 
     return MemoryCheckConfig(_memory_max_usage, _memory_usage_check_interval, _memory_usage_recovery_threshold)
 
 
 def _time_with_offset(check_interval):
     return time.time() - random.randint(0, check_interval/2.0)
 
 
 def pre_fork(server, worker):
     pass
 
 
 def post_fork(server, worker):
 
     memory_conf = get_memory_usage_params()
     _memory_max_usage = memory_conf.max_usage
     _memory_usage_check_interval = memory_conf.check_interval
     _memory_usage_recovery_threshold = memory_conf.recovery_threshold
 
     worker._memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
                                    or _memory_max_usage)
     worker._memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
                                               or _memory_usage_check_interval)
     worker._memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
                                                     or _memory_usage_recovery_threshold)
 
     # register memory last check time, with some random offset so we don't recycle all
     # at once
     worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
 
     if _memory_max_usage:
         server.log.info("pid=[%-10s] WORKER spawned with max memory set at %s", worker.pid,
                         _format_data_size(_memory_max_usage))
     else:
         server.log.info("pid=[%-10s] WORKER spawned", worker.pid)
 
 
 def pre_exec(server):
     server.log.info("Forked child, re-executing.")
 
 
 def on_starting(server):
     server_lbl = '{} {}'.format(server.proc_name, server.address)
     server.log.info("Server %s is starting.", server_lbl)
     server.log.info('Config:')
     server.log.info(f"\n{server.cfg}")
     server.log.info(get_memory_usage_params())
 
 
 def when_ready(server):
     server.log.info("Server %s is ready. Spawning workers", server)
 
 
 def on_reload(server):
     pass
 
 
 def _format_data_size(size, unit="B", precision=1, binary=True):
     """Format a number using SI units (kilo, mega, etc.).
 
     ``size``: The number as a float or int.
 
     ``unit``: The unit name in plural form. Examples: "bytes", "B".
 
     ``precision``: How many digits to the right of the decimal point. Default
     is 1.  0 suppresses the decimal point.
 
     ``binary``: If false, use base-10 decimal prefixes (kilo = K = 1000).
     If true, use base-2 binary prefixes (kibi = Ki = 1024).
 
     ``full_name``: If false (default), use the prefix abbreviation ("k" or
     "Ki").  If true, use the full prefix ("kilo" or "kibi"). If false,
     use abbreviation ("k" or "Ki").
 
     """
 
     if not binary:
         base = 1000
         multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
     else:
         base = 1024
         multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
 
     sign = ""
     if size > 0:
         m = int(math.log(size, base))
     elif size < 0:
         sign = "-"
         size = -size
         m = int(math.log(size, base))
     else:
         m = 0
     if m > 8:
         m = 8
 
     if m == 0:
         precision = '%.0f'
     else:
         precision = '%%.%df' % precision
 
     size = precision % (size / math.pow(base, m))
 
     return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
 
 
 def _check_memory_usage(worker):
     _memory_max_usage = worker._memory_max_usage
     if not _memory_max_usage:
         return
 
     _memory_usage_check_interval = worker._memory_usage_check_interval
     _memory_usage_recovery_threshold = memory_max_usage * worker._memory_usage_recovery_threshold
 
     elapsed = time.time() - worker._last_memory_check_time
     if elapsed > _memory_usage_check_interval:
         mem_usage = _get_process_rss()
         if mem_usage and mem_usage > _memory_max_usage:
             worker.log.info(
                 "memory usage %s > %s, forcing gc",
                 _format_data_size(mem_usage), _format_data_size(_memory_max_usage))
             # Try to clean it up by forcing a full collection.
             gc.collect()
             mem_usage = _get_process_rss()
             if mem_usage > _memory_usage_recovery_threshold:
                 # Didn't clean up enough, we'll have to terminate.
                 worker.log.warning(
                     "memory usage %s > %s after gc, quitting",
                     _format_data_size(mem_usage), _format_data_size(_memory_max_usage))
                 # This will cause worker to auto-restart itself
                 worker.alive = False
         worker._last_memory_check_time = time.time()
 
 
 def worker_int(worker):
     worker.log.info("pid=[%-10s] worker received INT or QUIT signal", worker.pid)
 
     # get traceback info, when a worker crashes
     def get_thread_id(t_id):
         id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
         return id2name.get(t_id, "unknown_thread_id")
 
     code = []
     for thread_id, stack in sys._current_frames().items():  # noqa
         code.append(
             "\n# Thread: %s(%d)" % (get_thread_id(thread_id), thread_id))
         for fname, lineno, name, line in traceback.extract_stack(stack):
             code.append('File: "%s", line %d, in %s' % (fname, lineno, name))
             if line:
                 code.append("  %s" % (line.strip()))
     worker.log.debug("\n".join(code))
 
 
 def worker_abort(worker):
     worker.log.info("pid=[%-10s] worker received SIGABRT signal", worker.pid)
 
 
 def worker_exit(server, worker):
     worker.log.info("pid=[%-10s] worker exit", worker.pid)
 
 
 def child_exit(server, worker):
     worker.log.info("pid=[%-10s] worker child exit", worker.pid)
 
 
 def pre_request(worker, req):
     worker.start_time = time.time()
     worker.log.debug(
         "GNCRN PRE  WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
 
 
 def post_request(worker, req, environ, resp):
     total_time = time.time() - worker.start_time
     # Gunicorn sometimes has problems with reading the status_code
     status_code = getattr(resp, 'status_code', '')
     worker.log.debug(
         "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
         worker.nr, req.method, req.path, status_code, total_time)
     _check_memory_usage(worker)
 
 
 def _filter_proxy(ip):
     """
     Passed in IP addresses in HEADERS can be in a special format of multiple
     ips. Those comma separated IPs are passed from various proxies in the
     chain of request processing. The left-most being the original client.
     We only care about the first IP which came from the org. client.
 
     :param ip: ip string from headers
     """
     if ',' in ip:
         _ips = ip.split(',')
         _first_ip = _ips[0].strip()
         return _first_ip
     return ip
 
 
 def _filter_port(ip):
     """
     Removes a port from ip, there are 4 main cases to handle here.
     - ipv4 eg. 127.0.0.1
     - ipv6 eg. ::1
     - ipv4+port eg. 127.0.0.1:8080
     - ipv6+port eg. [::1]:8080
 
     :param ip:
     """
     def is_ipv6(ip_addr):
         if hasattr(socket, 'inet_pton'):
             try:
                 socket.inet_pton(socket.AF_INET6, ip_addr)
             except socket.error:
                 return False
         else:
             return False
         return True
 
     if ':' not in ip:  # must be ipv4 pure ip
         return ip
 
     if '[' in ip and ']' in ip:  # ipv6 with port
         return ip.split(']')[0][1:].lower()
 
     # must be ipv6 or ipv4 with port
     if is_ipv6(ip):
         return ip
     else:
         ip, _port = ip.split(':')[:2]  # means ipv4+port
         return ip
 
 
 def get_ip_addr(environ):
     proxy_key = 'HTTP_X_REAL_IP'
     proxy_key2 = 'HTTP_X_FORWARDED_FOR'
     def_key = 'REMOTE_ADDR'
 
     def _filters(x):
         return _filter_port(_filter_proxy(x))
 
     ip = environ.get(proxy_key)
     if ip:
         return _filters(ip)
 
     ip = environ.get(proxy_key2)
     if ip:
         return _filters(ip)
 
     ip = environ.get(def_key, '0.0.0.0')
     return _filters(ip)
 
 
 class RhodeCodeLogger(Logger):
     """
     Custom Logger that allows some customization that gunicorn doesn't allow
     """
 
     datefmt = r"%Y-%m-%d %H:%M:%S"
 
     def __init__(self, cfg):
         Logger.__init__(self, cfg)
 
     def now(self):
         """ return date in RhodeCode Log format """
         now = time.time()
         msecs = int((now - int(now)) * 1000)
         return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
 
     def atoms(self, resp, req, environ, request_time):
         """ Gets atoms for log formatting.
         """
         status = resp.status
         if isinstance(status, str):
             status = status.split(None, 1)[0]
         atoms = {
             'h': get_ip_addr(environ),
             'l': '-',
             'u': self._get_user(environ) or '-',
             't': self.now(),
             'r': "%s %s %s" % (environ['REQUEST_METHOD'],
                                environ['RAW_URI'],
                                environ["SERVER_PROTOCOL"]),
             's': status,
             'm': environ.get('REQUEST_METHOD'),
             'U': environ.get('PATH_INFO'),
             'q': environ.get('QUERY_STRING'),
             'H': environ.get('SERVER_PROTOCOL'),
             'b': getattr(resp, 'sent', None) is not None and str(resp.sent) or '-',
             'B': getattr(resp, 'sent', None),
             'f': environ.get('HTTP_REFERER', '-'),
             'a': environ.get('HTTP_USER_AGENT', '-'),
             'T': request_time.seconds,
             'D': (request_time.seconds * 1000000) + request_time.microseconds,
             'M': (request_time.seconds * 1000) + int(request_time.microseconds/1000),
             'L': "%d.%06d" % (request_time.seconds, request_time.microseconds),
             'p': "<%s>" % os.getpid()
         }
 
         # add request headers
         if hasattr(req, 'headers'):
             req_headers = req.headers
         else:
             req_headers = req
 
         if hasattr(req_headers, "items"):
             req_headers = req_headers.items()
 
         atoms.update({"{%s}i" % k.lower(): v for k, v in req_headers})
 
         resp_headers = resp.headers
         if hasattr(resp_headers, "items"):
             resp_headers = resp_headers.items()
 
         # add response headers
         atoms.update({"{%s}o" % k.lower(): v for k, v in resp_headers})
 
         # add environ variables
         environ_variables = environ.items()
         atoms.update({"{%s}e" % k.lower(): v for k, v in environ_variables})
 
         return atoms
 
 
 logger_class = RhodeCodeLogger

configs/production.ini

@@ -1,824 +1,880 b''
 
 ; #########################################
 ; RHODECODE COMMUNITY EDITION CONFIGURATION
 ; #########################################
 
 [DEFAULT]
 ; Debug flag sets all loggers to debug, and enables request tracking
 debug = false
 
 ; ########################################################################
 ; EMAIL CONFIGURATION
 ; These settings will be used by the RhodeCode mailing system
 ; ########################################################################
 
 ; prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
 ; email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 
 [server:main]
 ; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
 ; Host port for gunicorn are controlled by gunicorn_conf.py
 host = 127.0.0.1
 port = 10020
 
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
 ; run with gunicorn  --config gunicorn_conf.py --paste rhodecode.ini
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
 
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.
 ; allows to set RhodeCode under a prefix in server.
 ; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ; And set your prefix like: `prefix = /custom_prefix`
 ; be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ; to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
 ; Each option in the app:main can be override by an environmental variable
 ;
 ;To override an option:
 ;
 ;RC_<KeyName>
 ;Everything should be uppercase, . and - should be replaced by _.
 ;For example, if you have these configuration settings:
 ;rc_cache.repo_object.backend = foo
 ;can be overridden by
 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
 
 use = egg:rhodecode-enterprise-ce
 
 ; enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 ; encryption key used to encrypt social plugin tokens,
 ; remote_urls with credentials etc, if not set it defaults to
 ; `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
 ; decryption strict mode (enabled by default). It controls if decryption raises
 ; `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
 ; Pick algorithm for encryption. Either fernet (more secure) or aes (default)
 ; fernet is safer, and we strongly recommend switching to it.
 ; Due to backward compatibility aes is used as default.
 #rhodecode.encrypted_values.algorithm = fernet
 
 ; Return gzipped responses from RhodeCode (static files/application)
 gzip_responses = false
 
 ; Auto-generate javascript routes file on startup
 generate_js_files = false
 
 ; System global default language.
 ; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ; Perform a full repository scan and import on each server start.
 ; Settings this to true could lead to very long startup time.
 startup.import_repos = false
 
 ; URL at which the application is running. This is used for Bootstrapping
 ; requests in context when no web request is available. Used in ishell, or
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
 ; Host at which the Service API is running.
 app.service_api.host = http://rhodecode.local:10020
 
 ; Secret for Service API authentication.
 app.service_api.token =
 
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
 ; Cut off limit for large diffs (size in bytes). If overall diff size on
 ; commit, or pull request exceeds this limit this diff will be displayed
 ; partially. E.g 512000 == 512Kb
 cut_off_limit_diff = 512000
 
 ; Cut off limit for large files inside diffs (size in bytes). Each individual
 ; file inside diff which exceeds this limit will be displayed partially.
 ;  E.g 128000 == 128Kb
 cut_off_limit_file = 128000
 
 ; Use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = true
 
 ; Force https in RhodeCode, fixes https redirects, assumes it's always https.
 ; Normally this is controlled by proper flags sent from http server such as Nginx or Apache
 force_https = false
 
 ; use Strict-Transport-Security headers
 use_htsts = false
 
 ; Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ; RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ; gist URL alias, used to create nicer urls for gist. This should be an
 ; url that does rewrites to _admin/gists/{gistid}.
 ; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ; List of views (using glob pattern syntax) that AUTH TOKENS could be
 ; used for access.
 ; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ; came from the the logged in user who own this authentication token.
 ; Additionally @TOKEN syntax can be used to bound the view to specific
 ; authentication token. Such view would be only accessible when used together
 ; with this authentication token
 ; list of all views can be found under `/_admin/permissions/auth_token_access`
 ; The list should be "," separated and on a single line.
 ; Most common views to enable:
 
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
 ; Default encoding used to convert from and to unicode
 ; can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ; instance-id prefix
 ; a prefix key for this instance used for cache invalidation when running
 ; multiple instances of RhodeCode, make sure it's globally unique for
 ; all running RhodeCode instances. Leave empty if you don't use it
 instance_id =
 
 ; Fallback authentication plugin. Set this to a plugin ID to force the usage
 ; of an authentication plugin also if it is disabled by it's settings.
 ; This could be useful if you are unable to log in to the system due to broken
 ; authentication settings. Then you can enable e.g. the internal RhodeCode auth
 ; module to log in again and fix the settings.
 ; Available builtin plugin IDs (hash is part of the ID):
 ; egg:rhodecode-enterprise-ce#rhodecode
 ; egg:rhodecode-enterprise-ce#pam
 ; egg:rhodecode-enterprise-ce#ldap
 ; egg:rhodecode-enterprise-ce#jasig_cas
 ; egg:rhodecode-enterprise-ce#headers
 ; egg:rhodecode-enterprise-ce#crowd
 
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
 ; Flag to control loading of legacy plugins in py:/path format
 auth_plugin.import_legacy_plugins = true
 
 ; alternative return HTTP header for failed authentication. Default HTTP
 ; response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ; handling that causing a series of failed authentication calls.
 ; Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ; This will be served instead of default 401 on bad authentication
 auth_ret_code =
 
 ; use special detection method when serving auth_ret_code, instead of serving
 ; ret_code directly, use 401 initially (Which triggers credentials prompt)
 ; and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ; locking return code. When repository is locked return this HTTP code. 2XX
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ; Filesystem location were repositories should be stored
 repo_store.path = /var/opt/rhodecode_repo_store
 
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ; Generated license token required for EE edition license.
 ; New generated token value can be found in Admin > settings > license page.
 license_token =
 
 ; This flag hides sensitive information on the license page such as token, and license data
 license.hide_license_info = false
 
+; Import EE license from this license path
+#license.import_path = %(here)s/rhodecode_enterprise.license
+
+; import license 'if-missing' or 'force' (always override)
+; if-missing means apply license if it doesn't exist. 'force' option always overrides it
+license.import_path_mode = if-missing
+
 ; supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 
 ; supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = prod
 
 ; Display extended labs settings
 labs_settings_active = true
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ; Send email with exception details when it happens
 #exception_tracker.send_email = false
 
 ; Comma separated list of recipients for exception emails,
 ; e.g [e-mail],[e-mail]
 ; Can be left empty, then emails will be sent to ALL super-admins
 #exception_tracker.send_email_recipients =
 
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
-; File store configuration. This is used to store and serve uploaded files
-file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store backend is always enabled
+#file_store.enabled = true
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead
 ; Storage backend, available options are: local
-file_store.backend = local
+#file_store.backend = local
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead
 ; path to store the uploaded binaries and artifacts
-file_store.storage_path = /var/opt/rhodecode_data/file_store
+#file_store.storage_path = /var/opt/rhodecode_data/file_store
+
+; Artifacts file-store, is used to store comment attachments and artifacts uploads.
+; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options
+; filesystem_v1 is backwards compat with pre 5.1 storage changes
+; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from
+; previous installations to keep the artifacts without a need of migration
+#file_store.backend.type = filesystem_v2
+
+; filesystem options...
+#file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+
+; filesystem_v2 options...
+#file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/artifacts_file_store
+#file_store.filesystem_v2.shards = 8
 
+; objectstore options...
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+#file_store.backend.type = objectstore
+#file_store.objectstore.url = http://s3-minio:9000
+
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number
+#file_store.objectstore.bucket = rhodecode-file-store
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+#file_store.objectstore.bucket_shards = 8
+
+; key for s3 auth
+#file_store.objectstore.key = s3admin
+
+; secret for s3 auth
+#file_store.objectstore.secret = s3secret4
+
+;region for s3 storage
+#file_store.objectstore.region = eu-central-1
 
 ; Redis url to acquire/check generation of archives locks
 archive_cache.locking.url = redis://redis:6379/1
 
 ; Storage backend, only 'filesystem' and 'objectstore' are available now
 archive_cache.backend.type = filesystem
 
 ; url for s3 compatible storage that allows to upload artifacts
 ; e.g http://minio:9000
 archive_cache.objectstore.url = http://s3-minio:9000
 
 ; key for s3 auth
 archive_cache.objectstore.key = key
 
 ; secret for s3 auth
 archive_cache.objectstore.secret = secret
 
 ;region for s3 storage
 archive_cache.objectstore.region = eu-central-1
 
 ; number of sharded buckets to create to distribute archives across
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8
 
 ; a top-level bucket to put all other shards in
 ; objects will be stored in rhodecode-archive-cache/shard-N based on the bucket_shards number
 archive_cache.objectstore.bucket = rhodecode-archive-cache
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.objectstore.retry = false
 
 ; number of seconds to wait for next try using retry
 archive_cache.objectstore.retry_backoff = 1
 
 ; how many tries do do a retry fetch from this backend
 archive_cache.objectstore.retry_attempts = 10
 
 ; Default is $cache_dir/archive_cache if not set
 ; Generated repo archives will be cached at this location
 ; and served from the cache during subsequent requests for the same archive of
 ; the repository. This path is important to be shared across filesystems and with
 ; RhodeCode and vcsserver
 archive_cache.filesystem.store_dir = /var/opt/rhodecode_data/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
 archive_cache.filesystem.cache_size_gb = 40
 
 ; Eviction policy used to clear out after cache_size_gb limit is reached
 archive_cache.filesystem.eviction_policy = least-recently-stored
 
 ; By default cache uses sharding technique, this specifies how many shards are there
 ; default is 8 shards
 archive_cache.filesystem.cache_shards = 8
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.filesystem.retry = false
 
 ; number of seconds to wait for next try using retry
 archive_cache.filesystem.retry_backoff = 1
 
 ; how many tries do do a retry fetch from this backend
 archive_cache.filesystem.retry_attempts = 10
 
 
 ; #############
 ; CELERY CONFIG
 ; #############
 
 ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
 
 use_celery = true
 
 ; path to store schedule database
 #celerybeat-schedule.path =
 
 ; connection url to the message broker (default redis)
 celery.broker_url = redis://redis:6379/8
 
 ; results backend to get results for (default redis)
 celery.result_backend = redis://redis:6379/8
 
 ; rabbitmq example
 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
 ; maximum tasks to execute before worker restart
 celery.max_tasks_per_child = 20
 
 ; tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = false
 
 ; #############
 ; DOGPILE CACHE
 ; #############
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
 cache_dir = /var/opt/rhodecode_data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
 ; Only supported backend is `memory_lru`
 ; *********************************************
 rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
 rc_cache.sql_cache_short.expiration_time = 30
 
 
 ; *****************************************************
 ; `cache_repo_longterm` cache for repo object instances
 ; Only supported backend is `memory_lru`
 ; *****************************************************
 rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
 ; by default we use 30 Days, cache is still invalidated on push
 rc_cache.cache_repo_longterm.expiration_time = 2592000
 ; max items in LRU cache, set to smaller number to save memory, and expire last used caches
 rc_cache.cache_repo_longterm.max_size = 10000
 
 
 ; *********************************************
 ; `cache_general` cache for general purpose use
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *********************************************
 rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_general.expiration_time = 43200
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_general.arguments.filename = /tmp/cache_general_db
 
 ; alternative `cache_general` redis backend with distributed lock
 #rc_cache.cache_general.backend = dogpile.cache.rc.redis
 #rc_cache.cache_general.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_general.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_general.arguments.host = localhost
 #rc_cache.cache_general.arguments.port = 6379
 #rc_cache.cache_general.arguments.db = 0
 #rc_cache.cache_general.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_general.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_general.arguments.lock_auto_renewal = true
 
 ; *************************************************
 ; `cache_perms` cache for permission tree, auth TTL
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *************************************************
 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_perms.expiration_time = 3600
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_perms.arguments.filename = /tmp/cache_perms_db
 
 ; alternative `cache_perms` redis backend with distributed lock
 #rc_cache.cache_perms.backend = dogpile.cache.rc.redis
 #rc_cache.cache_perms.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_perms.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_perms.arguments.host = localhost
 #rc_cache.cache_perms.arguments.port = 6379
 #rc_cache.cache_perms.arguments.db = 0
 #rc_cache.cache_perms.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_perms.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_perms.arguments.lock_auto_renewal = true
 
 ; ***************************************************
 ; `cache_repo` cache for file tree, Readme, RSS FEEDS
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; ***************************************************
 rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_repo.expiration_time = 2592000
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_repo.arguments.filename = /tmp/cache_repo_db
 
 ; alternative `cache_repo` redis backend with distributed lock
 #rc_cache.cache_repo.backend = dogpile.cache.rc.redis
 #rc_cache.cache_repo.expiration_time = 2592000
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
 
 #rc_cache.cache_repo.arguments.host = localhost
 #rc_cache.cache_repo.arguments.port = 6379
 #rc_cache.cache_repo.arguments.db = 1
 #rc_cache.cache_repo.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_repo.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_repo.arguments.lock_auto_renewal = true
 
 ; ##############
 ; BEAKER SESSION
 ; ##############
 
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
 ; types are file, ext:redis, ext:database, ext:memcached
 ; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session
 #beaker.session.type = file
 #beaker.session.data_dir = %(here)s/data/sessions
 
 ; Redis based sessions
 beaker.session.type = ext:redis
 beaker.session.url = redis://redis:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = production-rc-uytcxaz
 beaker.session.lock_dir = /data_ramdisk/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
 ; Sets session as invalid (also logging out user) if it haven not been
 ; accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 
 ; Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
 ; Set https secure cookie
 beaker.session.secure = false
 
 ; default cookie expiration time in seconds, set to `true` to set expire
 ; at browser close
 #beaker.session.cookie_expires = 3600
 
 ; #############################
 ; SEARCH INDEXING CONFIGURATION
 ; #############################
 
 ; Full text search indexer is available in rhodecode-tools under
 ; `rhodecode-tools index` command
 
 ; WHOOSH Backend, doesn't require additional services to run
 ; it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
 ; ####################
 ; CHANNELSTREAM CONFIG
 ; ####################
 
 ; channelstream enables persistent connections and live notification
 ; in the system. It's also used by the chat system
 
 channelstream.enabled = true
 
 ; server address for channelstream server on the backend
 channelstream.server = channelstream:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = ENV_GENERATED
 channelstream.history.location = /var/opt/rhodecode_data/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
 ; ##############################
 ; MAIN RHODECODE DATABASE CONFIG
 ; ##############################
 
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8
 ; pymysql is an alternative driver for MySQL, use in case of problems with default one
 #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
 sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 
 ; see sqlalchemy docs for other advanced settings
 ; print the sql statements to output
 sqlalchemy.db1.echo = false
 
 ; recycle the connections after this amount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 
 ; the number of connections to keep open inside the connection pool.
 ; 0 indicates no limit
 ; the general calculus with gevent is:
 ; if your system allows 500 concurrent greenlets (max_connections) that all do database access,
 ; then increase pool size + max overflow so that they add up to 500.
 #sqlalchemy.db1.pool_size = 5
 
 ; The number of connections to allow in connection pool "overflow", that is
 ; connections that can be opened above and beyond the pool_size setting,
 ; which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 ; Connection check ping, used to detect broken database connections
 ; could be enabled to better handle cases if MySQL has gone away errors
 #sqlalchemy.db1.ping_connection = true
 
 ; ##########
 ; VCS CONFIG
 ; ##########
 vcs.server.enable = true
 vcs.server = vcsserver:10010
 
 ; Web server connectivity protocol, responsible for web based VCS operations
 ; Available protocols are:
 ; `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
 ; Push/Pull operations protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
 ; `celery` - use celery based hooks
-vcs.hooks.protocol = http
+#DEPRECATED:vcs.hooks.protocol = http
+vcs.hooks.protocol.v2 = celery
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
 ; accessible via network.
 ; Use vcs.hooks.host = "*" to bind to current hostname (for Docker)
 vcs.hooks.host = *
 
 ; Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
 ; List of enabled VCS backends, available options are:
 ; `hg`  - mercurial
 ; `git` - git
 ; `svn` - subversion
 vcs.backends = hg, git, svn
 
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
 ; Cache flag to cache vcsserver remote calls locally
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = true
 
+; Filesystem location where Git lfs objects should be stored
+vcs.git.lfs.storage_location = /var/opt/rhodecode_repo_store/.cache/git_lfs_store
+
+; Filesystem location where Mercurial largefile objects should be stored
+vcs.hg.largefiles.storage_location = /var/opt/rhodecode_repo_store/.cache/hg_largefiles_store
+
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
 ; Maps RhodeCode repo groups into SVN paths for Apache
 ; ####################################################
 
 ; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ; Set a numeric version for your current SVN e.g 1.8, or 1.12
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
 ; Redis connection settings for svn integrations logic
 ; This connection string needs to be the same on ce and vcsserver
 vcs.svn.redis_conn = redis://redis:6379/0
 
 ; Enable SVN proxy of requests over HTTP
 vcs.svn.proxy.enabled = true
 
 ; host to connect to running SVN subsystem
 vcs.svn.proxy.host = http://svn:8090
 
 ; Enable or disable the config file generation.
 svn.proxy.generate_config = true
 
 ; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
 svn.proxy.config_file_path = /etc/rhodecode/conf/svn/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
 ; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako
 #svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako
 
 ; Used as a prefix to the `Location` block in the generated config file.
 ; In most cases it should be set to `/`.
 svn.proxy.location_root = /
 
 ; Command to reload the mod dav svn configuration on change.
 ; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh
 ; Make sure user who runs RhodeCode process is allowed to reload Apache
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
 
 ; If the timeout expires before the reload command finishes, the command will
 ; be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
 ; ####################
 ; SSH Support Settings
 ; ####################
 
 ; Defines if a custom authorized_keys file should be created and written on
 ; any change user ssh keys. Setting this to false also disables possibility
 ; of adding SSH keys by users from web interface. Super admins can still
 ; manage SSH Keys.
 ssh.generate_authorized_keyfile = true
 
 ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
 ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
-ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+#DEPRECATED: ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+ssh.wrapper_cmd.v2 = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
 ; Enables logging, and detailed output send back to the client during SSH
 ; operations. Useful for debugging, shouldn't be used in production.
 ssh.enable_debug_logging = false
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
 ssh.executable.hg = /usr/local/bin/rhodecode_bin/vcs_bin/hg
 ssh.executable.git = /usr/local/bin/rhodecode_bin/vcs_bin/git
 ssh.executable.svn = /usr/local/bin/rhodecode_bin/vcs_bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.
 ssh.enable_ui_key_generator = true
 
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Prometheus
 #statsd.enabled = false
 #statsd.statsd_host = 0.0.0.0
 #statsd.statsd_port = 8125
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
 ; configure logging automatically at server startup set to false
 ; to use the below custom logging config.
 ; RC_LOGGING_FORMATTER
 ; RC_LOGGING_LEVEL
 ; env variables can control the settings for logging in case of autoconfigure
 
 #logging.autoconfigure = true
 
 ; specify your own custom logging config file to configure logging
 #logging.logging_conf_file = /path/to/custom_logging.ini
 
 ; Dummy marker to add new entries after.
 ; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1
 
 
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
 
 [loggers]
 keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, json, color_formatter, color_formatter_sql
 
 ; #######
 ; LOGGERS
 ; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
 [logger_celery]
 level = DEBUG
 handlers =
 qualname = celery
 
 
 ; ########
 ; HANDLERS
 ; ########
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr, )
 level = INFO
 ; To enable JSON formatted logs replace 'generic/color_formatter' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 [handler_console_sql]
 ; "level = DEBUG" logs SQL queries and results.
 ; "level = INFO" logs SQL queries.
 ; "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
 args = (sys.stderr, )
 level = WARN
 ; To enable JSON formatted logs replace 'generic/color_formatter_sql' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_json]
 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
 class = rhodecode.lib._vendor.jsonlogger.JsonFormatter

docs/Dockerfile

@@ -1,33 +1,39 b''
 FROM python:3.12.0-bullseye
 
 WORKDIR /project
 
 RUN apt-get update \
  && apt-get install --no-install-recommends --yes \
     curl \
     zip \
     graphviz \
     dvipng \
     imagemagick \
     make \
     latexmk \
     texlive-latex-recommended \
     texlive-latex-extra \
     texlive-xetex \
     fonts-freefont-otf \
     texlive-fonts-recommended \
     texlive-lang-greek \
     tex-gyre \
  && apt-get autoremove \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*
 
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip && \
+    ./aws/install && \
+    rm -rf ./aws && \
+    rm awscliv2.zip
+
 RUN \
     python3 -m pip install --no-cache-dir --upgrade pip && \
     python3 -m pip install --no-cache-dir Sphinx Pillow
 
 ADD requirements_docs.txt /project
 RUN \
     python3 -m pip install -r requirements_docs.txt
 
 CMD ["sphinx-build", "-M", "html", ".", "_build"]

docs/admin/system-overview.rst

@@ -1,172 +1,168 b''
 .. _system-overview-ref:
 
 System Overview
 ===============
 
 Latest Version
 --------------
 
 * |release| on Unix and Windows systems.
 
 System Architecture
 -------------------
 
 The following diagram shows a typical production architecture.
 
 .. image:: ../images/architecture-diagram.png
   :align: center
 
 Supported Operating Systems
 ---------------------------
 
 Linux
 ^^^^^
 
 * Ubuntu 14.04+
 * CentOS 6.2, 7 and 8
 * RHEL 6.2, 7 and 8
 * Debian 7.8
 * RedHat Fedora
 * Arch Linux
 * SUSE Linux
 
 Windows
 ^^^^^^^
 
 * Windows Vista Ultimate 64bit
 * Windows 7 Ultimate 64bit
 * Windows 8 Professional 64bit
 * Windows 8.1 Enterprise 64bit
 * Windows Server 2008 64bit
 * Windows Server 2008-R2 64bit
 * Windows Server 2012 64bit
 
 Supported Databases
 -------------------
 
 * SQLite
 * MySQL
 * MariaDB
 * PostgreSQL
 
 Supported Browsers
 ------------------
 
 * Chrome
 * Safari
 * Firefox
 * Internet Explorer 10 & 11
 
 System Requirements
 -------------------
 
 |RCE| performs best on machines with ultra-fast hard disks. Generally disk
 performance is more important than CPU performance. In a corporate production
 environment handling 1000s of users and |repos| you should deploy on a 12+
 core 64GB RAM server. In short, the more RAM the better.
 
 
 For example:
 
  - for team of 1 - 5 active users you can run on 1GB RAM machine with 1CPU
  - above 250 active users, |RCE| needs at least 8GB of memory.
    Number of CPUs is less important, but recommended to have at least 2-3 CPUs
 
 
 .. _config-rce-files:
 
 Configuration Files
 -------------------
 
 * :file:`config/_shared/rhodecode.ini`
 * :file:`/home/{user}/.rccontrol/{instance-id}/search_mapping.ini`
 * :file:`/home/{user}/.rccontrol/{vcsserver-id}/vcsserver.ini`
 * :file:`/home/{user}/.rccontrol/supervisor/supervisord.ini`
 * :file:`/home/{user}/.rccontrol.ini`
 * :file:`/home/{user}/.rhoderc`
 * :file:`/home/{user}/.rccontrol/cache/MANIFEST`
 
 For more information, see the :ref:`config-files` section.
 
 Log Files
 ---------
 
 * :file:`/home/{user}/.rccontrol/{instance-id}/enterprise.log`
 * :file:`/home/{user}/.rccontrol/{vcsserver-id}/vcsserver.log`
 * :file:`/home/{user}/.rccontrol/supervisor/supervisord.log`
 * :file:`/tmp/rccontrol.log`
 * :file:`/tmp/rhodecode_tools.log`
 
 Storage Files
 -------------
 
 * :file:`/home/{user}/.rccontrol/{instance-id}/data/index/{index-file.toc}`
 * :file:`/home/{user}/repos/.rc_gist_store`
 * :file:`/home/{user}/.rccontrol/{instance-id}/rhodecode.db`
 * :file:`/opt/rhodecode/store/{unique-hash}`
 
 Default Repositories Location
 -----------------------------
 
 * :file:`/home/{user}/repos`
 
 Connection Methods
 ------------------
 
 * HTTPS
 * SSH
 * |RCE| API
 
 Internationalization Support
 ----------------------------
 
 Currently available in the following languages, see `Transifex`_ for the
 latest details. If you want a new language added, please contact us. To
 configure your language settings, see the :ref:`set-lang` section.
 
 .. hlist::
 
   * Belorussian
   * Chinese
   * French
   * German
   * Italian
   * Japanese
   * Portuguese
   * Polish
   * Russian
   * Spanish
 
 Licencing Information
 ---------------------
 
 * See licencing information `here`_
 
 Peer-to-peer Failover Support
 -----------------------------
 
 * Yes
 
-Additional Binaries
--------------------
-
-* Yes, see :ref:`rhodecode-nix-ref` for full details.
 
 Remote Connectivity
 -------------------
 
 * Available
 
 Executable Files
 ----------------
 
 Windows: :file:`RhodeCode-installer-{version}.exe`
 
 Deprecated Support
 ------------------
 
 - Internet Explorer 8 support deprecated since version 3.7.0.
 - Internet Explorer 9 support deprecated since version 3.8.0.
 
 .. _here: https://rhodecode.com/licenses/
 .. _Transifex: https://explore.transifex.com/rhodecode/RhodeCode/

docs/auth/auth-saml-bulk-enroll-users.rst

@@ -1,88 +1,90 b''
 .. _auth-saml-bulk-enroll-users-ref:
 
 
 Bulk enroll multiple existing users
 -----------------------------------
 
 
 RhodeCode Supports standard SAML 2.0 SSO for the web-application part.
 Below is an example how to enroll list of all or some users to use SAML authentication.
 This method simply enables SAML authentication for many users at once.
 
 
 From the server RhodeCode Enterprise is running run ishell on the instance which we
 want to apply the SAML migration::
 
-    rccontrol ishell enterprise-1
+    ./rcstack cli ishell
 
 Follow these steps to enable SAML authentication for multiple users.
 
 
 1) Create a user_id => attribute mapping
 
 
 `saml2user` is a mapping of external ID from SAML provider such as OneLogin, DuoSecurity, Google.
 This mapping consists of local rhodecode user_id mapped to set of required attributes needed to bind SAML
 account to internal rhodecode user.
 For example, 123 is local rhodecode user_id, and '48253211' is OneLogin ID.
 For other providers you'd have to figure out what would be the user-id, sometimes it's the email, i.e for Google
 The most important this id needs to be unique for each user.
 
 .. code-block:: python
 
     In [1]: saml2user = {
        ...: # OneLogin, uses externalID available to read from in the UI
        ...: 123: {'id': '48253211'},
        ...: # for Google/DuoSecurity email is also an option for unique ID
        ...: 124: {'id': '[e-mail]'},
        ...: }
 
 
 2) Import the plugin you want to run migration for.
 
 From available options pick only one and run the `import` statement
 
 .. code-block:: python
 
     # for Duo Security
     In [2]: from rc_auth_plugins.auth_duo_security import RhodeCodeAuthPlugin
+    # for Azure Entra
+    In [2]: from rc_auth_plugins.auth_azure import RhodeCodeAuthPlugin
     # for OneLogin
     In [2]: from rc_auth_plugins.auth_onelogin import RhodeCodeAuthPlugin
     # generic SAML plugin
     In [2]: from rc_auth_plugins.auth_saml import RhodeCodeAuthPlugin
 
 3) Run the migration based on saml2user mapping.
 
 Enter in the ishell prompt
 
 .. code-block:: python
 
     In [3]: for user in User.get_all():
        ...:     existing_identity = ExternalIdentity().query().filter(ExternalIdentity.local_user_id == user.user_id).scalar()
        ...:     attrs = saml2user.get(user.user_id)
        ...:     provider = RhodeCodeAuthPlugin.uid
        ...:     if existing_identity:
-       ...:         print('Identity for user `{}` already exists, skipping'.format(user.username))
+       ...:         print(f'Identity for user `{user.username}` already exists, skipping')
        ...:         continue
        ...:     if attrs:
        ...:         external_id = attrs['id']
        ...:         new_external_identity = ExternalIdentity()
        ...:         new_external_identity.external_id = external_id
-       ...:         new_external_identity.external_username = '{}-saml-{}'.format(user.username, user.user_id)
+       ...:         new_external_identity.external_username = f'{user.username}-saml-{user.user_id}'
        ...:         new_external_identity.provider_name = provider
        ...:         new_external_identity.local_user_id = user.user_id
        ...:         new_external_identity.access_token = ''
        ...:         new_external_identity.token_secret = ''
        ...:         new_external_identity.alt_token = ''
        ...:         Session().add(ex_identity)
        ...:         Session().commit()
-       ...:         print('Set user `{}` external identity bound to ExternalID:{}'.format(user.username, external_id))
+       ...:         print(f'Set user `{user.username}` external identity bound to ExternalID:{external_id}')
 
 .. note::
 
     saml2user can be really big and hard to maintain in ishell. It's also possible
     to load it as a JSON file prepared before and stored on disk. To do so run::
 
         import json
         saml2user = json.loads(open('/path/to/saml2user.json','rb').read())
 

docs/auth/auth-saml-duosecurity.rst

@@ -1,105 +1,161 b''
 .. _config-saml-duosecurity-ref:
 
 
 SAML 2.0 with Duo Security
 --------------------------
 
 **This plugin is available only in EE Edition.**
 
 |RCE| supports SAML 2.0 Authentication with Duo Security provider. This allows
 users to log-in to RhodeCode via SSO mechanism of external identity provider
 such as Duo. The login can be triggered either by the external IDP, or internally
 by clicking specific authentication button on the log-in page.
 
 
 Configuration steps
 ^^^^^^^^^^^^^^^^^^^
 
 To configure Duo Security SAML authentication, use the following steps:
 
 1. From the |RCE| interface, select
    :menuselection:`Admin --> Authentication`
 2. Activate the `Duo Security` plugin and select :guilabel:`Save`
 3. Go to newly available menu option called `Duo Security` on the left side.
 4. Check the `enabled` check box in the plugin configuration section,
    and fill in the required SAML information and :guilabel:`Save`, for more details,
    see :ref:`config-saml-duosecurity`
 
 
 .. _config-saml-duosecurity:
 
 
 Example SAML Duo Security configuration
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Example configuration for SAML 2.0 with Duo Security provider::
+Example configuration for SAML 2.0 with Duo Security provider
+
+
+Enabled
+    `True`:
 
-    *option*: `enabled` => `True`
-    # Enable or disable this authentication plugin.
+    .. note::
+      Enable or disable this authentication plugin.
+
+
+Auth Cache TTL
+    `30`:
 
-    *option*: `cache_ttl` => `0`
-    # Amount of seconds to cache the authentication and permissions check response call for this plugin.
-    # Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
+    .. note::
+      Amount of seconds to cache the authentication and permissions check response call for this plugin.
+      Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
+
+Debug
+    `True`:
 
-    *option*: `debug` => `True`
-    # Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
+    .. note::
+      Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
+
+
+Auth button name
+    `Azure Entra ID`:
 
-    *option*: `entity_id` => `http://rc-app.com/dag/saml2/idp/metadata.php`
-    # Identity Provider entity/metadata URI.
-    # E.g. https://duo-gateway.com/dag/saml2/idp/metadata.php
+    .. note::
+      Alternative authentication display name. E.g AzureAuth, CorporateID etc.
+
+
+Entity ID
+    `https://my-duo-gateway.com/dag/saml2/idp/metadata.php`:
+
+    .. note::
+      Identity Provider entity/metadata URI.
+      E.g. https://duo-gateway.com/dag/saml2/idp/metadata.php
+
+SSO URL
+    `https://duo-gateway.com/dag/saml2/idp/SSOService.php?spentityid=<metadata_entity_id>`:
 
-    *option*: `sso_service_url` => `http://rc-app.com/dag/saml2/idp/SSOService.php?spentityid=http://rc.local.pl/_admin/auth/duosecurity/saml-metadata`
-    # SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login
-    # E.g. https://duo-gateway.com/dag/saml2/idp/SSOService.php?spentityid=<metadata_entity_id>
+    .. note::
+      SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login, Known also as Login URL
+      E.g. http://rc-app.com/dag/saml2/idp/SSOService.php?spentityid=https://docker-dev/_admin/auth/duosecurity/saml-metadata
+
+SLO URL
+    `https://duo-gateway.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=<return_url>`:
 
-    *option*: `slo_service_url` => `http://rc-app.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=http://rc-app.com/dag/module.php/duosecurity/logout.php`
-    # SLO (SingleLogout) endpoint URL of the IdP.
-    # E.g. https://duo-gateway.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=http://duo-gateway.com/_admin/saml/sign-out-endpoint
+    .. note::
+      SLO (SingleLogout) endpoint URL of the IdP. , Known also as Logout URL
+      E.g. http://rc-app.com/dag/saml2/idp/SingleLogoutService.php?ReturnTo=https://docker-dev/_admin/auth/duosecurity/saml-sign-out-endpoint
 
-    *option*: `x509cert` => `<CERTIFICATE_STRING>`
-    # Identity provider public x509 certificate. It will be converted to single-line format without headers
+x509cert
+    `<CERTIFICATE_STRING>`:
 
-    *option*: `name_id_format` => `sha-1`
-    # The format that specifies how the NameID is sent to the service provider.
+    .. note::
+      Identity provider public x509 certificate. It will be converted to single-line format without headers.
+      Download the raw base64 encoded certificate from the Identity provider and paste it here.
+
+SAML Signature
+    `sha-256`:
+
+    .. note::
+      Type of Algorithm to use for verification of SAML signature on Identity provider side.
+
+SAML Digest
+    `sha-256`:
 
-    *option*: `signature_algo` => `sha-256`
-    # Type of Algorithm to use for verification of SAML signature on Identity provider side
+    .. note::
+      Type of Algorithm to use for verification of SAML digest on Identity provider side.
+
+Service Provider Cert Dir
+    `/etc/rhodecode/conf/saml_ssl/`:
 
-    *option*: `digest_algo` => `sha-256`
-    # Type of Algorithm to use for verification of SAML digest on Identity provider side
+    .. note::
+      Optional directory to store service provider certificate and private keys.
+      Expected certs for the SP should be stored in this folder as:
+
+      * sp.key     Private Key
+      * sp.crt     Public cert
+      * sp_new.crt Future Public cert
+
+      Also you can use other cert to sign the metadata of the SP using the:
 
-    *option*: `cert_dir` => `/etc/saml/`
-    # Optional directory to store service provider certificate and private keys.
-    # Expected certs for the SP should be stored in this folder as:
-    #  * sp.key     Private Key
-    #  * sp.crt     Public cert
-    #  * sp_new.crt Future Public cert
-    #
-    # Also you can use other cert to sign the metadata of the SP using the:
-    #  * metadata.key
-    #  * metadata.crt
+      * metadata.key
+      * metadata.crt
+
+Expected NameID Format
+    `nameid-format:emailAddress`:
+
+    .. note::
+      The format that specifies how the NameID is sent to the service provider.
+
+User ID Attribute
+    `PersonImmutableID`:
 
-    *option*: `user_id_attribute` => `PersonImmutableID`
-    # User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
-    # Ensure this is returned from DuoSecurity for example via duo_username
+    .. note::
+      User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
+      Ensure this is returned from DuoSecurity for example via duo_username.
+
+Username Attribute
+    `User.username`:
 
-    *option*: `username_attribute` => `User.username`
-    # Username Attribute name. This defines which attribute in SAML response will map to an username.
+    .. note::
+      Username Attribute name. This defines which attribute in SAML response will map to a username.
 
-    *option*: `email_attribute` => `User.email`
-    # Email Attribute name. This defines which attribute in SAML response will map to an email address.
+Email Attribute
+    `User.email`:
+
+    .. note::
+      Email Attribute name. This defines which attribute in SAML response will map to an email address.
+
 
 
 Below is example setup from DUO Administration page that can be used with above config.
 
 .. image:: ../images/saml-duosecurity-service-provider-example.png
    :alt: DUO Security SAML setup example
    :scale: 50 %
 
 
 Below is an example attribute mapping set for IDP provider required by the above config.
 
 
 .. image:: ../images/saml-duosecurity-attributes-example.png
    :alt: DUO Security SAML setup example
    :scale: 50 %
  No newline at end of file

docs/auth/auth-saml-generic.rst

@@ -1,19 +1,20 b''
 .. _config-saml-generic-ref:
 
 
 SAML 2.0 Authentication
 -----------------------
 
 
 **This plugin is available only in EE Edition.**
 
 RhodeCode Supports standard SAML 2.0 SSO for the web-application part.
 
 Please check for reference two example providers:
 
 .. toctree::
 
    auth-saml-duosecurity
    auth-saml-onelogin
+   auth-saml-azure
    auth-saml-bulk-enroll-users
 

docs/auth/auth-saml-onelogin.rst

@@ -1,106 +1,161 b''
 .. _config-saml-onelogin-ref:
 
 
 SAML 2.0 with One Login
 -----------------------
 
 **This plugin is available only in EE Edition.**
 
 |RCE| supports SAML 2.0 Authentication with OneLogin provider. This allows
 users to log-in to RhodeCode via SSO mechanism of external identity provider
 such as OneLogin. The login can be triggered either by the external IDP, or internally
 by clicking specific authentication button on the log-in page.
 
 
 Configuration steps
 ^^^^^^^^^^^^^^^^^^^
 
 To configure OneLogin SAML authentication, use the following steps:
 
 1. From the |RCE| interface, select
    :menuselection:`Admin --> Authentication`
 2. Activate the `OneLogin` plugin and select :guilabel:`Save`
 3. Go to newly available menu option called `OneLogin` on the left side.
 4. Check the `enabled` check box in the plugin configuration section,
    and fill in the required SAML information and :guilabel:`Save`, for more details,
    see :ref:`config-saml-onelogin`
 
 
 .. _config-saml-onelogin:
 
 
 Example SAML OneLogin configuration
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Example configuration for SAML 2.0 with OneLogin provider::
+Example configuration for SAML 2.0 with OneLogin provider
+
+
+Enabled
+    `True`:
 
-    *option*: `enabled` => `True`
-    # Enable or disable this authentication plugin.
+    .. note::
+      Enable or disable this authentication plugin.
+
+
+Auth Cache TTL
+    `30`:
 
-    *option*: `cache_ttl` => `0`
-    # Amount of seconds to cache the authentication and permissions check response call for this plugin.
-    # Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
+    .. note::
+      Amount of seconds to cache the authentication and permissions check response call for this plugin.
+      Useful for expensive calls like LDAP to improve the performance of the system (0 means disabled).
+
+Debug
+    `True`:
 
-    *option*: `debug` => `True`
-    # Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
+    .. note::
+      Enable or disable debug mode that shows SAML errors in the RhodeCode logs.
+
+
+Auth button name
+    `Azure Entra ID`:
 
-    *option*: `entity_id` => `https://app.onelogin.com/saml/metadata/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx`
-    # Identity Provider entity/metadata URI.
-    # E.g. https://app.onelogin.com/saml/metadata/<onelogin_connector_id>
+    .. note::
+      Alternative authentication display name. E.g AzureAuth, CorporateID etc.
+
+
+Entity ID
+    `https://app.onelogin.com/saml/metadata/<onelogin_connector_id>`:
+
+    .. note::
+      Identity Provider entity/metadata URI.
+      E.g. https://app.onelogin.com/saml/metadata/xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+
+SSO URL
+    `https://app.onelogin.com/trust/saml2/http-post/sso/<onelogin_connector_id>`:
 
-    *option*: `sso_service_url` => `https://customer-domain.onelogin.com/trust/saml2/http-post/sso/xxxxxx`
-    # SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login
-    # E.g. https://app.onelogin.com/trust/saml2/http-post/sso/<onelogin_connector_id>
+    .. note::
+      SSO (SingleSignOn) endpoint URL of the IdP. This can be used to initialize login, Known also as Login URL
+      E.g.  https://app.onelogin.com/trust/saml2/http-post/sso/<onelogin_connector_id>
+
+SLO URL
+    `https://app.onelogin.com/trust/saml2/http-redirect/slo/<onelogin_connector_id>`:
 
-    *option*: `slo_service_url` => `https://customer-domain.onelogin.com/trust/saml2/http-redirect/slo/xxxxxx`
-    # SLO (SingleLogout) endpoint URL of the IdP.
-    # E.g. https://app.onelogin.com/trust/saml2/http-redirect/slo/<onelogin_connector_id>
+    .. note::
+      SLO (SingleLogout) endpoint URL of the IdP. , Known also as Logout URL
+      E.g. https://app.onelogin.com/trust/saml2/http-redirect/slo/<onelogin_connector_id>
 
-    *option*: `x509cert` => `<CERTIFICATE_STRING>`
-    # Identity provider public x509 certificate. It will be converted to single-line format without headers
+x509cert
+    `<CERTIFICATE_STRING>`:
 
-    *option*: `name_id_format` => `sha-1`
-    # The format that specifies how the NameID is sent to the service provider.
+    .. note::
+      Identity provider public x509 certificate. It will be converted to single-line format without headers.
+      Download the raw base64 encoded certificate from the Identity provider and paste it here.
+
+SAML Signature
+    `sha-256`:
+
+    .. note::
+      Type of Algorithm to use for verification of SAML signature on Identity provider side.
+
+SAML Digest
+    `sha-256`:
 
-    *option*: `signature_algo` => `sha-256`
-    # Type of Algorithm to use for verification of SAML signature on Identity provider side
+    .. note::
+      Type of Algorithm to use for verification of SAML digest on Identity provider side.
+
+Service Provider Cert Dir
+    `/etc/rhodecode/conf/saml_ssl/`:
 
-    *option*: `digest_algo` => `sha-256`
-    # Type of Algorithm to use for verification of SAML digest on Identity provider side
+    .. note::
+      Optional directory to store service provider certificate and private keys.
+      Expected certs for the SP should be stored in this folder as:
+
+      * sp.key     Private Key
+      * sp.crt     Public cert
+      * sp_new.crt Future Public cert
 
-    *option*: `cert_dir` => `/etc/saml/`
-    # Optional directory to store service provider certificate and private keys.
-    # Expected certs for the SP should be stored in this folder as:
-    #  * sp.key     Private Key
-    #  * sp.crt     Public cert
-    #  * sp_new.crt Future Public cert
-    #
-    # Also you can use other cert to sign the metadata of the SP using the:
-    #  * metadata.key
-    #  * metadata.crt
+      Also you can use other cert to sign the metadata of the SP using the:
+
+      * metadata.key
+      * metadata.crt
+
+Expected NameID Format
+    `nameid-format:emailAddress`:
+
+    .. note::
+      The format that specifies how the NameID is sent to the service provider.
+
+User ID Attribute
+    `PersonImmutableID`:
 
-    *option*: `user_id_attribute` => `PersonImmutableID`
-    # User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
-    # Ensure this is returned from OneLogin for example via Internal ID
+    .. note::
+      User ID Attribute name. This defines which attribute in SAML response will be used to link accounts via unique id.
+      Ensure this is returned from DuoSecurity for example via duo_username.
+
+Username Attribute
+    `User.username`:
 
-    *option*: `username_attribute` => `User.username`
-    # Username Attribute name. This defines which attribute in SAML response will map to an username.
+    .. note::
+      Username Attribute name. This defines which attribute in SAML response will map to a username.
 
-    *option*: `email_attribute` => `User.email`
-    # Email Attribute name. This defines which attribute in SAML response will map to an email address.
+Email Attribute
+    `User.email`:
+
+    .. note::
+      Email Attribute name. This defines which attribute in SAML response will map to an email address.
 
 
 
 Below is example setup that can be used with OneLogin SAML authentication that can be used with above config..
 
 .. image:: ../images/saml-onelogin-config-example.png
    :alt: OneLogin SAML setup example
    :scale: 50 %
 
 
 Below is an example attribute mapping set for IDP provider required by the above config.
 
 
 .. image:: ../images/saml-onelogin-attributes-example.png
    :alt: OneLogin SAML setup example
    :scale: 50 %
  No newline at end of file

docs/auth/auth.rst

@@ -1,34 +1,35 b''
 .. _authentication-ref:
 
 Authentication Options
 ======================
 
 |RCE| provides a built in authentication against its own database. This is
 implemented using ``RhodeCode Internal`` plugin. This plugin is enabled by default.
 Additionally, |RCE| provides a Pluggable Authentication System. This gives the
 administrator greater control over how users authenticate with the system.
 
 .. important::
 
   You can disable the built in |RCE| authentication plugin
   ``RhodeCode Internal`` and force all authentication to go
   through your authentication plugin of choice e.g LDAP only.
   However, if you do this, and your external authentication tools fails,
   accessing |RCE| will be blocked unless a fallback plugin is
   enabled via :file: rhodecode.ini
 
 
 |RCE| comes with the following user authentication management plugins:
 
 
 .. toctree::
 
     auth-token
     auth-ldap
     auth-ldap-groups
     auth-saml-generic
     auth-saml-onelogin
     auth-saml-duosecurity
+    auth-saml-azure
     auth-crowd
     auth-pam
     ssh-connection

docs/contributing/dev-setup.rst

@@ -1,243 +1,14 b''
 .. _dev-setup:
 
 ===================
  Development setup
 ===================
 
-
-RhodeCode Enterprise runs inside a Nix managed environment. This ensures build
-environment dependencies are correctly declared and installed during setup.
-It also enables atomic upgrades, rollbacks, and multiple instances of RhodeCode
-Enterprise running with isolation.
-
-To set up RhodeCode Enterprise inside the Nix environment, use the following steps:
-
-
-
-Setup Nix Package Manager
--------------------------
-
-To install the Nix Package Manager, please run::
-
-   $ curl https://releases.nixos.org/nix/nix-2.3.4/install | sh
-
-or go to https://nixos.org/nix/ and follow the installation instructions.
-Once this is correctly set up on your system, you should be able to use the
-following commands:
-
-* `nix-env`
-
-* `nix-shell`
-
-
-.. tip::
-
-   Update your channels frequently by running ``nix-channel --update``.
-
-.. note::
-
-   To uninstall nix run the following:
-
-    remove the . "$HOME/.nix-profile/etc/profile.d/nix.sh" line in your ~/.profile or ~/.bash_profile
-    rm -rf $HOME/{.nix-channels,.nix-defexpr,.nix-profile,.config/nixpkgs}
-    sudo rm -rf /nix
-
-Switch nix to the latest STABLE channel
----------------------------------------
-
-run::
-
-   nix-channel --add https://nixos.org/channels/nixos-20.03 nixpkgs
-
-Followed by::
-
-   nix-channel --update
-   nix-env -i nix-2.3.4
-
-
-Install required binaries
--------------------------
-
-We need some handy tools first.
-
-run::
-
-    nix-env -i nix-prefetch-hg
-    nix-env -i nix-prefetch-git
-
-
-Speed up JS build by installing PhantomJS
------------------------------------------
-
-PhantomJS will be downloaded each time nix-shell is invoked. To speed this by
-setting already downloaded version do this::
-
-    nix-env -i phantomjs-2.1.1
-
-    # and set nix bin path
-    export PATH=$PATH:~/.nix-profile/bin
-
-
-Clone the required repositories
--------------------------------
-
-After Nix is set up, clone the RhodeCode Enterprise Community Edition and
-RhodeCode VCSServer repositories into the same directory.
-RhodeCode currently is using Mercurial Version Control System, please make sure
-you have it installed before continuing.
-
-To obtain the required sources, use the following commands::
-
-    mkdir rhodecode-develop && cd rhodecode-develop
-    hg clone -u default https://code.rhodecode.com/rhodecode-enterprise-ce
-    hg clone -u default https://code.rhodecode.com/rhodecode-vcsserver
-
-.. note::
-
-   If you cannot clone the repository, please contact us via [e-mail]
-
-
-Install some required libraries
--------------------------------
-
-There are some required drivers and dev libraries that we need to install to
-test RhodeCode under different types of databases. For example in Ubuntu we
-need to install the following.
-
-required libraries::
-
-    # svn related
-    sudo apt-get install libapr1-dev libaprutil1-dev
-    sudo apt-get install libsvn-dev
-    # libcurl required too
-    sudo apt-get install libcurl4-openssl-dev
-    # mysql/pg server for development, optional
-    sudo apt-get install mysql-server libmysqlclient-dev
-    sudo apt-get install postgresql postgresql-contrib libpq-dev
-
-
-
-Enter the Development Shell
----------------------------
-
-The final step is to start the development shells. To do this, run the
-following command from inside the cloned repository::
-
-   # first, the vcsserver
-   cd ~/rhodecode-vcsserver
-   nix-shell
-
-   # then enterprise sources
-   cd ~/rhodecode-enterprise-ce
-   nix-shell
-
-.. note::
-
-   On the first run, this will take a while to download and optionally compile
-   a few things. The following runs will be faster. The development shell works
-   fine on both MacOS and Linux platforms.
-
-
-Create config.nix for development
----------------------------------
-
-In order to run proper tests and setup linking across projects, a config.nix
-file needs to be setup::
-
-    # create config
-    mkdir -p ~/.nixpkgs
-    touch ~/.nixpkgs/config.nix
-
-    # put the below content into the ~/.nixpkgs/config.nix file
-    # adjusts, the path to where you cloned your repositories.
-
-    {
-      rc = {
-       sources = {
-        rhodecode-vcsserver = "/home/dev/rhodecode-vcsserver";
-        rhodecode-enterprise-ce = "/home/dev/rhodecode-enterprise-ce";
-        rhodecode-enterprise-ee = "/home/dev/rhodecode-enterprise-ee";
-       };
-      };
-    }
-
-
-
-Creating a Development Configuration
-------------------------------------
-
-To create a development environment for RhodeCode Enterprise,
-use the following steps:
-
-1. Create a copy of vcsserver config:
-    `cp ~/rhodecode-vcsserver/configs/development.ini ~/rhodecode-vcsserver/configs/dev.ini`
-2. Create a copy of rhodocode config:
-    `cp ~/rhodecode-enterprise-ce/configs/development.ini ~/rhodecode-enterprise-ce/configs/dev.ini`
-3. Adjust the configuration settings to your needs if needed.
-
-.. note::
-
-  It is recommended to use the name `dev.ini` since it's included in .hgignore file.
-
-
-Setup the Development Database
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-To create a development database, use the following example. This is a one
-time operation executed from the nix-shell of rhodecode-enterprise-ce sources ::
-
-    rc-setup-app dev.ini \
-        --user=admin --password=secret \
-        --email=[e-mail] \
-        --repos=~/my_dev_repos
-
-
-Compile CSS and JavaScript
-^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-To use the application's frontend and prepare it for production deployment,
-you will need to compile the CSS and JavaScript with Grunt.
-This is easily done from within the nix-shell using the following command::
-
-    make web-build
-
-When developing new features you will need to recompile following any
-changes made to the CSS or JavaScript files when developing the code::
-
-    grunt watch
-
-This prepares the development (with comments/whitespace) versions of files.
-
-Start the Development Servers
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-From the rhodecode-vcsserver directory, start the development server in another
-nix-shell, using the following command::
-
-      pserve configs/dev.ini
-
-In the adjacent nix-shell which you created for your development server, you may
-now start CE with the following command::
-
-
-      pserve --reload configs/dev.ini
-
-.. note::
-
-  `--reload` flag will automatically reload the server when source file changes.
-
-
-Run the Environment Tests
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Please make sure that the tests are passing to verify that your environment is
-set up correctly. RhodeCode uses py.test to run tests.
-While your instance is running, start a new nix-shell and simply run
-``make test`` to run the basic test suite.
-
+Please refer to RCstack installed documentation for instructions on setting up dev environment:
+https://docs.rhodecode.com/rcstack/dev/dev-setup.html
 
 Need Help?
 ^^^^^^^^^^
 
 Join us on Slack via https://rhodecode.com/join or post questions in our
 Community Portal at https://community.rhodecode.com

docs/index.rst

@@ -1,92 +1,104 b''
 |RCE|
 =====
 
 |RCE| is a high-performance source code management and collaboration system.
 It enables you to develop projects securely behind the firewall while
 providing collaboration tools that work with |git|, |hg|,
 and |svn| |repos|. The user interface allows you to create, edit,
 and commit files and |repos| while managing their security permissions.
 
 |RCE| provides the following features:
 
 * Source code management.
 * Extended permissions management.
 * Integrated code collaboration tools.
 * Integrated code review and notifications.
 * Scalability provided by multi-node setup.
 * Fully programmable automation API.
 * Web-based hook management.
 * Native |svn| support.
 * Migration from existing databases.
 * |RCE| SDK.
 * Built-in analytics
 * Built in integrations including: Slack, Webhooks (used for Jenkins/TeamCity and other CIs), Jira, Redmine, Hipchat
 * Pluggable authentication system.
 * Support for AD, |LDAP|, Crowd, CAS, PAM.
 * Support for external authentication via Oauth Google, Github, Bitbucket, Twitter.
 * Debug modes of operation.
 * Private and public gists.
 * Gists with limited lifetimes and within instance only sharing.
 * Fully integrated code search function.
 * Always on SSL connectivity.
 
 .. only:: html
 
    Table of Contents
    -----------------
 
 .. toctree::
    :maxdepth: 1
+   :caption: Documentation directory
+
+   Back to documentation directory <https://docs.rhodecode.com/>
+
+.. toctree::
+   :maxdepth: 1
+   :caption: RhodeCode RCstack Documentation
+
+   RhodeCode RCstack Installer <https://docs.rhodecode.com/rcstack/>
+
+.. toctree::
+   :maxdepth: 1
    :caption: Admin Documentation
 
    install/quick-start
    install/install-database
    install/install-steps
    admin/system-overview
    admin/system-admin
    admin/user-admin
    admin/repo-admin
    admin/security-tips
    auth/auth
    issue-trackers/issue-trackers
    admin/lab-settings
 
 .. toctree::
    :maxdepth: 1
    :caption: Feature Documentation
 
    collaboration/collaboration
    collaboration/review-notifications
    collaboration/pull-requests
    code-review/code-review
    integrations/integrations
 
 .. toctree::
    :maxdepth: 1
    :caption: User Documentation
 
    usage/basic-usage
    tutorials/tutorials
 
 .. toctree::
    :maxdepth: 1
    :caption: Developer Documentation
 
    api/api
    tools/rhodecode-tools
    extensions/extensions-hooks
    contributing/contributing
 
 .. toctree::
    :maxdepth: 2
    :caption: RhodeCode rcstack Documentation
 
    RhodeCode Installer <https://docs.rhodecode.com/rcstack/>
 
 .. toctree::
    :maxdepth: 1
    :caption: About
 
    release-notes/release-notes
    known-issues/known-issues
    admin/glossary

docs/install/quick-start.rst

@@ -1,92 +1,93 b''
 .. _quick-start:
 
 Quick Start Installation Guide
 ==============================
 
 .. important::
 
     These are quick start instructions. To optimize your |RCE|,
     |RCC|, and |RCT| usage, read the more detailed instructions in our guides.
     For detailed installation instructions, see
     :ref:`RhodeCode rcstack Documentation <rcstack:installation>`
 
 
 
 To get |RCE| up and running, run through the below steps:
 
 1. Register to get the latest |RCC| installer instruction from `rhodecode.com/download`_.
    If you don't have an account, sign up at `rhodecode.com/register`_.
 
 2. Run the |RCS| installer and start init process.
    following example:
 
 .. code-block:: bash
 
     mkdir docker-rhodecode && cd docker-rhodecode
     curl -L -s -o rcstack https://dls.rhodecode.com/get-rcstack && chmod +x rcstack
 
    ./rcstack init
 
 
 .. important::
 
    We recommend running RhodeCode as a non-root user, such as `rhodecode`;
    this user must have a proper home directory and sudo permissions (to start Docker)
    Either log in as that user to install the software, or do it as root
    with `sudo -i -u rhodecode ./rcstack init`
 
 
 3. Follow instructions on |RCS| documentation pages
 
    :ref:`Quick install tutorial <rcstack:quick_installation>`
 
 4. Check stack status
 
 .. code-block:: bash
 
     ./rcstack status
 
 
 Output should look similar to this:
 
 .. code-block:: bash
 
     ---
     CONTAINER ID   IMAGE                                   STATUS                        NAMES                                 PORTS
     ef54fc528e3a   traefik:v2.9.5                          Up 2 hours             rc_cluster_router-traefik-1           0.0.0.0:80->80/tcp, :::80->80/tcp
     f3ea0539e8b0   rhodecode/rhodecode-ee:4.28.0           Up 2 hours (healthy)   rc_cluster_apps-rhodecode-1           0.0.0.0:10020->10020/tcp, :::10020->10020/tcp
     2be52ba58ffe   rhodecode/rhodecode-ee:4.28.0           Up 2 hours (healthy)   rc_cluster_apps-vcsserver-1
     7cd730ad3263   rhodecode/rhodecode-ee:4.28.0           Up 2 hours (healthy)   rc_cluster_apps-celery-1
     dfa231342c87   rhodecode/rhodecode-ee:4.28.0           Up 2 hours (healthy)   rc_cluster_apps-celery-beat-1
     d3d76ce2de96   rhodecode/rhodecode-ee:4.28.0           Up 2 hours (healthy)   rc_cluster_apps-sshd-1
     daaac329414b   rhodecode/rhodecode-ee:4.28.0           Up 2 hours (healthy)   rc_cluster_apps-svn-1
     7b8504fb9acb   nginx:1.23.2                            Up 2 hours (healthy)   rc_cluster_services-nginx-1           80/tcp
     7279c25feb6b   elasticsearch:6.8.23                    Up 2 hours (healthy)   rc_cluster_services-elasticsearch-1   9200/tcp, 9300/tcp
     19fb93587493   redis:7.0.5                             Up 2 hours (healthy)   rc_cluster_services-redis-1           6379/tcp
     fb77fb6496c6   channelstream/channelstream:0.7.1       Up 2 hours (healthy)   rc_cluster_services-channelstream-1   8000/tcp
     cb6c5c022f5b   postgres:14.6                           Up 2 hours (healthy)   rc_cluster_services-database-1        5432/tcp
 
+
 At this point you should be able to access:
 
 - RhodeCode instance at your domain entered, e.g http://rhodecode.local, the default access
   credentials are generated and stored inside .runtime.env.
   For example::
 
     RHODECODE_USER_NAME=admin
     RHODECODE_USER_PASS=super-secret-password
 
 
+
 .. note::
 
    Recommended post quick start install instructions:
 
    * Read the documentation
    * Carry out the :ref:`rhodecode-post-install-ref`
    * Set up :ref:`indexing-ref`
    * Familiarise yourself with the :ref:`rhodecode-admin-ref` section.
 
-.. _rhodecode.com/download/: https://rhodecode.com/download/
 .. _rhodecode.com: https://rhodecode.com/
 .. _rhodecode.com/register: https://rhodecode.com/register/
 .. _rhodecode.com/download: https://rhodecode.com/download/
 

docs/install/using-sqllite.rst

@@ -1,32 +1,21 b''
 .. _install-sqlite-database:
 
-SQLite
-------
+SQLite (Deprecated)
+-------------------
 
 .. important::
 
-    We do not recommend using SQLite in a large development environment
-    as it has an internal locking mechanism which can become a performance
-    bottleneck when there are more than 5 concurrent users.
+    As of 5.x, SQLite is no longer supported, we advise to migrate to MySQL or PostgreSQL.
 
-|RCE| installs SQLite as the default database if you do not specify another
-during installation. SQLite is suitable for small teams,
-projects with a low load, and evaluation purposes since it is built into
-|RCE| and does not require any additional database server.
-
-Using MySQL or PostgreSQL in an large setup gives you much greater
-performance, and while migration tools exist to move from one database type
-to another, it is better to get it right first time and to immediately use
-MySQL or PostgreSQL when you deploy |RCE| in a production environment.
 
 Migrating From SQLite to PostgreSQL
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 If you started working with SQLite and now need to migrate your database
 to PostgreSQL, you can contact [e-mail] for some help. We have a
 set of scripts that enable SQLite to PostgreSQL migration. These scripts have
 been tested, and work with PostgreSQL 9.1+.
 
 .. note::
 
     There are no SQLite to MySQL or MariaDB scripts available.

docs/known-issues/known-issues.rst

@@ -1,95 +1,117 b''
 .. _known-issues:
 
 Known Issues
 ============
 
 Windows Upload
 --------------
 
 There can be an issue with uploading files from web interface on Windows,
 and afterwards users cannot properly clone or synchronize with the repository.
 
 Early testing shows that often uploading files via HTML forms on Windows
 includes the full path of the file being uploaded and not the name of the file.
 
 Old Format of Git Repositories
 ------------------------------
 
 There is an issue when trying to import old |git| format |repos| into recent
 versions of |RCE|. This issue can occur when importing from external |git|
 repositories or from older versions of |RCE| (<=2.2.7).
 
 To convert the old version into a current version, clone the old
 |repo| into a local machine using a recent |git| client, then push it to a new
 |repo| inside |RCE|.
 
 
 VCS Server Memory Consumption
 -----------------------------
 
 The VCS Server cache grows without limits if not configured correctly. This
 applies to |RCE| versions prior to the 3.3.2 releases, as 3.3.2
 shipped with the optimal configuration as default. See the
 :ref:`vcs-server-maintain` section for details.
 
 To fix this issue, upgrade to |RCE| 3.3.2 or greater, and if you discover
 memory consumption issues check the VCS Server settings.
 
 Newer Operating system locales
 ------------------------------
 
 |RCC| has a know problem with locales, due to changes in glibc 2.27+ which affects
 the local-archive format, which is now incompatible with our used glibc 2.26.
 
 Mostly affected are:
+
 - Fedora 23+
 - Ubuntu 18.04
 - CentOS / RHEL 8
 
 To work around this problem, you need set path to ``$LOCAL_ARCHIVE`` to the
 locale package in older pre glibc 2.27 format, or set `LC_ALL=C` in your enviroment.
 
 To use the pre 2.27 locale-archive fix follow these steps:
 
 1. Download the pre 2.27 locale-archive package
 
 .. code-block:: bash
 
     wget https://dls.rhodecode.com/assets/locale-archive
 
 
 2. Point ``$LOCAL_ARCHIVE`` to the locale package.
 
 .. code-block:: bash
 
     $ export LOCALE_ARCHIVE=/home/USER/locale-archive  # change to your path
 
 This should be added *both* in `enviroment` variable of `~/.rccontrol/supervisor/supervisord.ini`
 e.g
 
 ```
 [supervisord]
 environment = HOME=/home/user/rhodecode,LOCALE_ARCHIVE=/YOUR-PATH/locale-archive`
 ```
 
 and in user .bashrc/.zshrc etc, or via a startup script that
 runs `rccontrol self-init`
 
 If you happen to be running |RCC| from systemd, use the following
 example to pass the correct locale information on boot.
 
 .. code-block:: ini
 
     [Unit]
     Description=Rhodecode
     After=network.target
 
     [Service]
     Type=forking
     User=scm
     Environment="LOCALE_ARCHIVE=/YOUR-PATH/locale-archive"
     ExecStart=/YOUR-PATH/.rccontrol-profile/bin/rccontrol-self-init
 
     [Install]
     WantedBy=multi-user.target
 
+
+Merge stucks in "merging" status
+--------------------------------
+
+Similar issues:
+
+- Pull Request duplicated and/or stucks in "creating" status.
+
+Mostly affected are:
+
+- Kubernetes AWS EKS setup with NFS as shared storage
+- AWS EFS as shared storage
+
+Workaround:
+
+1. Manually clear the repo cache via UI: 
+:menuselection:`Repository Settings --> Caches --> Invalidate repository cache`
+
+1. Open problematic PR and reset status to "created"
+
+Now you can merge PR normally

docs/release-notes/release-notes-5.1.0.rst

@@ -1,59 +1,58 b''
 |RCE| 5.1.0 |RNS|
 -----------------
 
 Release Date
 ^^^^^^^^^^^^
 
 - 2024-07-18
 
 
 New Features
 ^^^^^^^^^^^^
 
-- We've introduced 2FA for users. Now alongside the external auth 2fa support RhodeCode allows to enable 2FA for users
+- We've introduced 2FA for users. Now alongside the external auth 2FA support RhodeCode allows to enable 2FA for users.
   2FA options will be available for each user individually, or enforced via authentication plugins like ldap, or internal.
 - Email based log-in. RhodeCode now allows to log-in using email as well as username for main authentication type.
 - Ability to replace a file using web UI. Now one can replace an existing file from the web-ui.
 - GIT LFS Sync automation. Remote push/pull commands now can also sync GIT LFS objects.
-- Added ability to remove or close branches from the web ui
-- Added ability to delete a branch automatically after merging PR for git repositories
-- Added support for S3 based archive_cache based that allows storing cached archives in S3 compatible object store.
+- Added ability to remove or close branches from the web ui.
+- Added ability to delete a branch automatically after merging PR for git repositories.
+- Added support for S3 based archive_cache that allows storing cached archives in S3 compatible object store.
 
 
 General
 ^^^^^^^
 
-- Upgraded all dependency libraries to their latest available versions
+- Upgraded all dependency libraries to their latest available versions.
 - Repository storage is no longer controlled via DB settings, but .ini file. This allows easier automated deployments.
 - Bumped mercurial to 6.7.4
 - Mercurial: enable httppostarguments for better support of large repositories with lots of heads.
 - Added explicit db-migrate step to update hooks for 5.X release.
 
 
 Security
 ^^^^^^^^
 
 
 
 Performance
 ^^^^^^^^^^^
 
 - Introduced a full rewrite of ssh backend for performance. The result is 2-5x speed improvement for operation with ssh.
-  enable new ssh wrapper by setting: `ssh.wrapper_cmd = /home/rhodecode/venv/bin/rc-ssh-wrapper-v2`
-- Introduced a new hooks subsystem that is more scalable and faster, enable it by settings: `vcs.hooks.protocol = celery`
+  Enable new ssh wrapper by setting: `ssh.wrapper_cmd = /home/rhodecode/venv/bin/rc-ssh-wrapper-v2`
+- Introduced a new hooks subsystem that is more scalable and faster, enable it by setting: `vcs.hooks.protocol = celery`
 
 
 Fixes
 ^^^^^
 
-- Archives: Zip archive download breaks when a gitmodules file is present
-- Branch permissions: fixed bug preventing to specify own rules from 4.X install
-- SVN: refactored svn events, thus fixing support for it in dockerized env
-- Fixed empty server url in PR link after push from cli
+- Archives: Zip archive download breaks when a gitmodules file is present.
+- Branch permissions: fixed bug preventing to specify own rules from 4.X install.
+- SVN: refactored svn events, thus fixing support for it in dockerized environment.
+- Fixed empty server url in PR link after push from cli.
 
 
 Upgrade notes
 ^^^^^^^^^^^^^
 
-- RhodeCode 5.1.0 is a mayor feature release after big 5.0.0 python3 migration. Happy to ship a first time feature
-  rich release
+- RhodeCode 5.1.0 is a major feature release after big 5.0.0 python3 migration. Happy to ship a first time feature-rich release.

docs/release-notes/release-notes.rst

@@ -1,173 +1,175 b''
 .. _rhodecode-release-notes-ref:
 
 Release Notes
 =============
 
 |RCE| 5.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
-
+   release-notes-5.2.0.rst
+   release-notes-5.1.2.rst
+   release-notes-5.1.1.rst
    release-notes-5.1.0.rst
    release-notes-5.0.3.rst
    release-notes-5.0.2.rst
    release-notes-5.0.1.rst
    release-notes-5.0.0.rst
 
 
 |RCE| 4.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
    release-notes-4.27.1.rst
    release-notes-4.27.0.rst
    release-notes-4.26.0.rst
    release-notes-4.25.2.rst
    release-notes-4.25.1.rst
    release-notes-4.25.0.rst
    release-notes-4.24.1.rst
    release-notes-4.24.0.rst
    release-notes-4.23.2.rst
    release-notes-4.23.1.rst
    release-notes-4.23.0.rst
    release-notes-4.22.0.rst
    release-notes-4.21.0.rst
    release-notes-4.20.1.rst
    release-notes-4.20.0.rst
    release-notes-4.19.3.rst
    release-notes-4.19.2.rst
    release-notes-4.19.1.rst
    release-notes-4.19.0.rst
    release-notes-4.18.3.rst
    release-notes-4.18.2.rst
    release-notes-4.18.1.rst
    release-notes-4.18.0.rst
    release-notes-4.17.4.rst
    release-notes-4.17.3.rst
    release-notes-4.17.2.rst
    release-notes-4.17.1.rst
    release-notes-4.17.0.rst
    release-notes-4.16.2.rst
    release-notes-4.16.1.rst
    release-notes-4.16.0.rst
    release-notes-4.15.2.rst
    release-notes-4.15.1.rst
    release-notes-4.15.0.rst
    release-notes-4.14.1.rst
    release-notes-4.14.0.rst
    release-notes-4.13.3.rst
    release-notes-4.13.2.rst
    release-notes-4.13.1.rst
    release-notes-4.13.0.rst
    release-notes-4.12.4.rst
    release-notes-4.12.3.rst
    release-notes-4.12.2.rst
    release-notes-4.12.1.rst
    release-notes-4.12.0.rst
    release-notes-4.11.6.rst
    release-notes-4.11.5.rst
    release-notes-4.11.4.rst
    release-notes-4.11.3.rst
    release-notes-4.11.2.rst
    release-notes-4.11.1.rst
    release-notes-4.11.0.rst
    release-notes-4.10.6.rst
    release-notes-4.10.5.rst
    release-notes-4.10.4.rst
    release-notes-4.10.3.rst
    release-notes-4.10.2.rst
    release-notes-4.10.1.rst
    release-notes-4.10.0.rst
    release-notes-4.9.1.rst
    release-notes-4.9.0.rst
    release-notes-4.8.0.rst
    release-notes-4.7.2.rst
    release-notes-4.7.1.rst
    release-notes-4.7.0.rst
    release-notes-4.6.1.rst
    release-notes-4.6.0.rst
    release-notes-4.5.2.rst
    release-notes-4.5.1.rst
    release-notes-4.5.0.rst
    release-notes-4.4.2.rst
    release-notes-4.4.1.rst
    release-notes-4.4.0.rst
    release-notes-4.3.1.rst
    release-notes-4.3.0.rst
    release-notes-4.2.1.rst
    release-notes-4.2.0.rst
    release-notes-4.1.2.rst
    release-notes-4.1.1.rst
    release-notes-4.1.0.rst
    release-notes-4.0.1.rst
    release-notes-4.0.0.rst
 
 |RCE| 3.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
    release-notes-3.8.4.rst
    release-notes-3.8.3.rst
    release-notes-3.8.2.rst
    release-notes-3.8.1.rst
    release-notes-3.8.0.rst
    release-notes-3.7.1.rst
    release-notes-3.7.0.rst
    release-notes-3.6.1.rst
    release-notes-3.6.0.rst
    release-notes-3.5.2.rst
    release-notes-3.5.1.rst
    release-notes-3.5.0.rst
    release-notes-3.4.1.rst
    release-notes-3.4.0.rst
    release-notes-3.3.4.rst
    release-notes-3.3.3.rst
    release-notes-3.3.2.rst
    release-notes-3.3.1.rst
    release-notes-3.3.0.rst
    release-notes-3.2.3.rst
    release-notes-3.2.2.rst
    release-notes-3.2.1.rst
    release-notes-3.2.0.rst
    release-notes-3.1.1.rst
    release-notes-3.1.0.rst
    release-notes-3.0.2.rst
    release-notes-3.0.1.rst
    release-notes-3.0.0.rst
 
 |RCE| 2.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
    release-notes-2.2.8.rst
    release-notes-2.2.7.rst
    release-notes-2.2.6.rst
    release-notes-2.2.5.rst
    release-notes-2.2.4.rst
    release-notes-2.2.3.rst
    release-notes-2.2.2.rst
    release-notes-2.2.1.rst
    release-notes-2.2.0.rst
    release-notes-2.1.0.rst
    release-notes-2.0.2.rst
    release-notes-2.0.1.rst
    release-notes-2.0.0.rst
 
 |RCE| 1.x Versions
 ------------------
 
 .. toctree::
    :maxdepth: 1
 
    release-notes-1.7.2.rst
    release-notes-1.7.1.rst
    release-notes-1.7.0.rst
    release-notes-1.6.0.rst

docs/requirements_docs.txt

@@ -1,11 +1,11 b''
 sphinx==7.2.6
 
 furo==2023.9.10
 sphinx-press-theme==0.8.0
 sphinx-rtd-theme==1.3.0
 
-pygments==2.16.1
+pygments==2.18.0
 
 docutils<0.19
 markupsafe==2.1.3
 jinja2==3.1.2

requirements.txt

@@ -1,313 +1,299 b''
 # deps, generated via pipdeptree --exclude setuptools,wheel,pipdeptree,pip -f | tr '[:upper:]' '[:lower:]'
 
 alembic==1.13.1
   mako==1.2.4
     markupsafe==2.1.2
   sqlalchemy==1.4.52
     greenlet==3.0.3
-  typing_extensions==4.9.0
+  typing_extensions==4.12.2
 async-timeout==4.0.3
 babel==2.12.1
 beaker==1.12.1
 celery==5.3.6
   billiard==4.2.0
   click==8.1.3
   click-didyoumean==0.3.0
     click==8.1.3
   click-plugins==1.1.1
     click==8.1.3
   click-repl==0.2.0
     click==8.1.3
-    prompt-toolkit==3.0.38
-      wcwidth==0.2.6
+    prompt_toolkit==3.0.47
+      wcwidth==0.2.13
     six==1.16.0
   kombu==5.3.5
     amqp==5.2.0
       vine==5.1.0
     vine==5.1.0
   python-dateutil==2.8.2
     six==1.16.0
   tzdata==2024.1
   vine==5.1.0
 channelstream==0.7.1
   gevent==24.2.1
     greenlet==3.0.3
     zope.event==5.0.0
-    zope.interface==6.3.0
+    zope.interface==7.0.3
   itsdangerous==1.1.0
   marshmallow==2.18.0
   pyramid==2.0.2
     hupper==1.12
     plaster==1.1.2
     plaster-pastedeploy==1.0.1
       pastedeploy==3.1.0
       plaster==1.1.2
     translationstring==1.4
     venusian==3.0.0
     webob==1.8.7
     zope.deprecation==5.0.0
-    zope.interface==6.3.0
+    zope.interface==7.0.3
   pyramid-jinja2==2.10
     jinja2==3.1.2
       markupsafe==2.1.2
     markupsafe==2.1.2
     pyramid==2.0.2
       hupper==1.12
       plaster==1.1.2
       plaster-pastedeploy==1.0.1
         pastedeploy==3.1.0
         plaster==1.1.2
       translationstring==1.4
       venusian==3.0.0
       webob==1.8.7
       zope.deprecation==5.0.0
-      zope.interface==6.3.0
+      zope.interface==7.0.3
     zope.deprecation==5.0.0
   python-dateutil==2.8.2
     six==1.16.0
   requests==2.28.2
     certifi==2022.12.7
     charset-normalizer==3.1.0
     idna==3.4
     urllib3==1.26.14
   ws4py==0.5.1
 deform==2.0.15
   chameleon==3.10.2
   colander==2.0
     iso8601==1.1.0
     translationstring==1.4
   iso8601==1.1.0
   peppercorn==0.6
   translationstring==1.4
   zope.deprecation==5.0.0
 docutils==0.19
 dogpile.cache==1.3.3
   decorator==5.1.1
   stevedore==5.1.0
     pbr==5.11.1
 formencode==2.1.0
   six==1.16.0
-fsspec==2024.6.0
-gunicorn==21.2.0
-  packaging==24.0
+fsspec==2024.9.0
+gunicorn==23.0.0
+  packaging==24.1
 gevent==24.2.1
   greenlet==3.0.3
   zope.event==5.0.0
-  zope.interface==6.3.0
-ipython==8.14.0
-  backcall==0.2.0
+  zope.interface==7.0.3
+ipython==8.26.0
   decorator==5.1.1
-  jedi==0.19.0
-    parso==0.8.3
-  matplotlib-inline==0.1.6
-    traitlets==5.9.0
-  pexpect==4.8.0
+  jedi==0.19.1
+    parso==0.8.4
+  matplotlib-inline==0.1.7
+    traitlets==5.14.3
+  pexpect==4.9.0
     ptyprocess==0.7.0
-  pickleshare==0.7.5
-  prompt-toolkit==3.0.38
-    wcwidth==0.2.6
-  pygments==2.15.1
-  stack-data==0.6.2
-    asttokens==2.2.1
+  prompt_toolkit==3.0.47
+    wcwidth==0.2.13
+  pygments==2.18.0
+  stack-data==0.6.3
+    asttokens==2.4.1
       six==1.16.0
-    executing==1.2.0
-    pure-eval==0.2.2
-  traitlets==5.9.0
+    executing==2.0.1
+    pure_eval==0.2.3
+  traitlets==5.14.3
+  typing_extensions==4.12.2
 markdown==3.4.3
 msgpack==1.0.8
 mysqlclient==2.1.1
 nbconvert==7.7.3
   beautifulsoup4==4.12.3
     soupsieve==2.5
   bleach==6.1.0
     six==1.16.0
     webencodings==0.5.1
   defusedxml==0.7.1
   jinja2==3.1.2
     markupsafe==2.1.2
   jupyter_core==5.3.1
     platformdirs==3.10.0
-    traitlets==5.9.0
+    traitlets==5.14.3
   jupyterlab-pygments==0.2.2
   markupsafe==2.1.2
   mistune==2.0.5
   nbclient==0.8.0
     jupyter_client==8.3.0
       jupyter_core==5.3.1
         platformdirs==3.10.0
-        traitlets==5.9.0
+        traitlets==5.14.3
       python-dateutil==2.8.2
         six==1.16.0
       pyzmq==25.0.0
       tornado==6.2
-      traitlets==5.9.0
+      traitlets==5.14.3
     jupyter_core==5.3.1
       platformdirs==3.10.0
-      traitlets==5.9.0
+      traitlets==5.14.3
     nbformat==5.9.2
       fastjsonschema==2.18.0
       jsonschema==4.18.6
         attrs==22.2.0
         pyrsistent==0.19.3
       jupyter_core==5.3.1
         platformdirs==3.10.0
-        traitlets==5.9.0
-      traitlets==5.9.0
-    traitlets==5.9.0
+        traitlets==5.14.3
+      traitlets==5.14.3
+    traitlets==5.14.3
   nbformat==5.9.2
     fastjsonschema==2.18.0
     jsonschema==4.18.6
       attrs==22.2.0
       pyrsistent==0.19.3
     jupyter_core==5.3.1
       platformdirs==3.10.0
-      traitlets==5.9.0
-    traitlets==5.9.0
+      traitlets==5.14.3
+    traitlets==5.14.3
   pandocfilters==1.5.0
-  pygments==2.15.1
+  pygments==2.18.0
   tinycss2==1.2.1
     webencodings==0.5.1
-  traitlets==5.9.0
-orjson==3.10.3
+  traitlets==5.14.3
+orjson==3.10.7
 paste==3.10.1
 premailer==3.10.0
   cachetools==5.3.3
   cssselect==1.2.0
   cssutils==2.6.0
-  lxml==4.9.3
+  lxml==5.3.0
   requests==2.28.2
     certifi==2022.12.7
     charset-normalizer==3.1.0
     idna==3.4
     urllib3==1.26.14
 psutil==5.9.8
 psycopg2==2.9.9
 py-bcrypt==0.4
 pycmarkgfm==1.2.0
   cffi==1.16.0
     pycparser==2.21
 pycryptodome==3.17
 pycurl==7.45.3
 pymysql==1.0.3
 pyotp==2.8.0
 pyparsing==3.1.1
-pyramid-debugtoolbar==4.12.1
-  pygments==2.15.1
-  pyramid==2.0.2
-    hupper==1.12
-    plaster==1.1.2
-    plaster-pastedeploy==1.0.1
-      pastedeploy==3.1.0
-      plaster==1.1.2
-    translationstring==1.4
-    venusian==3.0.0
-    webob==1.8.7
-    zope.deprecation==5.0.0
-    zope.interface==6.3.0
-  pyramid-mako==1.1.0
-    mako==1.2.4
-      markupsafe==2.1.2
-    pyramid==2.0.2
-      hupper==1.12
-      plaster==1.1.2
-      plaster-pastedeploy==1.0.1
-        pastedeploy==3.1.0
-        plaster==1.1.2
-      translationstring==1.4
-      venusian==3.0.0
-      webob==1.8.7
-      zope.deprecation==5.0.0
-      zope.interface==6.3.0
 pyramid-mailer==0.15.1
   pyramid==2.0.2
     hupper==1.12
     plaster==1.1.2
     plaster-pastedeploy==1.0.1
       pastedeploy==3.1.0
       plaster==1.1.2
     translationstring==1.4
     venusian==3.0.0
     webob==1.8.7
     zope.deprecation==5.0.0
-    zope.interface==6.3.0
+    zope.interface==7.0.3
   repoze.sendmail==4.4.1
-    transaction==3.1.0
-      zope.interface==6.3.0
-    zope.interface==6.3.0
-  transaction==3.1.0
-    zope.interface==6.3.0
+    transaction==5.0.0
+      zope.interface==7.0.3
+    zope.interface==7.0.3
+  transaction==5.0.0
+    zope.interface==7.0.3
+pyramid-mako==1.1.0
+  mako==1.2.4
+    markupsafe==2.1.2
+  pyramid==2.0.2
+    hupper==1.12
+    plaster==1.1.2
+    plaster-pastedeploy==1.0.1
+      pastedeploy==3.1.0
+      plaster==1.1.2
+    translationstring==1.4
+    venusian==3.0.0
+    webob==1.8.7
+    zope.deprecation==5.0.0
+    zope.interface==7.0.3
 python-ldap==3.4.3
   pyasn1==0.4.8
   pyasn1-modules==0.2.8
     pyasn1==0.4.8
 python-memcached==1.59
   six==1.16.0
 python-pam==2.0.2
-python3-saml==1.15.0
+python3-saml==1.16.0
   isodate==0.6.1
     six==1.16.0
-  lxml==4.9.3
-  xmlsec==1.3.13
-    lxml==4.9.3
+  lxml==5.3.0
+  xmlsec==1.3.14
+    lxml==5.3.0
 pyyaml==6.0.1
-redis==5.0.4
+redis==5.1.0
   async-timeout==4.0.3
 regex==2022.10.31
 routes==2.5.1
   repoze.lru==0.7
   six==1.16.0
-s3fs==2024.6.0
+s3fs==2024.9.0
   aiobotocore==2.13.0
     aiohttp==3.9.5
       aiosignal==1.3.1
         frozenlist==1.4.1
       attrs==22.2.0
       frozenlist==1.4.1
       multidict==6.0.5
       yarl==1.9.4
         idna==3.4
         multidict==6.0.5
     aioitertools==0.11.0
     botocore==1.34.106
       jmespath==1.0.1
       python-dateutil==2.8.2
         six==1.16.0
       urllib3==1.26.14
     wrapt==1.16.0
   aiohttp==3.9.5
     aiosignal==1.3.1
       frozenlist==1.4.1
     attrs==22.2.0
     frozenlist==1.4.1
     multidict==6.0.5
     yarl==1.9.4
       idna==3.4
       multidict==6.0.5
-  fsspec==2024.6.0
+  fsspec==2024.9.0
 simplejson==3.19.2
 sshpubkeys==3.3.1
   cryptography==40.0.2
     cffi==1.16.0
       pycparser==2.21
   ecdsa==0.18.0
     six==1.16.0
 sqlalchemy==1.4.52
   greenlet==3.0.3
-  typing_extensions==4.9.0
+  typing_extensions==4.12.2
 supervisor==4.2.5
 tzlocal==4.3
   pytz-deprecation-shim==0.1.0.post0
     tzdata==2024.1
 tempita==0.5.2
 unidecode==1.3.6
 urlobject==2.4.3
 waitress==3.0.0
 webhelpers2==2.1
   markupsafe==2.1.2
   six==1.16.0
 whoosh==2.7.4
 zope.cachedescriptors==5.0.0
 qrcode==7.4.2
 
 ## uncomment to add the debug libraries
 #-r requirements_debug.txt

requirements_debug.txt

@@ -1,28 +1,29 b''
 ## special libraries we could extend the requirements.txt file with to add some
 ## custom libraries usefull for debug and memory tracing
 
 objgraph
 memory-profiler
 pympler
 
 ## debug
 ipdb
 ipython
 rich
+pyramid-debugtoolbar
 
 # format
 flake8
 ruff
 
 pipdeptree==2.7.1
 invoke==2.0.0
 bumpversion==0.6.0
 bump2version==1.0.1
 
 docutils-stubs
 types-redis
 types-requests==2.31.0.6
 types-sqlalchemy
 types-psutil
 types-pycurl
 types-ujson

requirements_test.txt

@@ -1,48 +1,48 b''
 # test related requirements
 mock==5.1.0
 pytest-cov==4.1.0
   coverage==7.4.3
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
 pytest-env==1.1.3
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
 pytest-profiling==1.7.0
   gprof2dot==2022.7.29
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
   six==1.16.0
 pytest-rerunfailures==13.0
-  packaging==24.0
+  packaging==24.1
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
 pytest-runner==6.0.1
 pytest-sugar==1.0.0
-  packaging==24.0
+  packaging==24.1
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
   termcolor==2.4.0
 pytest-timeout==2.3.1
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
 webtest==3.0.0
   beautifulsoup4==4.12.3
     soupsieve==2.5
   waitress==3.0.0
   webob==1.8.7
 
 # RhodeCode test-data
 rc_testdata @ https://code.rhodecode.com/upstream/rc-testdata-dist/raw/77378e9097f700b4c1b9391b56199fe63566b5c9/rc_testdata-0.11.0.tar.gz#egg=rc_testdata
 rc_testdata==0.11.0

rhodecode/VERSION

@@ -1,1 +1,1 b''
-5.1.2
  No newline at end of file
+5.2.0

rhodecode/api/__init__.py

@@ -1,581 +1,581 b''
 # Copyright (C) 2011-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import itertools
 import logging
 import sys
 import fnmatch
 
 import decorator
 import venusian
 from collections import OrderedDict
 
 from pyramid.exceptions import ConfigurationError
 from pyramid.renderers import render
 from pyramid.response import Response
 from pyramid.httpexceptions import HTTPNotFound
 
 from rhodecode.api.exc import (
     JSONRPCBaseError, JSONRPCError, JSONRPCForbidden, JSONRPCValidationError)
 from rhodecode.apps._base import TemplateArgs
 from rhodecode.lib.auth import AuthUser
 from rhodecode.lib.base import get_ip_addr, attach_context_attributes
 from rhodecode.lib.exc_tracking import store_exception
 from rhodecode.lib import ext_json
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.plugins.utils import get_plugin_settings
 from rhodecode.model.db import User, UserApiKeys
+from rhodecode.config.patches import inspect_getargspec
 
 log = logging.getLogger(__name__)
 
 DEFAULT_RENDERER = 'jsonrpc_renderer'
 DEFAULT_URL = '/_admin/api'
 SERVICE_API_IDENTIFIER = 'service_'
 
 
 def find_methods(jsonrpc_methods, pattern):
     matches = OrderedDict()
     if not isinstance(pattern, (list, tuple)):
         pattern = [pattern]
 
     for single_pattern in pattern:
         for method_name, method in filter(
                 lambda x: not x[0].startswith(SERVICE_API_IDENTIFIER), jsonrpc_methods.items()
         ):
             if fnmatch.fnmatch(method_name, single_pattern):
                 matches[method_name] = method
     return matches
 
 
 class ExtJsonRenderer(object):
     """
     Custom renderer that makes use of our ext_json lib
 
     """
 
     def __init__(self):
         self.serializer = ext_json.formatted_json
 
     def __call__(self, info):
         """ Returns a plain JSON-encoded string with content-type
         ``application/json``. The content-type may be overridden by
         setting ``request.response.content_type``."""
 
         def _render(value, system):
             request = system.get('request')
             if request is not None:
                 response = request.response
                 ct = response.content_type
                 if ct == response.default_content_type:
                     response.content_type = 'application/json'
 
             return self.serializer(value)
 
         return _render
 
 
 def jsonrpc_response(request, result):
     rpc_id = getattr(request, 'rpc_id', None)
 
     ret_value = ''
     if rpc_id:
         ret_value = {'id': rpc_id, 'result': result, 'error': None}
 
         # fetch deprecation warnings, and store it inside results
         deprecation = getattr(request, 'rpc_deprecation', None)
         if deprecation:
             ret_value['DEPRECATION_WARNING'] = deprecation
 
     raw_body = render(DEFAULT_RENDERER, ret_value, request=request)
     content_type = 'application/json'
     content_type_header = 'Content-Type'
     headers = {
         content_type_header: content_type
     }
     return Response(
         body=raw_body,
         content_type=content_type,
         headerlist=[(k, v) for k, v in headers.items()]
     )
 
 
 def jsonrpc_error(request, message, retid=None, code: int | None = None, headers: dict | None = None):
     """
     Generate a Response object with a JSON-RPC error body
     """
     headers = headers or {}
     content_type = 'application/json'
     content_type_header = 'Content-Type'
     if content_type_header not in headers:
         headers[content_type_header] = content_type
 
     err_dict = {'id': retid, 'result': None, 'error': message}
     raw_body = render(DEFAULT_RENDERER, err_dict, request=request)
 
     return Response(
         body=raw_body,
         status=code,
         content_type=content_type,
         headerlist=[(k, v) for k, v in headers.items()]
     )
 
 
 def exception_view(exc, request):
     rpc_id = getattr(request, 'rpc_id', None)
 
     if isinstance(exc, JSONRPCError):
         fault_message = safe_str(exc)
         log.debug('json-rpc error rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, JSONRPCValidationError):
         colander_exc = exc.colander_exception
         # TODO(marcink): think maybe of nicer way to serialize errors ?
         fault_message = colander_exc.asdict()
         log.debug('json-rpc colander error rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, JSONRPCForbidden):
         fault_message = 'Access was denied to this resource.'
         log.warning('json-rpc forbidden call rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, HTTPNotFound):
         method = request.rpc_method
         log.debug('json-rpc method `%s` not found in list of '
                   'api calls: %s, rpc_id:%s',
                   method, list(request.registry.jsonrpc_methods.keys()), rpc_id)
 
         similar = 'none'
         try:
             similar_paterns = [f'*{x}*' for x in method.split('_')]
             similar_found = find_methods(
                 request.registry.jsonrpc_methods, similar_paterns)
             similar = ', '.join(similar_found.keys()) or similar
         except Exception:
             # make the whole above block safe
             pass
 
         fault_message = f"No such method: {method}. Similar methods: {similar}"
     else:
         fault_message = 'undefined error'
         exc_info = exc.exc_info()
         store_exception(id(exc_info), exc_info, prefix='rhodecode-api')
 
         statsd = request.registry.statsd
         if statsd:
             exc_type = f"{exc.__class__.__module__}.{exc.__class__.__name__}"
             statsd.incr('rhodecode_exception_total',
                         tags=["exc_source:api", f"type:{exc_type}"])
 
     return jsonrpc_error(request, fault_message, rpc_id)
 
 
 def request_view(request):
     """
     Main request handling method. It handles all logic to call a specific
     exposed method
     """
     # cython compatible inspect
-    from rhodecode.config.patches import inspect_getargspec
     inspect = inspect_getargspec()
 
     # check if we can find this session using api_key, get_by_auth_token
     # search not expired tokens only
     try:
         if not request.rpc_method.startswith(SERVICE_API_IDENTIFIER):
             api_user = User.get_by_auth_token(request.rpc_api_key)
 
             if api_user is None:
                 return jsonrpc_error(
                     request, retid=request.rpc_id, message='Invalid API KEY')
 
             if not api_user.active:
                 return jsonrpc_error(
                     request, retid=request.rpc_id,
                     message='Request from this user not allowed')
 
             # check if we are allowed to use this IP
             auth_u = AuthUser(
                 api_user.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr)
             if not auth_u.ip_allowed:
                 return jsonrpc_error(
                     request, retid=request.rpc_id,
                     message='Request from IP:{} not allowed'.format(
                         request.rpc_ip_addr))
             else:
                 log.info('Access for IP:%s allowed', request.rpc_ip_addr)
 
             # register our auth-user
             request.rpc_user = auth_u
             request.environ['rc_auth_user_id'] = str(auth_u.user_id)
 
             # now check if token is valid for API
             auth_token = request.rpc_api_key
             token_match = api_user.authenticate_by_token(
                 auth_token, roles=[UserApiKeys.ROLE_API])
             invalid_token = not token_match
 
             log.debug('Checking if API KEY is valid with proper role')
             if invalid_token:
                 return jsonrpc_error(
                     request, retid=request.rpc_id,
                     message='API KEY invalid or, has bad role for an API call')
         else:
             auth_u = 'service'
             if request.rpc_api_key != request.registry.settings['app.service_api.token']:
                 raise Exception("Provided service secret is not recognized!")
 
     except Exception:
         log.exception('Error on API AUTH')
         return jsonrpc_error(
             request, retid=request.rpc_id, message='Invalid API KEY')
 
     method = request.rpc_method
     func = request.registry.jsonrpc_methods[method]
 
     # now that we have a method, add request._req_params to
     # self.kargs and dispatch control to WGIController
 
     argspec = inspect.getargspec(func)
     arglist = argspec[0]
     defs = argspec[3] or []
     defaults = [type(a) for a in defs]
     default_empty = type(NotImplemented)
 
     # kw arguments required by this method
     func_kwargs = dict(itertools.zip_longest(
         reversed(arglist), reversed(defaults), fillvalue=default_empty))
 
     # This attribute will need to be first param of a method that uses
     # api_key, which is translated to instance of user at that name
     user_var = 'apiuser'
     request_var = 'request'
 
     for arg in [user_var, request_var]:
         if arg not in arglist:
             return jsonrpc_error(
                 request,
                 retid=request.rpc_id,
                 message='This method [%s] does not support '
                         'required parameter `%s`' % (func.__name__, arg))
 
     # get our arglist and check if we provided them as args
     for arg, default in func_kwargs.items():
         if arg in [user_var, request_var]:
             # user_var and request_var are pre-hardcoded parameters and we
             # don't need to do any translation
             continue
 
         # skip the required param check if it's default value is
         # NotImplementedType (default_empty)
         if default == default_empty and arg not in request.rpc_params:
             return jsonrpc_error(
                 request,
                 retid=request.rpc_id,
                 message=('Missing non optional `%s` arg in JSON DATA' % arg)
             )
 
     # sanitize extra passed arguments
     for k in list(request.rpc_params.keys()):
         if k not in func_kwargs:
             del request.rpc_params[k]
 
     call_params = request.rpc_params
     call_params.update({
         'request': request,
         'apiuser': auth_u
     })
 
     # register some common functions for usage
     rpc_user = request.rpc_user.user_id if hasattr(request, 'rpc_user') else None
     attach_context_attributes(TemplateArgs(), request, rpc_user)
 
     statsd = request.registry.statsd
 
     try:
         ret_value = func(**call_params)
         resp = jsonrpc_response(request, ret_value)
         if statsd:
             statsd.incr('rhodecode_api_call_success_total')
         return resp
     except JSONRPCBaseError:
         raise
     except Exception:
         log.exception('Unhandled exception occurred on api call: %s', func)
         exc_info = sys.exc_info()
         exc_id, exc_type_name = store_exception(
             id(exc_info), exc_info, prefix='rhodecode-api')
         error_headers = {
             'RhodeCode-Exception-Id': str(exc_id),
             'RhodeCode-Exception-Type': str(exc_type_name)
         }
         err_resp = jsonrpc_error(
             request, retid=request.rpc_id, message='Internal server error',
             headers=error_headers)
         if statsd:
             statsd.incr('rhodecode_api_call_fail_total')
         return err_resp
 
 
 def setup_request(request):
     """
     Parse a JSON-RPC request body. It's used inside the predicates method
     to validate and bootstrap requests for usage in rpc calls.
 
     We need to raise JSONRPCError here if we want to return some errors back to
     user.
     """
 
     log.debug('Executing setup request: %r', request)
     request.rpc_ip_addr = get_ip_addr(request.environ)
     # TODO(marcink): deprecate GET at some point
     if request.method not in ['POST', 'GET']:
         log.debug('unsupported request method "%s"', request.method)
         raise JSONRPCError(
             'unsupported request method "%s". Please use POST' % request.method)
 
     if 'CONTENT_LENGTH' not in request.environ:
         log.debug("No Content-Length")
         raise JSONRPCError("Empty body, No Content-Length in request")
 
     else:
         length = request.environ['CONTENT_LENGTH']
         log.debug('Content-Length: %s', length)
 
         if length == 0:
             log.debug("Content-Length is 0")
             raise JSONRPCError("Content-Length is 0")
 
     raw_body = request.body
     log.debug("Loading JSON body now")
     try:
         json_body = ext_json.json.loads(raw_body)
     except ValueError as e:
         # catch JSON errors Here
         raise JSONRPCError(f"JSON parse error ERR:{e} RAW:{raw_body!r}")
 
     request.rpc_id = json_body.get('id')
     request.rpc_method = json_body.get('method')
 
     # check required base parameters
     try:
         api_key = json_body.get('api_key')
         if not api_key:
             api_key = json_body.get('auth_token')
 
         if not api_key:
             raise KeyError('api_key or auth_token')
 
         # TODO(marcink): support passing in token in request header
 
         request.rpc_api_key = api_key
         request.rpc_id = json_body['id']
         request.rpc_method = json_body['method']
         request.rpc_params = json_body['args'] \
             if isinstance(json_body['args'], dict) else {}
 
         log.debug('method: %s, params: %.10240r', request.rpc_method, request.rpc_params)
     except KeyError as e:
         raise JSONRPCError(f'Incorrect JSON data. Missing {e}')
 
     log.debug('setup complete, now handling method:%s rpcid:%s',
               request.rpc_method, request.rpc_id, )
 
 
 class RoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return f'jsonrpc route = {self.val}'
 
     phash = text
 
     def __call__(self, info, request):
         if self.val:
             # potentially setup and bootstrap our call
             setup_request(request)
 
             # Always return True so that even if it isn't a valid RPC it
             # will fall through to the underlaying handlers like notfound_view
             return True
 
 
 class NotFoundPredicate(object):
     def __init__(self, val, config):
         self.val = val
         self.methods = config.registry.jsonrpc_methods
 
     def text(self):
         return f'jsonrpc method not found = {self.val}'
 
     phash = text
 
     def __call__(self, info, request):
         return hasattr(request, 'rpc_method')
 
 
 class MethodPredicate(object):
     def __init__(self, val, config):
         self.method = val
 
     def text(self):
         return f'jsonrpc method = {self.method}'
 
     phash = text
 
     def __call__(self, context, request):
         # we need to explicitly return False here, so pyramid doesn't try to
         # execute our view directly. We need our main handler to execute things
         return getattr(request, 'rpc_method') == self.method
 
 
 def add_jsonrpc_method(config, view, **kwargs):
     # pop the method name
     method = kwargs.pop('method', None)
 
     if method is None:
         raise ConfigurationError(
             'Cannot register a JSON-RPC method without specifying the "method"')
 
     # we define custom predicate, to enable to detect conflicting methods,
     # those predicates are kind of "translation" from the decorator variables
     # to internal predicates names
 
     kwargs['jsonrpc_method'] = method
 
     # register our view into global view store for validation
     config.registry.jsonrpc_methods[method] = view
 
     # we're using our main request_view handler, here, so each method
     # has a unified handler for itself
     config.add_view(request_view, route_name='apiv2', **kwargs)
 
 
 class jsonrpc_method(object):
     """
     decorator that works similar to @add_view_config decorator,
     but tailored for our JSON RPC
     """
 
     venusian = venusian  # for testing injection
 
     def __init__(self, method=None, **kwargs):
         self.method = method
         self.kwargs = kwargs
 
     def __call__(self, wrapped):
         kwargs = self.kwargs.copy()
         kwargs['method'] = self.method or wrapped.__name__
         depth = kwargs.pop('_depth', 0)
 
         def callback(context, name, ob):
             config = context.config.with_package(info.module)
             config.add_jsonrpc_method(view=ob, **kwargs)
 
         info = venusian.attach(wrapped, callback, category='pyramid',
                                depth=depth + 1)
         if info.scope == 'class':
             # ensure that attr is set if decorating a class method
             kwargs.setdefault('attr', wrapped.__name__)
 
         kwargs['_info'] = info.codeinfo  # fbo action_method
         return wrapped
 
 
 class jsonrpc_deprecated_method(object):
     """
     Marks method as deprecated, adds log.warning, and inject special key to
     the request variable to mark method as deprecated.
     Also injects special docstring that extract_docs will catch to mark
     method as deprecated.
 
     :param use_method: specify which method should be used instead of
         the decorated one
 
     Use like::
 
         @jsonrpc_method()
         @jsonrpc_deprecated_method(use_method='new_func', deprecated_at_version='3.0.0')
         def old_func(request, apiuser, arg1, arg2):
             ...
     """
 
     def __init__(self, use_method, deprecated_at_version):
         self.use_method = use_method
         self.deprecated_at_version = deprecated_at_version
         self.deprecated_msg = ''
 
     def __call__(self, func):
         self.deprecated_msg = 'Please use method `{method}` instead.'.format(
             method=self.use_method)
 
         docstring = """\n
         .. deprecated:: {version}
 
            {deprecation_message}
 
         {original_docstring}
         """
         func.__doc__ = docstring.format(
             version=self.deprecated_at_version,
             deprecation_message=self.deprecated_msg,
             original_docstring=func.__doc__)
         return decorator.decorator(self.__wrapper, func)
 
     def __wrapper(self, func, *fargs, **fkwargs):
         log.warning('DEPRECATED API CALL on function %s, please '
                     'use `%s` instead', func, self.use_method)
         # alter function docstring to mark as deprecated, this is picked up
         # via fabric file that generates API DOC.
         result = func(*fargs, **fkwargs)
 
         request = fargs[0]
         request.rpc_deprecation = 'DEPRECATED METHOD ' + self.deprecated_msg
         return result
 
 
 def add_api_methods(config):
     from rhodecode.api.views import (
         deprecated_api, gist_api, pull_request_api, repo_api, repo_group_api,
         server_api, search_api, testing_api, user_api, user_group_api)
 
     config.scan('rhodecode.api.views')
 
 
 def includeme(config):
     plugin_module = 'rhodecode.api'
     plugin_settings = get_plugin_settings(
         plugin_module, config.registry.settings)
 
     if not hasattr(config.registry, 'jsonrpc_methods'):
         config.registry.jsonrpc_methods = OrderedDict()
 
     # match filter by given method only
     config.add_view_predicate('jsonrpc_method', MethodPredicate)
     config.add_view_predicate('jsonrpc_method_not_found', NotFoundPredicate)
 
     config.add_renderer(DEFAULT_RENDERER, ExtJsonRenderer())
     config.add_directive('add_jsonrpc_method', add_jsonrpc_method)
 
     config.add_route_predicate(
         'jsonrpc_call', RoutePredicate)
 
     config.add_route(
         'apiv2', plugin_settings.get('url', DEFAULT_URL), jsonrpc_call=True)
 
     # register some exception handling view
     config.add_view(exception_view, context=JSONRPCBaseError)
     config.add_notfound_view(exception_view, jsonrpc_method_not_found=True)
 
     add_api_methods(config)

rhodecode/api/views/server_api.py

@@ -1,423 +1,423 b''
 # Copyright (C) 2011-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import itertools
 import base64
 
 from rhodecode.api import (
     jsonrpc_method, JSONRPCError, JSONRPCForbidden, find_methods)
 
 from rhodecode.api.utils import (
     Optional, OAttr, has_superadmin_permission, get_user_or_error)
 from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path
 from rhodecode.lib import system_info
 from rhodecode.lib import user_sessions
 from rhodecode.lib import exc_tracking
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.model.db import UserIpMap
 from rhodecode.model.scm import ScmModel
-from rhodecode.apps.file_store import utils
+from rhodecode.apps.file_store import utils as store_utils
 from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
     FileOverSizeException
 
 log = logging.getLogger(__name__)
 
 
 @jsonrpc_method()
 def get_server_info(request, apiuser):
     """
     Returns the |RCE| server information.
 
     This includes the running version of |RCE| and all installed
     packages. This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'modules': [<module name>,...]
         'py_version': <python version>,
         'platform': <platform type>,
         'rhodecode_version': <rhodecode version>
       }
       error :  null
     """
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     server_info = ScmModel().get_server_info(request.environ)
     # rhodecode-index requires those
 
     server_info['index_storage'] = server_info['search']['value']['location']
     server_info['storage'] = server_info['storage']['value']['path']
 
     return server_info
 
 
 @jsonrpc_method()
 def get_repo_store(request, apiuser):
     """
     Returns the |RCE| repository storage information.
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'modules': [<module name>,...]
         'py_version': <python version>,
         'platform': <platform type>,
         'rhodecode_version': <rhodecode version>
       }
       error :  null
     """
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     path = get_rhodecode_repo_store_path()
     return {"path": path}
 
 
 @jsonrpc_method()
 def get_ip(request, apiuser, userid=Optional(OAttr('apiuser'))):
     """
     Displays the IP Address as seen from the |RCE| server.
 
     * This command displays the IP Address, as well as all the defined IP
       addresses for the specified user. If the ``userid`` is not set, the
       data returned is for the user calling the method.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from |authtoken|.
     :type apiuser: AuthUser
     :param userid: Sets the userid for which associated IP Address data
         is returned.
     :type userid: Optional(str or int)
 
     Example output:
 
     .. code-block:: bash
 
         id : <id_given_in_input>
         result : {
                      "server_ip_addr": "<ip_from_clien>",
                      "user_ips": [
                                     {
                                        "ip_addr": "<ip_with_mask>",
                                        "ip_range": ["<start_ip>", "<end_ip>"],
                                     },
                                     ...
                                  ]
         }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     userid = Optional.extract(userid, evaluate_locals=locals())
     userid = getattr(userid, 'user_id', userid)
 
     user = get_user_or_error(userid)
     ips = UserIpMap.query().filter(UserIpMap.user == user).all()
     return {
         'server_ip_addr': request.rpc_ip_addr,
         'user_ips': ips
     }
 
 
 @jsonrpc_method()
 def rescan_repos(request, apiuser, remove_obsolete=Optional(False)):
     """
     Triggers a rescan of the specified repositories.
 
     * If the ``remove_obsolete`` option is set, it also deletes repositories
       that are found in the database but not on the file system, so called
       "clean zombies".
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param remove_obsolete: Deletes repositories from the database that
         are not found on the filesystem.
     :type remove_obsolete: Optional(``True`` | ``False``)
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : {
         'added': [<added repository name>,...]
         'removed': [<removed repository name>,...]
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         'Error occurred during rescan repositories action'
       }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     try:
         rm_obsolete = Optional.extract(remove_obsolete)
         added, removed = repo2db_mapper(ScmModel().repo_scan(),
                                         remove_obsolete=rm_obsolete, force_hooks_rebuild=True)
         return {'added': added, 'removed': removed}
     except Exception:
         log.exception('Failed to run repo rescann')
         raise JSONRPCError(
             'Error occurred during rescan repositories action'
         )
 
 
 @jsonrpc_method()
 def cleanup_sessions(request, apiuser, older_then=Optional(60)):
     """
     Triggers a session cleanup action.
 
     If the ``older_then`` option is set, only sessions that hasn't been
     accessed in the given number of days will be removed.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param older_then: Deletes session that hasn't been accessed
         in given number of days.
     :type older_then: Optional(int)
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result: {
         "backend": "<type of backend>",
         "sessions_removed": <number_of_removed_sessions>
       }
       error :  null
 
     Example error output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       result : null
       error :  {
         'Error occurred during session cleanup'
       }
 
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     older_then = safe_int(Optional.extract(older_then)) or 60
     older_than_seconds = 60 * 60 * 24 * older_then
 
     config = system_info.rhodecode_config().get_value()['value']['config']
     session_model = user_sessions.get_session_handler(
         config.get('beaker.session.type', 'memory'))(config)
 
     backend = session_model.SESSION_TYPE
     try:
         cleaned = session_model.clean_sessions(
             older_than_seconds=older_than_seconds)
         return {'sessions_removed': cleaned, 'backend': backend}
     except user_sessions.CleanupCommand as msg:
         return {'cleanup_command': str(msg), 'backend': backend}
     except Exception as e:
         log.exception('Failed session cleanup')
         raise JSONRPCError(
             'Error occurred during session cleanup'
         )
 
 
 @jsonrpc_method()
 def get_method(request, apiuser, pattern=Optional('*')):
     """
     Returns list of all available API methods. By default match pattern
     os "*" but any other pattern can be specified. eg *comment* will return
     all methods with comment inside them. If just single method is matched
     returned data will also include method specification
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
     :param pattern: pattern to match method names against
     :type pattern: Optional("*")
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": [
         "changeset_comment",
         "comment_pull_request",
         "comment_commit"
       ]
       error :  null
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": [
         "comment_commit",
         {
           "apiuser": "<RequiredType>",
           "comment_type": "<Optional:u'note'>",
           "commit_id": "<RequiredType>",
           "message": "<RequiredType>",
           "repoid": "<RequiredType>",
           "request": "<RequiredType>",
           "resolves_comment_id": "<Optional:None>",
           "status": "<Optional:None>",
           "userid": "<Optional:<OptionalAttr:apiuser>>"
         }
       ]
       error :  null
     """
-    from rhodecode.config.patches import inspect_getargspec
-    inspect = inspect_getargspec()
+    from rhodecode.config import patches
+    inspect = patches.inspect_getargspec()
 
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     pattern = Optional.extract(pattern)
 
     matches = find_methods(request.registry.jsonrpc_methods, pattern)
 
     args_desc = []
     matches_keys = list(matches.keys())
     if len(matches_keys) == 1:
         func = matches[matches_keys[0]]
 
         argspec = inspect.getargspec(func)
         arglist = argspec[0]
         defaults = list(map(repr, argspec[3] or []))
 
         default_empty = '<RequiredType>'
 
         # kw arguments required by this method
         func_kwargs = dict(itertools.zip_longest(
             reversed(arglist), reversed(defaults), fillvalue=default_empty))
         args_desc.append(func_kwargs)
 
     return matches_keys + args_desc
 
 
 @jsonrpc_method()
 def store_exception(request, apiuser, exc_data_json, prefix=Optional('rhodecode')):
     """
     Stores sent exception inside the built-in exception tracker in |RCE| server.
 
     This command can only be run using an |authtoken| with admin rights to
     the specified repository.
 
     This command takes the following options:
 
     :param apiuser: This is filled automatically from the |authtoken|.
     :type apiuser: AuthUser
 
     :param exc_data_json: JSON data with exception e.g
         {"exc_traceback": "Value `1` is not allowed", "exc_type_name": "ValueError"}
     :type exc_data_json: JSON data
 
     :param prefix: prefix for error type, e.g 'rhodecode', 'vcsserver', 'rhodecode-tools'
     :type prefix: Optional("rhodecode")
 
     Example output:
 
     .. code-block:: bash
 
       id : <id_given_in_input>
       "result": {
         "exc_id": 139718459226384,
         "exc_url": "http://localhost:8080/_admin/settings/exceptions/139718459226384"
       }
       error :  null
     """
     if not has_superadmin_permission(apiuser):
         raise JSONRPCForbidden()
 
     prefix = Optional.extract(prefix)
     exc_id = exc_tracking.generate_id()
 
     try:
         exc_data = json.loads(exc_data_json)
     except Exception:
         log.error('Failed to parse JSON: %r', exc_data_json)
         raise JSONRPCError('Failed to parse JSON data from exc_data_json field. '
                            'Please make sure it contains a valid JSON.')
 
     try:
         exc_traceback = exc_data['exc_traceback']
         exc_type_name = exc_data['exc_type_name']
         exc_value = ''
     except KeyError as err:
         raise JSONRPCError(
             f'Missing exc_traceback, or exc_type_name '
             f'in exc_data_json field. Missing: {err}')
 
     class ExcType:
         __name__ = exc_type_name
 
     exc_info = (ExcType(), exc_value, exc_traceback)
 
     exc_tracking._store_exception(
         exc_id=exc_id, exc_info=exc_info, prefix=prefix)
 
     exc_url = request.route_url(
         'admin_settings_exception_tracker_show', exception_id=exc_id)
     return {'exc_id': exc_id, 'exc_url': exc_url}

rhodecode/apps/admin/__init__.py

@@ -1,1093 +1,1124 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 from rhodecode.apps._base import ADMIN_PREFIX
 from rhodecode.apps._base.navigation import includeme as nav_includeme
 from rhodecode.apps.admin.views.main_views import AdminMainView
 
 
 def admin_routes(config):
     """
     Admin prefixed routes
     """
     from rhodecode.apps.admin.views.audit_logs import AdminAuditLogsView
     from rhodecode.apps.admin.views.artifacts import AdminArtifactsView
     from rhodecode.apps.admin.views.automation import AdminAutomationView
     from rhodecode.apps.admin.views.scheduler import AdminSchedulerView
     from rhodecode.apps.admin.views.defaults import AdminDefaultSettingsView
     from rhodecode.apps.admin.views.exception_tracker import ExceptionsTrackerView
     from rhodecode.apps.admin.views.open_source_licenses import OpenSourceLicensesAdminSettingsView
     from rhodecode.apps.admin.views.permissions import AdminPermissionsView
     from rhodecode.apps.admin.views.process_management import AdminProcessManagementView
     from rhodecode.apps.admin.views.repo_groups import AdminRepoGroupsView 
     from rhodecode.apps.admin.views.repositories import AdminReposView
     from rhodecode.apps.admin.views.sessions import AdminSessionSettingsView
     from rhodecode.apps.admin.views.settings import AdminSettingsView
     from rhodecode.apps.admin.views.svn_config import AdminSvnConfigView
     from rhodecode.apps.admin.views.system_info import AdminSystemInfoSettingsView
     from rhodecode.apps.admin.views.user_groups import AdminUserGroupsView
     from rhodecode.apps.admin.views.users import AdminUsersView, UsersView
-    
+    from rhodecode.apps.admin.views.security import AdminSecurityView
+
+    # Security EE feature
+
+    config.add_route(
+        'admin_security',
+        pattern='/security')
+    config.add_view(
+        AdminSecurityView,
+        attr='security',
+        route_name='admin_security', request_method='GET',
+        renderer='rhodecode:templates/admin/security/security.mako')
+
+    config.add_route(
+        name='admin_security_update',
+        pattern='/security/update')
+    config.add_view(
+        AdminSecurityView,
+        attr='security_update',
+        route_name='admin_security_update', request_method='POST',
+        renderer='rhodecode:templates/admin/security/security.mako')
+
+    config.add_route(
+        name='admin_security_modify_allowed_vcs_client_versions',
+        pattern=ADMIN_PREFIX + '/security/modify/allowed_vcs_client_versions')
+    config.add_view(
+        AdminSecurityView,
+        attr='vcs_whitelisted_client_versions_edit',
+        route_name='admin_security_modify_allowed_vcs_client_versions', request_method=('GET', 'POST'),
+        renderer='rhodecode:templates/admin/security/edit_allowed_vcs_client_versions.mako')
+
+
     config.add_route(
         name='admin_audit_logs',
         pattern='/audit_logs')
     config.add_view(
         AdminAuditLogsView,
         attr='admin_audit_logs',
         route_name='admin_audit_logs', request_method='GET',
         renderer='rhodecode:templates/admin/admin_audit_logs.mako')
 
     config.add_route(
         name='admin_audit_log_entry',
         pattern='/audit_logs/{audit_log_id}')
     config.add_view(
         AdminAuditLogsView,
         attr='admin_audit_log_entry',
         route_name='admin_audit_log_entry', request_method='GET',
         renderer='rhodecode:templates/admin/admin_audit_log_entry.mako')
 
     # Artifacts EE feature
     config.add_route(
         'admin_artifacts',
         pattern=ADMIN_PREFIX + '/artifacts')
     config.add_route(
         'admin_artifacts_show_all',
         pattern=ADMIN_PREFIX + '/artifacts')
     config.add_view(
         AdminArtifactsView,
         attr='artifacts',
         route_name='admin_artifacts', request_method='GET',
         renderer='rhodecode:templates/admin/artifacts/artifacts.mako')
     config.add_view(
         AdminArtifactsView,
         attr='artifacts',
         route_name='admin_artifacts_show_all', request_method='GET',
         renderer='rhodecode:templates/admin/artifacts/artifacts.mako')
 
     # EE views
     config.add_route(
         name='admin_artifacts_show_info',
         pattern=ADMIN_PREFIX + '/artifacts/{uid}')
     config.add_route(
         name='admin_artifacts_delete',
         pattern=ADMIN_PREFIX + '/artifacts/{uid}/delete')
     config.add_route(
         name='admin_artifacts_update',
         pattern=ADMIN_PREFIX + '/artifacts/{uid}/update')
 
     # Automation EE feature
     config.add_route(
         'admin_automation',
         pattern=ADMIN_PREFIX + '/automation')
     config.add_view(
         AdminAutomationView,
         attr='automation',
         route_name='admin_automation', request_method='GET',
         renderer='rhodecode:templates/admin/automation/automation.mako')
 
     # Scheduler EE feature
     config.add_route(
         'admin_scheduler',
         pattern=ADMIN_PREFIX + '/scheduler')
     config.add_view(
         AdminSchedulerView,
         attr='scheduler',
         route_name='admin_scheduler', request_method='GET',
         renderer='rhodecode:templates/admin/scheduler/scheduler.mako')
 
     config.add_route(
         name='admin_settings_open_source',
         pattern='/settings/open_source')
     config.add_view(
         OpenSourceLicensesAdminSettingsView,
         attr='open_source_licenses',
         route_name='admin_settings_open_source', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_vcs_svn_generate_cfg',
         pattern='/settings/vcs/svn_generate_cfg')
     config.add_view(
         AdminSvnConfigView,
         attr='vcs_svn_generate_config',
         route_name='admin_settings_vcs_svn_generate_cfg',
         request_method='POST', renderer='json')
 
     config.add_route(
         name='admin_settings_system',
         pattern='/settings/system')
     config.add_view(
         AdminSystemInfoSettingsView,
         attr='settings_system_info',
         route_name='admin_settings_system', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_system_update',
         pattern='/settings/system/updates')
     config.add_view(
         AdminSystemInfoSettingsView,
         attr='settings_system_info_check_update',
         route_name='admin_settings_system_update', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings_system_update.mako')
 
     config.add_route(
         name='admin_settings_exception_tracker',
         pattern='/settings/exceptions')
     config.add_view(
         ExceptionsTrackerView,
         attr='browse_exceptions',
         route_name='admin_settings_exception_tracker', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_exception_tracker_delete_all',
         pattern='/settings/exceptions_delete_all')
     config.add_view(
         ExceptionsTrackerView,
         attr='exception_delete_all',
         route_name='admin_settings_exception_tracker_delete_all', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_exception_tracker_show',
         pattern='/settings/exceptions/{exception_id}')
     config.add_view(
         ExceptionsTrackerView,
         attr='exception_show',
         route_name='admin_settings_exception_tracker_show', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_exception_tracker_delete',
         pattern='/settings/exceptions/{exception_id}/delete')
     config.add_view(
         ExceptionsTrackerView,
         attr='exception_delete',
         route_name='admin_settings_exception_tracker_delete', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_sessions',
         pattern='/settings/sessions')
     config.add_view(
         AdminSessionSettingsView,
         attr='settings_sessions',
         route_name='admin_settings_sessions', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_sessions_cleanup',
         pattern='/settings/sessions/cleanup')
     config.add_view(
         AdminSessionSettingsView,
         attr='settings_sessions_cleanup',
         route_name='admin_settings_sessions_cleanup', request_method='POST')
 
     config.add_route(
         name='admin_settings_process_management',
         pattern='/settings/process_management')
     config.add_view(
         AdminProcessManagementView,
         attr='process_management',
         route_name='admin_settings_process_management', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_process_management_data',
         pattern='/settings/process_management/data')
     config.add_view(
         AdminProcessManagementView,
         attr='process_management_data',
         route_name='admin_settings_process_management_data', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings_process_management_data.mako')
 
     config.add_route(
         name='admin_settings_process_management_signal',
         pattern='/settings/process_management/signal')
     config.add_view(
         AdminProcessManagementView,
         attr='process_management_signal',
         route_name='admin_settings_process_management_signal',
         request_method='POST', renderer='json_ext')
 
     config.add_route(
         name='admin_settings_process_management_master_signal',
         pattern='/settings/process_management/master_signal')
     config.add_view(
         AdminProcessManagementView,
         attr='process_management_master_signal',
         route_name='admin_settings_process_management_master_signal',
         request_method='POST', renderer='json_ext')
 
     # default settings
     config.add_route(
         name='admin_defaults_repositories',
         pattern='/defaults/repositories')
     config.add_view(
         AdminDefaultSettingsView,
         attr='defaults_repository_show',
         route_name='admin_defaults_repositories', request_method='GET',
         renderer='rhodecode:templates/admin/defaults/defaults.mako')
 
     config.add_route(
         name='admin_defaults_repositories_update',
         pattern='/defaults/repositories/update')
     config.add_view(
         AdminDefaultSettingsView,
         attr='defaults_repository_update',
         route_name='admin_defaults_repositories_update', request_method='POST',
         renderer='rhodecode:templates/admin/defaults/defaults.mako')
 
     # admin settings
 
     config.add_route(
         name='admin_settings',
         pattern='/settings')
     config.add_view(
         AdminSettingsView,
         attr='settings_global',
         route_name='admin_settings', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_update',
         pattern='/settings/update')
     config.add_view(
         AdminSettingsView,
         attr='settings_global_update',
         route_name='admin_settings_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_global',
         pattern='/settings/global')
     config.add_view(
         AdminSettingsView,
         attr='settings_global',
         route_name='admin_settings_global', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_global_update',
         pattern='/settings/global/update')
     config.add_view(
         AdminSettingsView,
         attr='settings_global_update',
         route_name='admin_settings_global_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_vcs',
         pattern='/settings/vcs')
     config.add_view(
         AdminSettingsView,
         attr='settings_vcs',
         route_name='admin_settings_vcs', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_vcs_update',
         pattern='/settings/vcs/update')
     config.add_view(
         AdminSettingsView,
         attr='settings_vcs_update',
         route_name='admin_settings_vcs_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_vcs_svn_pattern_delete',
         pattern='/settings/vcs/svn_pattern_delete')
     config.add_view(
         AdminSettingsView,
         attr='settings_vcs_delete_svn_pattern',
         route_name='admin_settings_vcs_svn_pattern_delete', request_method='POST',
         renderer='json_ext', xhr=True)
 
     config.add_route(
         name='admin_settings_mapping',
         pattern='/settings/mapping')
     config.add_view(
         AdminSettingsView,
         attr='settings_mapping',
         route_name='admin_settings_mapping', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_mapping_update',
         pattern='/settings/mapping/update')
     config.add_view(
         AdminSettingsView,
         attr='settings_mapping_update',
         route_name='admin_settings_mapping_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_visual',
         pattern='/settings/visual')
     config.add_view(
         AdminSettingsView,
         attr='settings_visual',
         route_name='admin_settings_visual', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_visual_update',
         pattern='/settings/visual/update')
     config.add_view(
         AdminSettingsView,
         attr='settings_visual_update',
         route_name='admin_settings_visual_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_issuetracker',
         pattern='/settings/issue-tracker')
     config.add_view(
         AdminSettingsView,
         attr='settings_issuetracker',
         route_name='admin_settings_issuetracker', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_issuetracker_update',
         pattern='/settings/issue-tracker/update')
     config.add_view(
         AdminSettingsView,
         attr='settings_issuetracker_update',
         route_name='admin_settings_issuetracker_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_issuetracker_test',
         pattern='/settings/issue-tracker/test')
     config.add_view(
         AdminSettingsView,
         attr='settings_issuetracker_test',
         route_name='admin_settings_issuetracker_test', request_method='POST',
         renderer='string', xhr=True)
 
     config.add_route(
         name='admin_settings_issuetracker_delete',
         pattern='/settings/issue-tracker/delete')
     config.add_view(
         AdminSettingsView,
         attr='settings_issuetracker_delete',
         route_name='admin_settings_issuetracker_delete', request_method='POST',
         renderer='json_ext', xhr=True)
 
     config.add_route(
         name='admin_settings_email',
         pattern='/settings/email')
     config.add_view(
         AdminSettingsView,
         attr='settings_email',
         route_name='admin_settings_email', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_email_update',
         pattern='/settings/email/update')
     config.add_view(
         AdminSettingsView,
         attr='settings_email_update',
         route_name='admin_settings_email_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_hooks',
         pattern='/settings/hooks')
     config.add_view(
         AdminSettingsView,
         attr='settings_hooks',
         route_name='admin_settings_hooks', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_hooks_update',
         pattern='/settings/hooks/update')
     config.add_view(
         AdminSettingsView,
         attr='settings_hooks_update',
         route_name='admin_settings_hooks_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_hooks_delete',
         pattern='/settings/hooks/delete')
     config.add_view(
         AdminSettingsView,
         attr='settings_hooks_update',
         route_name='admin_settings_hooks_delete', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_search',
         pattern='/settings/search')
     config.add_view(
         AdminSettingsView,
         attr='settings_search',
         route_name='admin_settings_search', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_labs',
         pattern='/settings/labs')
     config.add_view(
         AdminSettingsView,
         attr='settings_labs',
         route_name='admin_settings_labs', request_method='GET',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     config.add_route(
         name='admin_settings_labs_update',
         pattern='/settings/labs/update')
     config.add_view(
         AdminSettingsView,
         attr='settings_labs_update',
         route_name='admin_settings_labs_update', request_method='POST',
         renderer='rhodecode:templates/admin/settings/settings.mako')
 
     # global permissions
 
     config.add_route(
         name='admin_permissions_application',
         pattern='/permissions/application')
     config.add_view(
         AdminPermissionsView,
         attr='permissions_application',
         route_name='admin_permissions_application', request_method='GET',
         renderer='rhodecode:templates/admin/permissions/permissions.mako')
 
     config.add_route(
         name='admin_permissions_application_update',
         pattern='/permissions/application/update')
     config.add_view(
         AdminPermissionsView,
         attr='permissions_application_update',
         route_name='admin_permissions_application_update', request_method='POST',
         renderer='rhodecode:templates/admin/permissions/permissions.mako')
 
     config.add_route(
         name='admin_permissions_global',
         pattern='/permissions/global')
     config.add_view(
         AdminPermissionsView,
         attr='permissions_global',
         route_name='admin_permissions_global', request_method='GET',
         renderer='rhodecode:templates/admin/permissions/permissions.mako')
     
     config.add_route(
         name='admin_permissions_global_update',
         pattern='/permissions/global/update')
     config.add_view(
         AdminPermissionsView,
         attr='permissions_global_update',
         route_name='admin_permissions_global_update', request_method='POST',
         renderer='rhodecode:templates/admin/permissions/permissions.mako')
 
     config.add_route(
         name='admin_permissions_object',
         pattern='/permissions/object')
     config.add_view(
         AdminPermissionsView,
         attr='permissions_objects',
         route_name='admin_permissions_object', request_method='GET',
         renderer='rhodecode:templates/admin/permissions/permissions.mako')
 
     config.add_route(
         name='admin_permissions_object_update',
         pattern='/permissions/object/update')
     config.add_view(
         AdminPermissionsView,
         attr='permissions_objects_update',
         route_name='admin_permissions_object_update', request_method='POST',
         renderer='rhodecode:templates/admin/permissions/permissions.mako')
 
     # Branch perms EE feature
     config.add_route(
         name='admin_permissions_branch',
         pattern='/permissions/branch')
     config.add_view(
         AdminPermissionsView,
         attr='permissions_branch',
         route_name='admin_permissions_branch', request_method='GET',
         renderer='rhodecode:templates/admin/permissions/permissions.mako')
 
     config.add_route(
         name='admin_permissions_ips',
         pattern='/permissions/ips')
     config.add_view(
         AdminPermissionsView,
         attr='permissions_ips',
         route_name='admin_permissions_ips', request_method='GET',
         renderer='rhodecode:templates/admin/permissions/permissions.mako')
 
     config.add_route(
         name='admin_permissions_overview',
         pattern='/permissions/overview')
     config.add_view(
         AdminPermissionsView,
         attr='permissions_overview',
         route_name='admin_permissions_overview', request_method='GET',
         renderer='rhodecode:templates/admin/permissions/permissions.mako')
 
     config.add_route(
         name='admin_permissions_auth_token_access',
         pattern='/permissions/auth_token_access')
     config.add_view(
         AdminPermissionsView,
         attr='auth_token_access',
         route_name='admin_permissions_auth_token_access', request_method='GET',
         renderer='rhodecode:templates/admin/permissions/permissions.mako')
 
     config.add_route(
         name='admin_permissions_ssh_keys',
         pattern='/permissions/ssh_keys')
     config.add_view(
         AdminPermissionsView,
         attr='ssh_keys',
         route_name='admin_permissions_ssh_keys', request_method='GET',
         renderer='rhodecode:templates/admin/permissions/permissions.mako')
 
     config.add_route(
         name='admin_permissions_ssh_keys_data',
         pattern='/permissions/ssh_keys/data')
     config.add_view(
         AdminPermissionsView,
         attr='ssh_keys_data',
         route_name='admin_permissions_ssh_keys_data', request_method='GET',
         renderer='json_ext', xhr=True)
 
     config.add_route(
         name='admin_permissions_ssh_keys_update',
         pattern='/permissions/ssh_keys/update')
     config.add_view(
         AdminPermissionsView,
         attr='ssh_keys_update',
         route_name='admin_permissions_ssh_keys_update', request_method='POST',
         renderer='rhodecode:templates/admin/permissions/permissions.mako')
 
     # users admin
     config.add_route(
         name='users',
         pattern='/users')
     config.add_view(
         AdminUsersView,
         attr='users_list',
         route_name='users', request_method='GET',
         renderer='rhodecode:templates/admin/users/users.mako')
 
     config.add_route(
         name='users_data',
         pattern='/users_data')
     config.add_view(
         AdminUsersView,
         attr='users_list_data',
         # renderer defined below
         route_name='users_data', request_method='GET',
         renderer='json_ext', xhr=True)
 
     config.add_route(
         name='users_create',
         pattern='/users/create')
     config.add_view(
         AdminUsersView,
         attr='users_create',
         route_name='users_create', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_add.mako')
 
     config.add_route(
         name='users_new',
         pattern='/users/new')
     config.add_view(
         AdminUsersView,
         attr='users_new',
         route_name='users_new', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_add.mako')
 
     # user management
     config.add_route(
         name='user_edit',
         pattern=r'/users/{user_id:\d+}/edit',
         user_route=True)
     config.add_view(
         UsersView,
         attr='user_edit',
         route_name='user_edit', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='user_edit_advanced',
         pattern=r'/users/{user_id:\d+}/edit/advanced',
         user_route=True)
     config.add_view(
         UsersView,
         attr='user_edit_advanced',
         route_name='user_edit_advanced', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='user_edit_global_perms',
         pattern=r'/users/{user_id:\d+}/edit/global_permissions',
         user_route=True)
     config.add_view(
         UsersView,
         attr='user_edit_global_perms',
         route_name='user_edit_global_perms', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='user_edit_global_perms_update',
         pattern=r'/users/{user_id:\d+}/edit/global_permissions/update',
         user_route=True)
     config.add_view(
         UsersView,
         attr='user_edit_global_perms_update',
         route_name='user_edit_global_perms_update', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='user_update',
         pattern=r'/users/{user_id:\d+}/update',
         user_route=True)
     config.add_view(
         UsersView,
         attr='user_update',
         route_name='user_update', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='user_delete',
         pattern=r'/users/{user_id:\d+}/delete',
         user_route=True)
     config.add_view(
         UsersView,
         attr='user_delete',
         route_name='user_delete', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='user_enable_force_password_reset',
         pattern=r'/users/{user_id:\d+}/password_reset_enable',
         user_route=True)
     config.add_view(
         UsersView,
         attr='user_enable_force_password_reset',
         route_name='user_enable_force_password_reset', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='user_disable_force_password_reset',
         pattern=r'/users/{user_id:\d+}/password_reset_disable',
         user_route=True)
     config.add_view(
         UsersView,
         attr='user_disable_force_password_reset',
         route_name='user_disable_force_password_reset', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='user_create_personal_repo_group',
         pattern=r'/users/{user_id:\d+}/create_repo_group',
         user_route=True)
     config.add_view(
         UsersView,
         attr='user_create_personal_repo_group',
         route_name='user_create_personal_repo_group', request_method='POST',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     # user notice
     config.add_route(
         name='user_notice_dismiss',
         pattern=r'/users/{user_id:\d+}/notice_dismiss',
         user_route=True)
     config.add_view(
         UsersView,
         attr='user_notice_dismiss',
         route_name='user_notice_dismiss', request_method='POST',
         renderer='json_ext', xhr=True)
 
     # user auth tokens
     config.add_route(
         name='edit_user_auth_tokens',
         pattern=r'/users/{user_id:\d+}/edit/auth_tokens',
         user_route=True)
     config.add_view(
         UsersView,
         attr='auth_tokens',
         route_name='edit_user_auth_tokens', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='edit_user_auth_tokens_view',
         pattern=r'/users/{user_id:\d+}/edit/auth_tokens/view',
         user_route=True)
     config.add_view(
         UsersView,
         attr='auth_tokens_view',
         route_name='edit_user_auth_tokens_view', request_method='POST',
         renderer='json_ext', xhr=True)
 
     config.add_route(
         name='edit_user_auth_tokens_add',
         pattern=r'/users/{user_id:\d+}/edit/auth_tokens/new',
         user_route=True)
     config.add_view(
         UsersView,
         attr='auth_tokens_add',
         route_name='edit_user_auth_tokens_add', request_method='POST')
 
     config.add_route(
         name='edit_user_auth_tokens_delete',
         pattern=r'/users/{user_id:\d+}/edit/auth_tokens/delete',
         user_route=True)
     config.add_view(
         UsersView,
         attr='auth_tokens_delete',
         route_name='edit_user_auth_tokens_delete', request_method='POST')
 
     # user ssh keys
     config.add_route(
         name='edit_user_ssh_keys',
         pattern=r'/users/{user_id:\d+}/edit/ssh_keys',
         user_route=True)
     config.add_view(
         UsersView,
         attr='ssh_keys',
         route_name='edit_user_ssh_keys', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='edit_user_ssh_keys_generate_keypair',
         pattern=r'/users/{user_id:\d+}/edit/ssh_keys/generate',
         user_route=True)
     config.add_view(
         UsersView,
         attr='ssh_keys_generate_keypair',
         route_name='edit_user_ssh_keys_generate_keypair', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='edit_user_ssh_keys_add',
         pattern=r'/users/{user_id:\d+}/edit/ssh_keys/new',
         user_route=True)
     config.add_view(
         UsersView,
         attr='ssh_keys_add',
         route_name='edit_user_ssh_keys_add', request_method='POST')
 
     config.add_route(
         name='edit_user_ssh_keys_delete',
         pattern=r'/users/{user_id:\d+}/edit/ssh_keys/delete',
         user_route=True)
     config.add_view(
         UsersView,
         attr='ssh_keys_delete',
         route_name='edit_user_ssh_keys_delete', request_method='POST')
 
     # user emails
     config.add_route(
         name='edit_user_emails',
         pattern=r'/users/{user_id:\d+}/edit/emails',
         user_route=True)
     config.add_view(
         UsersView,
         attr='emails',
         route_name='edit_user_emails', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='edit_user_emails_add',
         pattern=r'/users/{user_id:\d+}/edit/emails/new',
         user_route=True)
     config.add_view(
         UsersView,
         attr='emails_add',
         route_name='edit_user_emails_add', request_method='POST')
 
     config.add_route(
         name='edit_user_emails_delete',
         pattern=r'/users/{user_id:\d+}/edit/emails/delete',
         user_route=True)
     config.add_view(
         UsersView,
         attr='emails_delete',
         route_name='edit_user_emails_delete', request_method='POST')
 
     # user IPs
     config.add_route(
         name='edit_user_ips',
         pattern=r'/users/{user_id:\d+}/edit/ips',
         user_route=True)
     config.add_view(
         UsersView,
         attr='ips',
         route_name='edit_user_ips', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='edit_user_ips_add',
         pattern=r'/users/{user_id:\d+}/edit/ips/new',
         user_route_with_default=True)  # enabled for default user too
     config.add_view(
         UsersView,
         attr='ips_add',
         route_name='edit_user_ips_add', request_method='POST')
 
     config.add_route(
         name='edit_user_ips_delete',
         pattern=r'/users/{user_id:\d+}/edit/ips/delete',
         user_route_with_default=True)  # enabled for default user too
     config.add_view(
         UsersView,
         attr='ips_delete',
         route_name='edit_user_ips_delete', request_method='POST')
 
     # user perms
     config.add_route(
         name='edit_user_perms_summary',
         pattern=r'/users/{user_id:\d+}/edit/permissions_summary',
         user_route=True)
     config.add_view(
         UsersView,
         attr='user_perms_summary',
         route_name='edit_user_perms_summary', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='edit_user_perms_summary_json',
         pattern=r'/users/{user_id:\d+}/edit/permissions_summary/json',
         user_route=True)
     config.add_view(
         UsersView,
         attr='user_perms_summary_json',
         route_name='edit_user_perms_summary_json', request_method='GET',
         renderer='json_ext')
 
     # user user groups management
     config.add_route(
         name='edit_user_groups_management',
         pattern=r'/users/{user_id:\d+}/edit/groups_management',
         user_route=True)
     config.add_view(
         UsersView,
         attr='groups_management',
         route_name='edit_user_groups_management', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='edit_user_groups_management_updates',
         pattern=r'/users/{user_id:\d+}/edit/edit_user_groups_management/updates',
         user_route=True)
     config.add_view(
         UsersView,
         attr='groups_management_updates',
         route_name='edit_user_groups_management_updates', request_method='POST')
 
     # user audit logs
     config.add_route(
         name='edit_user_audit_logs',
         pattern=r'/users/{user_id:\d+}/edit/audit', user_route=True)
     config.add_view(
         UsersView,
         attr='user_audit_logs',
         route_name='edit_user_audit_logs', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='edit_user_audit_logs_download',
         pattern=r'/users/{user_id:\d+}/edit/audit/download', user_route=True)
     config.add_view(
         UsersView,
         attr='user_audit_logs_download',
         route_name='edit_user_audit_logs_download', request_method='GET',
         renderer='string')
 
     # user caches
     config.add_route(
         name='edit_user_caches',
         pattern=r'/users/{user_id:\d+}/edit/caches',
         user_route=True)
     config.add_view(
         UsersView,
         attr='user_caches',
         route_name='edit_user_caches', request_method='GET',
         renderer='rhodecode:templates/admin/users/user_edit.mako')
 
     config.add_route(
         name='edit_user_caches_update',
         pattern=r'/users/{user_id:\d+}/edit/caches/update',
         user_route=True)
     config.add_view(
         UsersView,
         attr='user_caches_update',
         route_name='edit_user_caches_update', request_method='POST')
 
     # user-groups admin
     config.add_route(
         name='user_groups',
         pattern='/user_groups')
     config.add_view(
         AdminUserGroupsView,
         attr='user_groups_list',
         route_name='user_groups', request_method='GET',
         renderer='rhodecode:templates/admin/user_groups/user_groups.mako')
 
     config.add_route(
         name='user_groups_data',
         pattern='/user_groups_data')
     config.add_view(
         AdminUserGroupsView,
         attr='user_groups_list_data',
         route_name='user_groups_data', request_method='GET',
         renderer='json_ext', xhr=True)
 
     config.add_route(
         name='user_groups_new',
         pattern='/user_groups/new')
     config.add_view(
         AdminUserGroupsView,
         attr='user_groups_new',
         route_name='user_groups_new', request_method='GET',
         renderer='rhodecode:templates/admin/user_groups/user_group_add.mako')
 
     config.add_route(
         name='user_groups_create',
         pattern='/user_groups/create')
     config.add_view(
         AdminUserGroupsView,
         attr='user_groups_create',
         route_name='user_groups_create', request_method='POST',
         renderer='rhodecode:templates/admin/user_groups/user_group_add.mako')
 
     # repos admin
     config.add_route(
         name='repos',
         pattern='/repos')
     config.add_view(
         AdminReposView,
         attr='repository_list',
         route_name='repos', request_method='GET',
         renderer='rhodecode:templates/admin/repos/repos.mako')
 
     config.add_route(
         name='repos_data',
         pattern='/repos_data')
     config.add_view(
         AdminReposView,
         attr='repository_list_data',
         route_name='repos_data', request_method='GET',
         renderer='json_ext', xhr=True)
 
     config.add_route(
         name='repo_new',
         pattern='/repos/new')
     config.add_view(
         AdminReposView,
         attr='repository_new',
         route_name='repo_new', request_method='GET',
         renderer='rhodecode:templates/admin/repos/repo_add.mako')
 
     config.add_route(
         name='repo_create',
         pattern='/repos/create')
     config.add_view(
         AdminReposView,
         attr='repository_create',
         route_name='repo_create', request_method='POST',
         renderer='rhodecode:templates/admin/repos/repos.mako')
 
     # repo groups admin
     config.add_route(
         name='repo_groups',
         pattern='/repo_groups')
     config.add_view(
         AdminRepoGroupsView,
         attr='repo_group_list',
         route_name='repo_groups', request_method='GET',
         renderer='rhodecode:templates/admin/repo_groups/repo_groups.mako')
 
     config.add_route(
         name='repo_groups_data',
         pattern='/repo_groups_data')
     config.add_view(
         AdminRepoGroupsView,
         attr='repo_group_list_data',
         route_name='repo_groups_data', request_method='GET',
         renderer='json_ext', xhr=True)
 
     config.add_route(
         name='repo_group_new',
         pattern='/repo_group/new')
     config.add_view(
         AdminRepoGroupsView,
         attr='repo_group_new',
         route_name='repo_group_new', request_method='GET',
         renderer='rhodecode:templates/admin/repo_groups/repo_group_add.mako')
 
     config.add_route(
         name='repo_group_create',
         pattern='/repo_group/create')
     config.add_view(
         AdminRepoGroupsView,
         attr='repo_group_create',
         route_name='repo_group_create', request_method='POST',
         renderer='rhodecode:templates/admin/repo_groups/repo_group_add.mako')
 
 
 def includeme(config):
     # Create admin navigation registry and add it to the pyramid registry.
     nav_includeme(config)
 
     # main admin routes
     config.add_route(
         name='admin_home', pattern=ADMIN_PREFIX)
     config.add_view(
         AdminMainView,
         attr='admin_main',
         route_name='admin_home', request_method='GET',
         renderer='rhodecode:templates/admin/main.mako')
 
     # pr global redirect
     config.add_route(
         name='pull_requests_global_0',  # backward compat
         pattern=ADMIN_PREFIX + r'/pull_requests/{pull_request_id:\d+}')
     config.add_view(
         AdminMainView,
         attr='pull_requests',
         route_name='pull_requests_global_0', request_method='GET')
 
     config.add_route(
         name='pull_requests_global_1',  # backward compat
         pattern=ADMIN_PREFIX + r'/pull-requests/{pull_request_id:\d+}')
     config.add_view(
         AdminMainView,
         attr='pull_requests',
         route_name='pull_requests_global_1', request_method='GET')
 
     config.add_route(
         name='pull_requests_global',
         pattern=ADMIN_PREFIX + r'/pull-request/{pull_request_id:\d+}')
     config.add_view(
         AdminMainView,
         attr='pull_requests',
         route_name='pull_requests_global', request_method='GET')
 
     config.include(admin_routes, route_prefix=ADMIN_PREFIX)

rhodecode/apps/admin/views/settings.py

@@ -1,708 +1,715 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import logging
 import collections
 
 import datetime
 import formencode
 import formencode.htmlfill
 
 import rhodecode
 
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.apps._base import BaseAppView
 from rhodecode.apps._base.navigation import navigation_list
 from rhodecode.apps.svn_support import config_keys
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (
     LoginRequired, HasPermissionAllDecorator, CSRFRequired)
 from rhodecode.lib.celerylib import tasks, run_task
 from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.utils import repo2db_mapper, get_rhodecode_repo_store_path
 from rhodecode.lib.utils2 import str2bool, AttributeDict
 from rhodecode.lib.index import searcher_from_config
 
 from rhodecode.model.db import RhodeCodeUi, Repository
 from rhodecode.model.forms import (ApplicationSettingsForm,
     ApplicationUiSettingsForm, ApplicationVisualisationForm,
     LabsSettingsForm, IssueTrackerPatternsForm)
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo_group import RepoGroupModel
 
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.notification import EmailNotificationModel
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import (
     IssueTrackerSettingsModel, VcsSettingsModel, SettingNotFound,
     SettingsModel)
 
 
 log = logging.getLogger(__name__)
 
 
 class AdminSettingsView(BaseAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         c.labs_active = str2bool(
             rhodecode.CONFIG.get('labs_settings_active', 'true'))
         c.navlist = navigation_list(self.request)
         return c
 
     @classmethod
     def _get_ui_settings(cls):
         ret = RhodeCodeUi.query().all()
 
         if not ret:
             raise Exception('Could not get application ui settings !')
-        settings = {}
+        settings = {
+            # legacy param that needs to be kept
+            'web_push_ssl': False
+        }
         for each in ret:
             k = each.ui_key
             v = each.ui_value
+            # skip some options if they are defined
+            if k in ['push_ssl']:
+                continue
+
             if k == '/':
                 k = 'root_path'
 
-            if k in ['push_ssl', 'publish', 'enabled']:
+            if k in ['publish', 'enabled']:
                 v = str2bool(v)
 
             if k.find('.') != -1:
                 k = k.replace('.', '_')
 
             if each.ui_section in ['hooks', 'extensions']:
                 v = each.ui_active
 
             settings[each.ui_section + '_' + k] = v
+
         return settings
 
     @classmethod
     def _form_defaults(cls):
         defaults = SettingsModel().get_all_settings()
         defaults.update(cls._get_ui_settings())
 
         defaults.update({
             'new_svn_branch': '',
             'new_svn_tag': '',
         })
         return defaults
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_vcs(self):
         c = self.load_default_context()
         c.active = 'vcs'
         model = VcsSettingsModel()
         c.svn_branch_patterns = model.get_global_svn_branch_patterns()
         c.svn_tag_patterns = model.get_global_svn_tag_patterns()
         c.svn_generate_config = rhodecode.ConfigGet().get_bool(config_keys.generate_config)
         c.svn_config_path = rhodecode.ConfigGet().get_str(config_keys.config_file_path)
         defaults = self._form_defaults()
 
         model.create_largeobjects_dirs_if_needed(defaults['paths_root_path'])
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=defaults,
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_vcs_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'vcs'
 
         model = VcsSettingsModel()
         c.svn_branch_patterns = model.get_global_svn_branch_patterns()
         c.svn_tag_patterns = model.get_global_svn_tag_patterns()
 
         c.svn_generate_config = rhodecode.ConfigGet().get_bool(config_keys.generate_config)
         c.svn_config_path = rhodecode.ConfigGet().get_str(config_keys.config_file_path)
         application_form = ApplicationUiSettingsForm(self.request.translate)()
 
         try:
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
             h.flash(
                 _("Some form inputs contain invalid data."),
                 category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.unpack_errors() or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         try:
-            model.update_global_ssl_setting(form_result['web_push_ssl'])
             model.update_global_hook_settings(form_result)
 
             model.create_or_update_global_svn_settings(form_result)
             model.create_or_update_global_hg_settings(form_result)
             model.create_or_update_global_git_settings(form_result)
             model.create_or_update_global_pr_settings(form_result)
         except Exception:
             log.exception("Exception while updating settings")
             h.flash(_('Error occurred during updating '
                       'application settings'), category='error')
         else:
             Session().commit()
             h.flash(_('Updated VCS settings'), category='success')
             raise HTTPFound(h.route_path('admin_settings_vcs'))
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_vcs_delete_svn_pattern(self):
         delete_pattern_id = self.request.POST.get('delete_svn_pattern')
         model = VcsSettingsModel()
         try:
             model.delete_global_svn_pattern(delete_pattern_id)
         except SettingNotFound:
             log.exception(
                 'Failed to delete svn_pattern with id %s', delete_pattern_id)
             raise HTTPNotFound()
 
         Session().commit()
         return True
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_mapping(self):
         c = self.load_default_context()
         c.active = 'mapping'
         c.storage_path = get_rhodecode_repo_store_path()
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_mapping_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'mapping'
         rm_obsolete = self.request.POST.get('destroy', False)
         invalidate_cache = self.request.POST.get('invalidate', False)
         log.debug('rescanning repo location with destroy obsolete=%s', rm_obsolete)
 
         if invalidate_cache:
             log.debug('invalidating all repositories cache')
             for repo in Repository.get_all():
                 ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
 
         filesystem_repos = ScmModel().repo_scan()
         added, removed = repo2db_mapper(filesystem_repos, rm_obsolete, force_hooks_rebuild=True)
         PermissionModel().trigger_permission_flush()
 
         def _repr(rm_repo):
             return ', '.join(map(safe_str, rm_repo)) or '-'
 
         h.flash(_('Repositories successfully '
                   'rescanned added: %s ; removed: %s') %
                 (_repr(added), _repr(removed)),
                 category='success')
         raise HTTPFound(h.route_path('admin_settings_mapping'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_global(self):
         c = self.load_default_context()
         c.active = 'global'
         c.personal_repo_group_default_pattern = RepoGroupModel()\
             .get_personal_group_name_pattern()
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_global_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'global'
         c.personal_repo_group_default_pattern = RepoGroupModel()\
             .get_personal_group_name_pattern()
         application_form = ApplicationSettingsForm(self.request.translate)()
         try:
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
             h.flash(
                 _("Some form inputs contain invalid data."),
                 category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.unpack_errors() or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         settings = [
             ('title', 'rhodecode_title', 'unicode'),
             ('realm', 'rhodecode_realm', 'unicode'),
             ('pre_code', 'rhodecode_pre_code', 'unicode'),
             ('post_code', 'rhodecode_post_code', 'unicode'),
             ('captcha_public_key', 'rhodecode_captcha_public_key', 'unicode'),
             ('captcha_private_key', 'rhodecode_captcha_private_key', 'unicode'),
             ('create_personal_repo_group', 'rhodecode_create_personal_repo_group', 'bool'),
             ('personal_repo_group_pattern', 'rhodecode_personal_repo_group_pattern', 'unicode'),
         ]
 
         try:
             for setting, form_key, type_ in settings:
                 sett = SettingsModel().create_or_update_setting(
                     setting, form_result[form_key], type_)
                 Session().add(sett)
 
             Session().commit()
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated application settings'), category='success')
         except Exception:
             log.exception("Exception while updating application settings")
             h.flash(
                 _('Error occurred during updating application settings'),
                 category='error')
 
         raise HTTPFound(h.route_path('admin_settings_global'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_visual(self):
         c = self.load_default_context()
         c.active = 'visual'
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_visual_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'visual'
         application_form = ApplicationVisualisationForm(self.request.translate)()
         try:
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
             h.flash(
                 _("Some form inputs contain invalid data."),
                 category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.unpack_errors() or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         try:
             settings = [
                 ('show_public_icon', 'rhodecode_show_public_icon', 'bool'),
                 ('show_private_icon', 'rhodecode_show_private_icon', 'bool'),
                 ('stylify_metatags', 'rhodecode_stylify_metatags', 'bool'),
                 ('repository_fields', 'rhodecode_repository_fields', 'bool'),
                 ('dashboard_items', 'rhodecode_dashboard_items', 'int'),
                 ('admin_grid_items', 'rhodecode_admin_grid_items', 'int'),
                 ('show_version', 'rhodecode_show_version', 'bool'),
                 ('use_gravatar', 'rhodecode_use_gravatar', 'bool'),
                 ('markup_renderer', 'rhodecode_markup_renderer', 'unicode'),
                 ('gravatar_url', 'rhodecode_gravatar_url', 'unicode'),
                 ('clone_uri_tmpl', 'rhodecode_clone_uri_tmpl', 'unicode'),
                 ('clone_uri_id_tmpl', 'rhodecode_clone_uri_id_tmpl', 'unicode'),
                 ('clone_uri_ssh_tmpl', 'rhodecode_clone_uri_ssh_tmpl', 'unicode'),
                 ('support_url', 'rhodecode_support_url', 'unicode'),
                 ('show_revision_number', 'rhodecode_show_revision_number', 'bool'),
                 ('show_sha_length', 'rhodecode_show_sha_length', 'int'),
             ]
             for setting, form_key, type_ in settings:
                 sett = SettingsModel().create_or_update_setting(
                     setting, form_result[form_key], type_)
                 Session().add(sett)
 
             Session().commit()
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated visualisation settings'), category='success')
         except Exception:
             log.exception("Exception updating visualization settings")
             h.flash(_('Error occurred during updating '
                       'visualisation settings'),
                     category='error')
 
         raise HTTPFound(h.route_path('admin_settings_visual'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_issuetracker(self):
         c = self.load_default_context()
         c.active = 'issuetracker'
         defaults = c.rc_config
 
         entry_key = 'rhodecode_issuetracker_pat_'
 
         c.issuetracker_entries = {}
         for k, v in defaults.items():
             if k.startswith(entry_key):
                 uid = k[len(entry_key):]
                 c.issuetracker_entries[uid] = None
 
         for uid in c.issuetracker_entries:
             c.issuetracker_entries[uid] = AttributeDict({
                 'pat': defaults.get('rhodecode_issuetracker_pat_' + uid),
                 'url': defaults.get('rhodecode_issuetracker_url_' + uid),
                 'pref': defaults.get('rhodecode_issuetracker_pref_' + uid),
                 'desc': defaults.get('rhodecode_issuetracker_desc_' + uid),
             })
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_issuetracker_test(self):
         error_container = []
 
         urlified_commit = h.urlify_commit_message(
             self.request.POST.get('test_text', ''),
             'repo_group/test_repo1', error_container=error_container)
         if error_container:
             def converter(inp):
                 return h.html_escape(inp)
 
             return 'ERRORS: ' + '\n'.join(map(converter, error_container))
 
         return urlified_commit
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_issuetracker_update(self):
         _ = self.request.translate
         self.load_default_context()
         settings_model = IssueTrackerSettingsModel()
 
         try:
             form = IssueTrackerPatternsForm(self.request.translate)()
             data = form.to_python(self.request.POST)
         except formencode.Invalid as errors:
             log.exception('Failed to add new pattern')
             error = errors
             h.flash(_(f'Invalid issue tracker pattern: {error}'),
                     category='error')
             raise HTTPFound(h.route_path('admin_settings_issuetracker'))
 
         if data:
             for uid in data.get('delete_patterns', []):
                 settings_model.delete_entries(uid)
 
             for pattern in data.get('patterns', []):
                 for setting, value, type_ in pattern:
                     sett = settings_model.create_or_update_setting(
                         setting, value, type_)
                     Session().add(sett)
 
                 Session().commit()
 
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated issue tracker entries'), category='success')
         raise HTTPFound(h.route_path('admin_settings_issuetracker'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_issuetracker_delete(self):
         _ = self.request.translate
         self.load_default_context()
         uid = self.request.POST.get('uid')
         try:
             IssueTrackerSettingsModel().delete_entries(uid)
         except Exception:
             log.exception('Failed to delete issue tracker setting %s', uid)
             raise HTTPNotFound()
 
         SettingsModel().invalidate_settings_cache()
         h.flash(_('Removed issue tracker entry.'), category='success')
 
         return {'deleted': uid}
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_email(self):
         c = self.load_default_context()
         c.active = 'email'
         c.rhodecode_ini = rhodecode.CONFIG
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_email_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'email'
 
         test_email = self.request.POST.get('test_email')
 
         if not test_email:
             h.flash(_('Please enter email address'), category='error')
             raise HTTPFound(h.route_path('admin_settings_email'))
 
         email_kwargs = {
             'date': datetime.datetime.now(),
             'user': self._rhodecode_db_user
         }
 
         (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
             EmailNotificationModel.TYPE_EMAIL_TEST, **email_kwargs)
 
         recipients = [test_email] if test_email else None
 
         run_task(tasks.send_email, recipients, subject,
                  email_body_plaintext, email_body)
 
         h.flash(_('Send email task created'), category='success')
         raise HTTPFound(h.route_path('admin_settings_email'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_hooks(self):
         c = self.load_default_context()
         c.active = 'hooks'
 
         model = SettingsModel()
         c.hooks = model.get_builtin_hooks()
         c.custom_hooks = model.get_custom_hooks()
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_hooks_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'hooks'
         if c.visual.allow_custom_hooks_settings:
             ui_key = self.request.POST.get('new_hook_ui_key')
             ui_value = self.request.POST.get('new_hook_ui_value')
 
             hook_id = self.request.POST.get('hook_id')
             new_hook = False
 
             model = SettingsModel()
             try:
                 if ui_value and ui_key:
                     model.create_or_update_hook(ui_key, ui_value)
                     h.flash(_('Added new hook'), category='success')
                     new_hook = True
                 elif hook_id:
                     RhodeCodeUi.delete(hook_id)
                     Session().commit()
 
                 # check for edits
                 update = False
                 _d = self.request.POST.dict_of_lists()
                 for k, v in zip(_d.get('hook_ui_key', []),
                                 _d.get('hook_ui_value_new', [])):
                     model.create_or_update_hook(k, v)
                     update = True
 
                 if update and not new_hook:
                     h.flash(_('Updated hooks'), category='success')
                 Session().commit()
             except Exception:
                 log.exception("Exception during hook creation")
                 h.flash(_('Error occurred during hook creation'),
                         category='error')
 
         raise HTTPFound(h.route_path('admin_settings_hooks'))
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_search(self):
         c = self.load_default_context()
         c.active = 'search'
 
         c.searcher = searcher_from_config(self.request.registry.settings)
         c.statistics = c.searcher.statistics(self.request.translate)
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_labs(self):
         c = self.load_default_context()
         if not c.labs_active:
             raise HTTPFound(h.route_path('admin_settings'))
 
         c.active = 'labs'
         c.lab_settings = _LAB_SETTINGS
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     @CSRFRequired()
     def settings_labs_update(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'labs'
 
         application_form = LabsSettingsForm(self.request.translate)()
         try:
             form_result = application_form.to_python(dict(self.request.POST))
         except formencode.Invalid as errors:
             h.flash(
                 _("Some form inputs contain invalid data."),
                 category='error')
             data = render('rhodecode:templates/admin/settings/settings.mako',
                           self._get_template_context(c), self.request)
             html = formencode.htmlfill.render(
                 data,
                 defaults=errors.value,
                 errors=errors.unpack_errors() or {},
                 prefix_error=False,
                 encoding="UTF-8",
                 force_defaults=False
             )
             return Response(html)
 
         try:
             session = Session()
             for setting in _LAB_SETTINGS:
                 setting_name = setting.key[len('rhodecode_'):]
                 sett = SettingsModel().create_or_update_setting(
                     setting_name, form_result[setting.key], setting.type)
                 session.add(sett)
 
         except Exception:
             log.exception('Exception while updating lab settings')
             h.flash(_('Error occurred during updating labs settings'),
                     category='error')
         else:
             Session().commit()
             SettingsModel().invalidate_settings_cache()
             h.flash(_('Updated Labs settings'), category='success')
             raise HTTPFound(h.route_path('admin_settings_labs'))
 
         data = render('rhodecode:templates/admin/settings/settings.mako',
                       self._get_template_context(c), self.request)
         html = formencode.htmlfill.render(
             data,
             defaults=self._form_defaults(),
             encoding="UTF-8",
             force_defaults=False
         )
         return Response(html)
 
 
 # :param key: name of the setting including the 'rhodecode_' prefix
 # :param type: the RhodeCodeSetting type to use.
 # :param group: the i18ned group in which we should dispaly this setting
 # :param label: the i18ned label we should display for this setting
 # :param help: the i18ned help we should dispaly for this setting
 LabSetting = collections.namedtuple(
     'LabSetting', ('key', 'type', 'group', 'label', 'help'))
 
 
 # This list has to be kept in sync with the form
 # rhodecode.model.forms.LabsSettingsForm.
 _LAB_SETTINGS = [
 
 ]

rhodecode/apps/admin/views/system_info.py

@@ -1,243 +1,249 b''
 
 
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import urllib.request
 import urllib.error
 import urllib.parse
 import os
 
 import rhodecode
 from rhodecode.apps._base import BaseAppView
 from rhodecode.apps._base.navigation import navigation_list
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import (LoginRequired, HasPermissionAllDecorator)
 from rhodecode.lib.utils2 import str2bool
 from rhodecode.lib import system_info
 from rhodecode.model.update import UpdateModel
 
 log = logging.getLogger(__name__)
 
 
 class AdminSystemInfoSettingsView(BaseAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         return c
 
     def get_env_data(self):
         black_list = [
             'NIX_LDFLAGS',
             'NIX_CFLAGS_COMPILE',
             'propagatedBuildInputs',
             'propagatedNativeBuildInputs',
             'postInstall',
             'buildInputs',
             'buildPhase',
             'preShellHook',
             'preShellHook',
             'preCheck',
             'preBuild',
             'postShellHook',
             'postFixup',
             'postCheck',
             'nativeBuildInputs',
             'installPhase',
             'installCheckPhase',
             'checkPhase',
             'configurePhase',
             'shellHook'
         ]
         secret_list = [
             'RHODECODE_USER_PASS'
         ]
 
         for k, v in sorted(os.environ.items()):
             if k in black_list:
                 continue
             if k in secret_list:
                 v = '*****'
             yield k, v
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_system_info(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         c.active = 'system'
         c.navlist = navigation_list(self.request)
 
         # TODO(marcink), figure out how to allow only selected users to do this
         c.allowed_to_snapshot = self._rhodecode_user.admin
 
         snapshot = str2bool(self.request.params.get('snapshot'))
 
         c.rhodecode_update_url = UpdateModel().get_update_url()
         c.env_data = self.get_env_data()
         server_info = system_info.get_system_info(self.request.environ)
 
         for key, val in server_info.items():
             setattr(c, key, val)
 
         def val(name, subkey='human_value'):
             return server_info[name][subkey]
 
         def state(name):
             return server_info[name]['state']
 
         def val2(name):
             val = server_info[name]['human_value']
             state = server_info[name]['state']
             return val, state
 
         update_info_msg = _('Note: please make sure this server can '
                             'access `${url}` for the update link to work',
                             mapping=dict(url=c.rhodecode_update_url))
         version = UpdateModel().get_stored_version()
         is_outdated = UpdateModel().is_outdated(
             rhodecode.__version__, version)
         update_state = {
             'type': 'warning',
             'message': 'New version available: {}'.format(version)
             } \
             if is_outdated else {}
         c.data_items = [
             # update info
             (_('Update info'), h.literal(
                 '<span class="link" id="check_for_update" >%s.</span>' % (
                 _('Check for updates')) +
                 '<br/> <span >%s.</span>' % (update_info_msg)
             ), ''),
 
             # RhodeCode specific
             (_('RhodeCode Version'), val('rhodecode_app')['text'], state('rhodecode_app')),
             (_('Latest version'), version, update_state),
             (_('RhodeCode Base URL'), val('rhodecode_config')['config'].get('app.base_url'), state('rhodecode_config')),
             (_('RhodeCode Server IP'), val('server')['server_ip'], state('server')),
             (_('RhodeCode Server ID'), val('server')['server_id'], state('server')),
             (_('RhodeCode Configuration'), val('rhodecode_config')['path'], state('rhodecode_config')),
             (_('RhodeCode Certificate'), val('rhodecode_config')['cert_path'], state('rhodecode_config')),
             (_('Workers'), val('rhodecode_config')['config']['server:main'].get('workers', '?'), state('rhodecode_config')),
             (_('Worker Type'), val('rhodecode_config')['config']['server:main'].get('worker_class', 'sync'), state('rhodecode_config')),
             ('', '', ''),  # spacer
 
             # Database
             (_('Database'), val('database')['url'], state('database')),
             (_('Database version'), val('database')['version'], state('database')),
             ('', '', ''),  # spacer
 
             # Platform/Python
             (_('Platform'), val('platform')['name'], state('platform')),
             (_('Platform UUID'), val('platform')['uuid'], state('platform')),
             (_('Lang'), val('locale'), state('locale')),
             (_('Python version'), val('python')['version'], state('python')),
             (_('Python path'), val('python')['executable'], state('python')),
             ('', '', ''),  # spacer
 
             # Systems stats
             (_('CPU'), val('cpu')['text'], state('cpu')),
             (_('Load'), val('load')['text'], state('load')),
             (_('Memory'), val('memory')['text'], state('memory')),
             (_('Uptime'), val('uptime')['text'], state('uptime')),
             ('', '', ''),  # spacer
 
             # ulimit
             (_('Ulimit'), val('ulimit')['text'], state('ulimit')),
 
             # Repo storage
             (_('Storage location'), val('storage')['path'], state('storage')),
             (_('Storage info'), val('storage')['text'], state('storage')),
             (_('Storage inodes'), val('storage_inodes')['text'], state('storage_inodes')),
             ('', '', ''),  # spacer
 
             (_('Gist storage location'), val('storage_gist')['path'], state('storage_gist')),
             (_('Gist storage info'), val('storage_gist')['text'], state('storage_gist')),
             ('', '', ''),  # spacer
 
-            (_('Archive cache storage type'), val('storage_archive')['type'], state('storage_archive')),
+            (_('Artifacts storage backend'), val('storage_artifacts')['type'], state('storage_artifacts')),
+            (_('Artifacts storage location'), val('storage_artifacts')['path'], state('storage_artifacts')),
+            (_('Artifacts info'), val('storage_artifacts')['text'], state('storage_artifacts')),
+            ('', '', ''),  # spacer
+
+            (_('Archive cache storage backend'), val('storage_archive')['type'], state('storage_archive')),
             (_('Archive cache storage location'), val('storage_archive')['path'], state('storage_archive')),
             (_('Archive cache info'), val('storage_archive')['text'], state('storage_archive')),
             ('', '', ''),  # spacer
 
+
             (_('Temp storage location'), val('storage_temp')['path'], state('storage_temp')),
             (_('Temp storage info'), val('storage_temp')['text'], state('storage_temp')),
             ('', '', ''),  # spacer
 
             (_('Search info'), val('search')['text'], state('search')),
             (_('Search location'), val('search')['location'], state('search')),
             ('', '', ''),  # spacer
 
             # VCS specific
             (_('VCS Backends'), val('vcs_backends'), state('vcs_backends')),
             (_('VCS Server'), val('vcs_server')['text'], state('vcs_server')),
             (_('GIT'), val('git'), state('git')),
             (_('HG'), val('hg'), state('hg')),
             (_('SVN'), val('svn'), state('svn')),
 
         ]
 
         c.vcsserver_data_items = [
             (k, v) for k, v in (val('vcs_server_config') or {}).items()
         ]
 
         if snapshot:
             if c.allowed_to_snapshot:
                 c.data_items.pop(0)  # remove server info
                 self.request.override_renderer = 'admin/settings/settings_system_snapshot.mako'
             else:
                 h.flash('You are not allowed to do this', category='warning')
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasPermissionAllDecorator('hg.admin')
     def settings_system_info_check_update(self):
         _ = self.request.translate
         c = self.load_default_context()
 
         update_url = UpdateModel().get_update_url()
 
         def _err(s):
             return f'<div style="color:#ff8888; padding:4px 0px">{s}</div>'
 
         try:
             data = UpdateModel().get_update_data(update_url)
         except urllib.error.URLError as e:
             log.exception("Exception contacting upgrade server")
             self.request.override_renderer = 'string'
             return _err('Failed to contact upgrade server: %r' % e)
         except ValueError as e:
             log.exception("Bad data sent from update server")
             self.request.override_renderer = 'string'
             return _err('Bad data sent from update server')
 
         latest = data['versions'][0]
 
         c.update_url = update_url
         c.latest_data = latest
         c.latest_ver = (latest['version'] or '').strip()
         c.cur_ver = self.request.GET.get('ver') or rhodecode.__version__
         c.should_upgrade = False
 
         is_outdated = UpdateModel().is_outdated(c.cur_ver, c.latest_ver)
         if is_outdated:
             c.should_upgrade = True
         c.important_notices = latest['general']
         UpdateModel().store_version(latest['version'])
         return self._get_template_context(c)

rhodecode/apps/file_store/__init__.py

@@ -1,66 +1,97 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import os
-from rhodecode.apps.file_store import config_keys
+
+
 from rhodecode.config.settings_maker import SettingsMaker
 
 
 def _sanitize_settings_and_apply_defaults(settings):
     """
     Set defaults, convert to python types and validate settings.
     """
+    from rhodecode.apps.file_store import config_keys
+
+    # translate "legacy" params into new config
+    settings.pop(config_keys.deprecated_enabled, True)
+    if config_keys.deprecated_backend in settings:
+        # if legacy backend key is detected we use "legacy" backward compat setting
+        settings.pop(config_keys.deprecated_backend)
+        settings[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+
+    if config_keys.deprecated_store_path in settings:
+        store_path = settings.pop(config_keys.deprecated_store_path)
+        settings[config_keys.legacy_filesystem_storage_path] = store_path
+
     settings_maker = SettingsMaker(settings)
 
-    settings_maker.make_setting(config_keys.enabled, True, parser='bool')
-    settings_maker.make_setting(config_keys.backend, 'local')
+    default_cache_dir = settings['cache_dir']
+    default_store_dir = os.path.join(default_cache_dir, 'artifacts_filestore')
+
+    # set default backend
+    settings_maker.make_setting(config_keys.backend_type, config_keys.backend_legacy_filesystem)
+
+    # legacy filesystem defaults
+    settings_maker.make_setting(config_keys.legacy_filesystem_storage_path, default_store_dir, default_when_empty=True, )
 
-    default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store')
-    settings_maker.make_setting(config_keys.store_path, default_store)
+    # filesystem defaults
+    settings_maker.make_setting(config_keys.filesystem_storage_path, default_store_dir, default_when_empty=True,)
+    settings_maker.make_setting(config_keys.filesystem_shards, 8, parser='int')
+
+    # objectstore defaults
+    settings_maker.make_setting(config_keys.objectstore_url, 'http://s3-minio:9000')
+    settings_maker.make_setting(config_keys.objectstore_bucket, 'rhodecode-artifacts-filestore')
+    settings_maker.make_setting(config_keys.objectstore_bucket_shards, 8, parser='int')
+
+    settings_maker.make_setting(config_keys.objectstore_region, '')
+    settings_maker.make_setting(config_keys.objectstore_key, '')
+    settings_maker.make_setting(config_keys.objectstore_secret, '')
 
     settings_maker.env_expand()
 
 
 def includeme(config):
+
     from rhodecode.apps.file_store.views import FileStoreView
 
     settings = config.registry.settings
     _sanitize_settings_and_apply_defaults(settings)
 
     config.add_route(
         name='upload_file',
         pattern='/_file_store/upload')
     config.add_view(
         FileStoreView,
         attr='upload_file',
         route_name='upload_file', request_method='POST', renderer='json_ext')
 
     config.add_route(
         name='download_file',
         pattern='/_file_store/download/{fid:.*}')
     config.add_view(
         FileStoreView,
         attr='download_file',
         route_name='download_file')
 
     config.add_route(
         name='download_file_by_token',
         pattern='/_file_store/token-download/{_auth_token}/{fid:.*}')
     config.add_view(
         FileStoreView,
         attr='download_file_by_token',
         route_name='download_file_by_token')

rhodecode/apps/file_store/config_keys.py

@@ -1,25 +1,57 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 # Definition of setting keys used to configure this module. Defined here to
 # avoid repetition of keys throughout the module.
 
-enabled = 'file_store.enabled'
-backend = 'file_store.backend'
-store_path = 'file_store.storage_path'
+# OLD and deprecated keys not used anymore
+deprecated_enabled = 'file_store.enabled'
+deprecated_backend = 'file_store.backend'
+deprecated_store_path = 'file_store.storage_path'
+
+
+backend_type = 'file_store.backend.type'
+
+backend_legacy_filesystem = 'filesystem_v1'
+backend_filesystem = 'filesystem_v2'
+backend_objectstore = 'objectstore'
+
+backend_types = [
+    backend_legacy_filesystem,
+    backend_filesystem,
+    backend_objectstore,
+]
+
+# filesystem_v1 legacy
+legacy_filesystem_storage_path = 'file_store.filesystem_v1.storage_path'
+
+
+# filesystem_v2 new option
+filesystem_storage_path = 'file_store.filesystem_v2.storage_path'
+filesystem_shards = 'file_store.filesystem_v2.shards'
+
+# objectstore
+objectstore_url = 'file_store.objectstore.url'
+objectstore_bucket = 'file_store.objectstore.bucket'
+objectstore_bucket_shards = 'file_store.objectstore.bucket_shards'
+
+objectstore_region = 'file_store.objectstore.region'
+objectstore_key = 'file_store.objectstore.key'
+objectstore_secret = 'file_store.objectstore.secret'
+

rhodecode/apps/file_store/tests/__init__.py

@@ -1,18 +1,57 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
+import os
+import random
+import tempfile
+import string
+
+import pytest
+
+from rhodecode.apps.file_store import utils as store_utils
+
+
[e-mail]()
+def file_store_instance(ini_settings):
+    config = ini_settings
+    f_store = store_utils.get_filestore_backend(config=config, always_init=True)
+    return f_store
+
+
[e-mail]
+def random_binary_file():
+    # Generate random binary data
+    data = bytearray(random.getrandbits(8) for _ in range(1024 * 512))  # 512 KB of random data
+
+    # Create a temporary file
+    temp_file = tempfile.NamedTemporaryFile(delete=False)
+    filename = temp_file.name
+
+    try:
+        # Write the random binary data to the file
+        temp_file.write(data)
+        temp_file.seek(0)  # Rewind the file pointer to the beginning
+        yield filename, temp_file
+    finally:
+        # Close and delete the temporary file after the test
+        temp_file.close()
+        os.remove(filename)
+
+
+def generate_random_filename(length=10):
+    return ''.join(random.choices(string.ascii_letters + string.digits, k=length))
  No newline at end of file

rhodecode/apps/file_store/tests/test_upload_file.py

@@ -1,246 +1,253 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
+
 import os
+
 import pytest
 
 from rhodecode.lib.ext_json import json
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.db import Session, FileStore, Repository, User
-from rhodecode.apps.file_store import utils, config_keys
+from rhodecode.apps.file_store import utils as store_utils
+from rhodecode.apps.file_store import config_keys
 
 from rhodecode.tests import TestController
 from rhodecode.tests.routes import route_path
 
 
 class TestFileStoreViews(TestController):
 
+    @pytest.fixture()
+    def create_artifact_factory(self, tmpdir, ini_settings):
+
+        def factory(user_id, content, f_name='example.txt'):
+
+            config = ini_settings
+            config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+
+            f_store = store_utils.get_filestore_backend(config)
+
+            filesystem_file = os.path.join(str(tmpdir), f_name)
+            with open(filesystem_file, 'wt') as f:
+                f.write(content)
+
+            with open(filesystem_file, 'rb') as f:
+                store_uid, metadata = f_store.store(f_name, f, metadata={'filename': f_name})
+            os.remove(filesystem_file)
+
+            entry = FileStore.create(
+                file_uid=store_uid, filename=metadata["filename"],
+                file_hash=metadata["sha256"], file_size=metadata["size"],
+                file_display_name='file_display_name',
+                file_description='repo artifact `{}`'.format(metadata["filename"]),
+                check_acl=True, user_id=user_id,
+            )
+            Session().add(entry)
+            Session().commit()
+            return entry
+        return factory
+
     @pytest.mark.parametrize("fid, content, exists", [
         ('abcde-0.jpg', "xxxxx", True),
         ('abcde-0.exe', "1234567", True),
         ('abcde-0.jpg', "xxxxx", False),
     ])
-    def test_get_files_from_store(self, fid, content, exists, tmpdir, user_util):
+    def test_get_files_from_store(self, fid, content, exists, tmpdir, user_util, ini_settings):
         user = self.log_user()
         user_id = user['user_id']
         repo_id = user_util.create_repo().repo_id
-        store_path = self.app._pyramid_settings[config_keys.store_path]
+
+        config = ini_settings
+        config[config_keys.backend_type] = config_keys.backend_legacy_filesystem
+
         store_uid = fid
 
         if exists:
             status = 200
-            store = utils.get_file_storage({config_keys.store_path: store_path})
+            f_store = store_utils.get_filestore_backend(config)
             filesystem_file = os.path.join(str(tmpdir), fid)
             with open(filesystem_file, 'wt') as f:
                 f.write(content)
 
             with open(filesystem_file, 'rb') as f:
-                store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid})
+                store_uid, metadata = f_store.store(fid, f, metadata={'filename': fid})
+            os.remove(filesystem_file)
 
             entry = FileStore.create(
                 file_uid=store_uid, filename=metadata["filename"],
                 file_hash=metadata["sha256"], file_size=metadata["size"],
                 file_display_name='file_display_name',
                 file_description='repo artifact `{}`'.format(metadata["filename"]),
                 check_acl=True, user_id=user_id,
                 scope_repo_id=repo_id
             )
             Session().add(entry)
             Session().commit()
 
         else:
             status = 404
 
         response = self.app.get(route_path('download_file', fid=store_uid), status=status)
 
         if exists:
             assert response.text == content
-            file_store_path = os.path.dirname(store.resolve_name(store_uid, store_path)[1])
-            metadata_file = os.path.join(file_store_path, store_uid + '.meta')
-            assert os.path.exists(metadata_file)
-            with open(metadata_file, 'rb') as f:
-                json_data = json.loads(f.read())
 
-            assert json_data
-            assert 'size' in json_data
+            metadata = f_store.get_metadata(store_uid)
+
+            assert 'size' in metadata
 
     def test_upload_files_without_content_to_store(self):
         self.log_user()
         response = self.app.post(
             route_path('upload_file'),
             params={'csrf_token': self.csrf_token},
             status=200)
 
         assert response.json == {
             'error': 'store_file data field is missing',
             'access_path': None,
             'store_fid': None}
 
     def test_upload_files_bogus_content_to_store(self):
         self.log_user()
         response = self.app.post(
             route_path('upload_file'),
             params={'csrf_token': self.csrf_token, 'store_file': 'bogus'},
             status=200)
 
         assert response.json == {
             'error': 'filename cannot be read from the data field',
             'access_path': None,
             'store_fid': None}
 
     def test_upload_content_to_store(self):
         self.log_user()
         response = self.app.post(
             route_path('upload_file'),
             upload_files=[('store_file', b'myfile.txt', b'SOME CONTENT')],
             params={'csrf_token': self.csrf_token},
             status=200)
 
         assert response.json['store_fid']
 
-    @pytest.fixture()
-    def create_artifact_factory(self, tmpdir):
-        def factory(user_id, content):
-            store_path = self.app._pyramid_settings[config_keys.store_path]
-            store = utils.get_file_storage({config_keys.store_path: store_path})
-            fid = 'example.txt'
-
-            filesystem_file = os.path.join(str(tmpdir), fid)
-            with open(filesystem_file, 'wt') as f:
-                f.write(content)
-
-            with open(filesystem_file, 'rb') as f:
-                store_uid, metadata = store.save_file(f, fid, extra_metadata={'filename': fid})
-
-            entry = FileStore.create(
-                file_uid=store_uid, filename=metadata["filename"],
-                file_hash=metadata["sha256"], file_size=metadata["size"],
-                file_display_name='file_display_name',
-                file_description='repo artifact `{}`'.format(metadata["filename"]),
-                check_acl=True, user_id=user_id,
-            )
-            Session().add(entry)
-            Session().commit()
-            return entry
-        return factory
-
     def test_download_file_non_scoped(self, user_util, create_artifact_factory):
         user = self.log_user()
         user_id = user['user_id']
         content = 'HELLO MY NAME IS ARTIFACT !'
 
         artifact = create_artifact_factory(user_id, content)
         file_uid = artifact.file_uid
         response = self.app.get(route_path('download_file', fid=file_uid), status=200)
         assert response.text == content
 
         # log-in to new user and test download again
         user = user_util.create_user(password='qweqwe')
         self.log_user(user.username, 'qweqwe')
         response = self.app.get(route_path('download_file', fid=file_uid), status=200)
         assert response.text == content
 
     def test_download_file_scoped_to_repo(self, user_util, create_artifact_factory):
         user = self.log_user()
         user_id = user['user_id']
         content = 'HELLO MY NAME IS ARTIFACT !'
 
         artifact = create_artifact_factory(user_id, content)
         # bind to repo
         repo = user_util.create_repo()
         repo_id = repo.repo_id
         artifact.scope_repo_id = repo_id
         Session().add(artifact)
         Session().commit()
 
         file_uid = artifact.file_uid
         response = self.app.get(route_path('download_file', fid=file_uid), status=200)
         assert response.text == content
 
         # log-in to new user and test download again
         user = user_util.create_user(password='qweqwe')
         self.log_user(user.username, 'qweqwe')
         response = self.app.get(route_path('download_file', fid=file_uid), status=200)
         assert response.text == content
 
         # forbid user the rights to repo
         repo = Repository.get(repo_id)
         user_util.grant_user_permission_to_repo(repo, user, 'repository.none')
         self.app.get(route_path('download_file', fid=file_uid), status=404)
 
     def test_download_file_scoped_to_user(self, user_util, create_artifact_factory):
         user = self.log_user()
         user_id = user['user_id']
         content = 'HELLO MY NAME IS ARTIFACT !'
 
         artifact = create_artifact_factory(user_id, content)
         # bind to user
         user = user_util.create_user(password='qweqwe')
 
         artifact.scope_user_id = user.user_id
         Session().add(artifact)
         Session().commit()
 
         # artifact creator doesn't have access since it's bind to another user
         file_uid = artifact.file_uid
         self.app.get(route_path('download_file', fid=file_uid), status=404)
 
         # log-in to new user and test download again, should be ok since we're bind to this artifact
         self.log_user(user.username, 'qweqwe')
         response = self.app.get(route_path('download_file', fid=file_uid), status=200)
         assert response.text == content
 
     def test_download_file_scoped_to_repo_with_bad_token(self, user_util, create_artifact_factory):
         user_id = User.get_first_super_admin().user_id
         content = 'HELLO MY NAME IS ARTIFACT !'
 
         artifact = create_artifact_factory(user_id, content)
         # bind to repo
         repo = user_util.create_repo()
         repo_id = repo.repo_id
         artifact.scope_repo_id = repo_id
         Session().add(artifact)
         Session().commit()
 
         file_uid = artifact.file_uid
         self.app.get(route_path('download_file_by_token',
                                 _auth_token='bogus', fid=file_uid), status=302)
 
     def test_download_file_scoped_to_repo_with_token(self, user_util, create_artifact_factory):
         user = User.get_first_super_admin()
         AuthTokenModel().create(user, 'test artifact token',
                                 role=AuthTokenModel.cls.ROLE_ARTIFACT_DOWNLOAD)
 
         user = User.get_first_super_admin()
         artifact_token = user.artifact_token
 
         user_id = User.get_first_super_admin().user_id
         content = 'HELLO MY NAME IS ARTIFACT !'
 
         artifact = create_artifact_factory(user_id, content)
         # bind to repo
         repo = user_util.create_repo()
         repo_id = repo.repo_id
         artifact.scope_repo_id = repo_id
         Session().add(artifact)
         Session().commit()
 
         file_uid = artifact.file_uid
         response = self.app.get(
             route_path('download_file_by_token',
                        _auth_token=artifact_token, fid=file_uid), status=200)
         assert response.text == content

rhodecode/apps/file_store/utils.py

@@ -1,55 +1,145 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import io
 import uuid
 import pathlib
+import s3fs
+
+from rhodecode.lib.hash_utils import sha256_safe
+from rhodecode.apps.file_store import config_keys
+
+
+file_store_meta = None
+
+
+def get_filestore_config(config) -> dict:
+
+    final_config = {}
+
+    for k, v in config.items():
+        if k.startswith('file_store'):
+            final_config[k] = v
+
+    return final_config
 
 
-def get_file_storage(settings):
-    from rhodecode.apps.file_store.backends.local_store import LocalFileStorage
-    from rhodecode.apps.file_store import config_keys
-    store_path = settings.get(config_keys.store_path)
-    return LocalFileStorage(base_path=store_path)
+def get_filestore_backend(config, always_init=False):
+    """
+
+    usage::
+        from rhodecode.apps.file_store import get_filestore_backend
+        f_store = get_filestore_backend(config=CONFIG)
+
+    :param config:
+    :param always_init:
+    :return:
+    """
+
+    global file_store_meta
+    if file_store_meta is not None and not always_init:
+        return file_store_meta
+
+    config = get_filestore_config(config)
+    backend = config[config_keys.backend_type]
+
+    match backend:
+        case config_keys.backend_legacy_filesystem:
+            # Legacy backward compatible storage
+            from rhodecode.apps.file_store.backends.filesystem_legacy import LegacyFileSystemBackend
+            d_cache = LegacyFileSystemBackend(
+                settings=config
+            )
+        case config_keys.backend_filesystem:
+            from rhodecode.apps.file_store.backends.filesystem import FileSystemBackend
+            d_cache = FileSystemBackend(
+                settings=config
+            )
+        case config_keys.backend_objectstore:
+            from rhodecode.apps.file_store.backends.objectstore import ObjectStoreBackend
+            d_cache = ObjectStoreBackend(
+                settings=config
+            )
+        case _:
+            raise ValueError(
+                f'file_store.backend.type only supports "{config_keys.backend_types}" got {backend}'
+            )
+
+    cache_meta = d_cache
+    return cache_meta
 
 
 def splitext(filename):
-    ext = ''.join(pathlib.Path(filename).suffixes)
+    final_ext = []
+    for suffix in pathlib.Path(filename).suffixes:
+        if not suffix.isascii():
+            continue
+
+        suffix = " ".join(suffix.split()).replace(" ", "")
+        final_ext.append(suffix)
+    ext = ''.join(final_ext)
     return filename, ext
 
 
-def uid_filename(filename, randomized=True):
+def get_uid_filename(filename, randomized=True):
     """
     Generates a randomized or stable (uuid) filename,
     preserving the original extension.
 
     :param filename: the original filename
     :param randomized: define if filename should be stable (sha1 based) or randomized
     """
 
     _, ext = splitext(filename)
     if randomized:
         uid = uuid.uuid4()
     else:
-        hash_key = '{}.{}'.format(filename, 'store')
+        store_suffix = "store"
+        hash_key = f'{filename}.{store_suffix}'
         uid = uuid.uuid5(uuid.NAMESPACE_URL, hash_key)
     return str(uid) + ext.lower()
 
 
 def bytes_to_file_obj(bytes_data):
-    return io.StringIO(bytes_data)
+    return io.BytesIO(bytes_data)
+
+
+class ShardFileReader:
+
+    def __init__(self, file_like_reader):
+        self._file_like_reader = file_like_reader
+
+    def __getattr__(self, item):
+        if isinstance(self._file_like_reader, s3fs.core.S3File):
+            match item:
+                case 'name':
+                    # S3 FileWrapper doesn't support name attribute, and we use it
+                    return self._file_like_reader.full_name
+                case _:
+                    return getattr(self._file_like_reader, item)
+        else:
+            return getattr(self._file_like_reader, item)
+
+
+def archive_iterator(_reader, block_size: int = 4096 * 512):
+    # 4096 * 64 = 64KB
+    while 1:
+        data = _reader.read(block_size)
+        if not data:
+            break
+        yield data

rhodecode/apps/file_store/views.py

@@ -1,200 +1,197 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import logging
 
-
-from pyramid.response import FileResponse
+from pyramid.response import Response
 from pyramid.httpexceptions import HTTPFound, HTTPNotFound
 
 from rhodecode.apps._base import BaseAppView
-from rhodecode.apps.file_store import utils
+from rhodecode.apps.file_store import utils as store_utils
 from rhodecode.apps.file_store.exceptions import (
     FileNotAllowedException, FileOverSizeException)
 
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.auth import (
     CSRFRequired, NotAnonymous, HasRepoPermissionAny, HasRepoGroupPermissionAny,
     LoginRequired)
+from rhodecode.lib.str_utils import header_safe_str
 from rhodecode.lib.vcs.conf.mtypes import get_mimetypes_db
 from rhodecode.model.db import Session, FileStore, UserApiKeys
 
 log = logging.getLogger(__name__)
 
 
 class FileStoreView(BaseAppView):
     upload_key = 'store_file'
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
-        self.storage = utils.get_file_storage(self.request.registry.settings)
+        self.f_store = store_utils.get_filestore_backend(self.request.registry.settings)
         return c
 
     def _guess_type(self, file_name):
         """
         Our own type guesser for mimetypes using the rich DB
         """
         if not hasattr(self, 'db'):
             self.db = get_mimetypes_db()
         _content_type, _encoding = self.db.guess_type(file_name, strict=False)
         return _content_type, _encoding
 
     def _serve_file(self, file_uid):
-        if not self.storage.exists(file_uid):
-            store_path = self.storage.store_path(file_uid)
-            log.debug('File with FID:%s not found in the store under `%s`',
-                      file_uid, store_path)
+        if not self.f_store.filename_exists(file_uid):
+            store_path = self.f_store.store_path(file_uid)
+            log.warning('File with FID:%s not found in the store under `%s`',
+                        file_uid, store_path)
             raise HTTPNotFound()
 
         db_obj = FileStore.get_by_store_uid(file_uid, safe=True)
         if not db_obj:
             raise HTTPNotFound()
 
         # private upload for user
         if db_obj.check_acl and db_obj.scope_user_id:
             log.debug('Artifact: checking scope access for bound artifact user: `%s`',
                       db_obj.scope_user_id)
             user = db_obj.user
             if self._rhodecode_db_user.user_id != user.user_id:
                 log.warning('Access to file store object forbidden')
                 raise HTTPNotFound()
 
         # scoped to repository permissions
         if db_obj.check_acl and db_obj.scope_repo_id:
             log.debug('Artifact: checking scope access for bound artifact repo: `%s`',
                       db_obj.scope_repo_id)
             repo = db_obj.repo
             perm_set = ['repository.read', 'repository.write', 'repository.admin']
             has_perm = HasRepoPermissionAny(*perm_set)(repo.repo_name, 'FileStore check')
             if not has_perm:
                 log.warning('Access to file store object `%s` forbidden', file_uid)
                 raise HTTPNotFound()
 
         # scoped to repository group permissions
         if db_obj.check_acl and db_obj.scope_repo_group_id:
             log.debug('Artifact: checking scope access for bound artifact repo group: `%s`',
                       db_obj.scope_repo_group_id)
             repo_group = db_obj.repo_group
             perm_set = ['group.read', 'group.write', 'group.admin']
             has_perm = HasRepoGroupPermissionAny(*perm_set)(repo_group.group_name, 'FileStore check')
             if not has_perm:
                 log.warning('Access to file store object `%s` forbidden', file_uid)
                 raise HTTPNotFound()
 
         FileStore.bump_access_counter(file_uid)
 
-        file_path = self.storage.store_path(file_uid)
+        file_name = db_obj.file_display_name
         content_type = 'application/octet-stream'
-        content_encoding = None
 
-        _content_type, _encoding = self._guess_type(file_path)
+        _content_type, _encoding = self._guess_type(file_name)
         if _content_type:
             content_type = _content_type
 
         # For file store we don't submit any session data, this logic tells the
         # Session lib to skip it
         setattr(self.request, '_file_response', True)
-        response = FileResponse(
-            file_path, request=self.request,
-            content_type=content_type, content_encoding=content_encoding)
+        reader, _meta = self.f_store.fetch(file_uid)
 
-        file_name = db_obj.file_display_name
+        response = Response(app_iter=store_utils.archive_iterator(reader))
 
-        response.headers["Content-Disposition"] = (
-            f'attachment; filename="{str(file_name)}"'
-        )
+        response.content_type = str(content_type)
+        response.content_disposition = f'attachment; filename="{header_safe_str(file_name)}"'
+
         response.headers["X-RC-Artifact-Id"] = str(db_obj.file_store_id)
-        response.headers["X-RC-Artifact-Desc"] = str(db_obj.file_description)
+        response.headers["X-RC-Artifact-Desc"] = header_safe_str(db_obj.file_description)
         response.headers["X-RC-Artifact-Sha256"] = str(db_obj.file_hash)
         return response
 
     @LoginRequired()
     @NotAnonymous()
     @CSRFRequired()
     def upload_file(self):
         self.load_default_context()
         file_obj = self.request.POST.get(self.upload_key)
 
         if file_obj is None:
             return {'store_fid': None,
                     'access_path': None,
                     'error': f'{self.upload_key} data field is missing'}
 
         if not hasattr(file_obj, 'filename'):
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'filename cannot be read from the data field'}
 
         filename = file_obj.filename
 
         metadata = {
             'user_uploaded': {'username': self._rhodecode_user.username,
                               'user_id': self._rhodecode_user.user_id,
                               'ip': self._rhodecode_user.ip_addr}}
         try:
-            store_uid, metadata = self.storage.save_file(
-                file_obj.file, filename, extra_metadata=metadata)
+            store_uid, metadata = self.f_store.store(
+                filename, file_obj.file, extra_metadata=metadata)
         except FileNotAllowedException:
             return {'store_fid': None,
                     'access_path': None,
                     'error': f'File {filename} is not allowed.'}
 
         except FileOverSizeException:
             return {'store_fid': None,
                     'access_path': None,
                     'error': f'File {filename} is exceeding allowed limit.'}
 
         try:
             entry = FileStore.create(
                 file_uid=store_uid, filename=metadata["filename"],
                 file_hash=metadata["sha256"], file_size=metadata["size"],
                 file_description='upload attachment',
                 check_acl=False, user_id=self._rhodecode_user.user_id
             )
             Session().add(entry)
             Session().commit()
             log.debug('Stored upload in DB as %s', entry)
         except Exception:
             log.exception('Failed to store file %s', filename)
             return {'store_fid': None,
                     'access_path': None,
                     'error': f'File {filename} failed to store in DB.'}
 
         return {'store_fid': store_uid,
                 'access_path': h.route_path('download_file', fid=store_uid)}
 
     # ACL is checked by scopes, if no scope the file is accessible to all
     def download_file(self):
         self.load_default_context()
         file_uid = self.request.matchdict['fid']
-        log.debug('Requesting FID:%s from store %s', file_uid, self.storage)
+        log.debug('Requesting FID:%s from store %s', file_uid, self.f_store)
         return self._serve_file(file_uid)
 
     # in addition to @LoginRequired ACL is checked by scopes
     @LoginRequired(auth_token_access=[UserApiKeys.ROLE_ARTIFACT_DOWNLOAD])
     @NotAnonymous()
     def download_file_by_token(self):
         """
         Special view that allows to access the download file by special URL that
         is stored inside the URL.
 
         http://example.com/_file_store/token-download/TOKEN/FILE_UID
         """
         self.load_default_context()
         file_uid = self.request.matchdict['fid']
         return self._serve_file(file_uid)

rhodecode/apps/repository/views/repo_commits.py

@@ -1,830 +1,830 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import collections
 
 from pyramid.httpexceptions import (
     HTTPNotFound, HTTPBadRequest, HTTPFound, HTTPForbidden, HTTPConflict)
 from pyramid.renderers import render
 from pyramid.response import Response
 
 from rhodecode.apps._base import RepoAppView
 from rhodecode.apps.file_store import utils as store_utils
 from rhodecode.apps.file_store.exceptions import FileNotAllowedException, FileOverSizeException
 
 from rhodecode.lib import diffs, codeblocks, channelstream
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, NotAnonymous, CSRFRequired)
 from rhodecode.lib import ext_json
 from collections import OrderedDict
 from rhodecode.lib.diffs import (
     cache_diff, load_cached_diff, diff_cache_exist, get_diff_context,
     get_diff_whitespace_flag)
 from rhodecode.lib.exceptions import StatusChangeOnClosedPullRequestError, CommentVersionMismatch
 import rhodecode.lib.helpers as h
 from rhodecode.lib.utils2 import str2bool, StrictAttributeDict, safe_str
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.exceptions import (
     RepositoryError, CommitDoesNotExistError)
 from rhodecode.model.db import ChangesetComment, ChangesetStatus, FileStore, \
     ChangesetCommentHistory
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def _update_with_GET(params, request):
     for k in ['diff1', 'diff2', 'diff']:
         params[k] += request.GET.getall(k)
 
 
 class RepoCommitsView(RepoAppView):
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.rhodecode_repo = self.rhodecode_vcs_repo
 
         return c
 
     def _is_diff_cache_enabled(self, target_repo):
         caching_enabled = self._get_general_setting(
             target_repo, 'rhodecode_diff_cache')
         log.debug('Diff caching enabled: %s', caching_enabled)
         return caching_enabled
 
     def _commit(self, commit_id_range, method):
         _ = self.request.translate
         c = self.load_default_context()
         c.fulldiff = self.request.GET.get('fulldiff')
         redirect_to_combined = str2bool(self.request.GET.get('redirect_combined'))
 
         # fetch global flags of ignore ws or context lines
         diff_context = get_diff_context(self.request)
         hide_whitespace_changes = get_diff_whitespace_flag(self.request)
 
         # diff_limit will cut off the whole diff if the limit is applied
         # otherwise it will just hide the big files from the front-end
         diff_limit = c.visual.cut_off_limit_diff
         file_limit = c.visual.cut_off_limit_file
 
         # get ranges of commit ids if preset
         commit_range = commit_id_range.split('...')[:2]
 
         try:
             pre_load = ['affected_files', 'author', 'branch', 'date',
                         'message', 'parents']
             if self.rhodecode_vcs_repo.alias == 'hg':
                 pre_load += ['hidden', 'obsolete', 'phase']
 
             if len(commit_range) == 2:
                 commits = self.rhodecode_vcs_repo.get_commits(
                     start_id=commit_range[0], end_id=commit_range[1],
                     pre_load=pre_load, translate_tags=False)
                 commits = list(commits)
             else:
                 commits = [self.rhodecode_vcs_repo.get_commit(
                     commit_id=commit_id_range, pre_load=pre_load)]
 
             c.commit_ranges = commits
             if not c.commit_ranges:
                 raise RepositoryError('The commit range returned an empty result')
         except CommitDoesNotExistError as e:
             msg = _('No such commit exists. Org exception: `{}`').format(safe_str(e))
             h.flash(msg, category='error')
             raise HTTPNotFound()
         except Exception:
             log.exception("General failure")
             raise HTTPNotFound()
         single_commit = len(c.commit_ranges) == 1
 
         if redirect_to_combined and not single_commit:
             source_ref = getattr(c.commit_ranges[0].parents[0]
                                  if c.commit_ranges[0].parents else h.EmptyCommit(), 'raw_id')
             target_ref = c.commit_ranges[-1].raw_id
             next_url = h.route_path(
                 'repo_compare',
                 repo_name=c.repo_name,
                 source_ref_type='rev',
                 source_ref=source_ref,
                 target_ref_type='rev',
                 target_ref=target_ref)
             raise HTTPFound(next_url)
 
         c.changes = OrderedDict()
         c.lines_added = 0
         c.lines_deleted = 0
 
         # auto collapse if we have more than limit
         collapse_limit = diffs.DiffProcessor._collapse_commits_over
         c.collapse_all_commits = len(c.commit_ranges) > collapse_limit
 
         c.commit_statuses = ChangesetStatus.STATUSES
         c.inline_comments = []
         c.files = []
 
         c.comments = []
         c.unresolved_comments = []
         c.resolved_comments = []
 
         # Single commit
         if single_commit:
             commit = c.commit_ranges[0]
             c.comments = CommentsModel().get_comments(
                 self.db_repo.repo_id,
                 revision=commit.raw_id)
 
             # comments from PR
             statuses = ChangesetStatusModel().get_statuses(
                 self.db_repo.repo_id, commit.raw_id,
                 with_revisions=True)
 
             prs = set()
             reviewers = list()
             reviewers_duplicates = set()  # to not have duplicates from multiple votes
             for c_status in statuses:
 
                 # extract associated pull-requests from votes
                 if c_status.pull_request:
                     prs.add(c_status.pull_request)
 
                 # extract reviewers
                 _user_id = c_status.author.user_id
                 if _user_id not in reviewers_duplicates:
                     reviewers.append(
                         StrictAttributeDict({
                             'user': c_status.author,
 
                             # fake attributed for commit, page that we don't have
                             # but we share the display with PR page
                             'mandatory': False,
                             'reasons': [],
                             'rule_user_group_data': lambda: None
                         })
                     )
                     reviewers_duplicates.add(_user_id)
 
             c.reviewers_count = len(reviewers)
             c.observers_count = 0
 
             # from associated statuses, check the pull requests, and
             # show comments from them
             for pr in prs:
                 c.comments.extend(pr.comments)
 
             c.unresolved_comments = CommentsModel()\
                 .get_commit_unresolved_todos(commit.raw_id)
             c.resolved_comments = CommentsModel()\
                 .get_commit_resolved_todos(commit.raw_id)
 
             c.inline_comments_flat = CommentsModel()\
                 .get_commit_inline_comments(commit.raw_id)
 
             review_statuses = ChangesetStatusModel().aggregate_votes_by_user(
                 statuses, reviewers)
 
             c.commit_review_status = ChangesetStatus.STATUS_NOT_REVIEWED
 
             c.commit_set_reviewers_data_json = collections.OrderedDict({'reviewers': []})
 
             for review_obj, member, reasons, mandatory, status in review_statuses:
                 member_reviewer = h.reviewer_as_json(
                     member, reasons=reasons, mandatory=mandatory, role=None,
                     user_group=None
                 )
 
                 current_review_status = status[0][1].status if status else ChangesetStatus.STATUS_NOT_REVIEWED
                 member_reviewer['review_status'] = current_review_status
                 member_reviewer['review_status_label'] = h.commit_status_lbl(current_review_status)
                 member_reviewer['allowed_to_update'] = False
                 c.commit_set_reviewers_data_json['reviewers'].append(member_reviewer)
 
             c.commit_set_reviewers_data_json = ext_json.str_json(c.commit_set_reviewers_data_json)
 
             # NOTE(marcink): this uses the same voting logic as in pull-requests
             c.commit_review_status = ChangesetStatusModel().calculate_status(review_statuses)
             c.commit_broadcast_channel = channelstream.comment_channel(c.repo_name, commit_obj=commit)
 
         diff = None
         # Iterate over ranges (default commit view is always one commit)
         for commit in c.commit_ranges:
             c.changes[commit.raw_id] = []
 
             commit2 = commit
             commit1 = commit.first_parent
 
             if method == 'show':
                 inline_comments = CommentsModel().get_inline_comments(
                     self.db_repo.repo_id, revision=commit.raw_id)
                 c.inline_cnt = len(CommentsModel().get_inline_comments_as_list(
                     inline_comments))
                 c.inline_comments = inline_comments
 
                 cache_path = self.rhodecode_vcs_repo.get_create_shadow_cache_pr_path(
                     self.db_repo)
                 cache_file_path = diff_cache_exist(
                     cache_path, 'diff', commit.raw_id,
                     hide_whitespace_changes, diff_context, c.fulldiff)
 
                 caching_enabled = self._is_diff_cache_enabled(self.db_repo)
                 force_recache = str2bool(self.request.GET.get('force_recache'))
 
                 cached_diff = None
                 if caching_enabled:
                     cached_diff = load_cached_diff(cache_file_path)
 
                 has_proper_diff_cache = cached_diff and cached_diff.get('diff')
                 if not force_recache and has_proper_diff_cache:
                     diffset = cached_diff['diff']
                 else:
                     vcs_diff = self.rhodecode_vcs_repo.get_diff(
                         commit1, commit2,
                         ignore_whitespace=hide_whitespace_changes,
                         context=diff_context)
 
                     diff_processor = diffs.DiffProcessor(vcs_diff, diff_format='newdiff',
                                                          diff_limit=diff_limit,
                                                          file_limit=file_limit,
                                                          show_full_diff=c.fulldiff)
 
                     _parsed = diff_processor.prepare()
 
                     diffset = codeblocks.DiffSet(
                         repo_name=self.db_repo_name,
                         source_node_getter=codeblocks.diffset_node_getter(commit1),
                         target_node_getter=codeblocks.diffset_node_getter(commit2))
 
                     diffset = self.path_filter.render_patchset_filtered(
                         diffset, _parsed, commit1.raw_id, commit2.raw_id)
 
                     # save cached diff
                     if caching_enabled:
                         cache_diff(cache_file_path, diffset, None)
 
                 c.limited_diff = diffset.limited_diff
                 c.changes[commit.raw_id] = diffset
             else:
                 # TODO(marcink): no cache usage here...
                 _diff = self.rhodecode_vcs_repo.get_diff(
                     commit1, commit2,
                     ignore_whitespace=hide_whitespace_changes, context=diff_context)
                 diff_processor = diffs.DiffProcessor(_diff, diff_format='newdiff',
                                                      diff_limit=diff_limit,
                                                      file_limit=file_limit, show_full_diff=c.fulldiff)
                 # downloads/raw we only need RAW diff nothing else
                 diff = self.path_filter.get_raw_patch(diff_processor)
                 c.changes[commit.raw_id] = [None, None, None, None, diff, None, None]
 
         # sort comments by how they were generated
         c.comments = sorted(c.comments, key=lambda x: x.comment_id)
         c.at_version_num = None
 
         if len(c.commit_ranges) == 1:
             c.commit = c.commit_ranges[0]
             c.parent_tmpl = ''.join(
                 '# Parent  %s\n' % x.raw_id for x in c.commit.parents)
 
         if method == 'download':
             response = Response(diff)
             response.content_type = 'text/plain'
             response.content_disposition = (
                 'attachment; filename=%s.diff' % commit_id_range[:12])
             return response
         elif method == 'patch':
 
             c.diff = safe_str(diff)
             patch = render(
                 'rhodecode:templates/changeset/patch_changeset.mako',
                 self._get_template_context(c), self.request)
             response = Response(patch)
             response.content_type = 'text/plain'
             return response
         elif method == 'raw':
             response = Response(diff)
             response.content_type = 'text/plain'
             return response
         elif method == 'show':
             if len(c.commit_ranges) == 1:
                 html = render(
                     'rhodecode:templates/changeset/changeset.mako',
                     self._get_template_context(c), self.request)
                 return Response(html)
             else:
                 c.ancestor = None
                 c.target_repo = self.db_repo
                 html = render(
                     'rhodecode:templates/changeset/changeset_range.mako',
                     self._get_template_context(c), self.request)
                 return Response(html)
 
         raise HTTPBadRequest()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commit_show(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='show')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commit_raw(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='raw')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commit_patch(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='patch')
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commit_download(self):
         commit_id = self.request.matchdict['commit_id']
         return self._commit(commit_id, method='download')
 
     def _commit_comments_create(self, commit_id, comments):
         _ = self.request.translate
         data = {}
         if not comments:
             return
 
         commit = self.db_repo.get_commit(commit_id)
 
         all_drafts = len([x for x in comments if str2bool(x['is_draft'])]) == len(comments)
         for entry in comments:
             c = self.load_default_context()
             comment_type = entry['comment_type']
             text = entry['text']
             status = entry['status']
             is_draft = str2bool(entry['is_draft'])
             resolves_comment_id = entry['resolves_comment_id']
             f_path = entry['f_path']
             line_no = entry['line']
             target_elem_id = f'file-{h.safeid(h.safe_str(f_path))}'
 
             if status:
                 text = text or (_('Status change %(transition_icon)s %(status)s')
                                 % {'transition_icon': '>',
                                    'status': ChangesetStatus.get_status_lbl(status)})
 
             comment = CommentsModel().create(
                 text=text,
                 repo=self.db_repo.repo_id,
                 user=self._rhodecode_db_user.user_id,
                 commit_id=commit_id,
                 f_path=f_path,
                 line_no=line_no,
                 status_change=(ChangesetStatus.get_status_lbl(status)
                                if status else None),
                 status_change_type=status,
                 comment_type=comment_type,
                 is_draft=is_draft,
                 resolves_comment_id=resolves_comment_id,
                 auth_user=self._rhodecode_user,
                 send_email=not is_draft,  # skip notification for draft comments
             )
             is_inline = comment.is_inline
 
             # get status if set !
             if status:
                 # `dont_allow_on_closed_pull_request = True` means
                 # if latest status was from pull request and it's closed
                 # disallow changing status !
 
                 try:
                     ChangesetStatusModel().set_status(
                         self.db_repo.repo_id,
                         status,
                         self._rhodecode_db_user.user_id,
                         comment,
                         revision=commit_id,
                         dont_allow_on_closed_pull_request=True
                     )
                 except StatusChangeOnClosedPullRequestError:
                     msg = _('Changing the status of a commit associated with '
                             'a closed pull request is not allowed')
                     log.exception(msg)
                     h.flash(msg, category='warning')
                     raise HTTPFound(h.route_path(
                         'repo_commit', repo_name=self.db_repo_name,
                         commit_id=commit_id))
 
             Session().flush()
             # this is somehow required to get access to some relationship
             # loaded on comment
             Session().refresh(comment)
 
             # skip notifications for drafts
             if not is_draft:
                 CommentsModel().trigger_commit_comment_hook(
                     self.db_repo, self._rhodecode_user, 'create',
                     data={'comment': comment, 'commit': commit})
 
             comment_id = comment.comment_id
             data[comment_id] = {
                 'target_id': target_elem_id
             }
             Session().flush()
 
             c.co = comment
             c.at_version_num = 0
             c.is_new = True
             rendered_comment = render(
                 'rhodecode:templates/changeset/changeset_comment_block.mako',
                 self._get_template_context(c), self.request)
 
             data[comment_id].update(comment.get_dict())
             data[comment_id].update({'rendered_text': rendered_comment})
 
         # finalize, commit and redirect
         Session().commit()
 
         # skip channelstream for draft comments
         if not all_drafts:
             comment_broadcast_channel = channelstream.comment_channel(
                 self.db_repo_name, commit_obj=commit)
 
             comment_data = data
             posted_comment_type = 'inline' if is_inline else 'general'
             if len(data) == 1:
                 msg = _('posted {} new {} comment').format(len(data), posted_comment_type)
             else:
                 msg = _('posted {} new {} comments').format(len(data), posted_comment_type)
 
             channelstream.comment_channelstream_push(
                 self.request, comment_broadcast_channel, self._rhodecode_user, msg,
                 comment_data=comment_data)
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_commit_comment_create(self):
         _ = self.request.translate
         commit_id = self.request.matchdict['commit_id']
 
         multi_commit_ids = []
         for _commit_id in self.request.POST.get('commit_ids', '').split(','):
             if _commit_id not in ['', None, EmptyCommit.raw_id]:
                 if _commit_id not in multi_commit_ids:
                     multi_commit_ids.append(_commit_id)
 
         commit_ids = multi_commit_ids or [commit_id]
 
         data = []
         # Multiple comments for each passed commit id
         for current_id in filter(None, commit_ids):
             comment_data = {
                 'comment_type': self.request.POST.get('comment_type'),
                 'text': self.request.POST.get('text'),
                 'status': self.request.POST.get('changeset_status', None),
                 'is_draft': self.request.POST.get('draft'),
                 'resolves_comment_id': self.request.POST.get('resolves_comment_id', None),
                 'close_pull_request': self.request.POST.get('close_pull_request'),
                 'f_path': self.request.POST.get('f_path'),
                 'line': self.request.POST.get('line'),
             }
             comment = self._commit_comments_create(commit_id=current_id, comments=[comment_data])
             data.append(comment)
 
         return data if len(data) > 1 else data[0]
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_commit_comment_preview(self):
         # Technically a CSRF token is not needed as no state changes with this
         # call. However, as this is a POST is better to have it, so automated
         # tools don't flag it as potential CSRF.
         # Post is required because the payload could be bigger than the maximum
         # allowed by GET.
 
         text = self.request.POST.get('text')
         renderer = self.request.POST.get('renderer') or 'rst'
         if text:
             return h.render(text, renderer=renderer, mentions=True,
                             repo_name=self.db_repo_name)
         return ''
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_commit_comment_history_view(self):
         c = self.load_default_context()
         comment_id = self.request.matchdict['comment_id']
         comment_history_id = self.request.matchdict['comment_history_id']
 
         comment = ChangesetComment.get_or_404(comment_id)
         comment_owner = (comment.author.user_id == self._rhodecode_db_user.user_id)
         if comment.draft and not comment_owner:
             # if we see draft comments history, we only allow this for owner
             raise HTTPNotFound()
 
         comment_history = ChangesetCommentHistory.get_or_404(comment_history_id)
         is_repo_comment = comment_history.comment.repo.repo_id == self.db_repo.repo_id
 
         if is_repo_comment:
             c.comment_history = comment_history
 
             rendered_comment = render(
                 'rhodecode:templates/changeset/comment_history.mako',
                 self._get_template_context(c), self.request)
             return rendered_comment
         else:
             log.warning('No permissions for user %s to show comment_history_id: %s',
                         self._rhodecode_db_user, comment_history_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_commit_comment_attachment_upload(self):
         c = self.load_default_context()
         upload_key = 'attachment'
 
         file_obj = self.request.POST.get(upload_key)
 
         if file_obj is None:
             self.request.response.status = 400
             return {'store_fid': None,
                     'access_path': None,
                     'error': f'{upload_key} data field is missing'}
 
         if not hasattr(file_obj, 'filename'):
             self.request.response.status = 400
             return {'store_fid': None,
                     'access_path': None,
                     'error': 'filename cannot be read from the data field'}
 
         filename = file_obj.filename
         file_display_name = filename
 
         metadata = {
             'user_uploaded': {'username': self._rhodecode_user.username,
                               'user_id': self._rhodecode_user.user_id,
                               'ip': self._rhodecode_user.ip_addr}}
 
         # TODO(marcink): allow .ini configuration for allowed_extensions, and file-size
         allowed_extensions = [
             'gif', '.jpeg', '.jpg', '.png', '.docx', '.gz', '.log', '.pdf',
             '.pptx', '.txt', '.xlsx', '.zip']
         max_file_size = 10 * 1024 * 1024  # 10MB, also validated via dropzone.js
 
         try:
-            storage = store_utils.get_file_storage(self.request.registry.settings)
-            store_uid, metadata = storage.save_file(
-                file_obj.file, filename, extra_metadata=metadata,
+            f_store = store_utils.get_filestore_backend(self.request.registry.settings)
+            store_uid, metadata = f_store.store(
+                filename, file_obj.file, metadata=metadata,
                 extensions=allowed_extensions, max_filesize=max_file_size)
         except FileNotAllowedException:
             self.request.response.status = 400
             permitted_extensions = ', '.join(allowed_extensions)
-            error_msg = 'File `{}` is not allowed. ' \
-                        'Only following extensions are permitted: {}'.format(
-                            filename, permitted_extensions)
+            error_msg = f'File `{filename}` is not allowed. ' \
+                        f'Only following extensions are permitted: {permitted_extensions}'
+
             return {'store_fid': None,
                     'access_path': None,
                     'error': error_msg}
         except FileOverSizeException:
             self.request.response.status = 400
             limit_mb = h.format_byte_size_binary(max_file_size)
+            error_msg = f'File {filename} is exceeding allowed limit of {limit_mb}.'
             return {'store_fid': None,
                     'access_path': None,
-                    'error': 'File {} is exceeding allowed limit of {}.'.format(
-                        filename, limit_mb)}
+                    'error': error_msg}
 
         try:
             entry = FileStore.create(
                 file_uid=store_uid, filename=metadata["filename"],
                 file_hash=metadata["sha256"], file_size=metadata["size"],
                 file_display_name=file_display_name,
                 file_description=f'comment attachment `{safe_str(filename)}`',
                 hidden=True, check_acl=True, user_id=self._rhodecode_user.user_id,
                 scope_repo_id=self.db_repo.repo_id
             )
             Session().add(entry)
             Session().commit()
             log.debug('Stored upload in DB as %s', entry)
         except Exception:
             log.exception('Failed to store file %s', filename)
             self.request.response.status = 400
             return {'store_fid': None,
                     'access_path': None,
                     'error': f'File {filename} failed to store in DB.'}
 
         Session().commit()
 
         data = {
             'store_fid': store_uid,
             'access_path': h.route_path(
                 'download_file', fid=store_uid),
             'fqn_access_path': h.route_url(
                 'download_file', fid=store_uid),
             # for EE those are replaced by FQN links on repo-only like
             'repo_access_path': h.route_url(
                 'download_file', fid=store_uid),
             'repo_fqn_access_path': h.route_url(
                 'download_file', fid=store_uid),
         }
         # this data is a part of CE/EE additional code
         if c.rhodecode_edition_id == 'EE':
             data.update({
                 'repo_access_path': h.route_path(
                     'repo_artifacts_get', repo_name=self.db_repo_name, uid=store_uid),
                 'repo_fqn_access_path': h.route_url(
                     'repo_artifacts_get', repo_name=self.db_repo_name, uid=store_uid),
             })
 
         return data
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_commit_comment_delete(self):
         commit_id = self.request.matchdict['commit_id']
         comment_id = self.request.matchdict['comment_id']
 
         comment = ChangesetComment.get_or_404(comment_id)
         if not comment:
             log.debug('Comment with id:%s not found, skipping', comment_id)
             # comment already deleted in another call probably
             return True
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = (comment.author.user_id == self._rhodecode_db_user.user_id)
         is_repo_comment = comment.repo.repo_id == self.db_repo.repo_id
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if comment.draft and not comment_owner:
             # We never allow to delete draft comments for other than owners
             raise HTTPNotFound()
 
         if super_admin or comment_owner or comment_repo_admin:
             CommentsModel().delete(comment=comment, auth_user=self._rhodecode_user)
             Session().commit()
             return True
         else:
             log.warning('No permissions for user %s to delete comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @NotAnonymous()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_commit_comment_edit(self):
         self.load_default_context()
 
         commit_id = self.request.matchdict['commit_id']
         comment_id = self.request.matchdict['comment_id']
         comment = ChangesetComment.get_or_404(comment_id)
 
         if comment.immutable:
             # don't allow deleting comments that are immutable
             raise HTTPForbidden()
 
         is_repo_admin = h.HasRepoPermissionAny('repository.admin')(self.db_repo_name)
         super_admin = h.HasPermissionAny('hg.admin')()
         comment_owner = (comment.author.user_id == self._rhodecode_db_user.user_id)
         is_repo_comment = comment.repo.repo_id == self.db_repo.repo_id
         comment_repo_admin = is_repo_admin and is_repo_comment
 
         if super_admin or comment_owner or comment_repo_admin:
             text = self.request.POST.get('text')
             version = self.request.POST.get('version')
             if text == comment.text:
                 log.warning(
                     'Comment(repo): '
                     'Trying to create new version '
                     'with the same comment body {}'.format(
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             if version.isdigit():
                 version = int(version)
             else:
                 log.warning(
                     'Comment(repo): Wrong version type {} {} '
                     'for comment {}'.format(
                         version,
                         type(version),
                         comment_id,
                     )
                 )
                 raise HTTPNotFound()
 
             try:
                 comment_history = CommentsModel().edit(
                     comment_id=comment_id,
                     text=text,
                     auth_user=self._rhodecode_user,
                     version=version,
                 )
             except CommentVersionMismatch:
                 raise HTTPConflict()
 
             if not comment_history:
                 raise HTTPNotFound()
 
             if not comment.draft:
                 commit = self.db_repo.get_commit(commit_id)
                 CommentsModel().trigger_commit_comment_hook(
                     self.db_repo, self._rhodecode_user, 'edit',
                     data={'comment': comment, 'commit': commit})
 
             Session().commit()
             return {
                 'comment_history_id': comment_history.comment_history_id,
                 'comment_id': comment.comment_id,
                 'comment_version': comment_history.version,
                 'comment_author_username': comment_history.author.username,
                 'comment_author_gravatar': h.gravatar_url(comment_history.author.email, 16, request=self.request),
                 'comment_created_on': h.age_component(comment_history.created_on,
                                                       time_is_local=True),
             }
         else:
             log.warning('No permissions for user %s to edit comment_id: %s',
                         self._rhodecode_db_user, comment_id)
             raise HTTPNotFound()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commit_data(self):
         commit_id = self.request.matchdict['commit_id']
         self.load_default_context()
 
         try:
             return self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
         except CommitDoesNotExistError as e:
             return EmptyCommit(message=str(e))
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commit_children(self):
         commit_id = self.request.matchdict['commit_id']
         self.load_default_context()
 
         try:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
             children = commit.children
         except CommitDoesNotExistError:
             children = []
 
         result = {"results": children}
         return result
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_commit_parents(self):
         commit_id = self.request.matchdict['commit_id']
         self.load_default_context()
 
         try:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
             parents = commit.parents
         except CommitDoesNotExistError:
             parents = []
         result = {"results": parents}
         return result

rhodecode/apps/repository/views/repo_files.py

@@ -1,1716 +1,1716 b''
 # Copyright (C) 2011-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import itertools
 import logging
 import os
 import collections
 import urllib.request
 import urllib.parse
 import urllib.error
 import pathlib
 import time
 import random
 
 from pyramid.httpexceptions import HTTPNotFound, HTTPBadRequest, HTTPFound
 
 from pyramid.renderers import render
 from pyramid.response import Response
 
 import rhodecode
 from rhodecode.apps._base import RepoAppView
 
 
 from rhodecode.lib import diffs, helpers as h, rc_cache
 from rhodecode.lib import audit_logger
 from rhodecode.lib.hash_utils import sha1_safe
 from rhodecode.lib.archive_cache import (
     get_archival_cache_store, get_archival_config, ArchiveCacheGenerationLock, archive_iterator)
 from rhodecode.lib.str_utils import safe_bytes, convert_special_chars
 from rhodecode.lib.view_utils import parse_path_ref
 from rhodecode.lib.exceptions import NonRelativePathError
 from rhodecode.lib.codeblocks import (
     filenode_as_lines_tokens, filenode_as_annotated_lines_tokens)
 from rhodecode.lib.utils2 import convert_line_endings, detect_mode
 from rhodecode.lib.type_utils import str2bool
-from rhodecode.lib.str_utils import safe_str, safe_int
+from rhodecode.lib.str_utils import safe_str, safe_int, header_safe_str
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired)
 from rhodecode.lib.vcs import path as vcspath
 from rhodecode.lib.vcs.backends.base import EmptyCommit
 from rhodecode.lib.vcs.conf import settings
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.lib.vcs.exceptions import (
     RepositoryError, CommitDoesNotExistError, EmptyRepositoryError,
     ImproperArchiveTypeError, VCSError, NodeAlreadyExistsError,
     NodeDoesNotExistError, CommitError, NodeError)
 
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.db import Repository
 
 log = logging.getLogger(__name__)
 
 
 def get_archive_name(db_repo_id, db_repo_name, commit_sha, ext, subrepos=False, path_sha='', with_hash=True):
     # original backward compat name of archive
     clean_name = safe_str(convert_special_chars(db_repo_name).replace('/', '_'))
 
     # e.g vcsserver-id-abcd-sub-1-abcfdef-archive-all.zip
     # vcsserver-id-abcd-sub-0-abcfdef-COMMIT_SHA-PATH_SHA.zip
     id_sha = sha1_safe(str(db_repo_id))[:4]
     sub_repo = 'sub-1' if subrepos else 'sub-0'
     commit = commit_sha if with_hash else 'archive'
     path_marker = (path_sha if with_hash else '') or 'all'
     archive_name = f'{clean_name}-id-{id_sha}-{sub_repo}-{commit}-{path_marker}{ext}'
 
     return archive_name
 
 
 def get_path_sha(at_path):
     return safe_str(sha1_safe(at_path)[:8])
 
 
 def _get_archive_spec(fname):
     log.debug('Detecting archive spec for: `%s`', fname)
 
     fileformat = None
     ext = None
     content_type = None
     for a_type, content_type, extension in settings.ARCHIVE_SPECS:
 
         if fname.endswith(extension):
             fileformat = a_type
             log.debug('archive is of type: %s', fileformat)
             ext = extension
             break
 
     if not fileformat:
         raise ValueError()
 
     # left over part of whole fname is the commit
     commit_id = fname[:-len(ext)]
 
     return commit_id, ext, fileformat, content_type
 
 
 class RepoFilesView(RepoAppView):
 
     @staticmethod
     def adjust_file_path_for_svn(f_path, repo):
         """
         Computes the relative path of `f_path`.
 
         This is mainly based on prefix matching of the recognized tags and
         branches in the underlying repository.
         """
         tags_and_branches = itertools.chain(
             repo.branches.keys(),
             repo.tags.keys())
         tags_and_branches = sorted(tags_and_branches, key=len, reverse=True)
 
         for name in tags_and_branches:
             if f_path.startswith(f'{name}/'):
                 f_path = vcspath.relpath(f_path, name)
                 break
         return f_path
 
     def load_default_context(self):
         c = self._get_local_tmpl_context(include_app_defaults=True)
         c.rhodecode_repo = self.rhodecode_vcs_repo
         c.enable_downloads = self.db_repo.enable_downloads
         return c
 
     def _ensure_not_locked(self, commit_id='tip'):
         _ = self.request.translate
 
         repo = self.db_repo
         if repo.enable_locking and repo.locked[0]:
             h.flash(_('This repository has been locked by %s on %s')
                     % (h.person_by_id(repo.locked[0]),
                     h.format_date(h.time_to_datetime(repo.locked[1]))),
                     'warning')
             files_url = h.route_path(
                 'repo_files:default_path',
                 repo_name=self.db_repo_name, commit_id=commit_id)
             raise HTTPFound(files_url)
 
     def forbid_non_head(self, is_head, f_path, commit_id='tip', json_mode=False):
         _ = self.request.translate
 
         if not is_head:
             message = _('Cannot modify file. '
                         'Given commit `{}` is not head of a branch.').format(commit_id)
             h.flash(message, category='warning')
 
             if json_mode:
                 return message
 
             files_url = h.route_path(
                 'repo_files', repo_name=self.db_repo_name, commit_id=commit_id,
                 f_path=f_path)
             raise HTTPFound(files_url)
 
     def check_branch_permission(self, branch_name, commit_id='tip', json_mode=False):
         _ = self.request.translate
 
         rule, branch_perm = self._rhodecode_user.get_rule_and_branch_permission(
             self.db_repo_name, branch_name)
         if branch_perm and branch_perm not in ['branch.push', 'branch.push_force']:
             message = _('Branch `{}` changes forbidden by rule {}.').format(
                 h.escape(branch_name), h.escape(rule))
             h.flash(message, 'warning')
 
             if json_mode:
                 return message
 
             files_url = h.route_path(
                 'repo_files:default_path', repo_name=self.db_repo_name, commit_id=commit_id)
 
             raise HTTPFound(files_url)
 
     def _get_commit_and_path(self):
         default_commit_id = self.db_repo.landing_ref_name
         default_f_path = '/'
 
         commit_id = self.request.matchdict.get(
             'commit_id', default_commit_id)
         f_path = self._get_f_path(self.request.matchdict, default_f_path)
         return commit_id, f_path
 
     def _get_default_encoding(self, c):
         enc_list = getattr(c, 'default_encodings', [])
         return enc_list[0] if enc_list else 'UTF-8'
 
     def _get_commit_or_redirect(self, commit_id, redirect_after=True):
         """
         This is a safe way to get commit. If an error occurs it redirects to
         tip with proper message
 
         :param commit_id: id of commit to fetch
         :param redirect_after: toggle redirection
         """
         _ = self.request.translate
 
         try:
             return self.rhodecode_vcs_repo.get_commit(commit_id)
         except EmptyRepositoryError:
             if not redirect_after:
                 return None
 
             add_new = upload_new = ""
             if h.HasRepoPermissionAny(
                     'repository.write', 'repository.admin')(self.db_repo_name):
                 _url = h.route_path(
                     'repo_files_add_file',
                     repo_name=self.db_repo_name, commit_id=0, f_path='')
                 add_new = h.link_to(
                     _('add a new file'), _url, class_="alert-link")
 
                 _url_upld = h.route_path(
                     'repo_files_upload_file',
                     repo_name=self.db_repo_name, commit_id=0, f_path='')
                 upload_new = h.link_to(
                     _('upload a new file'), _url_upld, class_="alert-link")
 
             h.flash(h.literal(
                 _('There are no files yet. Click here to %s or %s.') % (add_new, upload_new)), category='warning')
             raise HTTPFound(
                 h.route_path('repo_summary', repo_name=self.db_repo_name))
 
         except (CommitDoesNotExistError, LookupError) as e:
             msg = _('No such commit exists for this repository. Commit: {}').format(commit_id)
             h.flash(msg, category='error')
             raise HTTPNotFound()
         except RepositoryError as e:
             h.flash(h.escape(safe_str(e)), category='error')
             raise HTTPNotFound()
 
     def _get_filenode_or_redirect(self, commit_obj, path, pre_load=None):
         """
         Returns file_node, if error occurs or given path is directory,
         it'll redirect to top level path
         """
         _ = self.request.translate
 
         try:
             file_node = commit_obj.get_node(path, pre_load=pre_load)
             if file_node.is_dir():
                 raise RepositoryError('The given path is a directory')
         except CommitDoesNotExistError:
             log.exception('No such commit exists for this repository')
             h.flash(_('No such commit exists for this repository'), category='error')
             raise HTTPNotFound()
         except RepositoryError as e:
             log.warning('Repository error while fetching filenode `%s`. Err:%s', path, e)
             h.flash(h.escape(safe_str(e)), category='error')
             raise HTTPNotFound()
 
         return file_node
 
     def _is_valid_head(self, commit_id, repo, landing_ref):
         branch_name = sha_commit_id = ''
         is_head = False
         log.debug('Checking if commit_id `%s` is a head for %s.', commit_id, repo)
 
         for _branch_name, branch_commit_id in repo.branches.items():
             # simple case we pass in branch name, it's a HEAD
             if commit_id == _branch_name:
                 is_head = True
                 branch_name = _branch_name
                 sha_commit_id = branch_commit_id
                 break
             # case when we pass in full sha commit_id, which is a head
             elif commit_id == branch_commit_id:
                 is_head = True
                 branch_name = _branch_name
                 sha_commit_id = branch_commit_id
                 break
 
         if h.is_svn(repo) and not repo.is_empty():
             # Note: Subversion only has one head.
             if commit_id == repo.get_commit(commit_idx=-1).raw_id:
                 is_head = True
                 return branch_name, sha_commit_id, is_head
 
         # checked branches, means we only need to try to get the branch/commit_sha
         if repo.is_empty():
             is_head = True
             branch_name = landing_ref
             sha_commit_id = EmptyCommit().raw_id
         else:
             commit = repo.get_commit(commit_id=commit_id)
             if commit:
                 branch_name = commit.branch
                 sha_commit_id = commit.raw_id
 
         return branch_name, sha_commit_id, is_head
 
     def _get_tree_at_commit(self, c, commit_id, f_path, full_load=False, at_rev=None):
 
         repo_id = self.db_repo.repo_id
         force_recache = self.get_recache_flag()
 
         cache_seconds = safe_int(
             rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time'))
         cache_on = not force_recache and cache_seconds > 0
         log.debug(
             'Computing FILE TREE for repo_id %s commit_id `%s` and path `%s`'
             'with caching: %s[TTL: %ss]' % (
                 repo_id, commit_id, f_path, cache_on, cache_seconds or 0))
 
         cache_namespace_uid = f'repo.{rc_cache.FILE_TREE_CACHE_VER}.{repo_id}'
         region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on)
         def compute_file_tree(_name_hash, _repo_id, _commit_id, _f_path, _full_load, _at_rev):
             log.debug('Generating cached file tree at for repo_id: %s, %s, %s',
                       _repo_id, _commit_id, _f_path)
 
             c.full_load = _full_load
             return render(
                 'rhodecode:templates/files/files_browser_tree.mako',
                 self._get_template_context(c), self.request, _at_rev)
 
         return compute_file_tree(
             self.db_repo.repo_name_hash, self.db_repo.repo_id, commit_id, f_path, full_load, at_rev)
 
     def create_pure_path(self, *parts):
         # Split paths and sanitize them, removing any ../ etc
         sanitized_path = [
             x for x in pathlib.PurePath(*parts).parts
             if x not in ['.', '..']]
 
         pure_path = pathlib.PurePath(*sanitized_path)
         return pure_path
 
     def _is_lf_enabled(self, target_repo):
         lf_enabled = False
 
         lf_key_for_vcs_map = {
             'hg': 'extensions_largefiles',
             'git': 'vcs_git_lfs_enabled'
         }
 
         lf_key_for_vcs = lf_key_for_vcs_map.get(target_repo.repo_type)
 
         if lf_key_for_vcs:
             lf_enabled = self._get_repo_setting(target_repo, lf_key_for_vcs)
 
         return lf_enabled
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_archivefile(self):
         # archive cache config
         from rhodecode import CONFIG
         _ = self.request.translate
         self.load_default_context()
         default_at_path = '/'
         fname = self.request.matchdict['fname']
         subrepos = self.request.GET.get('subrepos') == 'true'
         with_hash = str2bool(self.request.GET.get('with_hash', '1'))
         at_path = self.request.GET.get('at_path') or default_at_path
 
         if not self.db_repo.enable_downloads:
             return Response(_('Downloads disabled'))
 
         try:
             commit_id, ext, fileformat, content_type = \
                 _get_archive_spec(fname)
         except ValueError:
             return Response(_('Unknown archive type for: `{}`').format(
                 h.escape(fname)))
 
         try:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id)
         except CommitDoesNotExistError:
             return Response(_('Unknown commit_id {}').format(
                 h.escape(commit_id)))
         except EmptyRepositoryError:
             return Response(_('Empty repository'))
 
         # we used a ref, or a shorter version, lets redirect client ot use explicit hash
         if commit_id != commit.raw_id:
             fname=f'{commit.raw_id}{ext}'
             raise HTTPFound(self.request.current_route_path(fname=fname))
 
         try:
             at_path = commit.get_node(at_path).path or default_at_path
         except Exception:
             return Response(_('No node at path {} for this repository').format(h.escape(at_path)))
 
         path_sha = get_path_sha(at_path)
 
         # used for cache etc, consistent unique archive name
         archive_name_key = get_archive_name(
             self.db_repo.repo_id, self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos,
             path_sha=path_sha, with_hash=True)
 
         if not with_hash:
             path_sha = ''
 
         # what end client gets served
         response_archive_name = get_archive_name(
             self.db_repo.repo_id, self.db_repo_name, commit_sha=commit.short_id, ext=ext, subrepos=subrepos,
             path_sha=path_sha, with_hash=with_hash)
 
         # remove extension from our archive directory name
         archive_dir_name = response_archive_name[:-len(ext)]
 
         archive_cache_disable = self.request.GET.get('no_cache')
 
         d_cache = get_archival_cache_store(config=CONFIG)
 
         # NOTE: we get the config to pass to a call to lazy-init the SAME type of cache on vcsserver
         d_cache_conf = get_archival_config(config=CONFIG)
 
         # This is also a cache key, and lock key
         reentrant_lock_key = archive_name_key + '.lock'
 
         use_cached_archive = False
         if not archive_cache_disable and archive_name_key in d_cache:
             reader, metadata = d_cache.fetch(archive_name_key)
 
             use_cached_archive = True
             log.debug('Found cached archive as key=%s tag=%s, serving archive from cache reader=%s',
                       archive_name_key, metadata, reader.name)
         else:
             reader = None
             log.debug('Archive with key=%s is not yet cached, creating one now...', archive_name_key)
 
         if not reader:
             # generate new archive, as previous was not found in the cache
             try:
                 with d_cache.get_lock(reentrant_lock_key):
                     try:
                         commit.archive_repo(archive_name_key, archive_dir_name=archive_dir_name,
                                             kind=fileformat, subrepos=subrepos,
                                             archive_at_path=at_path, cache_config=d_cache_conf)
                     except ImproperArchiveTypeError:
                         return _('Unknown archive type')
 
             except ArchiveCacheGenerationLock:
                 retry_after = round(random.uniform(0.3, 3.0), 1)
                 time.sleep(retry_after)
 
                 location = self.request.url
                 response = Response(
                     f"archive {archive_name_key} generation in progress, Retry-After={retry_after}, Location={location}"
                 )
                 response.headers["Retry-After"] = str(retry_after)
                 response.status_code = 307  # temporary redirect
 
                 response.location = location
                 return response
 
             reader, metadata = d_cache.fetch(archive_name_key, retry=True, retry_attempts=30)
 
         response = Response(app_iter=archive_iterator(reader))
         response.content_disposition = f'attachment; filename={response_archive_name}'
         response.content_type = str(content_type)
 
         try:
             return response
         finally:
             # store download action
             audit_logger.store_web(
                 'repo.archive.download', action_data={
                     'user_agent': self.request.user_agent,
                     'archive_name': archive_name_key,
                     'archive_spec': fname,
                     'archive_cached': use_cached_archive},
                 user=self._rhodecode_user,
                 repo=self.db_repo,
                 commit=True
             )
 
     def _get_file_node(self, commit_id, f_path):
         if commit_id not in ['', None, 'None', '0' * 12, '0' * 40]:
             commit = self.rhodecode_vcs_repo.get_commit(commit_id=commit_id)
             try:
                 node = commit.get_node(f_path)
                 if node.is_dir():
                     raise NodeError(f'{node} path is a {type(node)} not a file')
             except NodeDoesNotExistError:
                 commit = EmptyCommit(
                     commit_id=commit_id,
                     idx=commit.idx,
                     repo=commit.repository,
                     alias=commit.repository.alias,
                     message=commit.message,
                     author=commit.author,
                     date=commit.date)
                 node = FileNode(safe_bytes(f_path), b'', commit=commit)
         else:
             commit = EmptyCommit(
                 repo=self.rhodecode_vcs_repo,
                 alias=self.rhodecode_vcs_repo.alias)
             node = FileNode(safe_bytes(f_path), b'', commit=commit)
         return node
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_files_diff(self):
         c = self.load_default_context()
         f_path = self._get_f_path(self.request.matchdict)
         diff1 = self.request.GET.get('diff1', '')
         diff2 = self.request.GET.get('diff2', '')
 
         path1, diff1 = parse_path_ref(diff1, default_path=f_path)
 
         ignore_whitespace = str2bool(self.request.GET.get('ignorews'))
         line_context = self.request.GET.get('context', 3)
 
         if not any((diff1, diff2)):
             h.flash(
                 'Need query parameter "diff1" or "diff2" to generate a diff.',
                 category='error')
             raise HTTPBadRequest()
 
         c.action = self.request.GET.get('diff')
         if c.action not in ['download', 'raw']:
             compare_url = h.route_path(
                 'repo_compare',
                 repo_name=self.db_repo_name,
                 source_ref_type='rev',
                 source_ref=diff1,
                 target_repo=self.db_repo_name,
                 target_ref_type='rev',
                 target_ref=diff2,
                 _query=dict(f_path=f_path))
             # redirect to new view if we render diff
             raise HTTPFound(compare_url)
 
         try:
             node1 = self._get_file_node(diff1, path1)
             node2 = self._get_file_node(diff2, f_path)
         except (RepositoryError, NodeError):
             log.exception("Exception while trying to get node from repository")
             raise HTTPFound(
                 h.route_path('repo_files', repo_name=self.db_repo_name,
                              commit_id='tip', f_path=f_path))
 
         if all(isinstance(node.commit, EmptyCommit)
                for node in (node1, node2)):
             raise HTTPNotFound()
 
         c.commit_1 = node1.commit
         c.commit_2 = node2.commit
 
         if c.action == 'download':
             _diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
             # NOTE: this was using diff_format='gitdiff'
             diff = diffs.DiffProcessor(_diff, diff_format='newdiff')
 
             response = Response(self.path_filter.get_raw_patch(diff))
             response.content_type = 'text/plain'
             response.content_disposition = (
                 f'attachment; filename={f_path}_{diff1}_vs_{diff2}.diff'
             )
             charset = self._get_default_encoding(c)
             if charset:
                 response.charset = charset
             return response
 
         elif c.action == 'raw':
             _diff = diffs.get_gitdiff(node1, node2,
                                       ignore_whitespace=ignore_whitespace,
                                       context=line_context)
             # NOTE: this was using diff_format='gitdiff'
             diff = diffs.DiffProcessor(_diff, diff_format='newdiff')
 
             response = Response(self.path_filter.get_raw_patch(diff))
             response.content_type = 'text/plain'
             charset = self._get_default_encoding(c)
             if charset:
                 response.charset = charset
             return response
 
         # in case we ever end up here
         raise HTTPNotFound()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_files_diff_2way_redirect(self):
         """
         Kept only to make OLD links work
         """
         f_path = self._get_f_path_unchecked(self.request.matchdict)
         diff1 = self.request.GET.get('diff1', '')
         diff2 = self.request.GET.get('diff2', '')
 
         if not any((diff1, diff2)):
             h.flash(
                 'Need query parameter "diff1" or "diff2" to generate a diff.',
                 category='error')
             raise HTTPBadRequest()
 
         compare_url = h.route_path(
             'repo_compare',
             repo_name=self.db_repo_name,
             source_ref_type='rev',
             source_ref=diff1,
             target_ref_type='rev',
             target_ref=diff2,
             _query=dict(f_path=f_path, diffmode='sideside',
                         target_repo=self.db_repo_name,))
         raise HTTPFound(compare_url)
 
     @LoginRequired()
     def repo_files_default_commit_redirect(self):
         """
         Special page that redirects to the landing page of files based on the default
         commit for repository
         """
         c = self.load_default_context()
         ref_name = c.rhodecode_db_repo.landing_ref_name
         landing_url = h.repo_files_by_ref_url(
             c.rhodecode_db_repo.repo_name,
             c.rhodecode_db_repo.repo_type,
             f_path='',
             ref_name=ref_name,
             commit_id='tip',
             query=dict(at=ref_name)
         )
 
         raise HTTPFound(landing_url)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_files(self):
         c = self.load_default_context()
 
         view_name = getattr(self.request.matched_route, 'name', None)
 
         c.annotate = view_name == 'repo_files:annotated'
         # default is false, but .rst/.md files later are auto rendered, we can
         # overwrite auto rendering by setting this GET flag
         c.renderer = view_name == 'repo_files:rendered' or not self.request.GET.get('no-render', False)
 
         commit_id, f_path = self._get_commit_and_path()
 
         c.commit = self._get_commit_or_redirect(commit_id)
         c.branch = self.request.GET.get('branch', None)
         c.f_path = f_path
         at_rev = self.request.GET.get('at')
 
         # files or dirs
         try:
             c.file = c.commit.get_node(f_path, pre_load=['is_binary', 'size', 'data'])
 
             c.file_author = True
             c.file_tree = ''
 
             # prev link
             try:
                 prev_commit = c.commit.prev(c.branch)
                 c.prev_commit = prev_commit
                 c.url_prev = h.route_path(
                     'repo_files', repo_name=self.db_repo_name,
                     commit_id=prev_commit.raw_id, f_path=f_path)
                 if c.branch:
                     c.url_prev += '?branch=%s' % c.branch
             except (CommitDoesNotExistError, VCSError):
                 c.url_prev = '#'
                 c.prev_commit = EmptyCommit()
 
             # next link
             try:
                 next_commit = c.commit.next(c.branch)
                 c.next_commit = next_commit
                 c.url_next = h.route_path(
                     'repo_files', repo_name=self.db_repo_name,
                     commit_id=next_commit.raw_id, f_path=f_path)
                 if c.branch:
                     c.url_next += '?branch=%s' % c.branch
             except (CommitDoesNotExistError, VCSError):
                 c.url_next = '#'
                 c.next_commit = EmptyCommit()
 
             # load file content
             if c.file.is_file():
 
                 c.lf_node = {}
 
                 has_lf_enabled = self._is_lf_enabled(self.db_repo)
                 if has_lf_enabled:
                     c.lf_node = c.file.get_largefile_node()
 
                 c.file_source_page = 'true'
                 c.file_last_commit = c.file.last_commit
 
                 c.file_size_too_big = c.file.size > c.visual.cut_off_limit_file
 
                 if not (c.file_size_too_big or c.file.is_binary):
                     if c.annotate:  # annotation has precedence over renderer
                         c.annotated_lines = filenode_as_annotated_lines_tokens(
                             c.file
                         )
                     else:
                         c.renderer = (
                             c.renderer and h.renderer_from_filename(c.file.path)
                         )
                         if not c.renderer:
                             c.lines = filenode_as_lines_tokens(c.file)
 
                 _branch_name, _sha_commit_id, is_head = \
                     self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
                                         landing_ref=self.db_repo.landing_ref_name)
                 c.on_branch_head = is_head
 
                 branch = c.commit.branch if (
                     c.commit.branch and '/' not in c.commit.branch) else None
                 c.branch_or_raw_id = branch or c.commit.raw_id
                 c.branch_name = c.commit.branch or h.short_id(c.commit.raw_id)
 
                 author = c.file_last_commit.author
                 c.authors = [[
                     h.email(author),
                     h.person(author, 'username_or_name_or_email'),
                     1
                 ]]
 
             else:  # load tree content at path
                 c.file_source_page = 'false'
                 c.authors = []
                 # this loads a simple tree without metadata to speed things up
                 # later via ajax we call repo_nodetree_full and fetch whole
                 c.file_tree = self._get_tree_at_commit(c, c.commit.raw_id, f_path, at_rev=at_rev)
 
                 c.readme_data, c.readme_file = \
                     self._get_readme_data(self.db_repo, c.visual.default_renderer,
                                           c.commit.raw_id, f_path)
 
         except RepositoryError as e:
             h.flash(h.escape(safe_str(e)), category='error')
             raise HTTPNotFound()
 
         if self.request.environ.get('HTTP_X_PJAX'):
             html = render('rhodecode:templates/files/files_pjax.mako',
                           self._get_template_context(c), self.request)
         else:
             html = render('rhodecode:templates/files/files.mako',
                           self._get_template_context(c), self.request)
         return Response(html)
 
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_files_annotated_previous(self):
         self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         prev_commit_id = commit.raw_id
         line_anchor = self.request.GET.get('line_anchor')
         is_file = False
         try:
             _file = commit.get_node(f_path)
             is_file = _file.is_file()
         except (NodeDoesNotExistError, CommitDoesNotExistError, VCSError):
             pass
 
         if is_file:
             history = commit.get_path_history(f_path)
             prev_commit_id = history[1].raw_id \
                 if len(history) > 1 else prev_commit_id
         prev_url = h.route_path(
             'repo_files:annotated', repo_name=self.db_repo_name,
             commit_id=prev_commit_id, f_path=f_path,
             _anchor=f'L{line_anchor}')
 
         raise HTTPFound(prev_url)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_nodetree_full(self):
         """
         Returns rendered html of file tree that contains commit date,
         author, commit_id for the specified combination of
         repo, commit_id and file path
         """
         c = self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         try:
             dir_node = commit.get_node(f_path)
         except RepositoryError as e:
             return Response(f'error: {h.escape(safe_str(e))}')
 
         if dir_node.is_file():
             return Response('')
 
         c.file = dir_node
         c.commit = commit
         at_rev = self.request.GET.get('at')
 
         html = self._get_tree_at_commit(
             c, commit.raw_id, dir_node.path, full_load=True, at_rev=at_rev)
 
         return Response(html)
 
     def _get_attachement_headers(self, f_path):
         f_name = safe_str(f_path.split(Repository.NAME_SEP)[-1])
         safe_path = f_name.replace('"', '\\"')
         encoded_path = urllib.parse.quote(f_name)
 
         headers = "attachment; " \
                   "filename=\"{}\"; " \
                   "filename*=UTF-8\'\'{}".format(safe_path, encoded_path)
 
-        return safe_bytes(headers).decode('latin-1', errors='replace')
+        return header_safe_str(headers)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_file_raw(self):
         """
         Action for show as raw, some mimetypes are "rendered",
         those include images, icons.
         """
         c = self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         file_node = self._get_filenode_or_redirect(commit, f_path)
 
         raw_mimetype_mapping = {
             # map original mimetype to a mimetype used for "show as raw"
             # you can also provide a content-disposition to override the
             # default "attachment" disposition.
             # orig_type: (new_type, new_dispo)
 
             # show images inline:
             # Do not re-add SVG: it is unsafe and permits XSS attacks. One can
             # for example render an SVG with javascript inside or even render
             # HTML.
             'image/x-icon': ('image/x-icon', 'inline'),
             'image/png': ('image/png', 'inline'),
             'image/gif': ('image/gif', 'inline'),
             'image/jpeg': ('image/jpeg', 'inline'),
             'application/pdf': ('application/pdf', 'inline'),
         }
 
         mimetype = file_node.mimetype
         try:
             mimetype, disposition = raw_mimetype_mapping[mimetype]
         except KeyError:
             # we don't know anything special about this, handle it safely
             if file_node.is_binary:
                 # do same as download raw for binary files
                 mimetype, disposition = 'application/octet-stream', 'attachment'
             else:
                 # do not just use the original mimetype, but force text/plain,
                 # otherwise it would serve text/html and that might be unsafe.
                 # Note: underlying vcs library fakes text/plain mimetype if the
                 # mimetype can not be determined and it thinks it is not
                 # binary.This might lead to erroneous text display in some
                 # cases, but helps in other cases, like with text files
                 # without extension.
                 mimetype, disposition = 'text/plain', 'inline'
 
         if disposition == 'attachment':
             disposition = self._get_attachement_headers(f_path)
 
         stream_content = file_node.stream_bytes()
 
         response = Response(app_iter=stream_content)
         response.content_disposition = disposition
         response.content_type = mimetype
 
         charset = self._get_default_encoding(c)
         if charset:
             response.charset = charset
 
         return response
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_file_download(self):
         c = self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         file_node = self._get_filenode_or_redirect(commit, f_path)
 
         if self.request.GET.get('lf'):
             # only if lf get flag is passed, we download this file
             # as LFS/Largefile
             lf_node = file_node.get_largefile_node()
             if lf_node:
                 # overwrite our pointer with the REAL large-file
                 file_node = lf_node
 
         disposition = self._get_attachement_headers(f_path)
 
         stream_content = file_node.stream_bytes()
 
         response = Response(app_iter=stream_content)
         response.content_disposition = disposition
         response.content_type = file_node.mimetype
 
         charset = self._get_default_encoding(c)
         if charset:
             response.charset = charset
 
         return response
 
     def _get_nodelist_at_commit(self, repo_name, repo_id, commit_id, f_path):
 
         cache_seconds = safe_int(
             rhodecode.CONFIG.get('rc_cache.cache_repo.expiration_time'))
         cache_on = cache_seconds > 0
         log.debug(
             'Computing FILE SEARCH for repo_id %s commit_id `%s` and path `%s`'
             'with caching: %s[TTL: %ss]' % (
                 repo_id, commit_id, f_path, cache_on, cache_seconds or 0))
 
         cache_namespace_uid = f'repo.{repo_id}'
         region = rc_cache.get_or_create_region('cache_repo', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid, condition=cache_on)
         def compute_file_search(_name_hash, _repo_id, _commit_id, _f_path):
             log.debug('Generating cached nodelist for repo_id:%s, %s, %s',
                       _repo_id, commit_id, f_path)
             try:
                 _d, _f = ScmModel().get_quick_filter_nodes(repo_name, _commit_id, _f_path)
             except (RepositoryError, CommitDoesNotExistError, Exception) as e:
                 log.exception(safe_str(e))
                 h.flash(h.escape(safe_str(e)), category='error')
                 raise HTTPFound(h.route_path(
                     'repo_files', repo_name=self.db_repo_name,
                     commit_id='tip', f_path='/'))
 
             return _d + _f
 
         result = compute_file_search(self.db_repo.repo_name_hash, self.db_repo.repo_id,
                                      commit_id, f_path)
         return filter(lambda n: self.path_filter.path_access_allowed(n['name']), result)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_nodelist(self):
         self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
 
         metadata = self._get_nodelist_at_commit(
             self.db_repo_name, self.db_repo.repo_id, commit.raw_id, f_path)
         return {'nodes': [x for x in metadata]}
 
     def _create_references(self, branches_or_tags, symbolic_reference, f_path, ref_type):
         items = []
         for name, commit_id in branches_or_tags.items():
             sym_ref = symbolic_reference(commit_id, name, f_path, ref_type)
             items.append((sym_ref, name, ref_type))
         return items
 
     def _symbolic_reference(self, commit_id, name, f_path, ref_type):
         return commit_id
 
     def _symbolic_reference_svn(self, commit_id, name, f_path, ref_type):
         return commit_id
 
         # NOTE(dan): old code we used in "diff" mode compare
         new_f_path = vcspath.join(name, f_path)
         return f'{new_f_path}@{commit_id}'
 
     def _get_node_history(self, commit_obj, f_path, commits=None):
         """
         get commit history for given node
 
         :param commit_obj: commit to calculate history
         :param f_path: path for node to calculate history for
         :param commits: if passed don't calculate history and take
             commits defined in this list
         """
         _ = self.request.translate
 
         # calculate history based on tip
         tip = self.rhodecode_vcs_repo.get_commit()
         if commits is None:
             pre_load = ["author", "branch"]
             try:
                 commits = tip.get_path_history(f_path, pre_load=pre_load)
             except (NodeDoesNotExistError, CommitError):
                 # this node is not present at tip!
                 commits = commit_obj.get_path_history(f_path, pre_load=pre_load)
 
         history = []
         commits_group = ([], _("Changesets"))
         for commit in commits:
             branch = ' (%s)' % commit.branch if commit.branch else ''
             n_desc = f'r{commit.idx}:{commit.short_id}{branch}'
             commits_group[0].append((commit.raw_id, n_desc, 'sha'))
         history.append(commits_group)
 
         symbolic_reference = self._symbolic_reference
 
         if self.rhodecode_vcs_repo.alias == 'svn':
             adjusted_f_path = RepoFilesView.adjust_file_path_for_svn(
                 f_path, self.rhodecode_vcs_repo)
             if adjusted_f_path != f_path:
                 log.debug(
                     'Recognized svn tag or branch in file "%s", using svn '
                     'specific symbolic references', f_path)
                 f_path = adjusted_f_path
                 symbolic_reference = self._symbolic_reference_svn
 
         branches = self._create_references(
             self.rhodecode_vcs_repo.branches, symbolic_reference, f_path, 'branch')
         branches_group = (branches, _("Branches"))
 
         tags = self._create_references(
             self.rhodecode_vcs_repo.tags, symbolic_reference, f_path, 'tag')
         tags_group = (tags, _("Tags"))
 
         history.append(branches_group)
         history.append(tags_group)
 
         return history, commits
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_file_history(self):
         self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         file_node = self._get_filenode_or_redirect(commit, f_path)
 
         if file_node.is_file():
             file_history, _hist = self._get_node_history(commit, f_path)
 
             res = []
             for section_items, section in file_history:
                 items = []
                 for obj_id, obj_text, obj_type in section_items:
                     at_rev = ''
                     if obj_type in ['branch', 'bookmark', 'tag']:
                         at_rev = obj_text
                     entry = {
                         'id': obj_id,
                         'text': obj_text,
                         'type': obj_type,
                         'at_rev': at_rev
                     }
 
                     items.append(entry)
 
                 res.append({
                     'text': section,
                     'children': items
                 })
 
             data = {
                 'more': False,
                 'results': res
             }
             return data
 
         log.warning('Cannot fetch history for directory')
         raise HTTPBadRequest()
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator(
         'repository.read', 'repository.write', 'repository.admin')
     def repo_file_authors(self):
         c = self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         commit = self._get_commit_or_redirect(commit_id)
         file_node = self._get_filenode_or_redirect(commit, f_path)
 
         if not file_node.is_file():
             raise HTTPBadRequest()
 
         c.file_last_commit = file_node.last_commit
         if self.request.GET.get('annotate') == '1':
             # use _hist from annotation if annotation mode is on
             commit_ids = {x[1] for x in file_node.annotate}
             _hist = (
                 self.rhodecode_vcs_repo.get_commit(commit_id)
                 for commit_id in commit_ids)
         else:
             _f_history, _hist = self._get_node_history(commit, f_path)
         c.file_author = False
 
         unique = collections.OrderedDict()
         for commit in _hist:
             author = commit.author
             if author not in unique:
                 unique[commit.author] = [
                     h.email(author),
                     h.person(author, 'username_or_name_or_email'),
                     1  # counter
                 ]
 
             else:
                 # increase counter
                 unique[commit.author][2] += 1
 
         c.authors = [val for val in unique.values()]
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     def repo_files_check_head(self):
         self.load_default_context()
 
         commit_id, f_path = self._get_commit_and_path()
         _branch_name, _sha_commit_id, is_head = \
             self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
                                 landing_ref=self.db_repo.landing_ref_name)
 
         new_path = self.request.POST.get('path')
         operation = self.request.POST.get('operation')
         path_exist = ''
 
         if new_path and operation in ['create', 'upload']:
             new_f_path = os.path.join(f_path.lstrip('/'), new_path)
             try:
                 commit_obj = self.rhodecode_vcs_repo.get_commit(commit_id)
                 # NOTE(dan): construct whole path without leading /
                 file_node = commit_obj.get_node(new_f_path)
                 if file_node is not None:
                     path_exist = new_f_path
             except EmptyRepositoryError:
                 pass
             except Exception:
                 pass
 
         return {
             'branch': _branch_name,
             'sha': _sha_commit_id,
             'is_head': is_head,
             'path_exists': path_exist
         }
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     def repo_files_remove_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
         _branch_name, _sha_commit_id, is_head = \
             self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
                                 landing_ref=self.db_repo.landing_ref_name)
 
         self.forbid_non_head(is_head, f_path)
         self.check_branch_permission(_branch_name)
 
         c.commit = self._get_commit_or_redirect(commit_id)
         c.file = self._get_filenode_or_redirect(c.commit, f_path)
 
         c.default_message = _(
             'Deleted file {} via RhodeCode Enterprise').format(f_path)
         c.f_path = f_path
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_files_delete_file(self):
         _ = self.request.translate
 
         c = self.load_default_context()
         commit_id, f_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
         _branch_name, _sha_commit_id, is_head = \
             self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
                                 landing_ref=self.db_repo.landing_ref_name)
 
         self.forbid_non_head(is_head, f_path)
         self.check_branch_permission(_branch_name)
 
         c.commit = self._get_commit_or_redirect(commit_id)
         c.file = self._get_filenode_or_redirect(c.commit, f_path)
 
         c.default_message = _(
             'Deleted file {} via RhodeCode Enterprise').format(f_path)
         c.f_path = f_path
         node_path = f_path
         author = self._rhodecode_db_user.full_contact
         message = self.request.POST.get('message') or c.default_message
         try:
             nodes = {
                 safe_bytes(node_path): {
                     'content': b''
                 }
             }
             ScmModel().delete_nodes(
                 user=self._rhodecode_db_user.user_id, repo=self.db_repo,
                 message=message,
                 nodes=nodes,
                 parent_commit=c.commit,
                 author=author,
             )
 
             h.flash(
                 _('Successfully deleted file `{}`').format(
                     h.escape(f_path)), category='success')
         except Exception:
             log.exception('Error during commit operation')
             h.flash(_('Error occurred during commit'), category='error')
         raise HTTPFound(
             h.route_path('repo_commit', repo_name=self.db_repo_name,
                          commit_id='tip'))
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     def repo_files_edit_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
         _branch_name, _sha_commit_id, is_head = \
             self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
                                 landing_ref=self.db_repo.landing_ref_name)
 
         self.forbid_non_head(is_head, f_path, commit_id=commit_id)
         self.check_branch_permission(_branch_name, commit_id=commit_id)
 
         c.commit = self._get_commit_or_redirect(commit_id)
         c.file = self._get_filenode_or_redirect(c.commit, f_path)
 
         if c.file.is_binary:
             files_url = h.route_path(
                 'repo_files',
                 repo_name=self.db_repo_name,
                 commit_id=c.commit.raw_id, f_path=f_path)
             raise HTTPFound(files_url)
 
         c.default_message = _('Edited file {} via RhodeCode Enterprise').format(f_path)
         c.f_path = f_path
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_files_update_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
 
         c.commit = self._get_commit_or_redirect(commit_id)
         c.file = self._get_filenode_or_redirect(c.commit, f_path)
 
         if c.file.is_binary:
             raise HTTPFound(h.route_path('repo_files', repo_name=self.db_repo_name,
                                          commit_id=c.commit.raw_id, f_path=f_path))
 
         _branch_name, _sha_commit_id, is_head = \
             self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
                                 landing_ref=self.db_repo.landing_ref_name)
 
         self.forbid_non_head(is_head, f_path, commit_id=commit_id)
         self.check_branch_permission(_branch_name, commit_id=commit_id)
 
         c.default_message = _('Edited file {} via RhodeCode Enterprise').format(f_path)
         c.f_path = f_path
 
         old_content = c.file.str_content
         sl = old_content.splitlines(1)
         first_line = sl[0] if sl else ''
 
         r_post = self.request.POST
         # line endings:  0 - Unix, 1 - Mac, 2 - DOS
         line_ending_mode = detect_mode(first_line, 0)
         content = convert_line_endings(r_post.get('content', ''), line_ending_mode)
 
         message = r_post.get('message') or c.default_message
 
         org_node_path = c.file.str_path
         filename = r_post['filename']
 
         root_path = c.file.dir_path
         pure_path = self.create_pure_path(root_path, filename)
         node_path = pure_path.as_posix()
 
         default_redirect_url = h.route_path('repo_commit', repo_name=self.db_repo_name,
                                             commit_id=commit_id)
         if content == old_content and node_path == org_node_path:
             h.flash(_('No changes detected on {}').format(h.escape(org_node_path)),
                     category='warning')
             raise HTTPFound(default_redirect_url)
 
         try:
             mapping = {
                 c.file.bytes_path: {
                     'org_filename': org_node_path,
                     'filename': safe_bytes(node_path),
                     'content': safe_bytes(content),
                     'lexer': '',
                     'op': 'mod',
                     'mode': c.file.mode
                 }
             }
 
             commit = ScmModel().update_nodes(
                 user=self._rhodecode_db_user.user_id,
                 repo=self.db_repo,
                 message=message,
                 nodes=mapping,
                 parent_commit=c.commit,
             )
 
             h.flash(_('Successfully committed changes to file `{}`').format(
                     h.escape(f_path)), category='success')
             default_redirect_url = h.route_path(
                 'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id)
 
         except Exception:
             log.exception('Error occurred during commit')
             h.flash(_('Error occurred during commit'), category='error')
 
         raise HTTPFound(default_redirect_url)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     def repo_files_add_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
 
         # Check if we need to use this page to upload binary
         upload_binary = str2bool(self.request.params.get('upload_binary', False))
 
         c.commit = self._get_commit_or_redirect(commit_id, redirect_after=False)
         if c.commit is None:
             c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias)
 
         if self.rhodecode_vcs_repo.is_empty():
             # for empty repository we cannot check for current branch, we rely on
             # c.commit.branch instead
             _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True
         else:
             _branch_name, _sha_commit_id, is_head = \
                 self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
                                     landing_ref=self.db_repo.landing_ref_name)
 
         self.forbid_non_head(is_head, f_path, commit_id=commit_id)
         self.check_branch_permission(_branch_name, commit_id=commit_id)
 
         c.default_message = (_('Added file via RhodeCode Enterprise')) \
             if not upload_binary else (_('Edited file {} via RhodeCode Enterprise').format(f_path))
         c.f_path = f_path.lstrip('/')  # ensure not relative path
         c.replace_binary = upload_binary
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_files_create_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
 
         c.commit = self._get_commit_or_redirect(commit_id, redirect_after=False)
         if c.commit is None:
             c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias)
 
         # calculate redirect URL
         if self.rhodecode_vcs_repo.is_empty():
             default_redirect_url = h.route_path(
                 'repo_summary', repo_name=self.db_repo_name)
         else:
             default_redirect_url = h.route_path(
                 'repo_commit', repo_name=self.db_repo_name, commit_id='tip')
 
         if self.rhodecode_vcs_repo.is_empty():
             # for empty repository we cannot check for current branch, we rely on
             # c.commit.branch instead
             _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True
         else:
             _branch_name, _sha_commit_id, is_head = \
                 self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
                                     landing_ref=self.db_repo.landing_ref_name)
 
         self.forbid_non_head(is_head, f_path, commit_id=commit_id)
         self.check_branch_permission(_branch_name, commit_id=commit_id)
 
         c.default_message = (_('Added file via RhodeCode Enterprise'))
         c.f_path = f_path
 
         r_post = self.request.POST
         message = r_post.get('message') or c.default_message
         filename = r_post.get('filename')
         unix_mode = 0
 
         if not filename:
             # If there's no commit, redirect to repo summary
             if type(c.commit) is EmptyCommit:
                 redirect_url = h.route_path(
                     'repo_summary', repo_name=self.db_repo_name)
             else:
                 redirect_url = default_redirect_url
             h.flash(_('No filename specified'), category='warning')
             raise HTTPFound(redirect_url)
 
         root_path = f_path
         pure_path = self.create_pure_path(root_path, filename)
         node_path = pure_path.as_posix().lstrip('/')
 
         author = self._rhodecode_db_user.full_contact
         content = convert_line_endings(r_post.get('content', ''), unix_mode)
         nodes = {
             safe_bytes(node_path): {
                 'content': safe_bytes(content)
             }
         }
 
         try:
 
             commit = ScmModel().create_nodes(
                 user=self._rhodecode_db_user.user_id,
                 repo=self.db_repo,
                 message=message,
                 nodes=nodes,
                 parent_commit=c.commit,
                 author=author,
             )
 
             h.flash(_('Successfully committed new file `{}`').format(
                     h.escape(node_path)), category='success')
 
             default_redirect_url = h.route_path(
                 'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id)
 
         except NonRelativePathError:
             log.exception('Non Relative path found')
             h.flash(_('The location specified must be a relative path and must not '
                       'contain .. in the path'), category='warning')
             raise HTTPFound(default_redirect_url)
         except (NodeError, NodeAlreadyExistsError) as e:
             h.flash(h.escape(safe_str(e)), category='error')
         except Exception:
             log.exception('Error occurred during commit')
             h.flash(_('Error occurred during commit'), category='error')
 
         raise HTTPFound(default_redirect_url)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_files_upload_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
 
         c.commit = self._get_commit_or_redirect(commit_id, redirect_after=False)
         if c.commit is None:
             c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias)
 
         # calculate redirect URL
         if self.rhodecode_vcs_repo.is_empty():
             default_redirect_url = h.route_path(
                 'repo_summary', repo_name=self.db_repo_name)
         else:
             default_redirect_url = h.route_path(
                 'repo_commit', repo_name=self.db_repo_name, commit_id='tip')
 
         if self.rhodecode_vcs_repo.is_empty():
             # for empty repository we cannot check for current branch, we rely on
             # c.commit.branch instead
             _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True
         else:
             _branch_name, _sha_commit_id, is_head = \
                 self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
                                     landing_ref=self.db_repo.landing_ref_name)
 
         error = self.forbid_non_head(is_head, f_path, json_mode=True)
         if error:
             return {
                 'error': error,
                 'redirect_url': default_redirect_url
             }
         error = self.check_branch_permission(_branch_name, json_mode=True)
         if error:
             return {
                 'error': error,
                 'redirect_url': default_redirect_url
             }
 
         c.default_message = (_('Added file via RhodeCode Enterprise'))
         c.f_path = f_path
 
         r_post = self.request.POST
 
         message = c.default_message
         user_message = r_post.getall('message')
         if isinstance(user_message, list) and user_message:
             # we take the first from duplicated results if it's not empty
             message = user_message[0] if user_message[0] else message
 
         nodes = {}
 
         for file_obj in r_post.getall('files_upload') or []:
             content = file_obj.file
             filename = file_obj.filename
 
             root_path = f_path
             pure_path = self.create_pure_path(root_path, filename)
             node_path = pure_path.as_posix().lstrip('/')
 
             nodes[safe_bytes(node_path)] = {
                 'content': content
             }
 
         if not nodes:
             error = 'missing files'
             return {
                 'error': error,
                 'redirect_url': default_redirect_url
             }
 
         author = self._rhodecode_db_user.full_contact
 
         try:
             commit = ScmModel().create_nodes(
                 user=self._rhodecode_db_user.user_id,
                 repo=self.db_repo,
                 message=message,
                 nodes=nodes,
                 parent_commit=c.commit,
                 author=author,
             )
             if len(nodes) == 1:
                 flash_message = _('Successfully committed {} new files').format(len(nodes))
             else:
                 flash_message = _('Successfully committed 1 new file')
 
             h.flash(flash_message, category='success')
 
             default_redirect_url = h.route_path(
                 'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id)
 
         except NonRelativePathError:
             log.exception('Non Relative path found')
             error = _('The location specified must be a relative path and must not '
                       'contain .. in the path')
             h.flash(error, category='warning')
 
             return {
                 'error': error,
                 'redirect_url': default_redirect_url
             }
         except (NodeError, NodeAlreadyExistsError) as e:
             error = h.escape(e)
             h.flash(error, category='error')
 
             return {
                 'error': error,
                 'redirect_url': default_redirect_url
             }
         except Exception:
             log.exception('Error occurred during commit')
             error = _('Error occurred during commit')
             h.flash(error, category='error')
             return {
                 'error': error,
                 'redirect_url': default_redirect_url
             }
 
         return {
             'error': None,
             'redirect_url': default_redirect_url
         }
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.write', 'repository.admin')
     @CSRFRequired()
     def repo_files_replace_file(self):
         _ = self.request.translate
         c = self.load_default_context()
         commit_id, f_path = self._get_commit_and_path()
 
         self._ensure_not_locked()
 
         c.commit = self._get_commit_or_redirect(commit_id, redirect_after=False)
         if c.commit is None:
             c.commit = EmptyCommit(alias=self.rhodecode_vcs_repo.alias)
 
         if self.rhodecode_vcs_repo.is_empty():
             default_redirect_url = h.route_path(
                 'repo_summary', repo_name=self.db_repo_name)
         else:
             default_redirect_url = h.route_path(
                 'repo_commit', repo_name=self.db_repo_name, commit_id='tip')
 
         if self.rhodecode_vcs_repo.is_empty():
             # for empty repository we cannot check for current branch, we rely on
             # c.commit.branch instead
             _branch_name, _sha_commit_id, is_head = c.commit.branch, '', True
         else:
             _branch_name, _sha_commit_id, is_head = \
                 self._is_valid_head(commit_id, self.rhodecode_vcs_repo,
                                     landing_ref=self.db_repo.landing_ref_name)
 
         error = self.forbid_non_head(is_head, f_path, json_mode=True)
         if error:
             return {
                 'error': error,
                 'redirect_url': default_redirect_url
             }
         error = self.check_branch_permission(_branch_name, json_mode=True)
         if error:
             return {
                 'error': error,
                 'redirect_url': default_redirect_url
             }
 
         c.default_message = (_('Edited file {} via RhodeCode Enterprise').format(f_path))
         c.f_path = f_path
 
         r_post = self.request.POST
 
         message = c.default_message
         user_message = r_post.getall('message')
         if isinstance(user_message, list) and user_message:
             # we take the first from duplicated results if it's not empty
             message = user_message[0] if user_message[0] else message
 
         data_for_replacement = r_post.getall('files_upload') or []
         if (objects_count := len(data_for_replacement)) > 1:
             return {
                 'error': 'too many files for replacement',
                 'redirect_url': default_redirect_url
             }
         elif not objects_count:
             return {
                 'error': 'missing files',
                 'redirect_url': default_redirect_url
             }
 
         content = data_for_replacement[0].file
         retrieved_filename = data_for_replacement[0].filename
 
         if retrieved_filename.split('.')[-1] != f_path.split('.')[-1]:
             return {
                 'error': 'file extension of uploaded file doesn\'t match an original file\'s extension',
                 'redirect_url': default_redirect_url
             }
 
         author = self._rhodecode_db_user.full_contact
 
         try:
             commit = ScmModel().update_binary_node(
                 user=self._rhodecode_db_user.user_id,
                 repo=self.db_repo,
                 message=message,
                 node={
                     'content': content,
                     'file_path': f_path.encode(),
                 },
                 parent_commit=c.commit,
                 author=author,
             )
 
             h.flash(_('Successfully committed 1 new file'), category='success')
 
             default_redirect_url = h.route_path(
                 'repo_commit', repo_name=self.db_repo_name, commit_id=commit.raw_id)
 
         except (NodeError, NodeAlreadyExistsError) as e:
             error = h.escape(e)
             h.flash(error, category='error')
 
             return {
                 'error': error,
                 'redirect_url': default_redirect_url
             }
         except Exception:
             log.exception('Error occurred during commit')
             error = _('Error occurred during commit')
             h.flash(error, category='error')
             return {
                 'error': error,
                 'redirect_url': default_redirect_url
             }
 
         return {
             'error': None,
             'redirect_url': default_redirect_url
         }

rhodecode/apps/repository/views/repo_settings_advanced.py

@@ -1,302 +1,309 b''
 # Copyright (C) 2011-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 
 from pyramid.httpexceptions import HTTPFound
 from packaging.version import Version
 
 from rhodecode import events
 from rhodecode.apps._base import RepoAppView
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAnyDecorator, CSRFRequired,
     HasRepoPermissionAny)
-from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError
+from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError
 from rhodecode.lib.utils2 import safe_int
 from rhodecode.lib.vcs import RepositoryError
 from rhodecode.model.db import Session, UserFollowing, User, Repository
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoSettingsAdvancedView(RepoAppView):
 
     def load_default_context(self):
         c = self._get_local_tmpl_context()
         return c
 
     def _get_users_with_permissions(self):
         user_permissions = {}
         for perm in self.db_repo.permissions():
             user_permissions[perm.user_id] = perm
 
         return user_permissions
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     def edit_advanced(self):
         _ = self.request.translate
         c = self.load_default_context()
         c.active = 'advanced'
 
         c.default_user_id = User.get_default_user_id()
         c.in_public_journal = UserFollowing.query() \
             .filter(UserFollowing.user_id == c.default_user_id) \
             .filter(UserFollowing.follows_repository == self.db_repo).scalar()
 
         c.ver_info_dict = self.rhodecode_vcs_repo.get_hooks_info()
         c.hooks_outdated = False
 
         try:
             if Version(c.ver_info_dict['pre_version']) < Version(c.rhodecode_version):
                 c.hooks_outdated = True
         except Exception:
             pass
 
         # update commit cache if GET flag is present
         if self.request.GET.get('update_commit_cache'):
             self.db_repo.update_commit_cache()
             h.flash(_('updated commit cache'), category='success')
 
         return self._get_template_context(c)
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     @CSRFRequired()
     def edit_advanced_archive(self):
         """
         Archives the repository. It will become read-only, and not visible in search
         or other queries. But still visible for super-admins.
         """
 
         _ = self.request.translate
 
         try:
             old_data = self.db_repo.get_api_data()
             RepoModel().archive(self.db_repo)
 
             repo = audit_logger.RepoWrap(repo_id=None, repo_name=self.db_repo.repo_name)
             audit_logger.store_web(
                 'repo.archive', action_data={'old_data': old_data},
                 user=self._rhodecode_user, repo=repo)
 
             ScmModel().mark_for_invalidation(self.db_repo_name, delete=True)
             h.flash(
                 _('Archived repository `%s`') % self.db_repo_name,
                 category='success')
             Session().commit()
         except Exception:
             log.exception("Exception during archiving of repository")
             h.flash(_('An error occurred during archiving of `%s`')
                     % self.db_repo_name, category='error')
             # redirect to advanced for more deletion options
             raise HTTPFound(
                 h.route_path('edit_repo_advanced', repo_name=self.db_repo_name,
                              _anchor='advanced-archive'))
 
         # flush permissions for all users defined in permissions
         affected_user_ids = self._get_users_with_permissions().keys()
         PermissionModel().trigger_permission_flush(affected_user_ids)
 
         raise HTTPFound(h.route_path('home'))
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     @CSRFRequired()
     def edit_advanced_delete(self):
         """
         Deletes the repository, or shows warnings if deletion is not possible
         because of attached forks or other errors.
         """
         _ = self.request.translate
         handle_forks = self.request.POST.get('forks', None)
         if handle_forks == 'detach_forks':
             handle_forks = 'detach'
         elif handle_forks == 'delete_forks':
             handle_forks = 'delete'
 
+        repo_advanced_url = h.route_path(
+            'edit_repo_advanced', repo_name=self.db_repo_name,
+            _anchor='advanced-delete')
         try:
             old_data = self.db_repo.get_api_data()
             RepoModel().delete(self.db_repo, forks=handle_forks)
 
             _forks = self.db_repo.forks.count()
             if _forks and handle_forks:
                 if handle_forks == 'detach_forks':
                     h.flash(_('Detached %s forks') % _forks, category='success')
                 elif handle_forks == 'delete_forks':
                     h.flash(_('Deleted %s forks') % _forks, category='success')
 
             repo = audit_logger.RepoWrap(repo_id=None, repo_name=self.db_repo.repo_name)
             audit_logger.store_web(
                 'repo.delete', action_data={'old_data': old_data},
                 user=self._rhodecode_user, repo=repo)
 
             ScmModel().mark_for_invalidation(self.db_repo_name, delete=True)
             h.flash(
                 _('Deleted repository `%s`') % self.db_repo_name,
                 category='success')
             Session().commit()
         except AttachedForksError:
-            repo_advanced_url = h.route_path(
-                'edit_repo_advanced', repo_name=self.db_repo_name,
-                _anchor='advanced-delete')
             delete_anchor = h.link_to(_('detach or delete'), repo_advanced_url)
             h.flash(_('Cannot delete `{repo}` it still contains attached forks. '
                       'Try using {delete_or_detach} option.')
                     .format(repo=self.db_repo_name, delete_or_detach=delete_anchor),
                     category='warning')
 
             # redirect to advanced for forks handle action ?
             raise HTTPFound(repo_advanced_url)
 
         except AttachedPullRequestsError:
-            repo_advanced_url = h.route_path(
-                'edit_repo_advanced', repo_name=self.db_repo_name,
-                _anchor='advanced-delete')
             attached_prs = len(self.db_repo.pull_requests_source +
                                self.db_repo.pull_requests_target)
             h.flash(
                 _('Cannot delete `{repo}` it still contains {num} attached pull requests. '
                   'Consider archiving the repository instead.').format(
                     repo=self.db_repo_name, num=attached_prs), category='warning')
 
             # redirect to advanced for forks handle action ?
             raise HTTPFound(repo_advanced_url)
 
+        except AttachedArtifactsError:
+
+            attached_artifacts = len(self.db_repo.artifacts)
+            h.flash(
+                _('Cannot delete `{repo}` it still contains {num} attached artifacts. '
+                  'Consider archiving the repository instead.').format(
+                    repo=self.db_repo_name, num=attached_artifacts), category='warning')
+
+            # redirect to advanced for forks handle action ?
+            raise HTTPFound(repo_advanced_url)
         except Exception:
             log.exception("Exception during deletion of repository")
             h.flash(_('An error occurred during deletion of `%s`')
                     % self.db_repo_name, category='error')
             # redirect to advanced for more deletion options
             raise HTTPFound(
                 h.route_path('edit_repo_advanced', repo_name=self.db_repo_name,
                              _anchor='advanced-delete'))
 
         raise HTTPFound(h.route_path('home'))
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     @CSRFRequired()
     def edit_advanced_journal(self):
         """
         Set's this repository to be visible in public journal,
         in other words making default user to follow this repo
         """
         _ = self.request.translate
 
         try:
             user_id = User.get_default_user_id()
             ScmModel().toggle_following_repo(self.db_repo.repo_id, user_id)
             h.flash(_('Updated repository visibility in public journal'),
                     category='success')
             Session().commit()
         except Exception:
             h.flash(_('An error occurred during setting this '
                       'repository in public journal'),
                     category='error')
 
         raise HTTPFound(
             h.route_path('edit_repo_advanced', repo_name=self.db_repo_name))
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     @CSRFRequired()
     def edit_advanced_fork(self):
         """
         Mark given repository as a fork of another
         """
         _ = self.request.translate
 
         new_fork_id = safe_int(self.request.POST.get('id_fork_of'))
 
         # valid repo, re-check permissions
         if new_fork_id:
             repo = Repository.get(new_fork_id)
             # ensure we have at least read access to the repo we mark
             perm_check = HasRepoPermissionAny(
                 'repository.read', 'repository.write', 'repository.admin')
 
             if repo and perm_check(repo_name=repo.repo_name):
                 new_fork_id = repo.repo_id
             else:
                 new_fork_id = None
 
         try:
             repo = ScmModel().mark_as_fork(
                 self.db_repo_name, new_fork_id, self._rhodecode_user.user_id)
             fork = repo.fork.repo_name if repo.fork else _('Nothing')
             Session().commit()
             h.flash(
                 _('Marked repo %s as fork of %s') % (self.db_repo_name, fork),
                 category='success')
         except RepositoryError as e:
             log.exception("Repository Error occurred")
             h.flash(str(e), category='error')
         except Exception:
             log.exception("Exception while editing fork")
             h.flash(_('An error occurred during this operation'),
                     category='error')
 
         raise HTTPFound(
             h.route_path('edit_repo_advanced', repo_name=self.db_repo_name))
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     @CSRFRequired()
     def edit_advanced_toggle_locking(self):
         """
         Toggle locking of repository
         """
         _ = self.request.translate
         set_lock = self.request.POST.get('set_lock')
         set_unlock = self.request.POST.get('set_unlock')
 
         try:
             if set_lock:
                 Repository.lock(self.db_repo, self._rhodecode_user.user_id,
                                 lock_reason=Repository.LOCK_WEB)
                 h.flash(_('Locked repository'), category='success')
             elif set_unlock:
                 Repository.unlock(self.db_repo)
                 h.flash(_('Unlocked repository'), category='success')
         except Exception as e:
             log.exception("Exception during unlocking")
             h.flash(_('An error occurred during unlocking'), category='error')
 
         raise HTTPFound(
             h.route_path('edit_repo_advanced', repo_name=self.db_repo_name))
 
     @LoginRequired()
     @HasRepoPermissionAnyDecorator('repository.admin')
     def edit_advanced_install_hooks(self):
         """
         Install Hooks for repository
         """
         _ = self.request.translate
         self.load_default_context()
         self.rhodecode_vcs_repo.install_hooks(force=True)
         h.flash(_('installed updated hooks into this repository'),
                 category='success')
         raise HTTPFound(
             h.route_path('edit_repo_advanced', repo_name=self.db_repo_name))

rhodecode/apps/ssh_support/__init__.py

@@ -1,60 +1,60 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 from . import config_keys
 
 from rhodecode.config.settings_maker import SettingsMaker
 
 log = logging.getLogger(__name__)
 
 
 def _sanitize_settings_and_apply_defaults(settings):
     """
     Set defaults, convert to python types and validate settings.
     """
     settings_maker = SettingsMaker(settings)
 
     settings_maker.make_setting(config_keys.generate_authorized_keyfile, False, parser='bool')
     settings_maker.make_setting(config_keys.wrapper_allow_shell, False, parser='bool')
     settings_maker.make_setting(config_keys.enable_debug_logging, False, parser='bool')
     settings_maker.make_setting(config_keys.ssh_key_generator_enabled, True, parser='bool')
 
     settings_maker.make_setting(config_keys.authorized_keys_file_path, '~/.ssh/authorized_keys_rhodecode')
-    settings_maker.make_setting(config_keys.wrapper_cmd, '')
+    settings_maker.make_setting(config_keys.wrapper_cmd, '/usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2')
     settings_maker.make_setting(config_keys.authorized_keys_line_ssh_opts, '')
 
     settings_maker.make_setting(config_keys.ssh_hg_bin, '/usr/local/bin/rhodecode_bin/vcs_bin/hg')
     settings_maker.make_setting(config_keys.ssh_git_bin, '/usr/local/bin/rhodecode_bin/vcs_bin/git')
     settings_maker.make_setting(config_keys.ssh_svn_bin, '/usr/local/bin/rhodecode_bin/vcs_bin/svnserve')
 
     settings_maker.env_expand()
 
 
 def includeme(config):
     settings = config.registry.settings
     _sanitize_settings_and_apply_defaults(settings)
 
     # if we have enable generation of file, subscribe to event
     if settings[config_keys.generate_authorized_keyfile]:
         # lazy import here for faster code reading... via sshwrapper-v2 mode
         from .subscribers import generate_ssh_authorized_keys_file_subscriber
         from .events import SshKeyFileChangeEvent
         config.add_subscriber(
             generate_ssh_authorized_keys_file_subscriber, SshKeyFileChangeEvent)

rhodecode/apps/ssh_support/config_keys.py

@@ -1,32 +1,32 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 # Definition of setting keys used to configure this module. Defined here to
 # avoid repetition of keys throughout the module.
 generate_authorized_keyfile = 'ssh.generate_authorized_keyfile'
 authorized_keys_file_path = 'ssh.authorized_keys_file_path'
 authorized_keys_line_ssh_opts = 'ssh.authorized_keys_ssh_opts'
 ssh_key_generator_enabled = 'ssh.enable_ui_key_generator'
-wrapper_cmd = 'ssh.wrapper_cmd'
+wrapper_cmd = 'ssh.wrapper_cmd.v2'
 wrapper_allow_shell = 'ssh.wrapper_cmd_allow_shell'
 enable_debug_logging = 'ssh.enable_debug_logging'
 
 ssh_hg_bin = 'ssh.executable.hg'
 ssh_git_bin = 'ssh.executable.git'
 ssh_svn_bin = 'ssh.executable.svn'

rhodecode/apps/ssh_support/lib/backends/base.py

@@ -1,175 +1,175 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import sys
 import logging
 
 from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.api_utils import call_service_api
 
 log = logging.getLogger(__name__)
 
 
 class SshVcsServer(object):
     repo_user_agent = None  # set in child classes
     _path = None  # set executable path for hg/git/svn binary
     backend = None  # set in child classes
     tunnel = None  # subprocess handling tunnel
     settings = None  # parsed settings module
     write_perms = ['repository.admin', 'repository.write']
     read_perms = ['repository.read', 'repository.admin', 'repository.write']
 
     def __init__(self, user, user_permissions, settings, env):
         self.user = user
         self.user_permissions = user_permissions
         self.settings = settings
         self.env = env
         self.stdin = sys.stdin
 
         self.repo_name = None
         self.repo_mode = None
         self.store = ''
         self.ini_path = ''
         self.hooks_protocol = None
 
     def _invalidate_cache(self, repo_name):
         """
         Set's cache for this repository for invalidation on next access
 
         :param repo_name: full repo name, also a cache key
         """
         # Todo: Leave only "celery" case after transition.
         match self.hooks_protocol:
             case 'http':
                 from rhodecode.model.scm import ScmModel
                 ScmModel().mark_for_invalidation(repo_name)
             case 'celery':
                 call_service_api(self.settings, {
                     "method": "service_mark_for_invalidation",
                     "args": {"repo_name": repo_name}
                 })
 
     def has_write_perm(self):
         permission = self.user_permissions.get(self.repo_name)
         if permission in ['repository.write', 'repository.admin']:
             return True
 
         return False
 
     def _check_permissions(self, action):
         permission = self.user_permissions.get(self.repo_name)
         user_info = f'{self.user["user_id"]}:{self.user["username"]}'
         log.debug('permission for %s on %s are: %s',
                   user_info, self.repo_name, permission)
 
         if not permission:
             log.error('user `%s` permissions to repo:%s are empty. Forbidding access.',
                       user_info, self.repo_name)
             return -2
 
         if action == 'pull':
             if permission in self.read_perms:
                 log.info(
                     'READ Permissions for User "%s" detected to repo "%s"!',
                     user_info, self.repo_name)
                 return 0
         else:
             if permission in self.write_perms:
                 log.info(
                     'WRITE, or Higher Permissions for User "%s" detected to repo "%s"!',
                     user_info, self.repo_name)
                 return 0
 
         log.error('Cannot properly fetch or verify user `%s` permissions. '
                   'Permissions: %s, vcs action: %s',
                   user_info, permission, action)
         return -2
 
     def update_environment(self, action, extras=None):
 
         scm_data = {
             'ip': os.environ['SSH_CLIENT'].split()[0],
             'username': self.user.username,
             'user_id': self.user.user_id,
             'action': action,
             'repository': self.repo_name,
             'scm': self.backend,
             'config': self.ini_path,
             'repo_store': self.store,
             'make_lock': None,
             'locked_by': [None, None],
             'server_url': None,
             'user_agent': f'{self.repo_user_agent}/ssh-user-agent',
             'hooks': ['push', 'pull'],
             'hooks_module': 'rhodecode.lib.hook_daemon.hook_module',
             'is_shadow_repo': False,
             'detect_force_push': False,
             'check_branch_perms': False,
 
             'SSH': True,
             'SSH_PERMISSIONS': self.user_permissions.get(self.repo_name),
         }
         if extras:
             scm_data.update(extras)
         os.putenv("RC_SCM_DATA", json.dumps(scm_data))
         return scm_data
 
     def get_root_store(self):
         root_store = self.store
         if not root_store.endswith('/'):
             # always append trailing slash
             root_store = root_store + '/'
         return root_store
 
     def _handle_tunnel(self, extras):
         # pre-auth
         action = 'pull'
         exit_code = self._check_permissions(action)
         if exit_code:
             return exit_code, False
 
         req = self.env.get('request')
         if req:
             server_url = req.host_url + req.script_name
             extras['server_url'] = server_url
 
         log.debug('Using %s binaries from path %s', self.backend, self._path)
         exit_code = self.tunnel.run(extras)
 
         return exit_code, action == "push"
 
     def run(self, tunnel_extras=None):
-        self.hooks_protocol = self.settings['vcs.hooks.protocol']
+        self.hooks_protocol = self.settings['vcs.hooks.protocol.v2']
         tunnel_extras = tunnel_extras or {}
         extras = {}
         extras.update(tunnel_extras)
 
         callback_daemon, extras = prepare_callback_daemon(
             extras, protocol=self.hooks_protocol,
             host=vcs_settings.HOOKS_HOST)
 
         with callback_daemon:
             try:
                 return self._handle_tunnel(extras)
             finally:
                 log.debug('Running cleanup with cache invalidation')
                 if self.repo_name:
                     self._invalidate_cache(self.repo_name)

rhodecode/apps/ssh_support/tests/test_server_git.py

@@ -1,151 +1,151 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 
 import mock
 import pytest
 
 from rhodecode.apps.ssh_support.lib.backends.git import GitServer
 from rhodecode.apps.ssh_support.tests.conftest import plain_dummy_env, plain_dummy_user
 from rhodecode.lib.ext_json import json
 
 
 class GitServerCreator(object):
     root = '/tmp/repo/path/'
     git_path = '/usr/local/bin/git'
     config_data = {
         'app:main': {
             'ssh.executable.git': git_path,
-            'vcs.hooks.protocol': 'http',
+            'vcs.hooks.protocol.v2': 'celery',
         }
     }
     repo_name = 'test_git'
     repo_mode = 'receive-pack'
     user = plain_dummy_user()
 
     def __init__(self):
         pass
 
     def create(self, **kwargs):
         parameters = {
             'store': self.root,
             'ini_path': '',
             'user': self.user,
             'repo_name': self.repo_name,
             'repo_mode': self.repo_mode,
             'user_permissions': {
                 self.repo_name: 'repository.admin'
             },
             'settings': self.config_data['app:main'],
             'env': plain_dummy_env()
         }
         parameters.update(kwargs)
         server = GitServer(**parameters)
         return server
 
 
 @pytest.fixture()
 def git_server(app):
     return GitServerCreator()
 
 
 class TestGitServer(object):
 
     def test_command(self, git_server):
         server = git_server.create()
         expected_command = (
             'cd {root}; {git_path} {repo_mode} \'{root}{repo_name}\''.format(
                 root=git_server.root, git_path=git_server.git_path,
                 repo_mode=git_server.repo_mode, repo_name=git_server.repo_name)
         )
         assert expected_command == server.tunnel.command()
 
     @pytest.mark.parametrize('permissions, action, code', [
         ({}, 'pull', -2),
         ({'test_git': 'repository.read'}, 'pull', 0),
         ({'test_git': 'repository.read'}, 'push', -2),
         ({'test_git': 'repository.write'}, 'push', 0),
         ({'test_git': 'repository.admin'}, 'push', 0),
 
     ])
     def test_permission_checks(self, git_server, permissions, action, code):
         server = git_server.create(user_permissions=permissions)
         result = server._check_permissions(action)
         assert result is code
 
     @pytest.mark.parametrize('permissions, value', [
         ({}, False),
         ({'test_git': 'repository.read'}, False),
         ({'test_git': 'repository.write'}, True),
         ({'test_git': 'repository.admin'}, True),
 
     ])
     def test_has_write_permissions(self, git_server, permissions, value):
         server = git_server.create(user_permissions=permissions)
         result = server.has_write_perm()
         assert result is value
 
     def test_run_returns_executes_command(self, git_server):
         server = git_server.create()
         from rhodecode.apps.ssh_support.lib.backends.git import GitTunnelWrapper
 
         os.environ['SSH_CLIENT'] = '127.0.0.1'
         with mock.patch.object(GitTunnelWrapper, 'create_hooks_env') as _patch:
             _patch.return_value = 0
             with mock.patch.object(GitTunnelWrapper, 'command', return_value='date'):
                 exit_code = server.run()
 
         assert exit_code == (0, False)
 
     @pytest.mark.parametrize(
         'repo_mode, action', [
             ['receive-pack', 'push'],
             ['upload-pack', 'pull']
         ])
     def test_update_environment(self, git_server, repo_mode, action):
         server = git_server.create(repo_mode=repo_mode)
         store = server.store
 
         with mock.patch('os.environ', {'SSH_CLIENT': '10.10.10.10 b'}):
             with mock.patch('os.putenv') as putenv_mock:
                 server.update_environment(action)
 
         expected_data = {
             'username': git_server.user.username,
             'user_id': git_server.user.user_id,
             'scm': 'git',
             'repository': git_server.repo_name,
             'make_lock': None,
             'action': action,
             'ip': '10.10.10.10',
             'locked_by': [None, None],
             'config': '',
             'repo_store': store,
             'server_url': None,
             'hooks': ['push', 'pull'],
             'is_shadow_repo': False,
             'hooks_module': 'rhodecode.lib.hook_daemon.hook_module',
             'check_branch_perms': False,
             'detect_force_push': False,
             'user_agent': u'git/ssh-user-agent',
             'SSH': True,
             'SSH_PERMISSIONS': 'repository.admin',
         }
         args, kwargs = putenv_mock.call_args
         assert json.loads(args[1]) == expected_data

rhodecode/apps/ssh_support/tests/test_server_hg.py

@@ -1,115 +1,115 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import mock
 import pytest
 
 from rhodecode.apps.ssh_support.lib.backends.hg import MercurialServer
 from rhodecode.apps.ssh_support.tests.conftest import plain_dummy_env, plain_dummy_user
 
 
 class MercurialServerCreator(object):
     root = '/tmp/repo/path/'
     hg_path = '/usr/local/bin/hg'
 
     config_data = {
         'app:main': {
             'ssh.executable.hg': hg_path,
-            'vcs.hooks.protocol': 'http',
+            'vcs.hooks.protocol.v2': 'celery',
         }
     }
     repo_name = 'test_hg'
     user = plain_dummy_user()
 
     def __init__(self):
         pass
 
     def create(self, **kwargs):
         parameters = {
             'store': self.root,
             'ini_path': '',
             'user': self.user,
             'repo_name': self.repo_name,
             'user_permissions': {
                 'test_hg': 'repository.admin'
             },
             'settings': self.config_data['app:main'],
             'env': plain_dummy_env()
         }
         parameters.update(kwargs)
         server = MercurialServer(**parameters)
         return server
 
 
 @pytest.fixture()
 def hg_server(app):
     return MercurialServerCreator()
 
 
 class TestMercurialServer(object):
 
     def test_command(self, hg_server, tmpdir):
         server = hg_server.create()
         custom_hgrc = os.path.join(str(tmpdir), 'hgrc')
         expected_command = (
             'cd {root}; HGRCPATH={custom_hgrc} {hg_path} -R {root}{repo_name} serve --stdio'.format(
                 root=hg_server.root, custom_hgrc=custom_hgrc, hg_path=hg_server.hg_path,
                 repo_name=hg_server.repo_name)
         )
         server_command = server.tunnel.command(custom_hgrc)
         assert expected_command == server_command
 
     @pytest.mark.parametrize('permissions, action, code', [
         ({}, 'pull', -2),
         ({'test_hg': 'repository.read'}, 'pull', 0),
         ({'test_hg': 'repository.read'}, 'push', -2),
         ({'test_hg': 'repository.write'}, 'push', 0),
         ({'test_hg': 'repository.admin'}, 'push', 0),
 
     ])
     def test_permission_checks(self, hg_server, permissions, action, code):
         server = hg_server.create(user_permissions=permissions)
         result = server._check_permissions(action)
         assert result is code
 
     @pytest.mark.parametrize('permissions, value', [
         ({}, False),
         ({'test_hg': 'repository.read'}, False),
         ({'test_hg': 'repository.write'}, True),
         ({'test_hg': 'repository.admin'}, True),
 
     ])
     def test_has_write_permissions(self, hg_server, permissions, value):
         server = hg_server.create(user_permissions=permissions)
         result = server.has_write_perm()
         assert result is value
 
     def test_run_returns_executes_command(self, hg_server):
         server = hg_server.create()
         from rhodecode.apps.ssh_support.lib.backends.hg import MercurialTunnelWrapper
         os.environ['SSH_CLIENT'] = '127.0.0.1'
         with mock.patch.object(MercurialTunnelWrapper, 'create_hooks_env') as _patch:
             _patch.return_value = 0
             with mock.patch.object(MercurialTunnelWrapper, 'command', return_value='date'):
                 exit_code = server.run()
 
         assert exit_code == (0, False)
 
 
 

rhodecode/apps/ssh_support/tests/test_server_svn.py

@@ -1,203 +1,203 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import os
 import mock
 import pytest
 
 from rhodecode.apps.ssh_support.lib.backends.svn import SubversionServer
 from rhodecode.apps.ssh_support.tests.conftest import plain_dummy_env, plain_dummy_user
 
 
 class SubversionServerCreator(object):
     root = '/tmp/repo/path/'
     svn_path = '/usr/local/bin/svnserve'
     config_data = {
         'app:main': {
             'ssh.executable.svn': svn_path,
-            'vcs.hooks.protocol': 'http',
+            'vcs.hooks.protocol.v2': 'celery',
         }
     }
     repo_name = 'test-svn'
     user = plain_dummy_user()
 
     def __init__(self):
         pass
 
     def create(self, **kwargs):
         parameters = {
             'store': self.root,
             'repo_name': self.repo_name,
             'ini_path': '',
             'user': self.user,
             'user_permissions': {
                 self.repo_name: 'repository.admin'
             },
             'settings': self.config_data['app:main'],
             'env': plain_dummy_env()
         }
 
         parameters.update(kwargs)
         server = SubversionServer(**parameters)
         return server
 
 
 @pytest.fixture()
 def svn_server(app):
     return SubversionServerCreator()
 
 
 class TestSubversionServer(object):
 
     def test_command(self, svn_server):
         server = svn_server.create()
         expected_command = [
             svn_server.svn_path, '-t',
             '--config-file', server.tunnel.svn_conf_path,
             '--tunnel-user', svn_server.user.username,
             '-r', svn_server.root
         ]
 
         assert expected_command == server.tunnel.command()
 
     @pytest.mark.parametrize('permissions, action, code', [
         ({}, 'pull', -2),
         ({'test-svn': 'repository.read'}, 'pull', 0),
         ({'test-svn': 'repository.read'}, 'push', -2),
         ({'test-svn': 'repository.write'}, 'push', 0),
         ({'test-svn': 'repository.admin'}, 'push', 0),
 
     ])
     def test_permission_checks(self, svn_server, permissions, action, code):
         server = svn_server.create(user_permissions=permissions)
         result = server._check_permissions(action)
         assert result is code
 
     @pytest.mark.parametrize('permissions, access_paths, expected_match', [
         # not matched repository name
         ({
              'test-svn': ''
          }, ['test-svn-1', 'test-svn-1/subpath'],
          None),
 
         # exact match
         ({
              'test-svn': ''
          },
          ['test-svn'],
          'test-svn'),
 
         # subdir commits
         ({
              'test-svn': ''
          },
          ['test-svn/foo',
           'test-svn/foo/test-svn',
           'test-svn/trunk/development.txt',
           ],
          'test-svn'),
 
         # subgroups + similar patterns
         ({
              'test-svn': '',
              'test-svn-1': '',
              'test-svn-subgroup/test-svn': '',
 
          },
          ['test-svn-1',
           'test-svn-1/foo/test-svn',
           'test-svn-1/test-svn',
           ],
          'test-svn-1'),
 
         # subgroups + similar patterns
         ({
              'test-svn-1': '',
              'test-svn-10': '',
              'test-svn-100': '',
          },
          ['test-svn-10',
           'test-svn-10/foo/test-svn',
           'test-svn-10/test-svn',
           ],
          'test-svn-10'),
 
         # subgroups + similar patterns
         ({
              'name': '',
              'nameContains': '',
              'nameContainsThis': '',
          },
          ['nameContains',
           'nameContains/This',
           'nameContains/This/test-svn',
           ],
          'nameContains'),
 
         # subgroups + similar patterns
         ({
              'test-svn': '',
              'test-svn-1': '',
              'test-svn-subgroup/test-svn': '',
 
          },
          ['test-svn-subgroup/test-svn',
           'test-svn-subgroup/test-svn/foo/test-svn',
           'test-svn-subgroup/test-svn/trunk/example.txt',
           ],
          'test-svn-subgroup/test-svn'),
     ])
     def test_repo_extraction_on_subdir(self, svn_server, permissions, access_paths, expected_match):
         server = svn_server.create(user_permissions=permissions)
         for path in access_paths:
             repo_name = server.tunnel._match_repo_name(path)
             assert repo_name == expected_match
 
     def test_run_returns_executes_command(self, svn_server):
         server = svn_server.create()
         from rhodecode.apps.ssh_support.lib.backends.svn import SubversionTunnelWrapper
         os.environ['SSH_CLIENT'] = '127.0.0.1'
         with mock.patch.object(
                 SubversionTunnelWrapper, 'get_first_client_response',
                 return_value={'url': 'http://server/test-svn'}):
             with mock.patch.object(
                     SubversionTunnelWrapper, 'patch_first_client_response',
                     return_value=0):
                 with mock.patch.object(
                         SubversionTunnelWrapper, 'sync',
                         return_value=0):
                     with mock.patch.object(
                             SubversionTunnelWrapper, 'command',
                             return_value=['date']):
 
                         exit_code = server.run()
         # SVN has this differently configured, and we get in our mock env
         # None as return code
         assert exit_code == (None, False)
 
     def test_run_returns_executes_command_that_cannot_extract_repo_name(self, svn_server):
         server = svn_server.create()
         from rhodecode.apps.ssh_support.lib.backends.svn import SubversionTunnelWrapper
         with mock.patch.object(
                 SubversionTunnelWrapper, 'command',
                 return_value=['date']):
             with mock.patch.object(
                     SubversionTunnelWrapper, 'get_first_client_response',
                     return_value=None):
                     exit_code = server.run()
 
         assert exit_code == (1, False)

rhodecode/authentication/routes.py

@@ -1,153 +1,154 b''
 # Copyright (C) 2012-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import collections
 
 from pyramid.exceptions import ConfigurationError
 
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.model.settings import SettingsModel
 from rhodecode.translation import _
 
 
 log = logging.getLogger(__name__)
 
 
 class AuthnResourceBase(object):
     __name__ = None
     __parent__ = None
 
     def get_root(self):
         current = self
         while current.__parent__ is not None:
             current = current.__parent__
         return current
 
 
 class AuthnPluginResourceBase(AuthnResourceBase):
 
     def __init__(self, plugin):
         self.plugin = plugin
         self.__name__ = plugin.get_url_slug()
         self.display_name = plugin.get_display_name()
 
 
 class AuthnRootResource(AuthnResourceBase):
     """
     This is the root traversal resource object for the authentication settings.
     """
+    is_root = True
 
     def __init__(self):
         self._store = collections.OrderedDict()
         self._resource_name_map = {}
         self.display_name = _('Authentication Plugins')
 
     def __getitem__(self, key):
         """
         Customized get item function to return only items (plugins) that are
         activated.
         """
         if self._is_item_active(key):
             return self._store[key]
         else:
             raise KeyError('Authentication plugin "{}" is not active.'.format(
                 key))
 
     def __iter__(self):
         for key in self._store.keys():
             if self._is_item_active(key):
                 yield self._store[key]
 
     def _is_item_active(self, key):
         activated_plugins = SettingsModel().get_auth_plugins()
         plugin_id = self.get_plugin_id(key)
         return plugin_id in activated_plugins
 
     def get_plugin_id(self, resource_name):
         """
         Return the plugin id for the given traversal resource name.
         """
         # TODO: Store this info in the resource element.
         return self._resource_name_map[resource_name]
 
     def get_sorted_list(self, sort_key=None):
         """
         Returns a sorted list of sub resources for displaying purposes.
         """
         def default_sort_key(resource):
             return str.lower(safe_str(resource.display_name))
 
         active = [item for item in self]
         return sorted(active, key=sort_key or default_sort_key)
 
     def get_nav_list(self, sort=True):
         """
         Returns a sorted list of resources for displaying the navigation.
         """
         if sort:
             nav_list = self.get_sorted_list()
         else:
             nav_list = [item for item in self]
 
         nav_list.insert(0, self)
         return nav_list
 
     def add_authn_resource(self, config, plugin_id, resource):
         """
         Register a traversal resource as a sub element to the authentication
         settings. This method is registered as a directive on the pyramid
         configurator object and called by plugins.
         """
 
         def _ensure_unique_name(name, limit=100):
             counter = 1
             current = name
             while current in self._store.keys():
                 current = f'{name}{counter}'
                 counter += 1
                 if counter > limit:
                     raise ConfigurationError(
                         'Cannot build unique name for traversal resource "%s" '
                         'registered by plugin "%s"', name, plugin_id)
             return current
 
         # Allow plugin resources with identical names by rename duplicates.
         unique_name = _ensure_unique_name(resource.__name__)
         if unique_name != resource.__name__:
             log.warning('Name collision for traversal resource "%s" registered '
                         'by authentication plugin "%s"', resource.__name__,
                         plugin_id)
             resource.__name__ = unique_name
 
         log.debug('Register traversal resource "%s" for plugin "%s"',
                   unique_name, plugin_id)
         self._resource_name_map[unique_name] = plugin_id
         resource.__parent__ = self
         self._store[unique_name] = resource
 
 
 root = AuthnRootResource()
 
 
 def root_factory(request=None):
     """
     Returns the root traversal resource instance used for the authentication
     settings route.
     """
     return root

rhodecode/config/config_maker.py

@@ -1,224 +1,237 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import tempfile
 import logging
 
 from pyramid.settings import asbool
 
 from rhodecode.config.settings_maker import SettingsMaker
 from rhodecode.config import utils as config_utils
 
 log = logging.getLogger(__name__)
 
 
 def sanitize_settings_and_apply_defaults(global_config, settings):
     """
     Applies settings defaults and does all type conversion.
 
     We would move all settings parsing and preparation into this place, so that
     we have only one place left which deals with this part. The remaining parts
     of the application would start to rely fully on well-prepared settings.
 
     This piece would later be split up per topic to avoid a big fat monster
     function.
     """
     jn = os.path.join
 
     global_settings_maker = SettingsMaker(global_config)
     global_settings_maker.make_setting('debug', default=False, parser='bool')
     debug_enabled = asbool(global_config.get('debug'))
 
     settings_maker = SettingsMaker(settings)
 
     settings_maker.make_setting(
         'logging.autoconfigure',
         default=False,
         parser='bool')
 
-    logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini')
+    ini_loc = os.path.dirname(global_config.get('__file__'))
+    logging_conf = jn(ini_loc, 'logging.ini')
     settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
 
     # Default includes, possible to change as a user
     pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
     log.debug(
         "Using the following pyramid.includes: %s",
         pyramid_includes)
 
     settings_maker.make_setting('rhodecode.edition', 'Community Edition')
     settings_maker.make_setting('rhodecode.edition_id', 'CE')
 
     if 'mako.default_filters' not in settings:
         # set custom default filters if we don't have it defined
         settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
         settings['mako.default_filters'] = 'h_filter'
 
     if 'mako.directories' not in settings:
         mako_directories = settings.setdefault('mako.directories', [
             # Base templates of the original application
             'rhodecode:templates',
         ])
         log.debug(
             "Using the following Mako template directories: %s",
             mako_directories)
 
     # NOTE(marcink): fix redis requirement for schema of connection since 3.X
     if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
         raw_url = settings['beaker.session.url']
         if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
             settings['beaker.session.url'] = 'redis://' + raw_url
 
     settings_maker.make_setting('__file__', global_config.get('__file__'))
 
     # TODO: johbo: Re-think this, usually the call to config.include
     # should allow to pass in a prefix.
     settings_maker.make_setting('rhodecode.api.url', '/_admin/api')
 
     # Sanitize generic settings.
     settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
     settings_maker.make_setting('gzip_responses', False, parser='bool')
     settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
 
+    # License settings.
+    settings_maker.make_setting('license.hide_license_info', False, parser='bool')
+    settings_maker.make_setting('license.import_path', '')
+    settings_maker.make_setting('license.import_path_mode', 'if-missing')
+
     # statsd
     settings_maker.make_setting('statsd.enabled', False, parser='bool')
     settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
     settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
     settings_maker.make_setting('statsd.statsd_prefix', '')
     settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
 
     settings_maker.make_setting('vcs.svn.compatible_version', '')
     settings_maker.make_setting('vcs.svn.redis_conn', 'redis://redis:6379/0')
     settings_maker.make_setting('vcs.svn.proxy.enabled', True, parser='bool')
     settings_maker.make_setting('vcs.svn.proxy.host', 'http://svn:8090', parser='string')
-    settings_maker.make_setting('vcs.hooks.protocol', 'http')
+    settings_maker.make_setting('vcs.hooks.protocol.v2', 'celery')
     settings_maker.make_setting('vcs.hooks.host', '*')
     settings_maker.make_setting('vcs.scm_app_implementation', 'http')
     settings_maker.make_setting('vcs.server', '')
     settings_maker.make_setting('vcs.server.protocol', 'http')
     settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
     settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
     settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
     settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
     settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
+    settings_maker.make_setting('vcs.git.lfs.storage_location', '/var/opt/rhodecode_repo_store/.cache/git_lfs_store')
+    settings_maker.make_setting('vcs.hg.largefiles.storage_location',
+                                '/var/opt/rhodecode_repo_store/.cache/hg_largefiles_store')
 
     settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
 
     # repo_store path
     settings_maker.make_setting('repo_store.path', '/var/opt/rhodecode_repo_store')
     # Support legacy values of vcs.scm_app_implementation. Legacy
     # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
     # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
     scm_app_impl = settings['vcs.scm_app_implementation']
     if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
         settings['vcs.scm_app_implementation'] = 'http'
 
     settings_maker.make_setting('appenlight', False, parser='bool')
 
     temp_store = tempfile.gettempdir()
     tmp_cache_dir = jn(temp_store, 'rc_cache')
 
     # save default, cache dir, and use it for all backends later.
     default_cache_dir = settings_maker.make_setting(
         'cache_dir',
         default=tmp_cache_dir, default_when_empty=True,
         parser='dir:ensured')
 
     # exception store cache
     settings_maker.make_setting(
         'exception_tracker.store_path',
         default=jn(default_cache_dir, 'exc_store'), default_when_empty=True,
         parser='dir:ensured'
     )
 
     settings_maker.make_setting(
         'celerybeat-schedule.path',
         default=jn(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
         parser='file:ensured'
     )
 
+    # celery
+    broker_url = settings_maker.make_setting('celery.broker_url', 'redis://redis:6379/8')
+    settings_maker.make_setting('celery.result_backend', broker_url)
+
     settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
     settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
 
     # sessions, ensure file since no-value is memory
     settings_maker.make_setting('beaker.session.type', 'file')
     settings_maker.make_setting('beaker.session.data_dir',  jn(default_cache_dir, 'session_data'))
 
     # cache_general
     settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
     settings_maker.make_setting('rc_cache.cache_general.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_general.db'))
 
     # cache_perms
     settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
     settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_perms_db'))
 
     # cache_repo
     settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
     settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_repo_db'))
 
     # cache_license
     settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
     settings_maker.make_setting('rc_cache.cache_license.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_license_db'))
 
     # cache_repo_longterm memory, 96H
     settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
     settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
     settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
 
     # sql_cache_short
     settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
     settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
     settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
 
     # archive_cache
     settings_maker.make_setting('archive_cache.locking.url', 'redis://redis:6379/1')
     settings_maker.make_setting('archive_cache.backend.type', 'filesystem')
 
     settings_maker.make_setting('archive_cache.filesystem.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
     settings_maker.make_setting('archive_cache.filesystem.cache_shards', 8, parser='int')
     settings_maker.make_setting('archive_cache.filesystem.cache_size_gb', 10, parser='float')
     settings_maker.make_setting('archive_cache.filesystem.eviction_policy', 'least-recently-stored')
 
     settings_maker.make_setting('archive_cache.filesystem.retry', False, parser='bool')
     settings_maker.make_setting('archive_cache.filesystem.retry_backoff', 1, parser='int')
     settings_maker.make_setting('archive_cache.filesystem.retry_attempts', 10, parser='int')
 
-    settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
+    settings_maker.make_setting('archive_cache.objectstore.url', 'http://s3-minio:9000', default_when_empty=True,)
     settings_maker.make_setting('archive_cache.objectstore.key', '')
     settings_maker.make_setting('archive_cache.objectstore.secret', '')
     settings_maker.make_setting('archive_cache.objectstore.region', 'eu-central-1')
     settings_maker.make_setting('archive_cache.objectstore.bucket', 'rhodecode-archive-cache', default_when_empty=True,)
     settings_maker.make_setting('archive_cache.objectstore.bucket_shards', 8, parser='int')
 
     settings_maker.make_setting('archive_cache.objectstore.cache_size_gb', 10, parser='float')
     settings_maker.make_setting('archive_cache.objectstore.eviction_policy', 'least-recently-stored')
 
     settings_maker.make_setting('archive_cache.objectstore.retry', False, parser='bool')
     settings_maker.make_setting('archive_cache.objectstore.retry_backoff', 1, parser='int')
     settings_maker.make_setting('archive_cache.objectstore.retry_attempts', 10, parser='int')
 
     settings_maker.env_expand()
 
     # configure instance id
     config_utils.set_instance_id(settings)
 
     return settings

rhodecode/config/environment.py

@@ -1,87 +1,99 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
-import os
 import logging
+
 import rhodecode
 import collections
 
 from rhodecode.config import utils
 
 from rhodecode.lib.utils import load_rcextensions
 from rhodecode.lib.utils2 import str2bool
 from rhodecode.lib.vcs import connect_vcs
 
 log = logging.getLogger(__name__)
 
 
+def propagate_rhodecode_config(global_config, settings, config):
+    # Store the settings to make them available to other modules.
+    settings_merged = global_config.copy()
+    settings_merged.update(settings)
+    if config:
+        settings_merged.update(config)
+
+    rhodecode.PYRAMID_SETTINGS = settings_merged
+    rhodecode.CONFIG = settings_merged
+
+    if 'default_user_id' not in rhodecode.CONFIG:
+        rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
+    log.debug('set rhodecode.CONFIG data')
+
+
 def load_pyramid_environment(global_config, settings):
     # Some parts of the code expect a merge of global and app settings.
     settings_merged = global_config.copy()
     settings_merged.update(settings)
 
     # TODO(marcink): probably not required anymore
     # configure channelstream,
     settings_merged['channelstream_config'] = {
         'enabled': str2bool(settings_merged.get('channelstream.enabled', False)),
         'server': settings_merged.get('channelstream.server'),
         'secret': settings_merged.get('channelstream.secret')
     }
 
     # If this is a test run we prepare the test environment like
     # creating a test database, test search index and test repositories.
     # This has to be done before the database connection is initialized.
     if rhodecode.is_test:
         rhodecode.disable_error_handler = True
         from rhodecode import authentication
         authentication.plugin_default_auth_ttl = 0
 
         utils.initialize_test_environment(settings_merged)
 
     # Initialize the database connection.
     utils.initialize_database(settings_merged)
 
     load_rcextensions(root_path=settings_merged['here'])
 
     # Limit backends to `vcs.backends` from configuration, and preserve the order
     for alias in list(rhodecode.BACKENDS.keys()):
         if alias not in settings['vcs.backends']:
             del rhodecode.BACKENDS[alias]
 
     _sorted_backend = sorted(rhodecode.BACKENDS.items(),
                              key=lambda item: settings['vcs.backends'].index(item[0]))
     rhodecode.BACKENDS = collections.OrderedDict(_sorted_backend)
 
     log.info('Enabled VCS backends: %s', list(rhodecode.BACKENDS.keys()))
 
     # initialize vcs client and optionally run the server if enabled
     vcs_server_uri = settings['vcs.server']
     vcs_server_enabled = settings['vcs.server.enable']
 
     utils.configure_vcs(settings)
 
-    # Store the settings to make them available to other modules.
-
-    rhodecode.PYRAMID_SETTINGS = settings_merged
-    rhodecode.CONFIG = settings_merged
-    rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
+    # first run, to store data...
+    propagate_rhodecode_config(global_config, settings, {})
 
     if vcs_server_enabled:
         connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))
     else:
         log.warning('vcs-server not enabled, vcs connection unavailable')

rhodecode/config/middleware.py

@@ -1,466 +1,471 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import sys
 import collections
 
 import time
 import logging.config
 
 from paste.gzipper import make_gzip_middleware
 import pyramid.events
 from pyramid.wsgi import wsgiapp
 from pyramid.config import Configurator
 from pyramid.settings import asbool, aslist
 from pyramid.httpexceptions import (
     HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
 from pyramid.renderers import render_to_response
 
 from rhodecode.model import meta
 from rhodecode.config import patches
 
-from rhodecode.config.environment import load_pyramid_environment
+from rhodecode.config.environment import load_pyramid_environment, propagate_rhodecode_config
 
 import rhodecode.events
 from rhodecode.config.config_maker import sanitize_settings_and_apply_defaults
 from rhodecode.lib.middleware.vcs import VCSMiddleware
 from rhodecode.lib.request import Request
 from rhodecode.lib.vcs import VCSCommunicationError
 from rhodecode.lib.exceptions import VCSServerUnavailable
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.lib.exc_tracking import store_exception, format_exc
 from rhodecode.subscribers import (
     scan_repositories_if_enabled, write_js_routes_if_enabled,
-    write_metadata_if_needed, write_usage_data)
+    write_metadata_if_needed, write_usage_data, import_license_if_present)
 from rhodecode.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
 
 
 def is_http_error(response):
     # error which should have traceback
     return response.status_code > 499
 
 
 def should_load_all():
     """
     Returns if all application components should be loaded. In some cases it's
     desired to skip apps loading for faster shell script execution
     """
     ssh_cmd = os.environ.get('RC_CMD_SSH_WRAPPER')
     if ssh_cmd:
         return False
 
     return True
 
 
 def make_pyramid_app(global_config, **settings):
     """
     Constructs the WSGI application based on Pyramid.
 
     Specials:
 
     * The application can also be integrated like a plugin via the call to
       `includeme`. This is accompanied with the other utility functions which
       are called. Changing this should be done with great care to not break
       cases when these fragments are assembled from another place.
 
     """
     start_time = time.time()
     log.info('Pyramid app config starting')
 
     sanitize_settings_and_apply_defaults(global_config, settings)
 
     # init and bootstrap StatsdClient
     StatsdClient.setup(settings)
 
     config = Configurator(settings=settings)
     # Init our statsd at very start
     config.registry.statsd = StatsdClient.statsd
 
     # Apply compatibility patches
     patches.inspect_getargspec()
+    patches.repoze_sendmail_lf_fix()
 
     load_pyramid_environment(global_config, settings)
 
     # Static file view comes first
     includeme_first(config)
 
     includeme(config)
 
     pyramid_app = config.make_wsgi_app()
     pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
     pyramid_app.config = config
 
     celery_settings = get_celery_config(settings)
     config.configure_celery(celery_settings)
 
+    # final config set...
+    propagate_rhodecode_config(global_config, settings, config.registry.settings)
+
     # creating the app uses a connection - return it after we are done
     meta.Session.remove()
 
     total_time = time.time() - start_time
     log.info('Pyramid app created and configured in %.2fs', total_time)
     return pyramid_app
 
 
 def get_celery_config(settings):
     """
     Converts basic ini configuration into celery 4.X options
     """
 
     def key_converter(key_name):
         pref = 'celery.'
         if key_name.startswith(pref):
             return key_name[len(pref):].replace('.', '_').lower()
 
     def type_converter(parsed_key, value):
         # cast to int
         if value.isdigit():
             return int(value)
 
         # cast to bool
         if value.lower() in ['true', 'false', 'True', 'False']:
             return value.lower() == 'true'
         return value
 
     celery_config = {}
     for k, v in settings.items():
         pref = 'celery.'
         if k.startswith(pref):
             celery_config[key_converter(k)] = type_converter(key_converter(k), v)
 
     # TODO:rethink if we want to support celerybeat based file config, probably NOT
     # beat_config = {}
     # for section in parser.sections():
     #     if section.startswith('celerybeat:'):
     #         name = section.split(':', 1)[1]
     #         beat_config[name] = get_beat_config(parser, section)
 
     # final compose of settings
     celery_settings = {}
 
     if celery_config:
         celery_settings.update(celery_config)
     # if beat_config:
     #     celery_settings.update({'beat_schedule': beat_config})
 
     return celery_settings
 
 
 def not_found_view(request):
     """
     This creates the view which should be registered as not-found-view to
     pyramid.
     """
 
     if not getattr(request, 'vcs_call', None):
         # handle like regular case with our error_handler
         return error_handler(HTTPNotFound(), request)
 
     # handle not found view as a vcs call
     settings = request.registry.settings
     ae_client = getattr(request, 'ae_client', None)
     vcs_app = VCSMiddleware(
         HTTPNotFound(), request.registry, settings,
         appenlight_client=ae_client)
 
     return wsgiapp(vcs_app)(None, request)
 
 
 def error_handler(exception, request):
     import rhodecode
     from rhodecode.lib import helpers
 
     rhodecode_title = rhodecode.CONFIG.get('rhodecode_title') or 'RhodeCode'
 
     base_response = HTTPInternalServerError()
     # prefer original exception for the response since it may have headers set
     if isinstance(exception, HTTPException):
         base_response = exception
     elif isinstance(exception, VCSCommunicationError):
         base_response = VCSServerUnavailable()
 
     if is_http_error(base_response):
         traceback_info = format_exc(request.exc_info)
         log.error(
             'error occurred handling this request for path: %s, \n%s',
             request.path, traceback_info)
 
     error_explanation = base_response.explanation or str(base_response)
     if base_response.status_code == 404:
         error_explanation += " Optionally you don't have permission to access this page."
     c = AttributeDict()
     c.error_message = base_response.status
     c.error_explanation = error_explanation
     c.visual = AttributeDict()
 
     c.visual.rhodecode_support_url = (
         request.registry.settings.get('rhodecode_support_url') or
         request.route_url('rhodecode_support')
     )
     c.redirect_time = 0
     c.rhodecode_name = rhodecode_title
     if not c.rhodecode_name:
         c.rhodecode_name = 'Rhodecode'
 
     c.causes = []
     if is_http_error(base_response):
         c.causes.append('Server is overloaded.')
         c.causes.append('Server database connection is lost.')
         c.causes.append('Server expected unhandled error.')
 
     if hasattr(base_response, 'causes'):
         c.causes = base_response.causes
 
     c.messages = helpers.flash.pop_messages(request=request)
     exc_info = sys.exc_info()
     c.exception_id = id(exc_info)
     c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \
                           or base_response.status_code > 499
     c.exception_id_url = request.route_url(
         'admin_settings_exception_tracker_show', exception_id=c.exception_id)
 
     debug_mode = rhodecode.ConfigGet().get_bool('debug')
     if c.show_exception_id:
         store_exception(c.exception_id, exc_info)
     c.exception_debug = debug_mode
     c.exception_config_ini = rhodecode.CONFIG.get('__file__')
 
     if debug_mode:
         try:
             from rich.traceback import install
             install(show_locals=True)
             log.debug('Installing rich tracebacks...')
         except ImportError:
             pass
 
     response = render_to_response(
         '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
         response=base_response)
 
     response.headers["X-RC-Exception-Id"] = str(c.exception_id)
 
     statsd = request.registry.statsd
     if statsd and base_response.status_code > 499:
         exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}"
         statsd.incr('rhodecode_exception_total',
                     tags=["exc_source:web",
                           f"http_code:{base_response.status_code}",
                           f"type:{exc_type}"])
 
     return response
 
 
 def includeme_first(config):
     # redirect automatic browser favicon.ico requests to correct place
     def favicon_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/images/favicon.ico'))
 
     config.add_view(favicon_redirect, route_name='favicon')
     config.add_route('favicon', '/favicon.ico')
 
     def robots_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/robots.txt'))
 
     config.add_view(robots_redirect, route_name='robots')
     config.add_route('robots', '/robots.txt')
 
     config.add_static_view(
         '_static/deform', 'deform:static')
     config.add_static_view(
         '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
 
 
 ce_auth_resources = [
     'rhodecode.authentication.plugins.auth_crowd',
     'rhodecode.authentication.plugins.auth_headers',
     'rhodecode.authentication.plugins.auth_jasig_cas',
     'rhodecode.authentication.plugins.auth_ldap',
     'rhodecode.authentication.plugins.auth_pam',
     'rhodecode.authentication.plugins.auth_rhodecode',
     'rhodecode.authentication.plugins.auth_token',
 ]
 
 
 def includeme(config, auth_resources=None):
     from rhodecode.lib.celerylib.loader import configure_celery
     log.debug('Initializing main includeme from %s', os.path.basename(__file__))
     settings = config.registry.settings
     config.set_request_factory(Request)
 
     # plugin information
     config.registry.rhodecode_plugins = collections.OrderedDict()
 
     config.add_directive(
         'register_rhodecode_plugin', register_rhodecode_plugin)
 
     config.add_directive('configure_celery', configure_celery)
 
     if settings.get('appenlight', False):
         config.include('appenlight_client.ext.pyramid_tween')
 
     load_all = should_load_all()
 
     # Includes which are required. The application would fail without them.
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
     config.include('rhodecode.lib.archive_cache')
 
     config.include('rhodecode.apps._base.navigation')
     config.include('rhodecode.apps._base.subscribers')
     config.include('rhodecode.tweens')
     config.include('rhodecode.authentication')
 
     if load_all:
 
         # load CE authentication plugins
 
         if auth_resources:
             ce_auth_resources.extend(auth_resources)
 
         for resource in ce_auth_resources:
             config.include(resource)
 
         # Auto discover authentication plugins and include their configuration.
         if asbool(settings.get('auth_plugin.import_legacy_plugins', 'true')):
             from rhodecode.authentication import discover_legacy_plugins
             discover_legacy_plugins(config)
 
     # apps
     if load_all:
         log.debug('Starting config.include() calls')
         config.include('rhodecode.api.includeme')
         config.include('rhodecode.apps._base.includeme')
         config.include('rhodecode.apps._base.navigation.includeme')
         config.include('rhodecode.apps._base.subscribers.includeme')
         config.include('rhodecode.apps.hovercards.includeme')
         config.include('rhodecode.apps.ops.includeme')
         config.include('rhodecode.apps.channelstream.includeme')
         config.include('rhodecode.apps.file_store.includeme')
         config.include('rhodecode.apps.admin.includeme')
         config.include('rhodecode.apps.login.includeme')
         config.include('rhodecode.apps.home.includeme')
         config.include('rhodecode.apps.journal.includeme')
 
         config.include('rhodecode.apps.repository.includeme')
         config.include('rhodecode.apps.repo_group.includeme')
         config.include('rhodecode.apps.user_group.includeme')
         config.include('rhodecode.apps.search.includeme')
         config.include('rhodecode.apps.user_profile.includeme')
         config.include('rhodecode.apps.user_group_profile.includeme')
         config.include('rhodecode.apps.my_account.includeme')
         config.include('rhodecode.apps.gist.includeme')
 
         config.include('rhodecode.apps.svn_support.includeme')
         config.include('rhodecode.apps.ssh_support.includeme')
         config.include('rhodecode.apps.debug_style')
 
     if load_all:
         config.include('rhodecode.integrations.includeme')
         config.include('rhodecode.integrations.routes.includeme')
 
     config.add_route('rhodecode_support', 'https://rhodecode.com/help/', static=True)
     settings['default_locale_name'] = settings.get('lang', 'en')
     config.add_translation_dirs('rhodecode:i18n/')
 
     # Add subscribers.
     if load_all:
         log.debug('Adding subscribers...')
         config.add_subscriber(scan_repositories_if_enabled,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_metadata_if_needed,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_usage_data,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_js_routes_if_enabled,
                               pyramid.events.ApplicationCreated)
-
+        config.add_subscriber(import_license_if_present,
+                              pyramid.events.ApplicationCreated)
 
     # Set the default renderer for HTML templates to mako.
     config.add_mako_renderer('.html')
 
     config.add_renderer(
         name='json_ext',
         factory='rhodecode.lib.ext_json_renderer.pyramid_ext_json')
 
     config.add_renderer(
         name='string_html',
         factory='rhodecode.lib.string_renderer.html')
 
     # include RhodeCode plugins
     includes = aslist(settings.get('rhodecode.includes', []))
     log.debug('processing rhodecode.includes data...')
     for inc in includes:
         config.include(inc)
 
     # custom not found view, if our pyramid app doesn't know how to handle
     # the request pass it to potential VCS handling ap
     config.add_notfound_view(not_found_view)
     if not settings.get('debugtoolbar.enabled', False):
         # disabled debugtoolbar handle all exceptions via the error_handlers
         config.add_view(error_handler, context=Exception)
 
     # all errors including 403/404/50X
     config.add_view(error_handler, context=HTTPError)
 
 
 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
     """
     Apply outer WSGI middlewares around the application.
     """
     registry = config.registry
     settings = registry.settings
 
     # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     pyramid_app = HttpsFixup(pyramid_app, settings)
 
     pyramid_app, _ae_client = wrap_in_appenlight_if_enabled(
         pyramid_app, settings)
     registry.ae_client = _ae_client
 
     if settings['gzip_responses']:
         pyramid_app = make_gzip_middleware(
             pyramid_app, settings, compress_level=1)
 
     # this should be the outer most middleware in the wsgi stack since
     # middleware like Routes make database calls
     def pyramid_app_with_cleanup(environ, start_response):
         start = time.time()
         try:
             return pyramid_app(environ, start_response)
         finally:
             # Dispose current database session and rollback uncommitted
             # transactions.
             meta.Session.remove()
 
             # In a single threaded mode server, on non sqlite db we should have
             # '0 Current Checked out connections' at the end of a request,
             # if not, then something, somewhere is leaving a connection open
             pool = meta.get_engine().pool
             log.debug('sa pool status: %s', pool.status())
             total = time.time() - start
             log.debug('Request processing finalized: %.4fs', total)
 
     return pyramid_app_with_cleanup

rhodecode/config/patches.py

@@ -1,160 +1,167 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Compatibility patches.
 
 Please keep the following principles in mind:
 
 * Keep imports local, so that importing this module does not cause too many
   side effects by itself.
 
 * Try to make patches idempotent, calling them multiple times should not do
   harm. If that is not possible, ensure that the second call explodes.
 
 """
 
 
 def inspect_formatargspec():
 
     import inspect
     from inspect import formatannotation
 
     def backport_inspect_formatargspec(
             args, varargs=None, varkw=None, defaults=None,
             kwonlyargs=(), kwonlydefaults={}, annotations={},
             formatarg=str,
             formatvarargs=lambda name: '*' + name,
             formatvarkw=lambda name: '**' + name,
             formatvalue=lambda value: '=' + repr(value),
             formatreturns=lambda text: ' -> ' + text,
             formatannotation=formatannotation):
         """Copy formatargspec from python 3.7 standard library.
         Python 3 has deprecated formatargspec and requested that Signature
         be used instead, however this requires a full reimplementation
         of formatargspec() in terms of creating Parameter objects and such.
         Instead of introducing all the object-creation overhead and having
         to reinvent from scratch, just copy their compatibility routine.
         Utimately we would need to rewrite our "decorator" routine completely
         which is not really worth it right now, until all Python 2.x support
         is dropped.
         """
 
         def formatargandannotation(arg):
             result = formatarg(arg)
             if arg in annotations:
                 result += ': ' + formatannotation(annotations[arg])
             return result
 
         specs = []
         if defaults:
             firstdefault = len(args) - len(defaults)
         for i, arg in enumerate(args):
             spec = formatargandannotation(arg)
             if defaults and i >= firstdefault:
                 spec = spec + formatvalue(defaults[i - firstdefault])
             specs.append(spec)
         if varargs is not None:
             specs.append(formatvarargs(formatargandannotation(varargs)))
         else:
             if kwonlyargs:
                 specs.append('*')
         if kwonlyargs:
             for kwonlyarg in kwonlyargs:
                 spec = formatargandannotation(kwonlyarg)
                 if kwonlydefaults and kwonlyarg in kwonlydefaults:
                     spec += formatvalue(kwonlydefaults[kwonlyarg])
                 specs.append(spec)
         if varkw is not None:
             specs.append(formatvarkw(formatargandannotation(varkw)))
         result = '(' + ', '.join(specs) + ')'
         if 'return' in annotations:
             result += formatreturns(formatannotation(annotations['return']))
         return result
 
     # NOTE: inject for python3.11
     inspect.formatargspec = backport_inspect_formatargspec
     return inspect
 
 
 def inspect_getargspec():
     """
     Pyramid rely on inspect.getargspec to lookup the signature of
     view functions. This is not compatible with cython, therefore we replace
     getargspec with a custom version.
     Code is inspired by the inspect module from Python-3.4
     """
     import inspect
 
     def _isCython(func):
         """
         Private helper that checks if a function is a cython function.
         """
         return func.__class__.__name__ == 'cython_function_or_method'
 
     def unwrap(func):
         """
         Get the object wrapped by *func*.
 
         Follows the chain of :attr:`__wrapped__` attributes returning the last
         object in the chain.
 
         *stop* is an optional callback accepting an object in the wrapper chain
         as its sole argument that allows the unwrapping to be terminated early
         if the callback returns a true value. If the callback never returns a
         true value, the last object in the chain is returned as usual. For
         example, :func:`signature` uses this to stop unwrapping if any object
         in the chain has a ``__signature__`` attribute defined.
 
         :exc:`ValueError` is raised if a cycle is encountered.
         """
         f = func  # remember the original func for error reporting
         memo = {id(f)}  # Memoise by id to tolerate non-hashable objects
         while hasattr(func, '__wrapped__'):
             func = func.__wrapped__
             id_func = id(func)
             if id_func in memo:
                 raise ValueError(f'wrapper loop when unwrapping {f!r}')
             memo.add(id_func)
         return func
 
     def custom_getargspec(func):
         """
         Get the names and default values of a function's arguments.
 
         A tuple of four things is returned: (args, varargs, varkw, defaults).
         'args' is a list of the argument names (it may contain nested lists).
         'varargs' and 'varkw' are the names of the * and ** arguments or None.
         'defaults' is an n-tuple of the default values of the last n arguments.
         """
 
         func = unwrap(func)
 
         if inspect.ismethod(func):
             func = func.im_func
         if not inspect.isfunction(func):
             if not _isCython(func):
                 raise TypeError('{!r} is not a Python or Cython function'
                                 .format(func))
         args, varargs, varkw = inspect.getargs(func.func_code)
         return inspect.ArgSpec(args, varargs, varkw, func.func_defaults)
 
     # NOTE: inject for python3.11
     inspect.getargspec = inspect.getfullargspec
 
     return inspect
+
+
+def repoze_sendmail_lf_fix():
+    from repoze.sendmail import encoding
+    from email.policy import SMTP
+
+    encoding.encode_message = lambda message, *args, **kwargs: message.as_bytes(policy=SMTP)

rhodecode/config/utils.py

@@ -1,110 +1,110 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import platform
 
 DEFAULT_USER = 'default'
 
 
 def configure_vcs(config):
     """
     Patch VCS config with some RhodeCode specific stuff
     """
     from rhodecode.lib.vcs import conf
     import rhodecode.lib.vcs.conf.settings
 
     conf.settings.BACKENDS = {
         'hg': 'rhodecode.lib.vcs.backends.hg.MercurialRepository',
         'git': 'rhodecode.lib.vcs.backends.git.GitRepository',
         'svn': 'rhodecode.lib.vcs.backends.svn.SubversionRepository',
     }
 
-    conf.settings.HOOKS_PROTOCOL = config['vcs.hooks.protocol']
+    conf.settings.HOOKS_PROTOCOL = config['vcs.hooks.protocol.v2']
     conf.settings.HOOKS_HOST = config['vcs.hooks.host']
     conf.settings.DEFAULT_ENCODINGS = config['default_encoding']
     conf.settings.ALIASES[:] = config['vcs.backends']
     conf.settings.SVN_COMPATIBLE_VERSION = config['vcs.svn.compatible_version']
 
 
 def initialize_database(config):
     from rhodecode.lib.utils2 import engine_from_config, get_encryption_key
     from rhodecode.model import init_model
     engine = engine_from_config(config, 'sqlalchemy.db1.')
     init_model(engine, encryption_key=get_encryption_key(config))
 
 
 def initialize_test_environment(settings, test_env=None):
     if test_env is None:
         test_env = not int(os.environ.get('RC_NO_TMP_PATH', 0))
 
     from rhodecode.lib.utils import (
         create_test_directory, create_test_database, create_test_repositories,
         create_test_index)
     from rhodecode.tests import TESTS_TMP_PATH
     from rhodecode.lib.vcs.backends.hg import largefiles_store
     from rhodecode.lib.vcs.backends.git import lfs_store
 
     # test repos
     if test_env:
         create_test_directory(TESTS_TMP_PATH)
         # large object stores
         create_test_directory(largefiles_store(TESTS_TMP_PATH))
         create_test_directory(lfs_store(TESTS_TMP_PATH))
 
         create_test_database(TESTS_TMP_PATH, settings)
         create_test_repositories(TESTS_TMP_PATH, settings)
         create_test_index(TESTS_TMP_PATH, settings)
 
 
 def get_vcs_server_protocol(config):
     return config['vcs.server.protocol']
 
 
 def set_instance_id(config):
     """
     Sets a dynamic generated config['instance_id'] if missing or '*'
     E.g instance_id = *cluster-1 or instance_id = *
     """
 
     config['instance_id'] = config.get('instance_id') or ''
     instance_id = config['instance_id']
     if instance_id.startswith('*') or not instance_id:
         prefix = instance_id.lstrip('*')
         _platform_id = platform.uname()[1] or 'instance'
         config['instance_id'] = '{prefix}uname:{platform}-pid:{pid}'.format(
             prefix=prefix,
             platform=_platform_id,
             pid=os.getpid())
 
 
 def get_default_user_id():
     from sqlalchemy import text
     from rhodecode.model import meta
 
     engine = meta.get_engine()
     with meta.SA_Session(engine) as session:
         result = session.execute(text(
             "SELECT user_id from users where username = :uname"
         ), {'uname': DEFAULT_USER})
         user = result.first()
         if not user:
             raise ValueError('Unable to retrieve default user data from DB')
         user_id = user[0]
 
     return user_id

rhodecode/lib/archive_cache/__init__.py

@@ -1,78 +1,80 b''
 # Copyright (C) 2015-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 from .backends.fanout_cache import FileSystemFanoutCache
 from .backends.objectstore_cache import ObjectStoreCache
 
 from .utils import archive_iterator  # noqa
 from .lock import ArchiveCacheGenerationLock # noqa
 
 log = logging.getLogger(__name__)
 
 
 cache_meta = None
 
 
 def includeme(config):
+    return  # don't init cache currently for faster startup time
+
     # init our cache at start
-    settings = config.get_settings()
-    get_archival_cache_store(settings)
+    # settings = config.get_settings()
+    # get_archival_cache_store(settings)
 
 
 def get_archival_config(config):
 
     final_config = {
 
     }
 
     for k, v in config.items():
         if k.startswith('archive_cache'):
             final_config[k] = v
 
     return final_config
 
 
 def get_archival_cache_store(config, always_init=False):
 
     global cache_meta
     if cache_meta is not None and not always_init:
         return cache_meta
 
     config = get_archival_config(config)
     backend = config['archive_cache.backend.type']
 
     archive_cache_locking_url = config['archive_cache.locking.url']
 
     match backend:
         case 'filesystem':
             d_cache = FileSystemFanoutCache(
                 locking_url=archive_cache_locking_url,
                 **config
             )
         case 'objectstore':
             d_cache = ObjectStoreCache(
                 locking_url=archive_cache_locking_url,
                 **config
             )
         case _:
             raise ValueError(f'archive_cache.backend.type only supports "filesystem" or "objectstore" got {backend} ')
 
     cache_meta = d_cache
     return cache_meta

rhodecode/lib/archive_cache/backends/objectstore_cache.py

@@ -1,173 +1,173 b''
 # Copyright (C) 2015-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import codecs
 import hashlib
 import logging
 import os
 import typing
 
 import fsspec
 
 from .base import BaseCache, BaseShard
 from ..utils import ShardFileReader, NOT_GIVEN
 from ...type_utils import str2bool
 
 log = logging.getLogger(__name__)
 
 
 class S3Shard(BaseShard):
 
     def __init__(self, index, bucket, bucket_folder, fs, **settings):
         self._index: int = index
         self._bucket_folder: str = bucket_folder
         self.storage_type: str = 'bucket'
         self._bucket_main: str = bucket
 
         self.fs = fs
 
     @property
     def bucket(self) -> str:
         """Cache bucket final path."""
         return os.path.join(self._bucket_main, self._bucket_folder)
 
     def _get_keyfile(self, archive_key) -> tuple[str, str]:
         key_file: str = f'{archive_key}-{self.key_suffix}'
         return key_file, os.path.join(self.bucket, key_file)
 
     def _get_writer(self, path, mode):
         return self.fs.open(path, 'wb')
 
     def _write_file(self, full_path, iterator, mode):
 
         # ensure folder in bucket exists
         destination = self.bucket
         if not self.fs.exists(destination):
-            self.fs.mkdir(destination, s3_additional_kwargs={})
+            self.fs.mkdir(destination)
 
         writer = self._get_writer(full_path, mode)
 
         digest = hashlib.sha256()
         with writer:
             size = 0
             for chunk in iterator:
                 size += len(chunk)
                 digest.update(chunk)
                 writer.write(chunk)
 
             sha256 = digest.hexdigest()
             log.debug('written new archive cache under %s, sha256: %s', full_path, sha256)
             return size, sha256
 
     def store(self, key, value_reader, metadata: dict | None = None):
         return self._store(key, value_reader, metadata, mode='wb')
 
     def fetch(self, key, retry=NOT_GIVEN,
               retry_attempts=NOT_GIVEN, retry_backoff=1,
               presigned_url_expires: int = 0) -> tuple[ShardFileReader, dict]:
         return self._fetch(key, retry, retry_attempts, retry_backoff, presigned_url_expires=presigned_url_expires)
 
     def remove(self, key):
         return self._remove(key)
 
     def random_filename(self):
         """Return filename and full-path tuple for file storage.
 
         Filename will be a randomly generated 28 character hexadecimal string
         with ".archive_cache" suffixed. Two levels of sub-directories will be used to
         reduce the size of directories. On older filesystems, lookups in
         directories with many files may be slow.
         """
 
         hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8')
 
         archive_name = hex_name[4:] + '.archive_cache'
         filename = f"{hex_name[:2]}-{hex_name[2:4]}-{archive_name}"
 
         full_path = os.path.join(self.bucket, filename)
         return archive_name, full_path
 
     def __repr__(self):
         return f'{self.__class__.__name__}(index={self._index}, bucket={self.bucket})'
 
 
 class ObjectStoreCache(BaseCache):
     shard_name: str = 'shard-{:03d}'
     shard_cls = S3Shard
 
     def __init__(self, locking_url, **settings):
         """
         Initialize objectstore cache instance.
 
         :param str locking_url: redis url for a lock
         :param settings: settings dict
 
         """
         self._locking_url = locking_url
         self._config = settings
 
         objectstore_url = self.get_conf('archive_cache.objectstore.url')
         self._storage_path = objectstore_url  # common path for all from BaseCache
 
         self._shard_count = int(self.get_conf('archive_cache.objectstore.bucket_shards', pop=True))
         if self._shard_count < 1:
             raise ValueError('cache_shards must be 1 or more')
 
         self._bucket = settings.pop('archive_cache.objectstore.bucket')
         if not self._bucket:
             raise ValueError('archive_cache.objectstore.bucket needs to have a value')
 
         self._eviction_policy = self.get_conf('archive_cache.objectstore.eviction_policy', pop=True)
         self._cache_size_limit = self.gb_to_bytes(int(self.get_conf('archive_cache.objectstore.cache_size_gb')))
 
         self.retry = str2bool(self.get_conf('archive_cache.objectstore.retry', pop=True))
         self.retry_attempts = int(self.get_conf('archive_cache.objectstore.retry_attempts', pop=True))
         self.retry_backoff = int(self.get_conf('archive_cache.objectstore.retry_backoff', pop=True))
 
         endpoint_url = settings.pop('archive_cache.objectstore.url')
         key = settings.pop('archive_cache.objectstore.key')
         secret = settings.pop('archive_cache.objectstore.secret')
         region = settings.pop('archive_cache.objectstore.region')
 
         log.debug('Initializing %s archival cache instance', self)
 
         fs = fsspec.filesystem(
             's3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret, client_kwargs={'region_name': region}
         )
 
         # init main bucket
         if not fs.exists(self._bucket):
             fs.mkdir(self._bucket)
 
         self._shards = tuple(
             self.shard_cls(
                 index=num,
                 bucket=self._bucket,
                 bucket_folder=self.shard_name.format(num),
                 fs=fs,
                 **settings,
             )
             for num in range(self._shard_count)
         )
         self._hash = self._shards[0].hash
 
     def _get_size(self, shard, archive_path):
         return shard.fs.info(archive_path)['size']
 
     def set_presigned_url_expiry(self, val: int) -> None:
         self.presigned_url_expires = val

rhodecode/lib/celerylib/loader.py

@@ -1,372 +1,373 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 """
 Celery loader, run with::
 
     celery worker \
         --task-events \
         --beat \
         --autoscale=20,2 \
         --max-tasks-per-child 1 \
         --app rhodecode.lib.celerylib.loader \
         --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler \
         --loglevel DEBUG --ini=.dev/dev.ini
 """
-from rhodecode.config.patches import inspect_getargspec, inspect_formatargspec
-inspect_getargspec()
-inspect_formatargspec()
+from rhodecode.config import patches
+patches.inspect_getargspec()
+patches.inspect_formatargspec()
 # python3.11 inspect patches for backward compat on `paste` code
+patches.repoze_sendmail_lf_fix()
 
 import sys
 import logging
 import importlib
 
 import click
 from celery import Celery
 from celery import signals
 from celery import Task
 from celery import exceptions  # noqa
 
 import rhodecode
 
 from rhodecode.lib.statsd_client import StatsdClient
 from rhodecode.lib.celerylib.utils import parse_ini_vars, ping_db
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.pyramid_utils import bootstrap, setup_logging
 from rhodecode.lib.utils2 import str2bool
 from rhodecode.model import meta
 
 log = logging.getLogger('celery.rhodecode.loader')
 
 
 imports = ['rhodecode.lib.celerylib.tasks']
 
 try:
     # try if we have EE tasks available
     importlib.import_module('rc_ee')
     imports.append('rc_ee.lib.celerylib.tasks')
 except ImportError:
     pass
 
 
 base_celery_config = {
     'result_backend': 'rpc://',
     'result_expires': 60 * 60 * 24,
     'result_persistent': True,
     'imports': imports,
     'worker_max_tasks_per_child': 100,
     'worker_hijack_root_logger': False,
     'worker_prefetch_multiplier': 1,
     'task_serializer': 'json',
     'accept_content': ['json', 'msgpack'],
     'result_serializer': 'json',
     'result_accept_content': ['json', 'msgpack'],
 
     'broker_connection_retry_on_startup': True,
     'database_table_names': {
         'task': 'beat_taskmeta',
         'group': 'beat_groupmeta',
     }
 }
 
 
 preload_option_ini = click.Option(
     ('--ini',),
     help='Path to ini configuration file.'
 )
 
 preload_option_ini_var = click.Option(
     ('--ini-var',),
     help='Comma separated list of key=value to pass to ini.'
 )
 
 
 def get_logger(obj):
     custom_log = logging.getLogger(
         'rhodecode.task.{}'.format(obj.__class__.__name__))
 
     if rhodecode.CELERY_ENABLED:
         try:
             custom_log = obj.get_logger()
         except Exception:
             pass
 
     return custom_log
 
 
 # init main celery app
 celery_app = Celery()
 celery_app.user_options['preload'].add(preload_option_ini)
 celery_app.user_options['preload'].add(preload_option_ini_var)
 
 
 @signals.setup_logging.connect
 def setup_logging_callback(**kwargs):
 
     if 'RC_INI_FILE' in celery_app.conf:
         ini_file = celery_app.conf['RC_INI_FILE']
     else:
         ini_file = celery_app.user_options['RC_INI_FILE']
 
     setup_logging(ini_file)
 
 
 @signals.user_preload_options.connect
 def on_preload_parsed(options, **kwargs):
 
     ini_file = options['ini']
     ini_vars = options['ini_var']
 
     if ini_file is None:
         print('You must provide the --ini argument to start celery')
         exit(-1)
 
     options = None
     if ini_vars is not None:
         options = parse_ini_vars(ini_vars)
 
     celery_app.conf['RC_INI_FILE'] = ini_file
     celery_app.user_options['RC_INI_FILE'] = ini_file
 
     celery_app.conf['RC_INI_OPTIONS'] = options
     celery_app.user_options['RC_INI_OPTIONS'] = options
 
     setup_logging(ini_file)
 
 
 def _init_celery(app_type=''):
     from rhodecode.config.middleware import get_celery_config
 
     log.debug('Bootstrapping RhodeCode application for %s...', app_type)
 
     ini_file = celery_app.conf['RC_INI_FILE']
     options = celery_app.conf['RC_INI_OPTIONS']
 
     env = None
     try:
         env = bootstrap(ini_file, options=options)
     except Exception:
         log.exception('Failed to bootstrap RhodeCode APP. '
                       'Probably there is another error present that prevents from running pyramid app')
 
     if not env:
         # we use sys.exit here since we need to signal app startup failure for docker to restart the container and re-try
         sys.exit(1)
 
     log.debug('Got Pyramid ENV: %s', env)
 
     settings = env['registry'].settings
     celery_settings = get_celery_config(settings)
 
     # init and bootstrap StatsdClient
     StatsdClient.setup(settings)
 
     setup_celery_app(
         app=env['app'], root=env['root'], request=env['request'],
         registry=env['registry'], closer=env['closer'],
         celery_settings=celery_settings)
 
 
 @signals.celeryd_init.connect
 def on_celeryd_init(sender=None, conf=None, **kwargs):
     _init_celery('celery worker')
 
     # fix the global flag even if it's disabled via .ini file because this
     # is a worker code that doesn't need this to be disabled.
     rhodecode.CELERY_ENABLED = True
 
 
 @signals.beat_init.connect
 def on_beat_init(sender=None, conf=None, **kwargs):
     _init_celery('celery beat')
 
 
 @signals.task_prerun.connect
 def task_prerun_signal(task_id, task, args, **kwargs):
     ping_db()
     statsd = StatsdClient.statsd
 
     if statsd:
         task_repr = getattr(task, 'name', task)
         statsd.incr('rhodecode_celery_task_total', tags=[
             f'task:{task_repr}',
             'mode:async'
         ])
 
 
 @signals.task_success.connect
 def task_success_signal(result, **kwargs):
     meta.Session.commit()
     closer = celery_app.conf['PYRAMID_CLOSER']
     if closer:
         closer()
 
 
 @signals.task_retry.connect
 def task_retry_signal(
         request, reason, einfo, **kwargs):
     meta.Session.remove()
     closer = celery_app.conf['PYRAMID_CLOSER']
     if closer:
         closer()
 
 
 @signals.task_failure.connect
 def task_failure_signal(
         task_id, exception, args, kwargs, traceback, einfo, **kargs):
 
     log.error('Task: %s failed !! exc_info: %s', task_id, einfo)
     from rhodecode.lib.exc_tracking import store_exception
     from rhodecode.lib.statsd_client import StatsdClient
 
     meta.Session.remove()
 
     # simulate sys.exc_info()
     exc_info = (einfo.type, einfo.exception, einfo.tb)
     store_exception(id(exc_info), exc_info, prefix='rhodecode-celery')
     statsd = StatsdClient.statsd
     if statsd:
         exc_type = "{}.{}".format(einfo.__class__.__module__, einfo.__class__.__name__)
         statsd.incr('rhodecode_exception_total',
                     tags=["exc_source:celery", "type:{}".format(exc_type)])
 
     closer = celery_app.conf['PYRAMID_CLOSER']
     if closer:
         closer()
 
 
 @signals.task_revoked.connect
 def task_revoked_signal(
         request, terminated, signum, expired, **kwargs):
     closer = celery_app.conf['PYRAMID_CLOSER']
     if closer:
         closer()
 
 
 class UNSET(object):
     pass
 
 
 _unset = UNSET()
 
 
 def set_celery_conf(app=_unset, root=_unset, request=_unset, registry=_unset, closer=_unset):
 
     if request is not UNSET:
         celery_app.conf.update({'PYRAMID_REQUEST': request})
 
     if registry is not UNSET:
         celery_app.conf.update({'PYRAMID_REGISTRY': registry})
 
 
 def setup_celery_app(app, root, request, registry, closer, celery_settings):
     log.debug('Got custom celery conf: %s', celery_settings)
     celery_config = base_celery_config
     celery_config.update({
         # store celerybeat scheduler db where the .ini file is
         'beat_schedule_filename': registry.settings['celerybeat-schedule.path'],
     })
 
     celery_config.update(celery_settings)
     celery_app.config_from_object(celery_config)
 
     celery_app.conf.update({'PYRAMID_APP': app})
     celery_app.conf.update({'PYRAMID_ROOT': root})
     celery_app.conf.update({'PYRAMID_REQUEST': request})
     celery_app.conf.update({'PYRAMID_REGISTRY': registry})
     celery_app.conf.update({'PYRAMID_CLOSER': closer})
 
 
 def configure_celery(config, celery_settings):
     """
     Helper that is called from our application creation logic. It gives
     connection info into running webapp and allows execution of tasks from
     RhodeCode itself
     """
     # store some globals into rhodecode
     rhodecode.CELERY_ENABLED = str2bool(
         config.registry.settings.get('use_celery'))
     if rhodecode.CELERY_ENABLED:
         log.info('Configuring celery based on `%s` settings', celery_settings)
         setup_celery_app(
             app=None, root=None, request=None, registry=config.registry,
             closer=None, celery_settings=celery_settings)
 
 
 def maybe_prepare_env(req):
     environ = {}
     try:
         environ.update({
             'PATH_INFO': req.environ['PATH_INFO'],
             'SCRIPT_NAME': req.environ['SCRIPT_NAME'],
             'HTTP_HOST': req.environ.get('HTTP_HOST', req.environ['SERVER_NAME']),
             'SERVER_NAME': req.environ['SERVER_NAME'],
             'SERVER_PORT': req.environ['SERVER_PORT'],
             'wsgi.url_scheme': req.environ['wsgi.url_scheme'],
         })
     except Exception:
         pass
 
     return environ
 
 
 class RequestContextTask(Task):
     """
     This is a celery task which will create a rhodecode app instance context
     for the task, patch pyramid with the original request
     that created the task and also add the user to the context.
     """
 
     def apply_async(self, args=None, kwargs=None, task_id=None, producer=None,
                     link=None, link_error=None, shadow=None, **options):
         """ queue the job to run (we are in web request context here) """
         from rhodecode.lib.base import get_ip_addr
 
         req = self.app.conf['PYRAMID_REQUEST']
         if not req:
             raise ValueError('celery_app.conf is having empty PYRAMID_REQUEST key')
 
         log.debug('Running Task with class: %s. Request Class: %s',
                   self.__class__, req.__class__)
 
         user_id = 0
 
         # web case
         if hasattr(req, 'user'):
             user_id = req.user.user_id
 
         # api case
         elif hasattr(req, 'rpc_user'):
             user_id = req.rpc_user.user_id
 
         # we hook into kwargs since it is the only way to pass our data to
         # the celery worker
         environ = maybe_prepare_env(req)
         options['headers'] = options.get('headers', {})
         options['headers'].update({
             'rhodecode_proxy_data': {
                 'environ': environ,
                 'auth_user': {
                     'ip_addr': get_ip_addr(req.environ),
                     'user_id': user_id
                 },
             }
         })
 
         return super(RequestContextTask, self).apply_async(
             args, kwargs, task_id, producer, link, link_error, shadow, **options)

rhodecode/lib/celerylib/tasks.py

@@ -1,454 +1,455 b''
-# Copyright (C) 2012-2023 RhodeCode GmbH
+# Copyright (C) 2012-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 RhodeCode task modules, containing all task that suppose to be run
 by celery daemon
 """
 
 import os
 import time
 
 from pyramid_mailer.mailer import Mailer
 from pyramid_mailer.message import Message
 from email.utils import formatdate
 
 import rhodecode
 from rhodecode.lib import audit_logger
 from rhodecode.lib.celerylib import get_logger, async_task, RequestContextTask, run_task
 from rhodecode.lib import hooks_base
 from rhodecode.lib.utils import adopt_for_celery
 from rhodecode.lib.utils2 import safe_int, str2bool, aslist
 from rhodecode.lib.statsd_client import StatsdClient
 from rhodecode.model.db import (
     true, null, Session, IntegrityError, Repository, RepoGroup, User)
 from rhodecode.model.permission import PermissionModel
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def send_email(recipients, subject, body='', html_body='', email_config=None,
                extra_headers=None):
     """
     Sends an email with defined parameters from the .ini files.
 
     :param recipients: list of recipients, it this is empty the defined email
         address from field 'email_to' is used instead
     :param subject: subject of the mail
     :param body: body of the mail
     :param html_body: html version of body
     :param email_config: specify custom configuration for mailer
     :param extra_headers: specify custom headers
     """
     log = get_logger(send_email)
 
     email_config = email_config or rhodecode.CONFIG
 
     mail_server = email_config.get('smtp_server') or None
     if mail_server is None:
         log.error("SMTP server information missing. Sending email failed. "
                   "Make sure that `smtp_server` variable is configured "
                   "inside the .ini file")
         return False
-
-    subject = "%s %s" % (email_config.get('email_prefix', ''), subject)
+    conf_prefix = email_config.get('email_prefix', None)
+    prefix = f'{conf_prefix} ' if conf_prefix else ''
+    subject = f"{prefix}{subject}"
 
     if recipients:
         if isinstance(recipients, str):
             recipients = recipients.split(',')
     else:
         # if recipients are not defined we send to email_config + all admins
         admins = []
         for u in User.query().filter(User.admin == true()).all():
             if u.email:
                 admins.append(u.email)
         recipients = []
         config_email = email_config.get('email_to')
         if config_email:
             recipients += [config_email]
         recipients += admins
 
     # translate our LEGACY config into the one that pyramid_mailer supports
     email_conf = dict(
         host=mail_server,
         port=email_config.get('smtp_port', 25),
-        username=email_config.get('smtp_username'),
-        password=email_config.get('smtp_password'),
+        username=email_config.get('smtp_username', None),
+        password=email_config.get('smtp_password', None),
 
         tls=str2bool(email_config.get('smtp_use_tls')),
         ssl=str2bool(email_config.get('smtp_use_ssl')),
 
         # SSL key file
         # keyfile='',
 
         # SSL certificate file
         # certfile='',
 
         # Location of maildir
         # queue_path='',
 
         default_sender=email_config.get('app_email_from', '[e-mail]'),
 
         debug=str2bool(email_config.get('smtp_debug')),
         # /usr/sbin/sendmail	Sendmail executable
         # sendmail_app='',
 
         # {sendmail_app} -t -i -f {sender}	Template for sendmail execution
         # sendmail_template='',
     )
 
     if extra_headers is None:
         extra_headers = {}
 
     extra_headers.setdefault('Date', formatdate(time.time()))
 
     if 'thread_ids' in extra_headers:
         thread_ids = extra_headers.pop('thread_ids')
         extra_headers['References'] = ' '.join('<{}>'.format(t) for t in thread_ids)
 
     try:
         mailer = Mailer(**email_conf)
 
         message = Message(subject=subject,
                           sender=email_conf['default_sender'],
                           recipients=recipients,
                           body=body, html=html_body,
                           extra_headers=extra_headers)
         mailer.send_immediately(message)
         statsd = StatsdClient.statsd
         if statsd:
             statsd.incr('rhodecode_email_sent_total')
 
     except Exception:
         log.exception('Mail sending failed')
         return False
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def create_repo(form_data, cur_user):
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.user import UserModel
     from rhodecode.model.scm import ScmModel
     from rhodecode.model.settings import SettingsModel
 
     log = get_logger(create_repo)
 
     cur_user = UserModel()._get_user(cur_user)
     owner = cur_user
 
     repo_name = form_data['repo_name']
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     description = form_data['repo_description']
     private = form_data['repo_private']
     clone_uri = form_data.get('clone_uri')
     repo_group = safe_int(form_data['repo_group'])
     copy_fork_permissions = form_data.get('copy_permissions')
     copy_group_permissions = form_data.get('repo_copy_permissions')
     fork_of = form_data.get('fork_parent_id')
     state = form_data.get('repo_state', Repository.STATE_PENDING)
 
     # repo creation defaults, private and repo_type are filled in form
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     enable_statistics = form_data.get(
         'enable_statistics', defs.get('repo_enable_statistics'))
     enable_locking = form_data.get(
         'enable_locking', defs.get('repo_enable_locking'))
     enable_downloads = form_data.get(
         'enable_downloads', defs.get('repo_enable_downloads'))
 
     # set landing rev based on default branches for SCM
     landing_ref, _label = ScmModel.backend_landing_ref(repo_type)
 
     try:
         RepoModel()._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_ref,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions,
             copy_group_permissions=copy_group_permissions,
             enable_statistics=enable_statistics,
             enable_locking=enable_locking,
             enable_downloads=enable_downloads,
             state=state
         )
 
         Session().commit()
 
         # now create this repo on Filesystem
         RepoModel()._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel()._get_repo_group(repo_group),
             clone_uri=clone_uri,
         )
         repo = Repository.get_by_repo_name(repo_name_full)
         hooks_base.create_repository(created_by=owner.username, **repo.get_dict())
 
         # update repo commit caches initially
-        repo.update_commit_cache()
+        repo.update_commit_cache(recursive=False)
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
         repo_id = repo.repo_id
         repo_data = repo.get_api_data()
 
         audit_logger.store(
             'repo.create', action_data={'data': repo_data},
             user=cur_user,
             repo=audit_logger.RepoWrap(repo_name=repo_name, repo_id=repo_id))
 
         Session().commit()
 
         PermissionModel().trigger_permission_flush()
 
     except Exception as e:
         log.warning('Exception occurred when creating repository, '
                     'doing cleanup...', exc_info=True)
         if isinstance(e, IntegrityError):
             Session().rollback()
 
         # rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             Session().commit()
             RepoModel()._delete_filesystem_repo(repo)
         log.info('Cleanup of repo %s finished', repo_name_full)
         raise
 
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def create_repo_fork(form_data, cur_user):
     """
     Creates a fork of repository using internal VCS methods
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.user import UserModel
 
     log = get_logger(create_repo_fork)
 
     cur_user = UserModel()._get_user(cur_user)
     owner = cur_user
 
     repo_name = form_data['repo_name']  # fork in this case
     repo_name_full = form_data['repo_name_full']
     repo_type = form_data['repo_type']
     description = form_data['description']
     private = form_data['private']
     clone_uri = form_data.get('clone_uri')
     repo_group = safe_int(form_data['repo_group'])
     landing_ref = form_data['landing_rev']
     copy_fork_permissions = form_data.get('copy_permissions')
     fork_id = safe_int(form_data.get('fork_parent_id'))
 
     try:
         fork_of = RepoModel()._get_repo(fork_id)
         RepoModel()._create_repo(
             repo_name=repo_name_full,
             repo_type=repo_type,
             description=description,
             owner=owner,
             private=private,
             clone_uri=clone_uri,
             repo_group=repo_group,
             landing_rev=landing_ref,
             fork_of=fork_of,
             copy_fork_permissions=copy_fork_permissions
         )
 
         Session().commit()
 
         base_path = Repository.base_path()
         source_repo_path = os.path.join(base_path, fork_of.repo_name)
 
         # now create this repo on Filesystem
         RepoModel()._create_filesystem_repo(
             repo_name=repo_name,
             repo_type=repo_type,
             repo_group=RepoModel()._get_repo_group(repo_group),
             clone_uri=source_repo_path,
         )
         repo = Repository.get_by_repo_name(repo_name_full)
         hooks_base.create_repository(created_by=owner.username, **repo.get_dict())
 
         # update repo commit caches initially
         config = repo._config
         config.set('extensions', 'largefiles', '')
-        repo.update_commit_cache(config=config)
+        repo.update_commit_cache(config=config, recursive=False)
 
         # set new created state
         repo.set_state(Repository.STATE_CREATED)
 
         repo_id = repo.repo_id
         repo_data = repo.get_api_data()
         audit_logger.store(
             'repo.fork', action_data={'data': repo_data},
             user=cur_user,
             repo=audit_logger.RepoWrap(repo_name=repo_name, repo_id=repo_id))
 
         Session().commit()
     except Exception as e:
         log.warning('Exception occurred when forking repository, '
                     'doing cleanup...', exc_info=True)
         if isinstance(e, IntegrityError):
             Session().rollback()
 
         # rollback things manually !
         repo = Repository.get_by_repo_name(repo_name_full)
         if repo:
             Repository.delete(repo.repo_id)
             Session().commit()
             RepoModel()._delete_filesystem_repo(repo)
         log.info('Cleanup of repo %s finished', repo_name_full)
         raise
 
     return True
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def repo_maintenance(repoid):
     from rhodecode.lib import repo_maintenance as repo_maintenance_lib
     log = get_logger(repo_maintenance)
     repo = Repository.get_by_id_or_repo_name(repoid)
     if repo:
         maintenance = repo_maintenance_lib.RepoMaintenance()
         tasks = maintenance.get_tasks_for_repo(repo)
         log.debug('Executing %s tasks on repo `%s`', tasks, repoid)
         executed_types = maintenance.execute(repo)
         log.debug('Got execution results %s', executed_types)
     else:
         log.debug('Repo `%s` not found or without a clone_url', repoid)
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def check_for_update(send_email_notification=True, email_recipients=None):
     from rhodecode.model.update import UpdateModel
     from rhodecode.model.notification import EmailNotificationModel
 
     log = get_logger(check_for_update)
     update_url = UpdateModel().get_update_url()
     cur_ver = rhodecode.__version__
 
     try:
         data = UpdateModel().get_update_data(update_url)
 
         current_ver = UpdateModel().get_stored_version(fallback=cur_ver)
         latest_ver = data['versions'][0]['version']
         UpdateModel().store_version(latest_ver)
 
         if send_email_notification:
             log.debug('Send email notification is enabled. '
                       'Current RhodeCode version: %s, latest known: %s', current_ver, latest_ver)
             if UpdateModel().is_outdated(current_ver, latest_ver):
 
                 email_kwargs = {
                     'current_ver': current_ver,
                     'latest_ver': latest_ver,
                 }
 
                 (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
                     EmailNotificationModel.TYPE_UPDATE_AVAILABLE, **email_kwargs)
 
                 email_recipients = aslist(email_recipients, sep=',') or \
                                    [user.email for user in User.get_all_super_admins()]
                 run_task(send_email, email_recipients, subject,
                          email_body_plaintext, email_body)
 
     except Exception:
         log.exception('Failed to check for update')
         raise
 
 
 def sync_last_update_for_objects(*args, **kwargs):
     skip_repos = kwargs.get('skip_repos')
     if not skip_repos:
         repos = Repository.query() \
             .order_by(Repository.group_id.asc())
 
         for repo in repos:
-            repo.update_commit_cache()
+            repo.update_commit_cache(recursive=False)
 
     skip_groups = kwargs.get('skip_groups')
     if not skip_groups:
         repo_groups = RepoGroup.query() \
             .filter(RepoGroup.group_parent_id == null())
 
         for root_gr in repo_groups:
             for repo_gr in reversed(root_gr.recursive_groups()):
                 repo_gr.update_commit_cache()
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def test_celery_exception(msg):
     raise Exception(f'Test exception: {msg}')
 
 
 @async_task(ignore_result=True, base=RequestContextTask)
 def sync_last_update(*args, **kwargs):
     sync_last_update_for_objects(*args, **kwargs)
 
 
 @async_task(ignore_result=False)
 def beat_check(*args, **kwargs):
     log = get_logger(beat_check)
     log.info('%r: Got args: %r and kwargs %r', beat_check, args, kwargs)
     return time.time()
 
 
 @async_task
 @adopt_for_celery
 def repo_size(extras):
     from rhodecode.lib.hooks_base import repo_size
     return repo_size(extras)
 
 
 @async_task
 @adopt_for_celery
 def pre_pull(extras):
     from rhodecode.lib.hooks_base import pre_pull
     return pre_pull(extras)
 
 
 @async_task
 @adopt_for_celery
 def post_pull(extras):
     from rhodecode.lib.hooks_base import post_pull
     return post_pull(extras)
 
 
 @async_task
 @adopt_for_celery
 def pre_push(extras):
     from rhodecode.lib.hooks_base import pre_push
     return pre_push(extras)
 
 
 @async_task
 @adopt_for_celery
 def post_push(extras):
     from rhodecode.lib.hooks_base import post_push
     return post_push(extras)

rhodecode/lib/db_manage.py

@@ -1,679 +1,678 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Database creation, and setup module for RhodeCode Enterprise. Used for creation
 of database as well as for migration operations
 """
 
 import os
 import sys
 import time
 import uuid
 import logging
 import getpass
 from os.path import dirname as dn, join as jn
 
 from sqlalchemy.engine import create_engine
 
 from rhodecode import __dbversion__
 from rhodecode.model import init_model
 from rhodecode.model.user import UserModel
 from rhodecode.model.db import (
     User, Permission, RhodeCodeUi, RhodeCodeSetting, UserToPerm,
     DbMigrateVersion, RepoGroup, UserRepoGroupToPerm, CacheKey, Repository)
 from rhodecode.model.meta import Session, Base
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import SettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 def notify(msg):
     """
     Notification for migrations messages
     """
     ml = len(msg) + (4 * 2)
     print((('\n%s\n*** %s ***\n%s' % ('*' * ml, msg, '*' * ml)).upper()))
 
 
 class DbManage(object):
 
     def __init__(self, log_sql, dbconf, root, tests=False,
                  SESSION=None, cli_args=None, enc_key=b''):
 
         self.dbname = dbconf.split('/')[-1]
         self.tests = tests
         self.root = root
         self.dburi = dbconf
         self.log_sql = log_sql
         self.cli_args = cli_args or {}
         self.sa = None
         self.engine = None
         self.enc_key = enc_key
         # sets .sa .engine
         self.init_db(SESSION=SESSION)
 
         self.ask_ok = self.get_ask_ok_func(self.cli_args.get('force_ask'))
 
     def db_exists(self):
         if not self.sa:
             self.init_db()
         try:
             self.sa.query(RhodeCodeUi)\
                 .filter(RhodeCodeUi.ui_key == '/')\
                 .scalar()
             return True
         except Exception:
             return False
         finally:
             self.sa.rollback()
 
     def get_ask_ok_func(self, param):
         if param not in [None]:
             # return a function lambda that has a default set to param
             return lambda *args, **kwargs: param
         else:
             from rhodecode.lib.utils import ask_ok
             return ask_ok
 
     def init_db(self, SESSION=None):
 
         if SESSION:
             self.sa = SESSION
             self.engine = SESSION.bind
         else:
             # init new sessions
             engine = create_engine(self.dburi, echo=self.log_sql)
             init_model(engine, encryption_key=self.enc_key)
             self.sa = Session()
             self.engine = engine
 
     def create_tables(self, override=False):
         """
         Create a auth database
         """
 
         log.info("Existing database with the same name is going to be destroyed.")
         log.info("Setup command will run DROP ALL command on that database.")
         engine = self.engine
 
         if self.tests:
             destroy = True
         else:
             destroy = self.ask_ok('Are you sure that you want to destroy the old database? [y/n]')
         if not destroy:
             log.info('db tables bootstrap: Nothing done.')
             sys.exit(0)
         if destroy:
             Base.metadata.drop_all(bind=engine)
 
         checkfirst = not override
         Base.metadata.create_all(bind=engine, checkfirst=checkfirst)
         log.info('Created tables for %s', self.dbname)
 
     def set_db_version(self):
         ver = DbMigrateVersion()
         ver.version = __dbversion__
         ver.repository_id = 'rhodecode_db_migrations'
         ver.repository_path = 'versions'
         self.sa.add(ver)
         log.info('db version set to: %s', __dbversion__)
 
     def run_post_migration_tasks(self):
         """
         Run various tasks before actually doing migrations
         """
         # delete cache keys on each upgrade
         total = CacheKey.query().count()
         log.info("Deleting (%s) cache keys now...", total)
         CacheKey.delete_all_cache()
 
     def upgrade(self, version=None):
         """
         Upgrades given database schema to given revision following
         all needed steps, to perform the upgrade
 
         """
 
         from rhodecode.lib.dbmigrate.migrate.versioning import api
         from rhodecode.lib.dbmigrate.migrate.exceptions import DatabaseNotControlledError
 
         if 'sqlite' in self.dburi:
             print(
                '********************** WARNING **********************\n'
                'Make sure your version of sqlite is at least 3.7.X.  \n'
                'Earlier versions are known to fail on some migrations\n'
                '*****************************************************\n')
 
         upgrade = self.ask_ok(
             'You are about to perform a database upgrade. Make '
             'sure you have backed up your database. '
             'Continue ? [y/n]')
         if not upgrade:
             log.info('No upgrade performed')
             sys.exit(0)
 
         repository_path = jn(dn(dn(dn(os.path.realpath(__file__)))),
                              'rhodecode/lib/dbmigrate')
         db_uri = self.dburi
 
         if version:
             DbMigrateVersion.set_version(version)
 
         try:
             curr_version = api.db_version(db_uri, repository_path)
             msg = (f'Found current database db_uri under version '
                    f'control with version {curr_version}')
 
         except (RuntimeError, DatabaseNotControlledError):
             curr_version = 1
             msg = f'Current database is not under version control. ' \
                   f'Setting as version {curr_version}'
             api.version_control(db_uri, repository_path, curr_version)
 
         notify(msg)
 
         if curr_version == __dbversion__:
             log.info('This database is already at the newest version')
             sys.exit(0)
 
         upgrade_steps = list(range(curr_version + 1, __dbversion__ + 1))
         notify(f'attempting to upgrade database from '
                f'version {curr_version} to version {__dbversion__}')
 
         # CALL THE PROPER ORDER OF STEPS TO PERFORM FULL UPGRADE
         final_step = 'latest'
         for step in upgrade_steps:
             notify(f'performing upgrade step {step}')
             time.sleep(0.5)
 
             api.upgrade(db_uri, repository_path, step)
             self.sa.rollback()
             notify(f'schema upgrade for step {step} completed')
 
             final_step = step
 
         self.run_post_migration_tasks()
         notify(f'upgrade to version {final_step} successful')
 
     def fix_repo_paths(self):
         """
         Fixes an old RhodeCode version path into new one without a '*'
         """
 
         paths = self.sa.query(RhodeCodeUi)\
                 .filter(RhodeCodeUi.ui_key == '/')\
                 .scalar()
 
         paths.ui_value = paths.ui_value.replace('*', '')
 
         try:
             self.sa.add(paths)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_default_user(self):
         """
         Fixes an old default user with some 'nicer' default values,
         used mostly for anonymous access
         """
         def_user = self.sa.query(User)\
             .filter(User.username == User.DEFAULT_USER)\
             .one()
 
         def_user.name = 'Anonymous'
         def_user.lastname = 'User'
         def_user.email = User.DEFAULT_USER_EMAIL
 
         try:
             self.sa.add(def_user)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def fix_settings(self):
         """
         Fixes rhodecode settings and adds ga_code key for google analytics
         """
 
         hgsettings3 = RhodeCodeSetting('ga_code', '')
 
         try:
             self.sa.add(hgsettings3)
             self.sa.commit()
         except Exception:
             self.sa.rollback()
             raise
 
     def create_admin_and_prompt(self):
 
         # defaults
         defaults = self.cli_args
         username = defaults.get('username')
         password = defaults.get('password')
         email = defaults.get('email')
 
         if username is None:
             username = input('Specify admin username:')
         if password is None:
             password = self._get_admin_password()
             if not password:
                 # second try
                 password = self._get_admin_password()
                 if not password:
                     sys.exit()
         if email is None:
             email = input('Specify admin email:')
         api_key = self.cli_args.get('api_key')
         self.create_user(username, password, email, True,
                          strict_creation_check=False,
                          api_key=api_key)
 
     def _get_admin_password(self):
         password = getpass.getpass('Specify admin password '
                                    '(min 6 chars):')
         confirm = getpass.getpass('Confirm password:')
 
         if password != confirm:
             log.error('passwords mismatch')
             return False
         if len(password) < 6:
             log.error('password is too short - use at least 6 characters')
             return False
 
         return password
 
     def create_test_admin_and_users(self):
         log.info('creating admin and regular test users')
         from rhodecode.tests import TEST_USER_ADMIN_LOGIN, \
             TEST_USER_ADMIN_PASS, TEST_USER_ADMIN_EMAIL, \
             TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS, \
             TEST_USER_REGULAR_EMAIL, TEST_USER_REGULAR2_LOGIN, \
             TEST_USER_REGULAR2_PASS, TEST_USER_REGULAR2_EMAIL
 
         self.create_user(TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS,
                          TEST_USER_ADMIN_EMAIL, True, api_key=True)
 
         self.create_user(TEST_USER_REGULAR_LOGIN, TEST_USER_REGULAR_PASS,
                          TEST_USER_REGULAR_EMAIL, False, api_key=True)
 
         self.create_user(TEST_USER_REGULAR2_LOGIN, TEST_USER_REGULAR2_PASS,
                          TEST_USER_REGULAR2_EMAIL, False, api_key=True)
 
     def create_ui_settings(self, repo_store_path):
         """
         Creates ui settings, fills out hooks
         and disables dotencode
         """
         settings_model = SettingsModel(sa=self.sa)
         from rhodecode.lib.vcs.backends.hg import largefiles_store
         from rhodecode.lib.vcs.backends.git import lfs_store
 
         # Build HOOKS
         hooks = [
             (RhodeCodeUi.HOOK_REPO_SIZE, 'python:vcsserver.hooks.repo_size'),
 
             # HG
             (RhodeCodeUi.HOOK_PRE_PULL, 'python:vcsserver.hooks.pre_pull'),
             (RhodeCodeUi.HOOK_PULL, 'python:vcsserver.hooks.log_pull_action'),
             (RhodeCodeUi.HOOK_PRE_PUSH, 'python:vcsserver.hooks.pre_push'),
             (RhodeCodeUi.HOOK_PRETX_PUSH, 'python:vcsserver.hooks.pre_push'),
             (RhodeCodeUi.HOOK_PUSH, 'python:vcsserver.hooks.log_push_action'),
             (RhodeCodeUi.HOOK_PUSH_KEY, 'python:vcsserver.hooks.key_push'),
 
         ]
 
         for key, value in hooks:
             hook_obj = settings_model.get_ui_by_key(key)
             hooks2 = hook_obj if hook_obj else RhodeCodeUi()
             hooks2.ui_section = 'hooks'
             hooks2.ui_key = key
             hooks2.ui_value = value
             self.sa.add(hooks2)
 
         # enable largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'extensions'
         largefiles.ui_key = 'largefiles'
         largefiles.ui_value = ''
         self.sa.add(largefiles)
 
         # set default largefiles cache dir, defaults to
         # /repo_store_location/.cache/largefiles
         largefiles = RhodeCodeUi()
         largefiles.ui_section = 'largefiles'
         largefiles.ui_key = 'usercache'
         largefiles.ui_value = largefiles_store(repo_store_path)
 
         self.sa.add(largefiles)
 
         # set default lfs cache dir, defaults to
         # /repo_store_location/.cache/lfs_store
         lfsstore = RhodeCodeUi()
         lfsstore.ui_section = 'vcs_git_lfs'
         lfsstore.ui_key = 'store_location'
         lfsstore.ui_value = lfs_store(repo_store_path)
 
         self.sa.add(lfsstore)
 
         # enable hgevolve disabled by default
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'extensions'
         hgevolve.ui_key = 'evolve'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'experimental'
         hgevolve.ui_key = 'evolution'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'experimental'
         hgevolve.ui_key = 'evolution.exchange'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         hgevolve = RhodeCodeUi()
         hgevolve.ui_section = 'extensions'
         hgevolve.ui_key = 'topic'
         hgevolve.ui_value = ''
         hgevolve.ui_active = False
         self.sa.add(hgevolve)
 
         # enable hggit disabled by default
         hggit = RhodeCodeUi()
         hggit.ui_section = 'extensions'
         hggit.ui_key = 'hggit'
         hggit.ui_value = ''
         hggit.ui_active = False
         self.sa.add(hggit)
 
         # set svn branch defaults
         branches = ["/branches/*", "/trunk"]
         tags = ["/tags/*"]
 
         for branch in branches:
             settings_model.create_ui_section_value(
                 RhodeCodeUi.SVN_BRANCH_ID, branch)
 
         for tag in tags:
             settings_model.create_ui_section_value(RhodeCodeUi.SVN_TAG_ID, tag)
 
     def create_auth_plugin_options(self, skip_existing=False):
         """
         Create default auth plugin settings, and make it active
 
         :param skip_existing:
         """
         defaults = [
             ('auth_plugins',
              'egg:rhodecode-enterprise-ce#token,egg:rhodecode-enterprise-ce#rhodecode',
              'list'),
 
             ('auth_authtoken_enabled',
              'True',
              'bool'),
 
             ('auth_rhodecode_enabled',
              'True',
              'bool'),
         ]
         for k, v, t in defaults:
             if (skip_existing and
                     SettingsModel().get_setting_by_name(k) is not None):
                 log.debug('Skipping option %s', k)
                 continue
             setting = RhodeCodeSetting(k, v, t)
             self.sa.add(setting)
 
     def create_default_options(self, skip_existing=False):
         """Creates default settings"""
 
         for k, v, t in [
             ('default_repo_enable_locking',  False, 'bool'),
             ('default_repo_enable_downloads', False, 'bool'),
             ('default_repo_enable_statistics', False, 'bool'),
             ('default_repo_private', False, 'bool'),
             ('default_repo_type', 'hg', 'unicode')]:
 
             if (skip_existing and
                     SettingsModel().get_setting_by_name(k) is not None):
                 log.debug('Skipping option %s', k)
                 continue
             setting = RhodeCodeSetting(k, v, t)
             self.sa.add(setting)
 
     def fixup_groups(self):
         def_usr = User.get_default_user()
         for g in RepoGroup.query().all():
             g.group_name = g.get_new_name(g.name)
             self.sa.add(g)
             # get default perm
             default = UserRepoGroupToPerm.query()\
                 .filter(UserRepoGroupToPerm.group == g)\
                 .filter(UserRepoGroupToPerm.user == def_usr)\
                 .scalar()
 
             if default is None:
                 log.debug('missing default permission for group %s adding', g)
                 perm_obj = RepoGroupModel()._create_default_perms(g)
                 self.sa.add(perm_obj)
 
     def reset_permissions(self, username):
         """
         Resets permissions to default state, useful when old systems had
         bad permissions, we must clean them up
 
         :param username:
         """
         default_user = User.get_by_username(username)
         if not default_user:
             return
 
         u2p = UserToPerm.query()\
             .filter(UserToPerm.user == default_user).all()
         fixed = False
         if len(u2p) != len(Permission.DEFAULT_USER_PERMISSIONS):
             for p in u2p:
                 Session().delete(p)
             fixed = True
             self.populate_default_permissions()
         return fixed
 
     def config_prompt(self, test_repo_path='', retries=3):
         defaults = self.cli_args
         _path = defaults.get('repos_location')
         if retries == 3:
             log.info('Setting up repositories config')
 
         if _path is not None:
             path = _path
         elif not self.tests and not test_repo_path:
             path = input(
                  'Enter a valid absolute path to store repositories. '
                  'All repositories in that path will be added automatically:'
             )
         else:
             path = test_repo_path
         path_ok = True
 
         # check proper dir
         if not os.path.isdir(path):
             path_ok = False
             log.error('Given path %s is not a valid directory', path)
 
         elif not os.path.isabs(path):
             path_ok = False
             log.error('Given path %s is not an absolute path', path)
 
         # check if path is at least readable.
         if not os.access(path, os.R_OK):
             path_ok = False
             log.error('Given path %s is not readable', path)
 
         # check write access, warn user about non writeable paths
         elif not os.access(path, os.W_OK) and path_ok:
             log.warning('No write permission to given path %s', path)
 
             q = (f'Given path {path} is not writeable, do you want to '
                  f'continue with read only mode ? [y/n]')
             if not self.ask_ok(q):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         if retries == 0:
             sys.exit('max retries reached')
         if not path_ok:
             retries -= 1
             return self.config_prompt(test_repo_path, retries)
 
         real_path = os.path.normpath(os.path.realpath(path))
 
         if real_path != os.path.normpath(path):
             q = (f'Path looks like a symlink, RhodeCode Enterprise will store '
                  f'given path as {real_path} ? [y/n]')
             if not self.ask_ok(q):
                 log.error('Canceled by user')
                 sys.exit(-1)
 
         return real_path
 
     def create_settings(self, path):
 
         self.create_ui_settings(path)
 
         ui_config = [
-            ('web', 'push_ssl', 'False'),
             ('web', 'allow_archive', 'gz zip bz2'),
             ('web', 'allow_push', '*'),
             ('web', 'baseurl', '/'),
             ('paths', '/', path),
             ('phases', 'publish', 'True')
         ]
         for section, key, value in ui_config:
             ui_conf = RhodeCodeUi()
             setattr(ui_conf, 'ui_section', section)
             setattr(ui_conf, 'ui_key', key)
             setattr(ui_conf, 'ui_value', value)
             self.sa.add(ui_conf)
 
         # rhodecode app settings
         settings = [
             ('realm', 'RhodeCode', 'unicode'),
             ('title', '', 'unicode'),
             ('pre_code', '', 'unicode'),
             ('post_code', '', 'unicode'),
 
             # Visual
             ('show_public_icon', True, 'bool'),
             ('show_private_icon', True, 'bool'),
             ('stylify_metatags', True, 'bool'),
             ('dashboard_items', 100, 'int'),
             ('admin_grid_items', 25, 'int'),
 
             ('markup_renderer', 'markdown', 'unicode'),
 
             ('repository_fields', True, 'bool'),
             ('show_version', True, 'bool'),
             ('show_revision_number', True, 'bool'),
             ('show_sha_length', 12, 'int'),
 
             ('use_gravatar', False, 'bool'),
             ('gravatar_url', User.DEFAULT_GRAVATAR_URL, 'unicode'),
 
             ('clone_uri_tmpl', Repository.DEFAULT_CLONE_URI, 'unicode'),
             ('clone_uri_id_tmpl', Repository.DEFAULT_CLONE_URI_ID, 'unicode'),
             ('clone_uri_ssh_tmpl', Repository.DEFAULT_CLONE_URI_SSH, 'unicode'),
             ('support_url', '', 'unicode'),
             ('update_url', RhodeCodeSetting.DEFAULT_UPDATE_URL, 'unicode'),
 
             # VCS Settings
             ('pr_merge_enabled', True, 'bool'),
             ('use_outdated_comments', True, 'bool'),
             ('diff_cache', True, 'bool'),
         ]
 
         for key, val, type_ in settings:
             sett = RhodeCodeSetting(key, val, type_)
             self.sa.add(sett)
 
         self.create_auth_plugin_options()
         self.create_default_options()
 
         log.info('created ui config')
 
     def create_user(self, username, password, email='', admin=False,
                     strict_creation_check=True, api_key=None):
         log.info('creating user `%s`', username)
         user = UserModel().create_or_update(
             username, password, email, firstname='RhodeCode', lastname='Admin',
             active=True, admin=admin, extern_type="rhodecode",
             strict_creation_check=strict_creation_check)
 
         if api_key:
             log.info('setting a new default auth token for user `%s`', username)
             UserModel().add_auth_token(
                 user=user, lifetime_minutes=-1,
                 role=UserModel.auth_token_role.ROLE_ALL,
                 description='BUILTIN TOKEN')
 
     def create_default_user(self):
         log.info('creating default user')
         # create default user for handling default permissions.
         user = UserModel().create_or_update(username=User.DEFAULT_USER,
                                             password=str(uuid.uuid1())[:20],
                                             email=User.DEFAULT_USER_EMAIL,
                                             firstname='Anonymous',
                                             lastname='User',
                                             strict_creation_check=False)
         # based on configuration options activate/de-activate this user which
         # controls anonymous access
         if self.cli_args.get('public_access') is False:
             log.info('Public access disabled')
             user.active = False
             Session().add(user)
             Session().commit()
 
     def create_permissions(self):
         """
         Creates all permissions defined in the system
         """
         # module.(access|create|change|delete)_[name]
         # module.(none|read|write|admin)
         log.info('creating permissions')
         PermissionModel(self.sa).create_permissions()
 
     def populate_default_permissions(self):
         """
         Populate default permissions. It will create only the default
         permissions that are missing, and not alter already defined ones
         """
         log.info('creating default user permissions')
         PermissionModel(self.sa).create_default_user_permissions(user=User.DEFAULT_USER)

rhodecode/lib/dbmigrate/versions/115_version_5_1_0.py

@@ -1,50 +1,53 b''
 
 
 import logging
 from sqlalchemy import *
 from sqlalchemy.engine import reflection
 
 from alembic.migration import MigrationContext
 from alembic.operations import Operations
 
 from rhodecode.lib.dbmigrate.versions import _reset_base
 from rhodecode.model import meta, init_model_encryption
 
 
 log = logging.getLogger(__name__)
 
 
 def _get_indexes_list(migrate_engine, table_name):
     inspector = reflection.Inspector.from_engine(migrate_engine)
     return inspector.get_indexes(table_name)
 
 
 def upgrade(migrate_engine):
     """
     Upgrade operations go here.
     Don't create your own engine; bind migrate_engine to your metadata
     """
     from rhodecode.model import db as db_5_1_0_0
 
     # issue fixups
     fixups(db_5_1_0_0, meta.Session)
 
 
 def downgrade(migrate_engine):
     pass
 
 
 def fixups(models, _SESSION):
+
     for db_repo in _SESSION.query(models.Repository).all():
 
-        config = db_repo._config
-        config.set('extensions', 'largefiles', '')
+        try:
+            config = db_repo._config
+            config.set('extensions', 'largefiles', '')
 
-        try:
-            scm = db_repo.scm_instance(cache=False, config=config)
+            scm = db_repo.scm_instance(cache=False, config=config, vcs_full_cache=False)
             if scm:
                 print(f'installing hook for repo: {db_repo}')
                 scm.install_hooks(force=True)
+            del scm  # force GC
+            del config
         except Exception as e:
             print(e)
             print('continue...')

rhodecode/lib/exceptions.py

@@ -1,205 +1,214 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Set of custom exceptions used in RhodeCode
 """
 
 from webob.exc import HTTPClientError
 from pyramid.httpexceptions import HTTPBadGateway
 
 
 class LdapUsernameError(Exception):
     pass
 
 
 class LdapPasswordError(Exception):
     pass
 
 
 class LdapConnectionError(Exception):
     pass
 
 
 class LdapImportError(Exception):
     pass
 
 
 class DefaultUserException(Exception):
     pass
 
 
 class UserOwnsReposException(Exception):
     pass
 
 
 class UserOwnsRepoGroupsException(Exception):
     pass
 
 
 class UserOwnsUserGroupsException(Exception):
     pass
 
 
 class UserOwnsPullRequestsException(Exception):
     pass
 
 
 class UserOwnsArtifactsException(Exception):
     pass
 
 
 class UserGroupAssignedException(Exception):
     pass
 
 
 class StatusChangeOnClosedPullRequestError(Exception):
     pass
 
 
 class AttachedForksError(Exception):
     pass
 
 
 class AttachedPullRequestsError(Exception):
     pass
 
 
+class AttachedArtifactsError(Exception):
+    pass
+
+
 class RepoGroupAssignmentError(Exception):
     pass
 
 
 class NonRelativePathError(Exception):
     pass
 
 
 class HTTPRequirementError(HTTPClientError):
     title = explanation = 'Repository Requirement Missing'
     reason = None
 
     def __init__(self, message, *args, **kwargs):
         self.title = self.explanation = message
         super().__init__(*args, **kwargs)
         self.args = (message, )
 
 
+class ClientNotSupportedError(HTTPRequirementError):
+    title = explanation = 'Client Not Supported'
+    reason = None
+
+
 class HTTPLockedRC(HTTPClientError):
     """
     Special Exception For locked Repos in RhodeCode, the return code can
     be overwritten by _code keyword argument passed into constructors
     """
     code = 423
     title = explanation = 'Repository Locked'
     reason = None
 
     def __init__(self, message, *args, **kwargs):
         import rhodecode
 
         self.code = rhodecode.ConfigGet().get_int('lock_ret_code', missing=self.code)
 
         self.title = self.explanation = message
         super().__init__(*args, **kwargs)
         self.args = (message, )
 
 
 class HTTPBranchProtected(HTTPClientError):
     """
     Special Exception For Indicating that branch is protected in RhodeCode, the
     return code can be overwritten by _code keyword argument passed into constructors
     """
     code = 403
     title = explanation = 'Branch Protected'
     reason = None
 
     def __init__(self, message, *args, **kwargs):
         self.title = self.explanation = message
         super().__init__(*args, **kwargs)
         self.args = (message, )
 
 
 class IMCCommitError(Exception):
     pass
 
 
 class UserCreationError(Exception):
     pass
 
 
 class NotAllowedToCreateUserError(Exception):
     pass
 
 
 class DuplicateUpdateUserError(Exception):
     pass
 
 
 class RepositoryCreationError(Exception):
     pass
 
 
 class VCSServerUnavailable(HTTPBadGateway):
     """ HTTP Exception class for VCS Server errors """
     code = 502
     title = 'VCS Server Error'
     causes = [
         'VCS Server is not running',
         'Incorrect vcs.server=host:port',
         'Incorrect vcs.server.protocol',
     ]
 
     def __init__(self, message=''):
         self.explanation = 'Could not connect to VCS Server'
         if message:
             self.explanation += ': ' + message
         super().__init__()
 
 
 class ArtifactMetadataDuplicate(ValueError):
 
     def __init__(self, *args, **kwargs):
         self.err_section = kwargs.pop('err_section', None)
         self.err_key = kwargs.pop('err_key', None)
         super().__init__(*args, **kwargs)
 
 
 class ArtifactMetadataBadValueType(ValueError):
     pass
 
 
 class CommentVersionMismatch(ValueError):
     pass
 
 
 class SignatureVerificationError(ValueError):
     pass
 
 
 def signature_verification_error(msg):
     details = """
 Encryption signature verification failed. 
 Please check your value of secret key, and/or encrypted value stored. 
 Secret key stored inside .ini file:
 `rhodecode.encrypted_values.secret` or defaults to
 `beaker.session.secret` 
 
 Probably the stored values were encrypted using a different secret then currently set in .ini file    
 """
 
     final_msg = f'{msg}\n{details}'
     return SignatureVerificationError(final_msg)
 

rhodecode/lib/helpers.py

@@ -1,2230 +1,2182 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Helper functions
 
 Consists of functions to typically be used within templates, but also
 available to Controllers. This module is available to both as 'h'.
 """
 import base64
 import collections
 
 import os
 import random
 import hashlib
 import io
 import textwrap
 import urllib.request
 import urllib.parse
 import urllib.error
 import math
 import logging
 import re
 import time
 import string
 import regex
 from collections import OrderedDict
 
 import pygments
 import itertools
 import fnmatch
 
 from datetime import datetime
 from functools import partial
 from pygments.formatters.html import HtmlFormatter
 from pygments.lexers import (
     get_lexer_by_name, get_lexer_for_filename, get_lexer_for_mimetype)
 
 from pyramid.threadlocal import get_current_request
 from tempita import looper
 from webhelpers2.html import literal, HTML, escape
 from webhelpers2.html._autolink import _auto_link_urls
 from webhelpers2.html.tools import (
     button_to, highlight, js_obfuscate, strip_links, strip_tags)
 
 from webhelpers2.text import (
     chop_at, collapse, convert_accented_entities,
     convert_misc_entities, lchop, plural, rchop, remove_formatting,
     replace_whitespace, urlify, truncate, wrap_paragraphs)
 from webhelpers2.date import time_ago_in_words
 
 from webhelpers2.html.tags import (
     _input, NotGiven, _make_safe_id_component as safeid,
     form as insecure_form,
     auto_discovery_link, checkbox, end_form, file,
     hidden, image, javascript_link, link_to, link_to_if, link_to_unless, ol,
     stylesheet_link, submit, text, password, textarea,
     ul, radio, Options)
 
 from webhelpers2.number import format_byte_size
 # python3.11 backport fixes for webhelpers2
 from rhodecode import ConfigGet
 from rhodecode.lib._vendor.webhelpers_backports import raw_select
 
 from rhodecode.lib.action_parser import action_parser
 from rhodecode.lib.html_filters import sanitize_html
 from rhodecode.lib.pagination import Page, RepoPage, SqlPage
 from rhodecode.lib import ext_json
-from rhodecode.lib.ext_json import json
+from rhodecode.lib.ext_json import json, formatted_str_json
 from rhodecode.lib.str_utils import safe_bytes, convert_special_chars, base64_to_str
 from rhodecode.lib.utils import repo_name_slug, get_custom_lexer
 from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.utils2 import (
     str2bool,
     get_commit_safe, datetime_to_time, time_to_datetime, time_to_utcdatetime,
     AttributeDict, safe_int, md5, md5_safe, get_host_info)
 from rhodecode.lib.markup_renderer import MarkupRenderer, relative_links
 from rhodecode.lib.vcs.exceptions import CommitDoesNotExistError
 from rhodecode.lib.vcs.backends.base import BaseChangeset, EmptyCommit
 from rhodecode.lib.vcs.conf.settings import ARCHIVE_SPECS
 from rhodecode.lib.index.search_utils import get_matching_line_offsets
 from rhodecode.config.conf import DATE_FORMAT, DATETIME_FORMAT
 from rhodecode.model.changeset_status import ChangesetStatusModel
 from rhodecode.model.db import Permission, User, Repository, UserApiKeys, FileStore
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import IssueTrackerSettingsModel
 
 
 log = logging.getLogger(__name__)
 
 
 DEFAULT_USER = User.DEFAULT_USER
 DEFAULT_USER_EMAIL = User.DEFAULT_USER_EMAIL
 
 
 def asset(path, ver=None, **kwargs):
     """
     Helper to generate a static asset file path for rhodecode assets
 
     eg. h.asset('images/image.png', ver='3923')
 
     :param path: path of asset
     :param ver: optional version query param to append as ?ver=
     """
     request = get_current_request()
     query = {}
     query.update(kwargs)
     if ver:
         query = {'ver': ver}
     return request.static_path(
         f'rhodecode:public/{path}', _query=query)
 
 
 default_html_escape_table = {
     ord('&'): '&amp;',
     ord('<'): '&lt;',
     ord('>'): '&gt;',
     ord('"'): '&quot;',
     ord("'"): '&#39;',
 }
 
 
 def html_escape(text, html_escape_table=default_html_escape_table):
     """Produce entities within text."""
     return text.translate(html_escape_table)
 
 
 def str_json(*args, **kwargs):
     return ext_json.str_json(*args, **kwargs)
 
 
 def formatted_str_json(*args, **kwargs):
     return ext_json.formatted_str_json(*args, **kwargs)
 
 
 def chop_at_smart(s, sub, inclusive=False, suffix_if_chopped=None):
     """
     Truncate string ``s`` at the first occurrence of ``sub``.
 
     If ``inclusive`` is true, truncate just after ``sub`` rather than at it.
     """
     suffix_if_chopped = suffix_if_chopped or ''
     pos = s.find(sub)
     if pos == -1:
         return s
 
     if inclusive:
         pos += len(sub)
 
     chopped = s[:pos]
     left = s[pos:].strip()
 
     if left and suffix_if_chopped:
         chopped += suffix_if_chopped
 
     return chopped
 
 
 def shorter(text, size=20, prefix=False):
     postfix = '...'
     if len(text) > size:
         if prefix:
             # shorten in front
             return postfix + text[-(size - len(postfix)):]
         else:
             return text[:size - len(postfix)] + postfix
     return text
 
 
 def reset(name, value=None, id=NotGiven, type="reset", **attrs):
     """
     Reset button
     """
     return _input(type, name, value, id, attrs)
 
 
 def select(name, selected_values, options, id=NotGiven, **attrs):
 
     if isinstance(options, (list, tuple)):
         options_iter = options
         # Handle old value,label lists ... where value also can be value,label lists
         options = Options()
         for opt in options_iter:
             if isinstance(opt, tuple) and len(opt) == 2:
                 value, label = opt
             elif isinstance(opt, str):
                 value = label = opt
             else:
                 raise ValueError('invalid select option type %r' % type(opt))
 
             if isinstance(value, (list, tuple)):
                 option_group = options.add_optgroup(label)
                 for opt2 in value:
                     if isinstance(opt2, tuple) and len(opt2) == 2:
                         group_value, group_label = opt2
                     elif isinstance(opt2, str):
                         group_value = group_label = opt2
                     else:
                         raise ValueError('invalid select option type %r' % type(opt2))
 
                     option_group.add_option(group_label, group_value)
             else:
                 options.add_option(label, value)
 
     return raw_select(name, selected_values, options, id=id, **attrs)
 
 
 def branding(name, length=40):
     return truncate(name, length, indicator="")
 
 
 def FID(raw_id, path):
     """
     Creates a unique ID for filenode based on it's hash of path and commit
     it's safe to use in urls
 
     :param raw_id:
     :param path:
     """
 
     return 'c-%s-%s' % (short_id(raw_id), md5_safe(path)[:12])
 
 
 class _GetError(object):
     """Get error from form_errors, and represent it as span wrapped error
     message
 
     :param field_name: field to fetch errors for
     :param form_errors: form errors dict
     """
 
     def __call__(self, field_name, form_errors):
         tmpl = """<span class="error_msg">%s</span>"""
         if form_errors and field_name in form_errors:
             return literal(tmpl % form_errors.get(field_name))
 
 
 get_error = _GetError()
 
 
 class _ToolTip(object):
 
     def __call__(self, tooltip_title, trim_at=50):
         """
         Special function just to wrap our text into nice formatted
         autowrapped text
 
         :param tooltip_title:
         """
         tooltip_title = escape(tooltip_title)
         tooltip_title = tooltip_title.replace('<', '&lt;').replace('>', '&gt;')
         return tooltip_title
 
 
 tooltip = _ToolTip()
 
 files_icon = '<i class="file-breadcrumb-copy tooltip icon-clipboard clipboard-action" data-clipboard-text="{}" title="Copy file path"></i>'
 
 
 def files_breadcrumbs(repo_name, repo_type, commit_id, file_path, landing_ref_name=None, at_ref=None,
                       limit_items=False, linkify_last_item=False, hide_last_item=False,
                       copy_path_icon=True):
 
     if at_ref:
         route_qry = {'at': at_ref}
         default_landing_ref = at_ref or landing_ref_name or commit_id
     else:
         route_qry = None
         default_landing_ref = commit_id
 
     # first segment is a `HOME` link to repo files root location
     root_name = literal('<i class="icon-home"></i>')
 
     url_segments = [
         link_to(
             root_name,
             repo_files_by_ref_url(
                 repo_name,
                 repo_type,
                 f_path=None,  # None here is a special case for SVN repos,
                               # that won't prefix with a ref
                 ref_name=default_landing_ref,
                 commit_id=commit_id,
                 query=route_qry
             )
         )]
 
     path_segments = file_path.split('/')
     last_cnt = len(path_segments) - 1
     for cnt, segment in enumerate(path_segments):
         if not segment:
             continue
         segment_html = escape(segment)
 
         last_item = cnt == last_cnt
 
         if last_item and hide_last_item:
             # iterate over and hide last element
             continue
 
         if last_item and linkify_last_item is False:
             # plain version
             url_segments.append(segment_html)
         else:
             url_segments.append(
                 link_to(
                     segment_html,
                     repo_files_by_ref_url(
                         repo_name,
                         repo_type,
                         f_path='/'.join(path_segments[:cnt + 1]),
                         ref_name=default_landing_ref,
                         commit_id=commit_id,
                         query=route_qry
                     ),
                 ))
 
     limited_url_segments = url_segments[:1] + ['...'] + url_segments[-5:]
     if limit_items and len(limited_url_segments) < len(url_segments):
         url_segments = limited_url_segments
 
     full_path = file_path
     if copy_path_icon:
         icon = files_icon.format(escape(full_path))
     else:
         icon = ''
 
     if file_path == '':
         return root_name
     else:
         return literal(' / '.join(url_segments) + icon)
 
 
 def files_url_data(request):
     matchdict = request.matchdict
 
     if 'f_path' not in matchdict:
         matchdict['f_path'] = ''
     else:
         matchdict['f_path'] = urllib.parse.quote(safe_str(matchdict['f_path']))
     if 'commit_id' not in matchdict:
         matchdict['commit_id'] = 'tip'
 
     return ext_json.str_json(matchdict)
 
 
 def repo_files_by_ref_url(db_repo_name, db_repo_type, f_path, ref_name, commit_id, query=None, ):
     _is_svn = is_svn(db_repo_type)
     final_f_path = f_path
 
     if _is_svn:
         """
         For SVN the ref_name cannot be used as a commit_id, it needs to be prefixed with
         actually commit_id followed by the ref_name. This should be done only in case
         This is a initial landing url, without additional paths.
 
         like: /1000/tags/1.0.0/?at=tags/1.0.0
         """
 
         if ref_name and ref_name != 'tip':
             # NOTE(marcink): for svn the ref_name is actually the stored path, so we prefix it
             # for SVN we only do this magic prefix if it's root, .eg landing revision
             # of files link. If we are in the tree we don't need this since we traverse the url
             # that has everything stored
             if f_path in ['', '/']:
                 final_f_path = '/'.join([ref_name, f_path])
 
         # SVN always needs a commit_id explicitly, without a named REF
         default_commit_id = commit_id
     else:
         """
         For git and mercurial we construct a new URL using the names instead of commit_id
         like: /master/some_path?at=master
         """
         # We currently do not support branches with slashes
         if '/' in ref_name:
             default_commit_id = commit_id
         else:
             default_commit_id = ref_name
 
     # sometimes we pass f_path as None, to indicate explicit no prefix,
     # we translate it to string to not have None
     final_f_path = final_f_path or ''
 
     files_url = route_path(
         'repo_files',
         repo_name=db_repo_name,
         commit_id=default_commit_id,
         f_path=final_f_path,
         _query=query
     )
     return files_url
 
 
 def code_highlight(code, lexer, formatter, use_hl_filter=False):
     """
     Lex ``code`` with ``lexer`` and format it with the formatter ``formatter``.
 
     If ``outfile`` is given and a valid file object (an object
     with a ``write`` method), the result will be written to it, otherwise
     it is returned as a string.
     """
     if use_hl_filter:
         # add HL filter
         from rhodecode.lib.index import search_utils
         lexer.add_filter(search_utils.ElasticSearchHLFilter())
     return pygments.format(pygments.lex(code, lexer), formatter)
 
 
 class CodeHtmlFormatter(HtmlFormatter):
     """
     My code Html Formatter for source codes
     """
 
     def wrap(self, source):
         return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = f'<div id="L{cnt+1}">{t}</div>'
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         dummyoutfile = io.StringIO()
         lncount = 0
         for t, line in inner:
             if t:
                 lncount += 1
             dummyoutfile.write(line)
 
         fl = self.linenostart
         mw = len(str(lncount + fl - 1))
         sp = self.linenospecial
         st = self.linenostep
         la = self.lineanchors
         aln = self.anchorlinenos
         nocls = self.noclasses
         if sp:
             lines = []
 
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if i % sp == 0:
                         if aln:
                             lines.append('<a href="#%s%d" class="special">%*d</a>' %
                                          (la, i, mw, i))
                         else:
                             lines.append('<span class="special">%*d</span>' % (mw, i))
                     else:
                         if aln:
                             lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                         else:
                             lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
         else:
             lines = []
             for i in range(fl, fl + lncount):
                 if i % st == 0:
                     if aln:
                         lines.append('<a href="#%s%d">%*d</a>' % (la, i, mw, i))
                     else:
                         lines.append('%*d' % (mw, i))
                 else:
                     lines.append('')
             ls = '\n'.join(lines)
 
         # in case you wonder about the seemingly redundant <div> here: since the
         # content in the other cell also is wrapped in a div, some browsers in
         # some configurations seem to mess up the formatting...
         if nocls:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td><div class="linenodiv" '
                       'style="background-color: #f0f0f0; padding-right: 10px">'
                       '<pre style="line-height: 125%">' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         else:
             yield 0, ('<table class="%stable">' % self.cssclass +
                       '<tr><td class="linenos"><div class="linenodiv"><pre>' +
                       ls + '</pre></div></td><td id="hlcode" class="code">')
         yield 0, dummyoutfile.getvalue()
         yield 0, '</td></tr></table>'
 
 
 class SearchContentCodeHtmlFormatter(CodeHtmlFormatter):
     def __init__(self, **kw):
         # only show these line numbers if set
         self.only_lines = kw.pop('only_line_numbers', [])
         self.query_terms = kw.pop('query_terms', [])
         self.max_lines = kw.pop('max_lines', 5)
         self.line_context = kw.pop('line_context', 3)
         self.url = kw.pop('url', None)
 
         super(CodeHtmlFormatter, self).__init__(**kw)
 
     def _wrap_code(self, source):
         for cnt, it in enumerate(source):
             i, t = it
             t = '<pre>%s</pre>' % t
             yield i, t
 
     def _wrap_tablelinenos(self, inner):
         yield 0, '<table class="code-highlight %stable">' % self.cssclass
 
         last_shown_line_number = 0
         current_line_number = 1
 
         for t, line in inner:
             if not t:
                 yield t, line
                 continue
 
             if current_line_number in self.only_lines:
                 if last_shown_line_number + 1 != current_line_number:
                     yield 0, '<tr>'
                     yield 0, '<td class="line">...</td>'
                     yield 0, '<td id="hlcode" class="code"></td>'
                     yield 0, '</tr>'
 
                 yield 0, '<tr>'
                 if self.url:
                     yield 0, '<td class="line"><a href="%s#L%i">%i</a></td>' % (
                         self.url, current_line_number, current_line_number)
                 else:
                     yield 0, '<td class="line"><a href="">%i</a></td>' % (
                         current_line_number)
                 yield 0, '<td id="hlcode" class="code">' + line + '</td>'
                 yield 0, '</tr>'
 
                 last_shown_line_number = current_line_number
 
             current_line_number += 1
 
         yield 0, '</table>'
 
 
 def hsv_to_rgb(h, s, v):
     """ Convert hsv color values to rgb """
 
     if s == 0.0:
         return v, v, v
     i = int(h * 6.0)  # XXX assume int() truncates!
     f = (h * 6.0) - i
     p = v * (1.0 - s)
     q = v * (1.0 - s * f)
     t = v * (1.0 - s * (1.0 - f))
     i = i % 6
     if i == 0:
         return v, t, p
     if i == 1:
         return q, v, p
     if i == 2:
         return p, v, t
     if i == 3:
         return p, q, v
     if i == 4:
         return t, p, v
     if i == 5:
         return v, p, q
 
 
 def unique_color_generator(n=10000, saturation=0.10, lightness=0.95):
     """
     Generator for getting n of evenly distributed colors using
     hsv color and golden ratio. It always return same order of colors
 
     :param n: number of colors to generate
     :param saturation: saturation of returned colors
     :param lightness: lightness of returned colors
     :returns: RGB tuple
     """
 
     golden_ratio = 0.618033988749895
     h = 0.22717784590367374
 
     for _ in range(n):
         h += golden_ratio
         h %= 1
         HSV_tuple = [h, saturation, lightness]
         RGB_tuple = hsv_to_rgb(*HSV_tuple)
         yield [str(int(x * 256)) for x in RGB_tuple]
 
 
 def color_hasher(n=10000, saturation=0.10, lightness=0.95):
     """
     Returns a function which when called with an argument returns a unique
     color for that argument, eg.
 
     :param n: number of colors to generate
     :param saturation: saturation of returned colors
     :param lightness: lightness of returned colors
     :returns: css RGB string
 
     >>> color_hash = color_hasher()
     >>> color_hash('hello')
     'rgb(34, 12, 59)'
     >>> color_hash('hello')
     'rgb(34, 12, 59)'
     >>> color_hash('other')
     'rgb(90, 224, 159)'
     """
 
     color_dict = {}
     cgenerator = unique_color_generator(
         saturation=saturation, lightness=lightness)
 
     def get_color_string(thing):
         if thing in color_dict:
             col = color_dict[thing]
         else:
             col = color_dict[thing] = next(cgenerator)
         return "rgb(%s)" % (', '.join(col))
 
     return get_color_string
 
 
 def get_lexer_safe(mimetype=None, filepath=None):
     """
     Tries to return a relevant pygments lexer using mimetype/filepath name,
     defaulting to plain text if none could be found
     """
     lexer = None
     try:
         if mimetype:
             lexer = get_lexer_for_mimetype(mimetype)
         if not lexer:
             lexer = get_lexer_for_filename(filepath)
     except pygments.util.ClassNotFound:
         pass
 
     if not lexer:
         lexer = get_lexer_by_name('text')
 
     return lexer
 
 
 def get_lexer_for_filenode(filenode):
     lexer = get_custom_lexer(filenode.extension) or filenode.lexer
     return lexer
 
 
 def pygmentize(filenode, **kwargs):
     """
     pygmentize function using pygments
 
     :param filenode:
     """
     lexer = get_lexer_for_filenode(filenode)
     return literal(code_highlight(filenode.content, lexer,
                                   CodeHtmlFormatter(**kwargs)))
 
 
 def is_following_repo(repo_name, user_id):
     from rhodecode.model.scm import ScmModel
     return ScmModel().is_following_repo(repo_name, user_id)
 
 
 class _Message(object):
     """A message returned by ``Flash.pop_messages()``.
 
     Converting the message to a string returns the message text. Instances
     also have the following attributes:
 
     * ``message``: the message text.
     * ``category``: the category specified when the message was created.
     """
 
     def __init__(self, category, message, sub_data=None):
         self.category = category
         self.message = message
         self.sub_data = sub_data or {}
 
     def __str__(self):
         return self.message
 
     __unicode__ = __str__
 
     def __html__(self):
         return escape(safe_str(self.message))
 
 
 class Flash(object):
     # List of allowed categories.  If None, allow any category.
     categories = ["warning", "notice", "error", "success"]
 
     # Default category if none is specified.
     default_category = "notice"
 
     def __init__(self, session_key="flash", categories=None,
                  default_category=None):
         """
         Instantiate a ``Flash`` object.
 
         ``session_key`` is the key to save the messages under in the user's
         session.
 
         ``categories`` is an optional list which overrides the default list
         of categories.
 
         ``default_category`` overrides the default category used for messages
         when none is specified.
         """
         self.session_key = session_key
         if categories is not None:
             self.categories = categories
         if default_category is not None:
             self.default_category = default_category
         if self.categories and self.default_category not in self.categories:
             raise ValueError(
                 "unrecognized default category %r" % (self.default_category,))
 
     def pop_messages(self, session=None, request=None):
         """
         Return all accumulated messages and delete them from the session.
 
         The return value is a list of ``Message`` objects.
         """
         messages = []
 
         if not session:
             if not request:
                 request = get_current_request()
             session = request.session
 
         # Pop the 'old' pylons flash messages. They are tuples of the form
         # (category, message)
         for cat, msg in session.pop(self.session_key, []):
             messages.append(_Message(cat, msg))
 
         # Pop the 'new' pyramid flash messages for each category as list
         # of strings.
         for cat in self.categories:
             for msg in session.pop_flash(queue=cat):
                 sub_data = {}
                 if hasattr(msg, 'rsplit'):
                     flash_data = msg.rsplit('|DELIM|', 1)
                     org_message = flash_data[0]
                     if len(flash_data) > 1:
                         sub_data = json.loads(flash_data[1])
                 else:
                     org_message = msg
 
                 messages.append(_Message(cat, org_message, sub_data=sub_data))
 
         # Map messages from the default queue to the 'notice' category.
         for msg in session.pop_flash():
             messages.append(_Message('notice', msg))
 
         session.save()
         return messages
 
     def json_alerts(self, session=None, request=None):
         payloads = []
         messages = flash.pop_messages(session=session, request=request) or []
         for message in messages:
             payloads.append({
                 'message': {
                     'message': '{}'.format(message.message),
                     'level': message.category,
                     'force': True,
                     'subdata': message.sub_data
                 }
             })
         return safe_str(json.dumps(payloads))
 
     def __call__(self, message, category=None, ignore_duplicate=True,
                  session=None, request=None):
 
         if not session:
             if not request:
                 request = get_current_request()
             session = request.session
 
         session.flash(
             message, queue=category, allow_duplicate=not ignore_duplicate)
 
 
 flash = Flash()
 
 #==============================================================================
 # SCM FILTERS available via h.
 #==============================================================================
 from rhodecode.lib.vcs.utils import author_name, author_email
 from rhodecode.lib.utils2 import age, age_from_seconds
 from rhodecode.model.db import User, ChangesetStatus
 
 
 email = author_email
 
 
 def capitalize(raw_text):
     return raw_text.capitalize()
 
 
 def short_id(long_id):
     return long_id[:12]
 
 
 def hide_credentials(url):
     from rhodecode.lib.utils2 import credentials_filter
     return credentials_filter(url)
 
 import zoneinfo
 import tzlocal
 local_timezone = tzlocal.get_localzone()
 
 
 def get_timezone(datetime_iso, time_is_local=False):
     tzinfo = '+00:00'
 
     # detect if we have a timezone info, otherwise, add it
     if time_is_local and isinstance(datetime_iso, datetime) and not datetime_iso.tzinfo:
         force_timezone = os.environ.get('RC_TIMEZONE', '')
         if force_timezone:
             force_timezone = zoneinfo.ZoneInfo(force_timezone)
         timezone = force_timezone or local_timezone
 
         offset = datetime_iso.replace(tzinfo=timezone).strftime('%z')
         tzinfo = '{}:{}'.format(offset[:-2], offset[-2:])
     return tzinfo
 
 
 def age_component(datetime_iso, value=None, time_is_local=False, tooltip=True):
     title = value or format_date(datetime_iso)
     tzinfo = get_timezone(datetime_iso, time_is_local=time_is_local)
 
     return literal(
         '<time class="timeago {cls}" title="{tt_title}" datetime="{dt}{tzinfo}">{title}</time>'.format(
             cls='tooltip' if tooltip else '',
             tt_title=('{title}{tzinfo}'.format(title=title, tzinfo=tzinfo)) if tooltip else '',
             title=title, dt=datetime_iso, tzinfo=tzinfo
         ))
 
 
 def _shorten_commit_id(commit_id, commit_len=None):
     if commit_len is None:
         request = get_current_request()
         commit_len = request.call_context.visual.show_sha_length
     return commit_id[:commit_len]
 
 
 def show_id(commit, show_idx=None, commit_len=None):
     """
     Configurable function that shows ID
     by default it's r123:fffeeefffeee
 
     :param commit: commit instance
     """
     if show_idx is None:
         request = get_current_request()
         show_idx = request.call_context.visual.show_revision_number
 
     raw_id = _shorten_commit_id(commit.raw_id, commit_len=commit_len)
     if show_idx:
         return 'r%s:%s' % (commit.idx, raw_id)
     else:
         return '%s' % (raw_id, )
 
 
 def format_date(date):
     """
     use a standardized formatting for dates used in RhodeCode
 
     :param date: date/datetime object
     :return: formatted date
     """
 
     if date:
         _fmt = "%a, %d %b %Y %H:%M:%S"
         return safe_str(date.strftime(_fmt))
 
     return ""
 
 
 class _RepoChecker(object):
 
     def __init__(self, backend_alias):
         self._backend_alias = backend_alias
 
     def __call__(self, repository):
         if hasattr(repository, 'alias'):
             _type = repository.alias
         elif hasattr(repository, 'repo_type'):
             _type = repository.repo_type
         else:
             _type = repository
         return _type == self._backend_alias
 
 
 is_git = _RepoChecker('git')
 is_hg = _RepoChecker('hg')
 is_svn = _RepoChecker('svn')
 
 
 def get_repo_type_by_name(repo_name):
     repo = Repository.get_by_repo_name(repo_name)
     if repo:
         return repo.repo_type
 
 
 def is_svn_without_proxy(repository):
     if is_svn(repository):
         return not ConfigGet().get_bool('vcs.svn.proxy.enabled')
     return False
 
 
 def discover_user(author):
     """
     Tries to discover RhodeCode User based on the author string. Author string
     is typically `FirstName LastName <[e-mail]>`
     """
 
     # if author is already an instance use it for extraction
     if isinstance(author, User):
         return author
 
     # Valid email in the attribute passed, see if they're in the system
     _email = author_email(author)
     if _email != '':
         user = User.get_by_email(_email, case_insensitive=True, cache=True)
         if user is not None:
             return user
 
     # Maybe it's a username, we try to extract it and fetch by username ?
     _author = author_name(author)
     user = User.get_by_username(_author, case_insensitive=True, cache=True)
     if user is not None:
         return user
 
     return None
 
 
 def email_or_none(author):
     # extract email from the commit string
     _email = author_email(author)
 
     # If we have an email, use it, otherwise
     # see if it contains a username we can get an email from
     if _email != '':
         return _email
     else:
         user = User.get_by_username(
             author_name(author), case_insensitive=True, cache=True)
 
     if user is not None:
             return user.email
 
     # No valid email, not a valid user in the system, none!
     return None
 
 
 def link_to_user(author, length=0, **kwargs):
     user = discover_user(author)
     # user can be None, but if we have it already it means we can re-use it
     # in the person() function, so we save 1 intensive-query
     if user:
         author = user
 
     display_person = person(author, 'username_or_name_or_email')
     if length:
         display_person = shorter(display_person, length)
 
     if user and user.username != user.DEFAULT_USER:
         return link_to(
             escape(display_person),
             route_path('user_profile', username=user.username),
             **kwargs)
     else:
         return escape(display_person)
 
 
 def link_to_group(users_group_name, **kwargs):
     return link_to(
         escape(users_group_name),
         route_path('user_group_profile', user_group_name=users_group_name),
         **kwargs)
 
 
 def person(author, show_attr="username_and_name"):
     user = discover_user(author)
     if user:
         return getattr(user, show_attr)
     else:
         _author = author_name(author)
         _email = email(author)
         return _author or _email
 
 
 def author_string(email):
     if email:
         user = User.get_by_email(email, case_insensitive=True, cache=True)
         if user:
             if user.first_name or user.last_name:
                 return '%s %s &lt;%s&gt;' % (
                     user.first_name, user.last_name, email)
             else:
                 return email
         else:
             return email
     else:
         return None
 
 
 def person_by_id(id_, show_attr="username_and_name"):
     # attr to return from fetched user
     def person_getter(usr):
         return getattr(usr, show_attr)
 
     #maybe it's an ID ?
     if str(id_).isdigit() or isinstance(id_, int):
         id_ = int(id_)
         user = User.get(id_)
         if user is not None:
             return person_getter(user)
     return id_
 
 
 def gravatar_with_user(request, author, show_disabled=False, tooltip=False):
     _render = request.get_partial_renderer('rhodecode:templates/base/base.mako')
     return _render('gravatar_with_user', author, show_disabled=show_disabled, tooltip=tooltip)
 
 
 tags_patterns = OrderedDict(
     (
         (
             "lang",
             (
                 re.compile(r"\[(lang|language)\ \=\&gt;\ *([a-zA-Z\-\/\#\+\.]*)\]"),
                 '<div class="metatag" tag="lang">\\2</div>',
             ),
         ),
         (
             "see",
             (
                 re.compile(r"\[see\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]"),
                 '<div class="metatag" tag="see">see: \\1 </div>',
             ),
         ),
         (
             "url",
             (
                 re.compile(
                     r"\[url\ \=\&gt;\ \[([a-zA-Z0-9\ \.\-\_]+)\]\((http://|https://|/)(.*?)\)\]"
                 ),
                 '<div class="metatag" tag="url"> <a href="\\2\\3">\\1</a> </div>',
             ),
         ),
         (
             "license",
             (
                 re.compile(
                     r"\[license\ \=\&gt;\ *([a-zA-Z0-9\/\=\?\&amp;\ \:\/\.\-]*)\]"
                 ),
                 # don't make it a raw string here...
                 '<div class="metatag" tag="license"><a href="http:\/\/www.opensource.org/licenses/\\1">\\1</a></div>',
             ),
         ),
         (
             "ref",
             (
                 re.compile(
                     r"\[(requires|recommends|conflicts|base)\ \=\&gt;\ *([a-zA-Z0-9\-\/]*)\]"
                 ),
                 '<div class="metatag" tag="ref \\1">\\1: <a href="/\\2">\\2</a></div>',
             ),
         ),
         (
             "state",
             (
                 re.compile(r"\[(stable|featured|stale|dead|dev|deprecated)\]"),
                 '<div class="metatag" tag="state \\1">\\1</div>',
             ),
         ),
         # label in grey
         (
             "label",
             (re.compile(r"\[([a-z]+)\]"), '<div class="metatag" tag="label">\\1</div>'),
         ),
         # generic catch all in grey
         (
             "generic",
             (
                 re.compile(r"\[([a-zA-Z0-9\.\-\_]+)\]"),
                 '<div class="metatag" tag="generic">\\1</div>',
             ),
         ),
     )
 )
 
 
 def extract_metatags(value):
     """
     Extract supported meta-tags from given text value
     """
     tags = []
     if not value:
         return tags, ''
 
     for key, val in list(tags_patterns.items()):
         pat, replace_html = val
         tags.extend([(key, x.group()) for x in pat.finditer(value)])
         value = pat.sub('', value)
 
     return tags, value
 
 
 def style_metatag(tag_type, value):
     """
     converts tags from value into html equivalent
     """
     if not value:
         return ''
 
     html_value = value
     tag_data = tags_patterns.get(tag_type)
     if tag_data:
         pat, replace_html = tag_data
         # convert to plain `str` instead of a markup tag to be used in
         # regex expressions. safe_str doesn't work here
         html_value = pat.sub(replace_html, value)
 
     return html_value
 
 
 def bool2icon(value, show_at_false=True):
     """
     Returns boolean value of a given value, represented as html element with
     classes that will represent icons
 
     :param value: given value to convert to html node
     """
 
     if value:  # does bool conversion
         return HTML.tag('i', class_="icon-true", title='True')
     else:  # not true as bool
         if show_at_false:
             return HTML.tag('i', class_="icon-false", title='False')
         return HTML.tag('i')
 
 
 def b64(inp):
     return base64.b64encode(safe_bytes(inp))
 
 #==============================================================================
 # PERMS
 #==============================================================================
 from rhodecode.lib.auth import (
     HasPermissionAny, HasPermissionAll,
     HasRepoPermissionAny, HasRepoPermissionAll, HasRepoGroupPermissionAll,
     HasRepoGroupPermissionAny, HasRepoPermissionAnyApi, get_csrf_token,
     csrf_token_key, AuthUser)
 
 
 #==============================================================================
 # GRAVATAR URL
 #==============================================================================
 class InitialsGravatar(object):
     def __init__(self, email_address, first_name, last_name, size=30,
                  background=None, text_color='#fff'):
         self.size = size
         self.first_name = first_name
         self.last_name = last_name
         self.email_address = email_address
         self.background = background or self.str2color(email_address)
         self.text_color = text_color
 
     def get_color_bank(self):
         """
         returns a predefined list of colors that gravatars can use.
         Those are randomized distinct colors that guarantee readability and
         uniqueness.
 
         generated with: http://phrogz.net/css/distinct-colors.html
         """
         return [
             '#bf3030', '#a67f53', '#00ff00', '#5989b3', '#392040', '#d90000',
             '#402910', '#204020', '#79baf2', '#a700b3', '#bf6060', '#7f5320',
             '#008000', '#003059', '#ee00ff', '#ff0000', '#8c4b00', '#007300',
             '#005fb3', '#de73e6', '#ff4040', '#ffaa00', '#3df255', '#203140',
             '#47004d', '#591616', '#664400', '#59b365', '#0d2133', '#83008c',
             '#592d2d', '#bf9f60', '#73e682', '#1d3f73', '#73006b', '#402020',
             '#b2862d', '#397341', '#597db3', '#e600d6', '#a60000', '#736039',
             '#00b318', '#79aaf2', '#330d30', '#ff8080', '#403010', '#16591f',
             '#002459', '#8c4688', '#e50000', '#ffbf40', '#00732e', '#102340',
             '#bf60ac', '#8c4646', '#cc8800', '#00a642', '#1d3473', '#b32d98',
             '#660e00', '#ffd580', '#80ffb2', '#7391e6', '#733967', '#d97b6c',
             '#8c5e00', '#59b389', '#3967e6', '#590047', '#73281d', '#665200',
             '#00e67a', '#2d50b3', '#8c2377', '#734139', '#b2982d', '#16593a',
             '#001859', '#ff00aa', '#a65e53', '#ffcc00', '#0d3321', '#2d3959',
             '#731d56', '#401610', '#4c3d00', '#468c6c', '#002ca6', '#d936a3',
             '#d94c36', '#403920', '#36d9a3', '#0d1733', '#592d4a', '#993626',
             '#cca300', '#00734d', '#46598c', '#8c005e', '#7f1100', '#8c7000',
             '#00a66f', '#7382e6', '#b32d74', '#d9896c', '#ffe680', '#1d7362',
             '#364cd9', '#73003d', '#d93a00', '#998a4d', '#59b3a1', '#5965b3',
             '#e5007a', '#73341d', '#665f00', '#00b38f', '#0018b3', '#59163a',
             '#b2502d', '#bfb960', '#00ffcc', '#23318c', '#a6537f', '#734939',
             '#b2a700', '#104036', '#3d3df2', '#402031', '#e56739', '#736f39',
             '#79f2ea', '#000059', '#401029', '#4c1400', '#ffee00', '#005953',
             '#101040', '#990052', '#402820', '#403d10', '#00ffee', '#0000d9',
             '#ff80c4', '#a66953', '#eeff00', '#00ccbe', '#8080ff', '#e673a1',
             '#a62c00', '#474d00', '#1a3331', '#46468c', '#733950', '#662900',
             '#858c23', '#238c85', '#0f0073', '#b20047', '#d9986c', '#becc00',
             '#396f73', '#281d73', '#ff0066', '#ff6600', '#dee673', '#59adb3',
             '#6559b3', '#590024', '#b2622d', '#98b32d', '#36ced9', '#332d59',
             '#40001a', '#733f1d', '#526600', '#005359', '#242040', '#bf6079',
             '#735039', '#cef23d', '#007780', '#5630bf', '#66001b', '#b24700',
             '#acbf60', '#1d6273', '#25008c', '#731d34', '#a67453', '#50592d',
             '#00ccff', '#6600ff', '#ff0044', '#4c1f00', '#8a994d', '#79daf2',
             '#a173e6', '#d93662', '#402310', '#aaff00', '#2d98b3', '#8c40ff',
             '#592d39', '#ff8c40', '#354020', '#103640', '#1a0040', '#331a20',
             '#331400', '#334d00', '#1d5673', '#583973', '#7f0022', '#4c3626',
             '#88cc00', '#36a3d9', '#3d0073', '#d9364c', '#33241a', '#698c23',
             '#5995b3', '#300059', '#e57382', '#7f3300', '#366600', '#00aaff',
             '#3a1659', '#733941', '#663600', '#74b32d', '#003c59', '#7f53a6',
             '#73000f', '#ff8800', '#baf279', '#79caf2', '#291040', '#a6293a',
             '#b2742d', '#587339', '#0077b3', '#632699', '#400009', '#d9a66c',
             '#294010', '#2d4a59', '#aa00ff', '#4c131b', '#b25f00', '#5ce600',
             '#267399', '#a336d9', '#990014', '#664e33', '#86bf60', '#0088ff',
             '#7700b3', '#593a16', '#073300', '#1d4b73', '#ac60bf', '#e59539',
             '#4f8c46', '#368dd9', '#5c0073'
         ]
 
     def rgb_to_hex_color(self, rgb_tuple):
         """
         Converts an rgb_tuple passed to an hex color.
 
         :param rgb_tuple: tuple with 3 ints represents rgb color space
         """
         return '#' + ("".join(map(chr, rgb_tuple)).encode('hex'))
 
     def email_to_int_list(self, email_str):
         """
         Get every byte of the hex digest value of email and turn it to integer.
         It's going to be always between 0-255
         """
         digest = md5_safe(email_str.lower())
         return [int(digest[i * 2:i * 2 + 2], 16) for i in range(16)]
 
     def pick_color_bank_index(self, email_str, color_bank):
         return self.email_to_int_list(email_str)[0] % len(color_bank)
 
     def str2color(self, email_str):
         """
         Tries to map in a stable algorithm an email to color
 
         :param email_str:
         """
         color_bank = self.get_color_bank()
         # pick position (module it's length so we always find it in the
         # bank even if it's smaller than 256 values
         pos = self.pick_color_bank_index(email_str, color_bank)
         return color_bank[pos]
 
     def normalize_email(self, email_address):
         # default host used to fill in the fake/missing email
         default_host = 'localhost'
 
         if not email_address:
             email_address = f'{User.DEFAULT_USER}@{default_host}'
 
         email_address = safe_str(email_address)
 
         if '@' not in email_address:
             email_address = f'{email_address}@{default_host}'
 
         if email_address.endswith('@'):
             email_address = f'{email_address}{default_host}'
 
         email_address = convert_special_chars(email_address)
 
         return email_address
 
     def get_initials(self):
         """
         Returns 2 letter initials calculated based on the input.
         The algorithm picks first given email address, and takes first letter
         of part before @, and then the first letter of server name. In case
         the part before @ is in a format of `somestring.somestring2` it replaces
         the server letter with first letter of somestring2
 
         In case function was initialized with both first and lastname, this
         overrides the extraction from email by first letter of the first and
         last name. We add special logic to that functionality, In case Full name
         is compound, like Guido Von Rossum, we use last part of the last name
         (Von Rossum) picking `R`.
 
         Function also normalizes the non-ascii characters to they ascii
         representation, eg Ą => A
         """
         # replace non-ascii to ascii
         first_name = convert_special_chars(self.first_name)
         last_name = convert_special_chars(self.last_name)
         # multi word last names, Guido Von Rossum, we take the last part only
         last_name = last_name.split(' ', 1)[-1]
 
         # do NFKD encoding, and also make sure email has proper format
         email_address = self.normalize_email(self.email_address)
 
         # first push the email initials
         prefix, server = email_address.split('@', 1)
 
         # check if prefix is maybe a 'first_name.last_name' syntax
         _dot_split = prefix.rsplit('.', 1)
         if len(_dot_split) == 2 and _dot_split[1]:
             initials = [_dot_split[0][0], _dot_split[1][0]]
         else:
             initials = [prefix[0], server[0]]
 
         # get first letter of first and last names to create initials
         fn_letter = (first_name or " ")[0].strip()
         ln_letter = (last_name or " ")[0].strip()
 
         if fn_letter:
             initials[0] = fn_letter
 
         if ln_letter:
             initials[1] = ln_letter
 
         return ''.join(initials).upper()
 
     def get_img_data_by_type(self, font_family, img_type):
         default_user = """
         <svg xmlns="http://www.w3.org/2000/svg"
         version="1.1" x="0px" y="0px" width="{size}" height="{size}"
         viewBox="-15 -10 439.165 429.164"
 
         xml:space="preserve"
         font-family="{font_family}"
         style="background:{background};" >
 
         <path d="M204.583,216.671c50.664,0,91.74-48.075,
                  91.74-107.378c0-82.237-41.074-107.377-91.74-107.377
                  c-50.668,0-91.74,25.14-91.74,107.377C112.844,
                  168.596,153.916,216.671,
                  204.583,216.671z" fill="{text_color}"/>
         <path d="M407.164,374.717L360.88,
                  270.454c-2.117-4.771-5.836-8.728-10.465-11.138l-71.83-37.392
                  c-1.584-0.823-3.502-0.663-4.926,0.415c-20.316,
                  15.366-44.203,23.488-69.076,23.488c-24.877,
                  0-48.762-8.122-69.078-23.488
                  c-1.428-1.078-3.346-1.238-4.93-0.415L58.75,
                  259.316c-4.631,2.41-8.346,6.365-10.465,11.138L2.001,374.717
                  c-3.191,7.188-2.537,15.412,1.75,22.005c4.285,
                  6.592,11.537,10.526,19.4,10.526h362.861c7.863,0,15.117-3.936,
                  19.402-10.527 C409.699,390.129,
                  410.355,381.902,407.164,374.717z" fill="{text_color}"/>
         </svg>""".format(
             size=self.size,
             background='#979797',  # @grey4
             text_color=self.text_color,
             font_family=font_family)
 
         return {
             "default_user": default_user
         }[img_type]
 
     def get_img_data(self, svg_type=None):
         """
         generates the svg metadata for image
         """
         fonts = [
             '-apple-system',
              'BlinkMacSystemFont',
              'Segoe UI',
              'Roboto',
              'Oxygen-Sans',
              'Ubuntu',
              'Cantarell',
              'Helvetica Neue',
              'sans-serif'
         ]
         font_family = ','.join(fonts)
         if svg_type:
             return self.get_img_data_by_type(font_family, svg_type)
 
         initials = self.get_initials()
         img_data = """
         <svg xmlns="http://www.w3.org/2000/svg" pointer-events="none"
              width="{size}" height="{size}"
              style="width: 100%; height: 100%; background-color: {background}"
              viewBox="0 0 {size} {size}">
             <text text-anchor="middle" y="50%" x="50%" dy="0.35em"
                   pointer-events="auto" fill="{text_color}"
                   font-family="{font_family}"
                   style="font-weight: 400; font-size: {f_size}px;">{text}
             </text>
         </svg>""".format(
             size=self.size,
             f_size=self.size/2.05,  # scale the text inside the box nicely
             background=self.background,
             text_color=self.text_color,
             text=initials.upper(),
             font_family=font_family)
 
         return img_data
 
     def generate_svg(self, svg_type=None):
         img_data = base64_to_str(self.get_img_data(svg_type))
         return "data:image/svg+xml;base64,{}".format(img_data)
 
 
-def initials_gravatar(request, email_address, first_name, last_name, size=30, store_on_disk=False):
+def initials_gravatar(request, email_address, first_name, last_name, size=30):
 
     svg_type = None
     if email_address == User.DEFAULT_USER_EMAIL:
         svg_type = 'default_user'
 
     klass = InitialsGravatar(email_address, first_name, last_name, size)
-
-    if store_on_disk:
-        from rhodecode.apps.file_store import utils as store_utils
-        from rhodecode.apps.file_store.exceptions import FileNotAllowedException, \
-            FileOverSizeException
-        from rhodecode.model.db import Session
-
-        image_key = md5_safe(email_address.lower()
-                             + first_name.lower() + last_name.lower())
-
-        storage = store_utils.get_file_storage(request.registry.settings)
-        filename = '{}.svg'.format(image_key)
-        subdir = 'gravatars'
-        # since final name has a counter, we apply the 0
-        uid = storage.apply_counter(0, store_utils.uid_filename(filename, randomized=False))
-        store_uid = os.path.join(subdir, uid)
-
-        db_entry = FileStore.get_by_store_uid(store_uid)
-        if db_entry:
-            return request.route_path('download_file', fid=store_uid)
-
-        img_data = klass.get_img_data(svg_type=svg_type)
-        img_file = store_utils.bytes_to_file_obj(img_data)
-
-        try:
-            store_uid, metadata = storage.save_file(
-                img_file, filename, directory=subdir,
-                extensions=['.svg'], randomized_name=False)
-        except (FileNotAllowedException, FileOverSizeException):
-            raise
-
-        try:
-            entry = FileStore.create(
-                file_uid=store_uid, filename=metadata["filename"],
-                file_hash=metadata["sha256"], file_size=metadata["size"],
-                file_display_name=filename,
-                file_description=f'user gravatar `{safe_str(filename)}`',
-                hidden=True, check_acl=False, user_id=1
-            )
-            Session().add(entry)
-            Session().commit()
-            log.debug('Stored upload in DB as %s', entry)
-        except Exception:
-            raise
-
-        return request.route_path('download_file', fid=store_uid)
-
-    else:
-        return klass.generate_svg(svg_type=svg_type)
+    return klass.generate_svg(svg_type=svg_type)
 
 
 def gravatar_external(request, gravatar_url_tmpl, email_address, size=30):
     return safe_str(gravatar_url_tmpl)\
         .replace('{email}', email_address) \
         .replace('{md5email}', md5_safe(email_address.lower())) \
         .replace('{netloc}', request.host) \
         .replace('{scheme}', request.scheme) \
         .replace('{size}', safe_str(size))
 
 
 def gravatar_url(email_address, size=30, request=None):
     request = request or get_current_request()
     _use_gravatar = request.call_context.visual.use_gravatar
 
     email_address = email_address or User.DEFAULT_USER_EMAIL
     if isinstance(email_address, str):
         # hashlib crashes on unicode items
         email_address = safe_str(email_address)
 
     # empty email or default user
     if not email_address or email_address == User.DEFAULT_USER_EMAIL:
         return initials_gravatar(request, User.DEFAULT_USER_EMAIL, '', '', size=size)
 
     if _use_gravatar:
         gravatar_url_tmpl = request.call_context.visual.gravatar_url \
                             or User.DEFAULT_GRAVATAR_URL
         return gravatar_external(request, gravatar_url_tmpl, email_address, size=size)
 
     else:
         return initials_gravatar(request, email_address, '', '', size=size)
 
 
 def breadcrumb_repo_link(repo):
     """
     Makes a breadcrumbs path link to repo
 
     ex::
         group >> subgroup >> repo
 
     :param repo: a Repository instance
     """
 
     path = [
         link_to(group.name, route_path('repo_group_home', repo_group_name=group.group_name),
                 title='last change:{}'.format(format_date(group.last_commit_change)))
         for group in repo.groups_with_parents
     ] + [
         link_to(repo.just_name, route_path('repo_summary', repo_name=repo.repo_name),
                 title='last change:{}'.format(format_date(repo.last_commit_change)))
     ]
 
     return literal(' &raquo; '.join(path))
 
 
 def breadcrumb_repo_group_link(repo_group):
     """
     Makes a breadcrumbs path link to repo
 
     ex::
         group >> subgroup
 
     :param repo_group: a Repository Group instance
     """
 
     path = [
         link_to(group.name,
                 route_path('repo_group_home', repo_group_name=group.group_name),
                 title='last change:{}'.format(format_date(group.last_commit_change)))
         for group in repo_group.parents
     ] + [
         link_to(repo_group.name,
                 route_path('repo_group_home', repo_group_name=repo_group.group_name),
                 title='last change:{}'.format(format_date(repo_group.last_commit_change)))
     ]
 
     return literal(' &raquo; '.join(path))
 
 
 def format_byte_size_binary(file_size):
     """
     Formats file/folder sizes to standard.
     """
     if file_size is None:
         file_size = 0
 
     formatted_size = format_byte_size(file_size, binary=True)
     return formatted_size
 
 
 def urlify_text(text_, safe=True, **href_attrs):
     """
     Extract urls from text and make html links out of them
     """
 
     url_pat = re.compile(r'''(http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@#.&+]'''
                          r'''|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+)''')
 
     def url_func(match_obj):
         url_full = match_obj.groups()[0]
         a_options = dict(href_attrs)
         a_options['href'] = url_full
         a_text = url_full
         return HTML.tag("a", a_text, **a_options)
 
     _new_text = url_pat.sub(url_func, text_)
 
     if safe:
         return literal(_new_text)
     return _new_text
 
 
 def urlify_commits(text_, repo_name):
     """
     Extract commit ids from text and make link from them
 
     :param text_:
     :param repo_name: repo name to build the URL with
     """
 
     url_pat = re.compile(r'(^|\s)([0-9a-fA-F]{12,40})($|\s)')
 
     def url_func(match_obj):
         commit_id = match_obj.groups()[1]
         pref = match_obj.groups()[0]
         suf = match_obj.groups()[2]
 
         tmpl = (
             '%(pref)s<a class="tooltip-hovercard %(cls)s" href="%(url)s" data-hovercard-alt="%(hovercard_alt)s" data-hovercard-url="%(hovercard_url)s">'
             '%(commit_id)s</a>%(suf)s'
         )
         return tmpl % {
             'pref': pref,
             'cls': 'revision-link',
             'url': route_url(
                 'repo_commit', repo_name=repo_name, commit_id=commit_id),
             'commit_id': commit_id,
             'suf': suf,
             'hovercard_alt': 'Commit: {}'.format(commit_id),
             'hovercard_url': route_url(
                 'hovercard_repo_commit', repo_name=repo_name, commit_id=commit_id)
         }
 
     new_text = url_pat.sub(url_func, text_)
 
     return new_text
 
 
 def _process_url_func(match_obj, repo_name, uid, entry,
                       return_raw_data=False, link_format='html'):
     pref = ''
     if match_obj.group().startswith(' '):
         pref = ' '
 
     issue_id = ''.join(match_obj.groups())
 
     if link_format == 'html':
         tmpl = (
             '%(pref)s<a class="tooltip %(cls)s" href="%(url)s" title="%(title)s">'
             '%(issue-prefix)s%(id-repr)s'
             '</a>')
     elif link_format == 'html+hovercard':
         tmpl = (
             '%(pref)s<a class="tooltip-hovercard %(cls)s" href="%(url)s" data-hovercard-url="%(hovercard_url)s">'
             '%(issue-prefix)s%(id-repr)s'
             '</a>')
     elif link_format in ['rst', 'rst+hovercard']:
         tmpl = '`%(issue-prefix)s%(id-repr)s <%(url)s>`_'
     elif link_format in ['markdown', 'markdown+hovercard']:
         tmpl = '[%(pref)s%(issue-prefix)s%(id-repr)s](%(url)s)'
     else:
         raise ValueError('Bad link_format:{}'.format(link_format))
 
     (repo_name_cleaned,
      parent_group_name) = RepoGroupModel()._get_group_name_and_parent(repo_name)
 
     # variables replacement
     named_vars = {
         'id': issue_id,
         'repo': repo_name,
         'repo_name': repo_name_cleaned,
         'group_name': parent_group_name,
         # set dummy keys so we always have them
         'hostname': '',
         'netloc': '',
         'scheme': ''
     }
 
     request = get_current_request()
     if request:
         # exposes, hostname, netloc, scheme
         host_data = get_host_info(request)
         named_vars.update(host_data)
 
     # named regex variables
     named_vars.update(match_obj.groupdict())
     _url = string.Template(entry['url']).safe_substitute(**named_vars)
     desc = string.Template(escape(entry['desc'])).safe_substitute(**named_vars)
     hovercard_url = string.Template(entry.get('hovercard_url', '')).safe_substitute(**named_vars)
 
     def quote_cleaner(input_str):
         """Remove quotes as it's HTML"""
         return input_str.replace('"', '')
 
     data = {
         'pref': pref,
         'cls': quote_cleaner('issue-tracker-link'),
         'url': quote_cleaner(_url),
         'id-repr': issue_id,
         'issue-prefix': entry['pref'],
         'serv': entry['url'],
         'title': sanitize_html(desc, strip=True),
         'hovercard_url': hovercard_url
     }
 
     if return_raw_data:
         return {
             'id': issue_id,
             'url': _url
         }
     return tmpl % data
 
 
 def get_active_pattern_entries(repo_name):
     repo = None
     if repo_name:
         # Retrieving repo_name to avoid invalid repo_name to explode on
         # IssueTrackerSettingsModel but still passing invalid name further down
         repo = Repository.get_by_repo_name(repo_name, cache=True)
 
     settings_model = IssueTrackerSettingsModel(repo=repo)
     active_entries = settings_model.get_settings(cache=True)
     return active_entries
 
 
 pr_pattern_re = regex.compile(r'(?:(?:^!)|(?: !))(\d+)')
 
 allowed_link_formats = [
     'html', 'rst', 'markdown', 'html+hovercard', 'rst+hovercard', 'markdown+hovercard']
 
 compile_cache = {
 
 }
 
 
 def process_patterns(text_string, repo_name, link_format='html', active_entries=None):
 
     if link_format not in allowed_link_formats:
         raise ValueError('Link format can be only one of:{} got {}'.format(
                          allowed_link_formats, link_format))
     issues_data = []
     errors = []
     new_text = text_string
 
     if active_entries is None:
         log.debug('Fetch active issue tracker patterns for repo: %s', repo_name)
         active_entries = get_active_pattern_entries(repo_name)
 
     log.debug('Got %s pattern entries to process', len(active_entries))
 
     for uid, entry in list(active_entries.items()):
 
         if not (entry['pat'] and entry['url']):
             log.debug('skipping due to missing data')
             continue
 
         log.debug('issue tracker entry: uid: `%s` PAT:%s URL:%s PREFIX:%s',
                   uid, entry['pat'], entry['url'], entry['pref'])
 
         if entry.get('pat_compiled'):
             pattern = entry['pat_compiled']
         elif entry['pat'] in compile_cache:
             pattern = compile_cache[entry['pat']]
         else:
             try:
                 pattern = regex.compile(r'%s' % entry['pat'])
             except regex.error as e:
                 regex_err = ValueError('{}:{}'.format(entry['pat'], e))
                 log.exception('issue tracker pattern: `%s` failed to compile', regex_err)
                 errors.append(regex_err)
                 continue
             compile_cache[entry['pat']] = pattern
 
         data_func = partial(
             _process_url_func, repo_name=repo_name, entry=entry, uid=uid,
             return_raw_data=True)
 
         for match_obj in pattern.finditer(text_string):
             issues_data.append(data_func(match_obj))
 
         url_func = partial(
             _process_url_func, repo_name=repo_name, entry=entry, uid=uid,
             link_format=link_format)
 
         new_text = pattern.sub(url_func, new_text)
         log.debug('processed prefix:uid `%s`', uid)
 
     # finally use global replace, eg !123 -> pr-link, those will not catch
     # if already similar pattern exists
     server_url = '${scheme}://${netloc}'
     pr_entry = {
         'pref': '!',
         'url': server_url + '/_admin/pull-requests/${id}',
         'desc': 'Pull Request !${id}',
         'hovercard_url': server_url + '/_hovercard/pull_request/${id}'
     }
     pr_url_func = partial(
         _process_url_func, repo_name=repo_name, entry=pr_entry, uid=None,
         link_format=link_format+'+hovercard')
     new_text = pr_pattern_re.sub(pr_url_func, new_text)
     log.debug('processed !pr pattern')
 
     return new_text, issues_data, errors
 
 
 def urlify_commit_message(commit_text, repository=None, active_pattern_entries=None,
                           issues_container_callback=None, error_container=None):
     """
     Parses given text message and makes proper links.
     issues are linked to given issue-server, and rest is a commit link
     """
 
     def escaper(_text):
         return _text.replace('<', '&lt;').replace('>', '&gt;')
 
     new_text = escaper(commit_text)
 
     # extract http/https links and make them real urls
     new_text = urlify_text(new_text, safe=False)
 
     # urlify commits - extract commit ids and make link out of them, if we have
     # the scope of repository present.
     if repository:
         new_text = urlify_commits(new_text, repository)
 
     # process issue tracker patterns
     new_text, issues, errors = process_patterns(
         new_text, repository or '', active_entries=active_pattern_entries)
 
     if issues_container_callback is not None:
         for issue in issues:
             issues_container_callback(issue)
 
     if error_container is not None:
         error_container.extend(errors)
 
     return literal(new_text)
 
 
 def render_binary(repo_name, file_obj):
     """
     Choose how to render a binary file
     """
 
     # unicode
     filename = file_obj.name
 
     # images
     for ext in ['*.png', '*.jpeg', '*.jpg', '*.ico', '*.gif']:
         if fnmatch.fnmatch(filename, pat=ext):
             src = route_path(
                 'repo_file_raw', repo_name=repo_name,
                 commit_id=file_obj.commit.raw_id,
                 f_path=file_obj.path)
 
             return literal(
                 '<img class="rendered-binary" alt="rendered-image" src="{}">'.format(src))
 
 
 def renderer_from_filename(filename, exclude=None):
     """
     choose a renderer based on filename, this works only for text based files
     """
 
     # ipython
     for ext in ['*.ipynb']:
         if fnmatch.fnmatch(filename, pat=ext):
             return 'jupyter'
 
     is_markup = MarkupRenderer.renderer_from_filename(filename, exclude=exclude)
     if is_markup:
         return is_markup
     return None
 
 
 def render(source, renderer='rst', mentions=False, relative_urls=None,
            repo_name=None, active_pattern_entries=None, issues_container_callback=None):
 
     def maybe_convert_relative_links(html_source):
         if relative_urls:
             return relative_links(html_source, relative_urls)
         return html_source
 
     if renderer == 'plain':
         return literal(
             MarkupRenderer.plain(source, leading_newline=False))
 
     elif renderer == 'rst':
         if repo_name:
             # process patterns on comments if we pass in repo name
             source, issues, errors = process_patterns(
                 source, repo_name, link_format='rst',
                 active_entries=active_pattern_entries)
             if issues_container_callback is not None:
                 for issue in issues:
                     issues_container_callback(issue)
 
         rendered_block = maybe_convert_relative_links(
                 MarkupRenderer.rst(source, mentions=mentions))
 
         return literal(f'<div class="rst-block">{rendered_block}</div>')
 
     elif renderer == 'markdown':
         if repo_name:
             # process patterns on comments if we pass in repo name
             source, issues, errors = process_patterns(
                 source, repo_name, link_format='markdown',
                 active_entries=active_pattern_entries)
             if issues_container_callback is not None:
                 for issue in issues:
                     issues_container_callback(issue)
 
         rendered_block = maybe_convert_relative_links(
             MarkupRenderer.markdown(source, flavored=True, mentions=mentions))
         return literal(f'<div class="markdown-block">{rendered_block}</div>')
 
     elif renderer == 'jupyter':
         rendered_block = maybe_convert_relative_links(
             MarkupRenderer.jupyter(source))
         return literal(f'<div class="ipynb">{rendered_block}</div>')
 
     # None means just show the file-source
     return None
 
 
 def commit_status(repo, commit_id):
     return ChangesetStatusModel().get_status(repo, commit_id)
 
 
 def commit_status_lbl(commit_status):
     return dict(ChangesetStatus.STATUSES).get(commit_status)
 
 
 def commit_time(repo_name, commit_id):
     repo = Repository.get_by_repo_name(repo_name)
     commit = repo.get_commit(commit_id=commit_id)
     return commit.date
 
 
 def get_permission_name(key):
     return dict(Permission.PERMS).get(key)
 
 
 def journal_filter_help(request):
     _ = request.translate
     from rhodecode.lib.audit_logger import ACTIONS
     actions = '\n'.join(textwrap.wrap(', '.join(sorted(ACTIONS.keys())), 80))
 
     return _(
         'Example filter terms:\n' +
         '     repository:vcs\n' +
         '     username:marcin\n' +
         '     username:(NOT marcin)\n' +
         '     action:*push*\n' +
         '     ip:127.0.0.1\n' +
         '     date:20120101\n' +
         '     date:[20120101100000 TO 20120102]\n' +
         '\n' +
         'Actions: {actions}\n' +
         '\n' +
         'Generate wildcards using \'*\' character:\n' +
         '     "repository:vcs*" - search everything starting with \'vcs\'\n' +
         '     "repository:*vcs*" - search for repository containing \'vcs\'\n' +
         '\n' +
         'Optional AND / OR operators in queries\n' +
         '     "repository:vcs OR repository:test"\n' +
         '     "username:test AND repository:test*"\n'
     ).format(actions=actions)
 
 
 def not_mapped_error(repo_name):
     from rhodecode.translation import _
     flash(_('%s repository is not mapped to db perhaps'
             ' it was created or renamed from the filesystem'
             ' please run the application again'
             ' in order to rescan repositories') % repo_name, category='error')
 
 
 def ip_range(ip_addr):
     from rhodecode.model.db import UserIpMap
     s, e = UserIpMap._get_ip_range(ip_addr)
     return '%s - %s' % (s, e)
 
 
 def form(url, method='post', needs_csrf_token=True, **attrs):
     """Wrapper around webhelpers.tags.form to prevent CSRF attacks."""
     if method.lower() != 'get' and needs_csrf_token:
         raise Exception(
             'Forms to POST/PUT/DELETE endpoints should have (in general) a ' +
             'CSRF token. If the endpoint does not require such token you can ' +
             'explicitly set the parameter needs_csrf_token to false.')
 
     return insecure_form(url, method=method, **attrs)
 
 
 def secure_form(form_url, method="POST", multipart=False, **attrs):
     """Start a form tag that points the action to an url. This
     form tag will also include the hidden field containing
     the auth token.
 
     The url options should be given either as a string, or as a
     ``url()`` function. The method for the form defaults to POST.
 
     Options:
 
     ``multipart``
         If set to True, the enctype is set to "multipart/form-data".
     ``method``
         The method to use when submitting the form, usually either
         "GET" or "POST". If "PUT", "DELETE", or another verb is used, a
         hidden input with name _method is added to simulate the verb
         over POST.
 
     """
 
     if 'request' in attrs:
         session = attrs['request'].session
         del attrs['request']
     else:
         raise ValueError(
             'Calling this form requires request= to be passed as argument')
 
     _form = insecure_form(form_url, method, multipart, **attrs)
     token = literal(
         '<input type="hidden" name="{}" value="{}">'.format(
             csrf_token_key, get_csrf_token(session)))
 
     return literal("%s\n%s" % (_form, token))
 
 
 def dropdownmenu(name, selected, options, enable_filter=False, **attrs):
     select_html = select(name, selected, options, **attrs)
 
     select2 = """
         <script>
             $(document).ready(function() {
                   $('#%s').select2({
                       containerCssClass: 'drop-menu %s',
                       dropdownCssClass: 'drop-menu-dropdown',
                       dropdownAutoWidth: true%s
                   });
             });
         </script>
     """
 
     filter_option = """,
                         minimumResultsForSearch: -1
     """
     input_id = attrs.get('id') or name
     extra_classes = ' '.join(attrs.pop('extra_classes', []))
     filter_enabled = "" if enable_filter else filter_option
     select_script = literal(select2 % (input_id, extra_classes, filter_enabled))
 
     return literal(select_html+select_script)
 
 
 def get_visual_attr(tmpl_context_var, attr_name):
     """
     A safe way to get a variable from visual variable of template context
 
     :param tmpl_context_var: instance of tmpl_context, usually present as `c`
     :param attr_name: name of the attribute we fetch from the c.visual
     """
     visual = getattr(tmpl_context_var, 'visual', None)
     if not visual:
         return
     else:
         return getattr(visual, attr_name, None)
 
 
 def get_last_path_part(file_node):
     if not file_node.path:
         return '/'
 
     path = safe_str(file_node.path.split('/')[-1])
     return '../' + path
 
 
 def route_url(*args, **kwargs):
     """
     Wrapper around pyramids `route_url` (fully qualified url) function.
     """
     req = get_current_request()
     return req.route_url(*args, **kwargs)
 
 
 def route_path(*args, **kwargs):
     """
     Wrapper around pyramids `route_path` function.
     """
     req = get_current_request()
     return req.route_path(*args, **kwargs)
 
 
 def route_path_or_none(*args, **kwargs):
     try:
         return route_path(*args, **kwargs)
     except KeyError:
         return None
 
 
 def current_route_path(request, **kw):
     new_args = request.GET.mixed()
     new_args.update(kw)
     return request.current_route_path(_query=new_args)
 
 
 def curl_api_example(method, args):
     args_json = json.dumps(OrderedDict([
         ('id', 1),
         ('auth_token', 'SECRET'),
         ('method', method),
         ('args', args)
     ]))
 
     return "curl {api_url} -X POST -H 'content-type:text/plain' --data-binary '{args_json}'".format(
         api_url=route_url('apiv2'),
         args_json=args_json
     )
 
 
 def api_call_example(method, args):
     """
     Generates an API call example via CURL
     """
     curl_call = curl_api_example(method, args)
 
     return literal(
         curl_call +
         "<br/><br/>SECRET can be found in <a href=\"{token_url}\">auth-tokens</a> page, "
         "and needs to be of `api calls` role."
         .format(token_url=route_url('my_account_auth_tokens')))
 
 
 def notification_description(notification, request):
     """
     Generate notification human readable description based on notification type
     """
     from rhodecode.model.notification import NotificationModel
     return NotificationModel().make_description(
         notification, translate=request.translate)
 
 
 def go_import_header(request, db_repo=None):
     """
     Creates a header for go-import functionality in Go Lang
     """
 
     if not db_repo:
         return
     if 'go-get' not in request.GET:
         return
 
     clone_url = db_repo.clone_url()
     prefix = re.split(r'^https?:\/\/', clone_url)[-1]
     # we have a repo and go-get flag,
     return literal('<meta name="go-import" content="{} {} {}">'.format(
         prefix, db_repo.repo_type, clone_url))
 
 
 def reviewer_as_json(*args, **kwargs):
     from rhodecode.apps.repository.utils import reviewer_as_json as _reviewer_as_json
     return _reviewer_as_json(*args, **kwargs)
 
 
 def get_repo_view_type(request):
     route_name = request.matched_route.name
     route_to_view_type = {
         'repo_changelog': 'commits',
         'repo_commits': 'commits',
         'repo_files': 'files',
         'repo_summary': 'summary',
         'repo_commit': 'commit'
     }
 
     return route_to_view_type.get(route_name)
 
 
 def is_active(menu_entry, selected):
     """
     Returns active class for selecting menus in templates
     <li class=${h.is_active('settings', current_active)}></li>
     """
     if not isinstance(menu_entry, list):
         menu_entry = [menu_entry]
 
     if selected in menu_entry:
         return "active"
 
 
 class IssuesRegistry(object):
     """
     issue_registry = IssuesRegistry()
     some_func(issues_callback=issues_registry(...))
     """
 
     def __init__(self):
         self.issues = []
         self.unique_issues = collections.defaultdict(lambda: [])
 
     def __call__(self, commit_dict=None):
         def callback(issue):
             if commit_dict and issue:
                 issue['commit'] = commit_dict
             self.issues.append(issue)
             self.unique_issues[issue['id']].append(issue)
         return callback
 
     def get_issues(self):
         return self.issues
 
     @property
     def issues_unique_count(self):
         return len(set(i['id'] for i in self.issues))
 
 
 def get_directory_statistics(start_path):
     """
     total_files, total_size, directory_stats = get_directory_statistics(start_path)
 
     print(f"Directory statistics for: {start_path}\n")
     print(f"Total files: {total_files}")
     print(f"Total size: {format_size(total_size)}\n")
 
     :param start_path:
     :return:
     """
 
     total_files = 0
     total_size = 0
     directory_stats = {}
 
     for dir_path, dir_names, file_names in os.walk(start_path):
         dir_size = 0
         file_count = len(file_names)
 
         for fname in file_names:
             filepath = os.path.join(dir_path, fname)
             file_size = os.path.getsize(filepath)
             dir_size += file_size
 
         directory_stats[dir_path] = {'file_count': file_count, 'size': dir_size}
         total_files += file_count
         total_size += dir_size
 
     return total_files, total_size, directory_stats

rhodecode/lib/hook_daemon/hook_module.py

@@ -1,104 +1,104 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 import traceback
 
 from rhodecode.model import meta
 
 from rhodecode.lib import hooks_base
 from rhodecode.lib.exceptions import HTTPLockedRC, HTTPBranchProtected
 from rhodecode.lib.utils2 import AttributeDict
 
 log = logging.getLogger(__name__)
 
 
 class Hooks(object):
     """
     Exposes the hooks for remote callbacks
     """
     def __init__(self, request=None, log_prefix=''):
         self.log_prefix = log_prefix
         self.request = request
 
     def repo_size(self, extras):
         log.debug("%sCalled repo_size of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.repo_size, extras)
 
     def pre_pull(self, extras):
         log.debug("%sCalled pre_pull of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.pre_pull, extras)
 
     def post_pull(self, extras):
         log.debug("%sCalled post_pull of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.post_pull, extras)
 
     def pre_push(self, extras):
         log.debug("%sCalled pre_push of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.pre_push, extras)
 
     def post_push(self, extras):
         log.debug("%sCalled post_push of %s object", self.log_prefix, self)
         return self._call_hook(hooks_base.post_push, extras)
 
     def _call_hook(self, hook, extras):
         extras = AttributeDict(extras)
         _server_url = extras['server_url']
 
         extras.request = self.request
 
         try:
             result = hook(extras)
             if result is None:
                 raise Exception(f'Failed to obtain hook result from func: {hook}')
-        except HTTPBranchProtected as handled_error:
+        except HTTPBranchProtected as error:
             # Those special cases don't need error reporting. It's a case of
             # locked repo or protected branch
             result = AttributeDict({
-                'status': handled_error.code,
-                'output': handled_error.explanation
+                'status': error.code,
+                'output': error.explanation
             })
         except (HTTPLockedRC, Exception) as error:
             # locked needs different handling since we need to also
             # handle PULL operations
             exc_tb = ''
             if not isinstance(error, HTTPLockedRC):
                 exc_tb = traceback.format_exc()
                 log.exception('%sException when handling hook %s', self.log_prefix, hook)
             error_args = error.args
             return {
                 'status': 128,
                 'output': '',
                 'exception': type(error).__name__,
                 'exception_traceback': exc_tb,
                 'exception_args': error_args,
             }
         finally:
             meta.Session.remove()
 
         log.debug('%sGot hook call response %s', self.log_prefix, result)
         return {
             'status': result.status,
             'output': result.output,
         }
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         pass

rhodecode/lib/hooks_base.py

@@ -1,534 +1,550 b''
 # Copyright (C) 2013-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Set of hooks run by RhodeCode Enterprise
 """
 
 import os
 import logging
 
 import rhodecode
 from rhodecode import events
 from rhodecode.lib import helpers as h
 from rhodecode.lib import audit_logger
 from rhodecode.lib.utils2 import safe_str, user_agent_normalizer
 from rhodecode.lib.exceptions import (
-    HTTPLockedRC, HTTPBranchProtected, UserCreationError)
+    HTTPLockedRC, HTTPBranchProtected, UserCreationError, ClientNotSupportedError)
 from rhodecode.model.db import Repository, User
 from rhodecode.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
 
 
 class HookResponse(object):
     def __init__(self, status, output):
         self.status = status
         self.output = output
 
     def __add__(self, other):
         other_status = getattr(other, 'status', 0)
         new_status = max(self.status, other_status)
         other_output = getattr(other, 'output', '')
         new_output = self.output + other_output
 
         return HookResponse(new_status, new_output)
 
     def __bool__(self):
         return self.status == 0
 
     def to_json(self):
         return {'status': self.status, 'output': self.output}
 
 
 def is_shadow_repo(extras):
     """
     Returns ``True`` if this is an action executed against a shadow repository.
     """
     return extras['is_shadow_repo']
 
 
+def check_vcs_client(extras):
+    """
+    Checks if vcs client is allowed (Only works in enterprise edition)
+    """
+    try:
+        from rc_ee.lib.security.utils import is_vcs_client_whitelisted
+    except ModuleNotFoundError:
+        is_vcs_client_whitelisted = lambda *x: True
+    backend = extras.get('scm')
+    if not is_vcs_client_whitelisted(extras.get('user_agent'), backend):
+        raise ClientNotSupportedError(f"Your {backend} client is forbidden")
+
 def _get_scm_size(alias, root_path):
 
     if not alias.startswith('.'):
         alias += '.'
 
     size_scm, size_root = 0, 0
     for path, unused_dirs, files in os.walk(safe_str(root_path)):
         if path.find(alias) != -1:
             for f in files:
                 try:
                     size_scm += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
         else:
             for f in files:
                 try:
                     size_root += os.path.getsize(os.path.join(path, f))
                 except OSError:
                     pass
 
     size_scm_f = h.format_byte_size_binary(size_scm)
     size_root_f = h.format_byte_size_binary(size_root)
     size_total_f = h.format_byte_size_binary(size_root + size_scm)
 
     return size_scm_f, size_root_f, size_total_f
 
 
 # actual hooks called by Mercurial internally, and GIT by our Python Hooks
 def repo_size(extras):
     """Present size of repository after push."""
     repo = Repository.get_by_repo_name(extras.repository)
     vcs_part = f'.{repo.repo_type}'
     size_vcs, size_root, size_total = _get_scm_size(vcs_part, repo.repo_full_path)
     msg = (f'RhodeCode: `{repo.repo_name}` size summary {vcs_part}:{size_vcs} repo:{size_root} total:{size_total}\n')
     return HookResponse(0, msg)
 
 
 def pre_push(extras):
     """
     Hook executed before pushing code.
 
     It bans pushing when the repository is locked.
     """
 
+    check_vcs_client(extras)
     user = User.get_by_username(extras.username)
     output = ''
     if extras.locked_by[0] and user.user_id != int(extras.locked_by[0]):
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output = _http_ret.title
         else:
             raise _http_ret
 
     hook_response = ''
     if not is_shadow_repo(extras):
 
         if extras.commit_ids and extras.check_branch_perms:
             auth_user = user.AuthUser()
             repo = Repository.get_by_repo_name(extras.repository)
+            if not repo:
+                raise ValueError(f'Repo for {extras.repository} not found')
             affected_branches = []
             if repo.repo_type == 'hg':
                 for entry in extras.commit_ids:
                     if entry['type'] == 'branch':
                         is_forced = bool(entry['multiple_heads'])
                         affected_branches.append([entry['name'], is_forced])
             elif repo.repo_type == 'git':
                 for entry in extras.commit_ids:
                     if entry['type'] == 'heads':
                         is_forced = bool(entry['pruned_sha'])
                         affected_branches.append([entry['name'], is_forced])
 
             for branch_name, is_forced in affected_branches:
 
                 rule, branch_perm = auth_user.get_rule_and_branch_permission(
                     extras.repository, branch_name)
                 if not branch_perm:
                     # no branch permission found for this branch, just keep checking
                     continue
 
                 if branch_perm == 'branch.push_force':
                     continue
                 elif branch_perm == 'branch.push' and is_forced is False:
                     continue
                 elif branch_perm == 'branch.push' and is_forced is True:
                     halt_message = f'Branch `{branch_name}` changes rejected by rule {rule}. ' \
                                    f'FORCE PUSH FORBIDDEN.'
                 else:
                     halt_message = f'Branch `{branch_name}` changes rejected by rule {rule}.'
 
                 if halt_message:
                     _http_ret = HTTPBranchProtected(halt_message)
                     raise _http_ret
 
         # Propagate to external components. This is done after checking the
         # lock, for consistent behavior.
         hook_response = pre_push_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPrePushEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def pre_pull(extras):
     """
     Hook executed before pulling the code.
 
     It bans pulling when the repository is locked.
     """
 
+    check_vcs_client(extras)
     output = ''
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         # this exception is interpreted in git/hg middlewares and based
         # on that proper return code is server to client
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output = _http_ret.title
         else:
             raise _http_ret
 
     # Propagate to external components. This is done after checking the
     # lock, for consistent behavior.
     hook_response = ''
     if not is_shadow_repo(extras):
         extras.hook_type = extras.hook_type or 'pre_pull'
         hook_response = pre_pull_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPrePullEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def post_pull(extras):
     """Hook executed after client pulls the code."""
 
     audit_user = audit_logger.UserWrap(
         username=extras.username,
         ip_addr=extras.ip)
     repo = audit_logger.RepoWrap(repo_name=extras.repository)
     audit_logger.store(
         'user.pull', action_data={'user_agent': extras.user_agent},
         user=audit_user, repo=repo, commit=True)
 
     statsd = StatsdClient.statsd
     if statsd:
         statsd.incr('rhodecode_pull_total', tags=[
             f'user-agent:{user_agent_normalizer(extras.user_agent)}',
         ])
     output = ''
     # make lock is a tri state False, True, None. We only make lock on True
     if extras.make_lock is True and not is_shadow_repo(extras):
         user = User.get_by_username(extras.username)
         Repository.lock(Repository.get_by_repo_name(extras.repository),
                         user.user_id,
                         lock_reason=Repository.LOCK_PULL)
         msg = 'Made lock on repo `{}`'.format(extras.repository)
         output += msg
 
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output += _http_ret.title
 
     # Propagate to external components.
     hook_response = ''
     if not is_shadow_repo(extras):
         extras.hook_type = extras.hook_type or 'post_pull'
         hook_response = post_pull_extension(
             repo_store_path=Repository.base_path(), **extras)
         events.trigger(events.RepoPullEvent(
             repo_name=extras.repository, extras=extras))
 
     return HookResponse(0, output) + hook_response
 
 
 def post_push(extras):
     """Hook executed after user pushes to the repository."""
     commit_ids = extras.commit_ids
 
     # log the push call
     audit_user = audit_logger.UserWrap(
         username=extras.username, ip_addr=extras.ip)
     repo = audit_logger.RepoWrap(repo_name=extras.repository)
     audit_logger.store(
         'user.push', action_data={
             'user_agent': extras.user_agent,
             'commit_ids': commit_ids[:400]},
         user=audit_user, repo=repo, commit=True)
 
     statsd = StatsdClient.statsd
     if statsd:
         statsd.incr('rhodecode_push_total', tags=[
             f'user-agent:{user_agent_normalizer(extras.user_agent)}',
         ])
 
     # Propagate to external components.
     output = ''
     # make lock is a tri state False, True, None. We only release lock on False
     if extras.make_lock is False and not is_shadow_repo(extras):
         Repository.unlock(Repository.get_by_repo_name(extras.repository))
         msg = f'Released lock on repo `{extras.repository}`\n'
         output += msg
 
     if extras.locked_by[0]:
         locked_by = User.get(extras.locked_by[0]).username
         reason = extras.locked_by[2]
         _http_ret = HTTPLockedRC(
             _locked_by_explanation(extras.repository, locked_by, reason))
         # TODO: johbo: if not?
         if str(_http_ret.code).startswith('2'):
             # 2xx Codes don't raise exceptions
             output += _http_ret.title
 
     if extras.new_refs:
         tmpl = '{}/{}/pull-request/new?{{ref_type}}={{ref_name}}'.format(
             safe_str(extras.server_url), safe_str(extras.repository))
 
         for branch_name in extras.new_refs['branches']:
             pr_link = tmpl.format(ref_type='branch', ref_name=safe_str(branch_name))
             output += f'RhodeCode: open pull request link: {pr_link}\n'
 
         for book_name in extras.new_refs['bookmarks']:
             pr_link = tmpl.format(ref_type='bookmark', ref_name=safe_str(book_name))
             output += f'RhodeCode: open pull request link: {pr_link}\n'
 
     hook_response = ''
     if not is_shadow_repo(extras):
         hook_response = post_push_extension(
             repo_store_path=Repository.base_path(),
             **extras)
         events.trigger(events.RepoPushEvent(
             repo_name=extras.repository, pushed_commit_ids=commit_ids, extras=extras))
 
     output += 'RhodeCode: push completed\n'
     return HookResponse(0, output) + hook_response
 
 
 def _locked_by_explanation(repo_name, user_name, reason):
     message = f'Repository `{repo_name}` locked by user `{user_name}`. Reason:`{reason}`'
     return message
 
 
 def check_allowed_create_user(user_dict, created_by, **kwargs):
     # pre create hooks
     if pre_create_user.is_active():
         hook_result = pre_create_user(created_by=created_by, **user_dict)
         allowed = hook_result.status == 0
         if not allowed:
             reason = hook_result.output
             raise UserCreationError(reason)
 
 
 class ExtensionCallback(object):
     """
     Forwards a given call to rcextensions, sanitizes keyword arguments.
 
     Does check if there is an extension active for that hook. If it is
     there, it will forward all `kwargs_keys` keyword arguments to the
     extension callback.
     """
 
     def __init__(self, hook_name, kwargs_keys):
         self._hook_name = hook_name
         self._kwargs_keys = set(kwargs_keys)
 
     def __call__(self, *args, **kwargs):
         log.debug('Calling extension callback for `%s`', self._hook_name)
         callback = self._get_callback()
         if not callback:
             log.debug('extension callback `%s` not found, skipping...', self._hook_name)
             return
 
         kwargs_to_pass = {}
         for key in self._kwargs_keys:
             try:
                 kwargs_to_pass[key] = kwargs[key]
             except KeyError:
                 log.error('Failed to fetch %s key from given kwargs. '
                           'Expected keys: %s', key, self._kwargs_keys)
                 raise
 
         # backward compat for removed api_key for old hooks. This was it works
         # with older rcextensions that require api_key present
         if self._hook_name in ['CREATE_USER_HOOK', 'DELETE_USER_HOOK']:
             kwargs_to_pass['api_key'] = '_DEPRECATED_'
         return callback(**kwargs_to_pass)
 
     def is_active(self):
         return hasattr(rhodecode.EXTENSIONS, self._hook_name)
 
     def _get_callback(self):
         return getattr(rhodecode.EXTENSIONS, self._hook_name, None)
 
 
 pre_pull_extension = ExtensionCallback(
     hook_name='PRE_PULL_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'hook_type', 'user_agent', 'repo_store_path',))
 
 
 post_pull_extension = ExtensionCallback(
     hook_name='PULL_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'hook_type', 'user_agent', 'repo_store_path',))
 
 
 pre_push_extension = ExtensionCallback(
     hook_name='PRE_PUSH_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'repo_store_path', 'commit_ids', 'hook_type', 'user_agent',))
 
 
 post_push_extension = ExtensionCallback(
     hook_name='PUSH_HOOK',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'repo_store_path', 'commit_ids', 'hook_type', 'user_agent',))
 
 
 pre_create_user = ExtensionCallback(
     hook_name='PRE_CREATE_USER_HOOK',
     kwargs_keys=(
         'username', 'password', 'email', 'firstname', 'lastname', 'active',
         'admin', 'created_by'))
 
 
 create_pull_request = ExtensionCallback(
     hook_name='CREATE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 merge_pull_request = ExtensionCallback(
     hook_name='MERGE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 close_pull_request = ExtensionCallback(
     hook_name='CLOSE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 review_pull_request = ExtensionCallback(
     hook_name='REVIEW_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 comment_pull_request = ExtensionCallback(
     hook_name='COMMENT_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'comment', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 comment_edit_pull_request = ExtensionCallback(
     hook_name='COMMENT_EDIT_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'comment', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 update_pull_request = ExtensionCallback(
     hook_name='UPDATE_PULL_REQUEST',
     kwargs_keys=(
         'server_url', 'config', 'scm', 'username', 'ip', 'action',
         'repository', 'pull_request_id', 'url', 'title', 'description',
         'status', 'created_on', 'updated_on', 'commit_ids', 'review_status',
         'mergeable', 'source', 'target', 'author', 'reviewers'))
 
 
 create_user = ExtensionCallback(
     hook_name='CREATE_USER_HOOK',
     kwargs_keys=(
         'username', 'full_name_or_username', 'full_contact', 'user_id',
         'name', 'firstname', 'short_contact', 'admin', 'lastname',
         'ip_addresses', 'extern_type', 'extern_name',
         'email', 'api_keys', 'last_login',
         'full_name', 'active', 'password', 'emails',
         'inherit_default_permissions', 'created_by', 'created_on'))
 
 
 delete_user = ExtensionCallback(
     hook_name='DELETE_USER_HOOK',
     kwargs_keys=(
         'username', 'full_name_or_username', 'full_contact', 'user_id',
         'name', 'firstname', 'short_contact', 'admin', 'lastname',
         'ip_addresses',
         'email', 'last_login',
         'full_name', 'active', 'password', 'emails',
         'inherit_default_permissions', 'deleted_by'))
 
 
 create_repository = ExtensionCallback(
     hook_name='CREATE_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id', 'created_by'))
 
 
 delete_repository = ExtensionCallback(
     hook_name='DELETE_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id', 'deleted_by', 'deleted_on'))
 
 
 comment_commit_repository = ExtensionCallback(
     hook_name='COMMENT_COMMIT_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id',
         'repository', 'created_by', 'comment', 'commit'))
 
 comment_edit_commit_repository = ExtensionCallback(
     hook_name='COMMENT_EDIT_COMMIT_REPO_HOOK',
     kwargs_keys=(
         'repo_name', 'repo_type', 'description', 'private', 'created_on',
         'enable_downloads', 'repo_id', 'user_id', 'enable_statistics',
         'clone_uri', 'fork_id', 'group_id',
         'repository', 'created_by', 'comment', 'commit'))
 
 
 create_repository_group = ExtensionCallback(
     hook_name='CREATE_REPO_GROUP_HOOK',
     kwargs_keys=(
         'group_name', 'group_parent_id', 'group_description',
         'group_id', 'user_id', 'created_by', 'created_on',
         'enable_locking'))

rhodecode/lib/middleware/request_wrapper.py

@@ -1,96 +1,96 b''
 
 
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import time
 import logging
 
 import rhodecode
 from rhodecode.lib.auth import AuthUser
 from rhodecode.lib.base import get_ip_addr, get_user_agent
 from rhodecode.lib.middleware.utils import get_path_info
 from rhodecode.lib.utils2 import safe_str, get_current_rhodecode_user
 
 
 log = logging.getLogger(__name__)
 
 
 class RequestWrapperTween(object):
     def __init__(self, handler, registry):
         self.handler = handler
         self.registry = registry
 
         # one-time configuration code goes here
 
     def _get_user_info(self, request):
         user = get_current_rhodecode_user(request)
         if not user:
             user = AuthUser.repr_user(ip=get_ip_addr(request.environ))
         return user
 
     def __call__(self, request):
         start = time.time()
-        log.debug('Starting request time measurement')
+        log.debug('Starting request processing')
         response = None
         request.req_wrapper_start = start
 
         try:
             response = self.handler(request)
         finally:
             count = request.request_count()
             _ver_ = rhodecode.__version__
             _path = get_path_info(request.environ)
             _auth_user = self._get_user_info(request)
             ip = get_ip_addr(request.environ)
             match_route = request.matched_route.name if request.matched_route else "NOT_FOUND"
             resp_code = getattr(response, 'status_code', 'UNDEFINED')
 
             total = time.time() - start
             log.info(
-                'Req[%4s] %s %s Request to %s time: %.4fs [%s], RhodeCode %s',
+                'Finished request processing: req[%4s] %s %s Request to %s time: %.4fs [%s], RhodeCode %s',
                 count, _auth_user, request.environ.get('REQUEST_METHOD'),
                 _path, total, get_user_agent(request. environ), _ver_,
                 extra={"time": total, "ver": _ver_, "ip": ip,
                        "path": _path, "view_name": match_route, "code": resp_code}
             )
 
             statsd = request.registry.statsd
             if statsd:
                 elapsed_time_ms = round(1000.0 * total)  # use ms only
                 statsd.timing(
                     "rhodecode_req_timing.histogram", elapsed_time_ms,
                     tags=[
                         "view_name:{}".format(match_route),
                         "code:{}".format(resp_code)
                     ],
                     use_decimals=False
                 )
                 statsd.incr(
                     'rhodecode_req_total', tags=[
                         "view_name:{}".format(match_route),
                         "code:{}".format(resp_code)
                     ])
 
         return response
 
 
 def includeme(config):
     config.add_tween(
         'rhodecode.lib.middleware.request_wrapper.RequestWrapperTween',
     )

rhodecode/lib/middleware/simplehg.py

@@ -1,161 +1,167 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleHG middleware for handling mercurial protocol request
 (push/clone etc.). It's implemented with basic auth function
 """
 
 import logging
 import urllib.parse
 import urllib.request
 import urllib.parse
 import urllib.error
 
 from rhodecode.lib import utils
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.middleware import simplevcs
 from rhodecode.lib.middleware.utils import get_path_info
 
 log = logging.getLogger(__name__)
 
 
 class SimpleHg(simplevcs.SimpleVCS):
 
     SCM = 'hg'
 
     def _get_repository_name(self, environ):
         """
         Gets repository name out of PATH_INFO header
 
         :param environ: environ where PATH_INFO is stored
         """
         repo_name = get_path_info(environ)
         if repo_name and repo_name.startswith('/'):
             # remove only the first leading /
             repo_name = repo_name[1:]
         return repo_name.rstrip('/')
 
     _ACTION_MAPPING = {
+        'between': 'pull',
+        'branches': 'pull',
+        'branchmap': 'pull',
+        'capabilities': 'pull',
         'changegroup': 'pull',
         'changegroupsubset': 'pull',
+        'changesetdata': 'pull',
+        'clonebundles': 'pull',
+        'clonebundles_manifest': 'pull',
+        'debugwireargs': 'pull',
+        'filedata': 'pull',
         'getbundle': 'pull',
-        'stream_out': 'pull',
-        'listkeys': 'pull',
-        'between': 'pull',
-        'branchmap': 'pull',
-        'branches': 'pull',
-        'clonebundles': 'pull',
-        'capabilities': 'pull',
-        'debugwireargs': 'pull',
         'heads': 'pull',
-        'lookup': 'pull',
         'hello': 'pull',
         'known': 'pull',
+        'listkeys': 'pull',
+        'lookup': 'pull',
+        'manifestdata': 'pull',
+        'narrow_widen': 'pull',
+        'protocaps': 'pull',
+        'stream_out': 'pull',
 
         # largefiles
+        'getlfile': 'pull',
         'putlfile': 'push',
-        'getlfile': 'pull',
         'statlfile': 'pull',
         'lheads': 'pull',
 
         # evolve
         'evoext_obshashrange_v1': 'pull',
         'evoext_obshash': 'pull',
         'evoext_obshash1': 'pull',
 
         'unbundle': 'push',
         'pushkey': 'push',
     }
 
     @classmethod
     def _get_xarg_headers(cls, environ):
         i = 1
         chunks = []  # gather chunks stored in multiple 'hgarg_N'
         while True:
             head = environ.get('HTTP_X_HGARG_{}'.format(i))
             if not head:
                 break
             i += 1
             chunks.append(urllib.parse.unquote_plus(head))
         full_arg = ''.join(chunks)
         pref = 'cmds='
         if full_arg.startswith(pref):
             # strip the cmds= header defining our batch commands
             full_arg = full_arg[len(pref):]
         cmds = full_arg.split(';')
         return cmds
 
     @classmethod
     def _get_batch_cmd(cls, environ):
         """
         Handle batch command send commands. Those are ';' separated commands
         sent by batch command that server needs to execute. We need to extract
         those, and map them to our ACTION_MAPPING to get all push/pull commands
         specified in the batch
         """
         default = 'push'
         batch_cmds = []
         try:
             cmds = cls._get_xarg_headers(environ)
             for pair in cmds:
                 parts = pair.split(' ', 1)
                 if len(parts) != 2:
                     continue
                 # entry should be in a format `key ARGS`
                 cmd, args = parts
                 action = cls._ACTION_MAPPING.get(cmd, default)
                 batch_cmds.append(action)
         except Exception:
             log.exception('Failed to extract batch commands operations')
 
         # in case we failed, (e.g malformed data) assume it's PUSH sub-command
         # for safety
         return batch_cmds or [default]
 
     def _get_action(self, environ):
         """
         Maps mercurial request commands into a pull or push command.
         In case of unknown/unexpected data, it returns 'push' to be safe.
 
         :param environ:
         """
         default = 'push'
         query = urllib.parse.parse_qs(environ['QUERY_STRING'], keep_blank_values=True)
 
         if 'cmd' in query:
             cmd = query['cmd'][0]
             if cmd == 'batch':
                 cmds = self._get_batch_cmd(environ)
                 if 'push' in cmds:
                     return 'push'
                 else:
                     return 'pull'
             return self._ACTION_MAPPING.get(cmd, default)
 
         return default
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         return self.scm_app.create_hg_wsgi_app(repo_path, repo_name, config)
 
     def _create_config(self, extras, repo_name, scheme='http'):
         config = utils.make_db_config(repo=repo_name)
         config.set('rhodecode', 'RC_SCM_DATA', json.dumps(extras))
 
         return config.serialize()

rhodecode/lib/middleware/simplevcs.py

@@ -1,683 +1,662 b''
 
 
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 SimpleVCS middleware for handling protocol request (push/clone etc.)
 It's implemented with basic auth function
 """
 
 import os
 import re
 import logging
 import importlib
 from functools import wraps
 
 import time
 from paste.httpheaders import REMOTE_USER, AUTH_TYPE
 
 from pyramid.httpexceptions import (
     HTTPNotFound, HTTPForbidden, HTTPNotAcceptable, HTTPInternalServerError)
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 import rhodecode
 from rhodecode.authentication.base import authenticate, VCS_TYPE, loadplugin
 from rhodecode.lib import rc_cache
 from rhodecode.lib.svn_txn_utils import store_txn_id_data
 from rhodecode.lib.auth import AuthUser, HasPermissionAnyMiddleware
 from rhodecode.lib.base import (
     BasicAuth, get_ip_addr, get_user_agent, vcs_operation_context)
 from rhodecode.lib.exceptions import (UserCreationError, NotAllowedToCreateUserError)
 from rhodecode.lib.hook_daemon.base import prepare_callback_daemon
 from rhodecode.lib.middleware import appenlight
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.lib.str_utils import safe_bytes, safe_int
 from rhodecode.lib.utils import is_valid_repo, SLUG_RE
 from rhodecode.lib.utils2 import safe_str, fix_PATH, str2bool
 from rhodecode.lib.vcs.conf import settings as vcs_settings
 from rhodecode.lib.vcs.backends import base
 
 from rhodecode.model import meta
 from rhodecode.model.db import User, Repository, PullRequest
 from rhodecode.model.scm import ScmModel
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.settings import SettingsModel, VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def initialize_generator(factory):
     """
     Initializes the returned generator by draining its first element.
 
     This can be used to give a generator an initializer, which is the code
     up to the first yield statement. This decorator enforces that the first
     produced element has the value ``"__init__"`` to make its special
     purpose very explicit in the using code.
     """
 
     @wraps(factory)
     def wrapper(*args, **kwargs):
         gen = factory(*args, **kwargs)
         try:
             init = next(gen)
         except StopIteration:
             raise ValueError('Generator must yield at least one element.')
         if init != "__init__":
             raise ValueError('First yielded element must be "__init__".')
         return gen
     return wrapper
 
 
 class SimpleVCS(object):
     """Common functionality for SCM HTTP handlers."""
 
     SCM = 'unknown'
 
     acl_repo_name = None
     url_repo_name = None
     vcs_repo_name = None
     rc_extras = {}
 
     # We have to handle requests to shadow repositories different than requests
     # to normal repositories. Therefore we have to distinguish them. To do this
     # we use this regex which will match only on URLs pointing to shadow
     # repositories.
     shadow_repo_re = re.compile(
         '(?P<groups>(?:{slug_pat}/)*)'   # repo groups
         '(?P<target>{slug_pat})/'        # target repo
         'pull-request/(?P<pr_id>\\d+)/'  # pull request
         'repository$'                    # shadow repo
         .format(slug_pat=SLUG_RE.pattern))
 
     def __init__(self, config, registry):
         self.registry = registry
         self.config = config
         # re-populated by specialized middleware
         self.repo_vcs_config = base.Config()
 
         rc_settings = SettingsModel().get_all_settings(cache=True, from_request=False)
         realm = rc_settings.get('rhodecode_realm') or 'RhodeCode AUTH'
 
         # authenticate this VCS request using authfunc
         auth_ret_code_detection = \
             str2bool(self.config.get('auth_ret_code_detection', False))
         self.authenticate = BasicAuth(
             '', authenticate, registry, config.get('auth_ret_code'),
             auth_ret_code_detection, rc_realm=realm)
         self.ip_addr = '0.0.0.0'
 
     @LazyProperty
     def global_vcs_config(self):
         try:
             return VcsSettingsModel().get_ui_settings_as_config_obj()
         except Exception:
             return base.Config()
 
     @property
     def base_path(self):
         settings_path = self.config.get('repo_store.path')
 
         if not settings_path:
             raise ValueError('FATAL: repo_store.path is empty')
         return settings_path
 
     def set_repo_names(self, environ):
         """
         This will populate the attributes acl_repo_name, url_repo_name,
         vcs_repo_name and is_shadow_repo. In case of requests to normal (non
         shadow) repositories all names are equal. In case of requests to a
         shadow repository the acl-name points to the target repo of the pull
         request and the vcs-name points to the shadow repo file system path.
         The url-name is always the URL used by the vcs client program.
 
         Example in case of a shadow repo:
             acl_repo_name = RepoGroup/MyRepo
             url_repo_name = RepoGroup/MyRepo/pull-request/3/repository
             vcs_repo_name = /repo/base/path/RepoGroup/.__shadow_MyRepo_pr-3'
         """
         # First we set the repo name from URL for all attributes. This is the
         # default if handling normal (non shadow) repo requests.
         self.url_repo_name = self._get_repository_name(environ)
         self.acl_repo_name = self.vcs_repo_name = self.url_repo_name
         self.is_shadow_repo = False
 
         # Check if this is a request to a shadow repository.
         match = self.shadow_repo_re.match(self.url_repo_name)
         if match:
             match_dict = match.groupdict()
 
             # Build acl repo name from regex match.
             acl_repo_name = safe_str('{groups}{target}'.format(
                 groups=match_dict['groups'] or '',
                 target=match_dict['target']))
 
             # Retrieve pull request instance by ID from regex match.
             pull_request = PullRequest.get(match_dict['pr_id'])
 
             # Only proceed if we got a pull request and if acl repo name from
             # URL equals the target repo name of the pull request.
             if pull_request and (acl_repo_name == pull_request.target_repo.repo_name):
 
                 # Get file system path to shadow repository.
                 workspace_id = PullRequestModel()._workspace_id(pull_request)
                 vcs_repo_name = pull_request.target_repo.get_shadow_repository_path(workspace_id)
 
                 # Store names for later usage.
                 self.vcs_repo_name = vcs_repo_name
                 self.acl_repo_name = acl_repo_name
                 self.is_shadow_repo = True
 
         log.debug('Setting all VCS repository names: %s', {
             'acl_repo_name': self.acl_repo_name,
             'url_repo_name': self.url_repo_name,
             'vcs_repo_name': self.vcs_repo_name,
         })
 
     @property
     def scm_app(self):
         custom_implementation = self.config['vcs.scm_app_implementation']
         if custom_implementation == 'http':
             log.debug('Using HTTP implementation of scm app.')
             scm_app_impl = scm_app_http
         else:
             log.debug('Using custom implementation of scm_app: "{}"'.format(
                 custom_implementation))
             scm_app_impl = importlib.import_module(custom_implementation)
         return scm_app_impl
 
     def _get_by_id(self, repo_name):
         """
         Gets a special pattern _<ID> from clone url and tries to replace it
         with a repository_name for support of _<ID> non changeable urls
         """
 
         data = repo_name.split('/')
         if len(data) >= 2:
             from rhodecode.model.repo import RepoModel
             by_id_match = RepoModel().get_repo_by_id(repo_name)
             if by_id_match:
                 data[1] = by_id_match.repo_name
 
         # Because PEP-3333-WSGI uses bytes-tunneled-in-latin-1 as PATH_INFO
         # and we use this data
         maybe_new_path = '/'.join(data)
         return safe_bytes(maybe_new_path).decode('latin1')
 
     def _invalidate_cache(self, repo_name):
         """
         Set's cache for this repository for invalidation on next access
 
         :param repo_name: full repo name, also a cache key
         """
         ScmModel().mark_for_invalidation(repo_name)
 
     def is_valid_and_existing_repo(self, repo_name, base_path, scm_type):
         db_repo = Repository.get_by_repo_name(repo_name)
         if not db_repo:
             log.debug('Repository `%s` not found inside the database.',
                       repo_name)
             return False
 
         if db_repo.repo_type != scm_type:
             log.warning(
                 'Repository `%s` have incorrect scm_type, expected %s got %s',
                 repo_name, db_repo.repo_type, scm_type)
             return False
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         return is_valid_repo(
             repo_name, base_path,
             explicit_scm=scm_type, expect_scm=scm_type, config=config)
 
     def valid_and_active_user(self, user):
         """
         Checks if that user is not empty, and if it's actually object it checks
         if he's active.
 
         :param user: user object or None
         :return: boolean
         """
         if user is None:
             return False
 
         elif user.active:
             return True
 
         return False
 
     @property
     def is_shadow_repo_dir(self):
         return os.path.isdir(self.vcs_repo_name)
 
     def _check_permission(self, action, user, auth_user, repo_name, ip_addr=None,
                           plugin_id='', plugin_cache_active=False, cache_ttl=0):
         """
         Checks permissions using action (push/pull) user and repository
         name. If plugin_cache and ttl is set it will use the plugin which
         authenticated the user to store the cached permissions result for N
         amount of seconds as in cache_ttl
 
         :param action: push or pull action
         :param user: user instance
         :param repo_name: repository name
         """
 
         log.debug('AUTH_CACHE_TTL for permissions `%s` active: %s (TTL: %s)',
                   plugin_id, plugin_cache_active, cache_ttl)
 
         user_id = user.user_id
         cache_namespace_uid = f'cache_user_auth.{rc_cache.PERMISSIONS_CACHE_VER}.{user_id}'
         region = rc_cache.get_or_create_region('cache_perms', cache_namespace_uid)
 
         @region.conditional_cache_on_arguments(namespace=cache_namespace_uid,
                                                expiration_time=cache_ttl,
                                                condition=plugin_cache_active)
         def compute_perm_vcs(
                 cache_name, plugin_id, action, user_id, repo_name, ip_addr):
 
-            log.debug('auth: calculating permission access now...')
+            log.debug('auth: calculating permission access now for vcs operation: %s', action)
             # check IP
             inherit = user.inherit_default_permissions
             ip_allowed = AuthUser.check_ip_allowed(
                 user_id, ip_addr, inherit_from_default=inherit)
             if ip_allowed:
                 log.info('Access for IP:%s allowed', ip_addr)
             else:
                 return False
 
             if action == 'push':
                 perms = ('repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             else:
                 # any other action need at least read permission
                 perms = (
                     'repository.read', 'repository.write', 'repository.admin')
                 if not HasPermissionAnyMiddleware(*perms)(auth_user, repo_name):
                     return False
 
             return True
 
         start = time.time()
         log.debug('Running plugin `%s` permissions check', plugin_id)
 
         # for environ based auth, password can be empty, but then the validation is
         # on the server that fills in the env data needed for authentication
         perm_result = compute_perm_vcs(
             'vcs_permissions', plugin_id, action, user.user_id, repo_name, ip_addr)
 
         auth_time = time.time() - start
         log.debug('Permissions for plugin `%s` completed in %.4fs, '
                   'expiration time of fetched cache %.1fs.',
                   plugin_id, auth_time, cache_ttl)
 
         return perm_result
 
     def _get_http_scheme(self, environ):
         try:
             return environ['wsgi.url_scheme']
         except Exception:
             log.exception('Failed to read http scheme')
             return 'http'
 
-    def _check_ssl(self, environ, start_response):
-        """
-        Checks the SSL check flag and returns False if SSL is not present
-        and required True otherwise
-        """
-        org_proto = environ['wsgi._org_proto']
-        # check if we have SSL required  ! if not it's a bad request !
-        require_ssl = str2bool(self.repo_vcs_config.get('web', 'push_ssl'))
-        if require_ssl and org_proto == 'http':
-            log.debug(
-                'Bad request: detected protocol is `%s` and '
-                'SSL/HTTPS is required.', org_proto)
-            return False
-        return True
-
     def _get_default_cache_ttl(self):
         # take AUTH_CACHE_TTL from the `rhodecode` auth plugin
         plugin = loadplugin('egg:rhodecode-enterprise-ce#rhodecode')
         plugin_settings = plugin.get_settings()
         plugin_cache_active, cache_ttl = plugin.get_ttl_cache(
             plugin_settings) or (False, 0)
         return plugin_cache_active, cache_ttl
 
     def __call__(self, environ, start_response):
         try:
             return self._handle_request(environ, start_response)
         except Exception:
             log.exception("Exception while handling request")
             appenlight.track_exception(environ)
             return HTTPInternalServerError()(environ, start_response)
         finally:
             meta.Session.remove()
 
     def _handle_request(self, environ, start_response):
-        if not self._check_ssl(environ, start_response):
-            reason = ('SSL required, while RhodeCode was unable '
-                      'to detect this as SSL request')
-            log.debug('User not allowed to proceed, %s', reason)
-            return HTTPNotAcceptable(reason)(environ, start_response)
-
         if not self.url_repo_name:
             log.warning('Repository name is empty: %s', self.url_repo_name)
             # failed to get repo name, we fail now
             return HTTPNotFound()(environ, start_response)
         log.debug('Extracted repo name is %s', self.url_repo_name)
 
         ip_addr = get_ip_addr(environ)
         user_agent = get_user_agent(environ)
         username = None
 
         # skip passing error to error controller
         environ['pylons.status_code_redirect'] = True
 
         # ======================================================================
         # GET ACTION PULL or PUSH
         # ======================================================================
         action = self._get_action(environ)
 
         # ======================================================================
         # Check if this is a request to a shadow repository of a pull request.
         # In this case only pull action is allowed.
         # ======================================================================
         if self.is_shadow_repo and action != 'pull':
             reason = 'Only pull action is allowed for shadow repositories.'
             log.debug('User not allowed to proceed, %s', reason)
             return HTTPNotAcceptable(reason)(environ, start_response)
 
         # Check if the shadow repo actually exists, in case someone refers
         # to it, and it has been deleted because of successful merge.
         if self.is_shadow_repo and not self.is_shadow_repo_dir:
             log.debug(
                 'Shadow repo detected, and shadow repo dir `%s` is missing',
                 self.is_shadow_repo_dir)
             return HTTPNotFound()(environ, start_response)
 
         # ======================================================================
         # CHECK ANONYMOUS PERMISSION
         # ======================================================================
         detect_force_push = False
         check_branch_perms = False
         if action in ['pull', 'push']:
             user_obj = anonymous_user = User.get_default_user()
             auth_user = user_obj.AuthUser()
             username = anonymous_user.username
             if anonymous_user.active:
                 plugin_cache_active, cache_ttl = self._get_default_cache_ttl()
                 # ONLY check permissions if the user is activated
                 anonymous_perm = self._check_permission(
                     action, anonymous_user, auth_user, self.acl_repo_name, ip_addr,
                     plugin_id='anonymous_access',
                     plugin_cache_active=plugin_cache_active,
                     cache_ttl=cache_ttl,
                 )
             else:
                 anonymous_perm = False
 
             if not anonymous_user.active or not anonymous_perm:
                 if not anonymous_user.active:
                     log.debug('Anonymous access is disabled, running '
                               'authentication')
 
                 if not anonymous_perm:
                     log.debug('Not enough credentials to access repo: `%s` '
                               'repository as anonymous user', self.acl_repo_name)
 
                 username = None
                 # ==============================================================
                 # DEFAULT PERM FAILED OR ANONYMOUS ACCESS IS DISABLED SO WE
                 # NEED TO AUTHENTICATE AND ASK FOR AUTH USER PERMISSIONS
                 # ==============================================================
 
                 # try to auth based on environ, container auth methods
                 log.debug('Running PRE-AUTH for container|headers based authentication')
 
                 # headers auth, by just reading special headers and bypass the auth with user/passwd
                 pre_auth = authenticate(
                     '', '', environ, VCS_TYPE, registry=self.registry,
                     acl_repo_name=self.acl_repo_name)
 
                 if pre_auth and pre_auth.get('username'):
                     username = pre_auth['username']
                 log.debug('PRE-AUTH got `%s` as username', username)
                 if pre_auth:
                     log.debug('PRE-AUTH successful from %s',
                               pre_auth.get('auth_data', {}).get('_plugin'))
 
                 # If not authenticated by the container, running basic auth
                 # before inject the calling repo_name for special scope checks
                 self.authenticate.acl_repo_name = self.acl_repo_name
 
                 plugin_cache_active, cache_ttl = False, 0
                 plugin = None
 
                 # regular auth chain
                 if not username:
                     self.authenticate.realm = self.authenticate.get_rc_realm()
 
                     try:
                         auth_result = self.authenticate(environ)
                     except (UserCreationError, NotAllowedToCreateUserError) as e:
                         log.error(e)
                         reason = safe_str(e)
                         return HTTPNotAcceptable(reason)(environ, start_response)
 
                     if isinstance(auth_result, dict):
                         AUTH_TYPE.update(environ, 'basic')
                         REMOTE_USER.update(environ, auth_result['username'])
                         username = auth_result['username']
                         plugin = auth_result.get('auth_data', {}).get('_plugin')
                         log.info(
                             'MAIN-AUTH successful for user `%s` from %s plugin',
                             username, plugin)
 
                         plugin_cache_active, cache_ttl = auth_result.get(
                             'auth_data', {}).get('_ttl_cache') or (False, 0)
                     else:
                         return auth_result.wsgi_application(environ, start_response)
 
                 # ==============================================================
                 # CHECK PERMISSIONS FOR THIS REQUEST USING GIVEN USERNAME
                 # ==============================================================
                 user = User.get_by_username(username)
                 if not self.valid_and_active_user(user):
                     return HTTPForbidden()(environ, start_response)
                 username = user.username
                 user_id = user.user_id
 
                 # check user attributes for password change flag
                 user_obj = user
                 auth_user = user_obj.AuthUser()
                 if user_obj and user_obj.username != User.DEFAULT_USER and \
                         user_obj.user_data.get('force_password_change'):
                     reason = 'password change required'
                     log.debug('User not allowed to authenticate, %s', reason)
                     return HTTPNotAcceptable(reason)(environ, start_response)
 
                 # check permissions for this repository
                 perm = self._check_permission(
                     action, user, auth_user, self.acl_repo_name, ip_addr,
                     plugin, plugin_cache_active, cache_ttl)
                 if not perm:
                     return HTTPForbidden()(environ, start_response)
                 environ['rc_auth_user_id'] = str(user_id)
 
             if action == 'push':
                 perms = auth_user.get_branch_permissions(self.acl_repo_name)
                 if perms:
                     check_branch_perms = True
                     detect_force_push = True
 
         # extras are injected into UI object and later available
         # in hooks executed by RhodeCode
         check_locking = _should_check_locking(environ.get('QUERY_STRING'))
 
         extras = vcs_operation_context(
             environ, repo_name=self.acl_repo_name, username=username,
             action=action, scm=self.SCM, check_locking=check_locking,
             is_shadow_repo=self.is_shadow_repo, check_branch_perms=check_branch_perms,
             detect_force_push=detect_force_push
         )
 
         # ======================================================================
         # REQUEST HANDLING
         # ======================================================================
         repo_path = os.path.join(
             safe_str(self.base_path), safe_str(self.vcs_repo_name))
         log.debug('Repository path is %s', repo_path)
 
         fix_PATH()
 
         log.info(
             '%s action on %s repo "%s" by "%s" from %s %s',
             action, self.SCM, safe_str(self.url_repo_name),
             safe_str(username), ip_addr, user_agent)
 
         return self._generate_vcs_response(
             environ, start_response, repo_path, extras, action)
 
     def _get_txn_id(self, environ):
 
         for k in ['RAW_URI', 'HTTP_DESTINATION']:
             url = environ.get(k)
             if not url:
                 continue
 
             # regex to search for svn-txn-id
             pattern = r'/!svn/txr/([^/]+)/'
 
             # Search for the pattern in the URL
             match = re.search(pattern, url)
 
             # Check if a match is found and extract the captured group
             if match:
                 txn_id = match.group(1)
                 return txn_id
 
     @initialize_generator
     def _generate_vcs_response(
             self, environ, start_response, repo_path, extras, action):
         """
         Returns a generator for the response content.
 
         This method is implemented as a generator, so that it can trigger
         the cache validation after all content sent back to the client. It
         also handles the locking exceptions which will be triggered when
         the first chunk is produced by the underlying WSGI application.
         """
         svn_txn_id = ''
         if action == 'push':
             svn_txn_id = self._get_txn_id(environ)
 
         callback_daemon, extras = self._prepare_callback_daemon(
             extras, environ, action, txn_id=svn_txn_id)
 
         if svn_txn_id:
 
             port = safe_int(extras['hooks_uri'].split(':')[-1])
             txn_id_data = extras.copy()
             txn_id_data.update({'port': port})
             txn_id_data.update({'req_method': environ['REQUEST_METHOD']})
 
             full_repo_path = repo_path
             store_txn_id_data(full_repo_path, svn_txn_id, txn_id_data)
 
         log.debug('HOOKS extras is %s', extras)
 
         http_scheme = self._get_http_scheme(environ)
 
         config = self._create_config(extras, self.acl_repo_name, scheme=http_scheme)
         app = self._create_wsgi_app(repo_path, self.url_repo_name, config)
         with callback_daemon:
             app.rc_extras = extras
 
             try:
                 response = app(environ, start_response)
             finally:
                 # This statement works together with the decorator
                 # "initialize_generator" above. The decorator ensures that
                 # we hit the first yield statement before the generator is
                 # returned back to the WSGI server. This is needed to
                 # ensure that the call to "app" above triggers the
                 # needed callback to "start_response" before the
                 # generator is actually used.
                 yield "__init__"
 
             # iter content
             for chunk in response:
                 yield chunk
 
             try:
                 # invalidate cache on push
                 if action == 'push':
                     self._invalidate_cache(self.url_repo_name)
             finally:
                 meta.Session.remove()
 
     def _get_repository_name(self, environ):
         """Get repository name out of the environmnent
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _get_action(self, environ):
         """Map request commands into a pull or push command.
 
         :param environ: WSGI environment
         """
         raise NotImplementedError()
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         """Return the WSGI app that will finally handle the request."""
         raise NotImplementedError()
 
     def _create_config(self, extras, repo_name, scheme='http'):
         """Create a safe config representation."""
         raise NotImplementedError()
 
     def _should_use_callback_daemon(self, extras, environ, action):
         if extras.get('is_shadow_repo'):
             # we don't want to execute hooks, and callback daemon for shadow repos
             return False
         return True
 
     def _prepare_callback_daemon(self, extras, environ, action, txn_id=None):
         protocol = vcs_settings.HOOKS_PROTOCOL
 
         if not self._should_use_callback_daemon(extras, environ, action):
             # disable callback daemon for actions that don't require it
             protocol = 'local'
 
         return prepare_callback_daemon(
             extras, protocol=protocol,
             host=vcs_settings.HOOKS_HOST, txn_id=txn_id)
 
 
 def _should_check_locking(query_string):
     # this is kind of hacky, but due to how mercurial handles client-server
     # server see all operation on commit; bookmarks, phases and
     # obsolescence marker in different transaction, we don't want to check
     # locking on those
     return query_string not in ['cmd=listkeys']

rhodecode/lib/middleware/vcs.py

@@ -1,308 +1,312 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import gzip
 import shutil
 import logging
 import tempfile
 import urllib.parse
 
 from webob.exc import HTTPNotFound
 
 import rhodecode
 from rhodecode.apps._base import ADMIN_PREFIX
 from rhodecode.lib.middleware.utils import get_path_info
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.simplegit import SimpleGit, GIT_PROTO_PAT
 from rhodecode.lib.middleware.simplehg import SimpleHg
 from rhodecode.lib.middleware.simplesvn import SimpleSvn
 from rhodecode.lib.str_utils import safe_str
 from rhodecode.model.settings import VcsSettingsModel
 
 
 log = logging.getLogger(__name__)
 
 VCS_TYPE_KEY = '_rc_vcs_type'
 VCS_TYPE_SKIP = '_rc_vcs_skip'
 
 
 def is_git(environ):
     """
     Returns True if requests should be handled by GIT wsgi middleware
     """
     path_info = get_path_info(environ)
     is_git_path = GIT_PROTO_PAT.match(path_info)
     log.debug(
         'request path: `%s` detected as GIT PROTOCOL %s', path_info,
         is_git_path is not None)
 
     return is_git_path
 
 
 def is_hg(environ):
     """
     Returns True if requests target is mercurial server - header
     ``HTTP_ACCEPT`` of such request would start with ``application/mercurial``.
     """
     is_hg_path = False
 
     http_accept = environ.get('HTTP_ACCEPT')
 
     if http_accept and http_accept.startswith('application/mercurial'):
         query = urllib.parse.parse_qs(environ['QUERY_STRING'])
         if 'cmd' in query:
             is_hg_path = True
 
     path_info = get_path_info(environ)
     log.debug(
         'request path: `%s` detected as HG PROTOCOL %s', path_info,
         is_hg_path)
 
     return is_hg_path
 
 
 def is_svn(environ):
     """
     Returns True if requests target is Subversion server
     """
 
     http_dav = environ.get('HTTP_DAV', '')
     magic_path_segment = rhodecode.CONFIG.get(
         'rhodecode_subversion_magic_path', '/!svn')
     path_info = get_path_info(environ)
     req_method = environ['REQUEST_METHOD']
 
     is_svn_path = (
         'subversion' in http_dav or
         magic_path_segment in path_info
         or req_method in ['PROPFIND', 'PROPPATCH', 'HEAD']
     )
     log.debug(
         'request path: `%s` detected as SVN PROTOCOL %s', path_info,
         is_svn_path)
 
     return is_svn_path
 
 
 class GunzipMiddleware(object):
     """
     WSGI middleware that unzips gzip-encoded requests before
     passing on to the underlying application.
     """
 
     def __init__(self, application):
         self.app = application
 
     def __call__(self, environ, start_response):
         accepts_encoding_header = safe_str(environ.get('HTTP_CONTENT_ENCODING', ''))
 
         if 'gzip' in accepts_encoding_header:
             log.debug('gzip detected, now running gunzip wrapper')
             wsgi_input = environ['wsgi.input']
 
             if not hasattr(environ['wsgi.input'], 'seek'):
                 # The gzip implementation in the standard library of Python 2.x
                 # requires the '.seek()' and '.tell()' methods to be available
                 # on the input stream.  Read the data into a temporary file to
                 # work around this limitation.
 
                 wsgi_input = tempfile.SpooledTemporaryFile(64 * 1024 * 1024)
                 shutil.copyfileobj(environ['wsgi.input'], wsgi_input)
                 wsgi_input.seek(0)
 
             environ['wsgi.input'] = gzip.GzipFile(fileobj=wsgi_input, mode='r')
             # since we "Ungzipped" the content we say now it's no longer gzip
             # content encoding
             del environ['HTTP_CONTENT_ENCODING']
 
             # content length has changes ? or i'm not sure
             if 'CONTENT_LENGTH' in environ:
                 del environ['CONTENT_LENGTH']
         else:
             log.debug('content not gzipped, gzipMiddleware passing '
                       'request further')
         return self.app(environ, start_response)
 
 
 def is_vcs_call(environ):
     if VCS_TYPE_KEY in environ:
         raw_type = environ[VCS_TYPE_KEY]
         return raw_type and raw_type != VCS_TYPE_SKIP
     return False
 
 
 def detect_vcs_request(environ, backends):
     checks = {
         'hg': (is_hg, SimpleHg),
         'git': (is_git, SimpleGit),
         'svn': (is_svn, SimpleSvn),
     }
     handler = None
     # List of path views first chunk we don't do any checks
     white_list = [
         # favicon often requested by browsers
         'favicon.ico',
 
+        # static files no detection
+        '_static++',
+
+        # debug-toolbar
+        '_debug_toolbar++',
+
         # e.g /_file_store/download
         '_file_store++',
 
         # login
-        "_admin/login",
+        f"{ADMIN_PREFIX}/login",
+        f"{ADMIN_PREFIX}/logout",
 
         # 2fa
         f"{ADMIN_PREFIX}/check_2fa",
         f"{ADMIN_PREFIX}/setup_2fa",
 
         # _admin/api is safe too
         f'{ADMIN_PREFIX}/api',
 
         # _admin/gist is safe too
         f'{ADMIN_PREFIX}/gists++',
 
         # _admin/my_account is safe too
         f'{ADMIN_PREFIX}/my_account++',
 
-        # static files no detection
-        '_static++',
-
-        # debug-toolbar
-        '_debug_toolbar++',
-
         # skip ops ping, status
         f'{ADMIN_PREFIX}/ops/ping',
         f'{ADMIN_PREFIX}/ops/status',
 
         # full channelstream connect should be VCS skipped
         f'{ADMIN_PREFIX}/channelstream/connect',
 
         '++/repo_creating_check'
     ]
+
     path_info = get_path_info(environ)
     path_url = path_info.lstrip('/')
     req_method = environ.get('REQUEST_METHOD')
 
     for item in white_list:
+        item = item.lstrip('/')
+
         if item.endswith('++') and path_url.startswith(item[:-2]):
             log.debug('path `%s` in whitelist (match:%s), skipping...', path_url, item)
             return handler
         if item.startswith('++') and path_url.endswith(item[2:]):
             log.debug('path `%s` in whitelist (match:%s), skipping...', path_url, item)
             return handler
         if item == path_url:
             log.debug('path `%s` in whitelist (match:%s), skipping...', path_url, item)
             return handler
 
     if VCS_TYPE_KEY in environ:
         raw_type = environ[VCS_TYPE_KEY]
         if raw_type == VCS_TYPE_SKIP:
             log.debug('got `skip` marker for vcs detection, skipping...')
             return handler
 
         _check, handler = checks.get(raw_type) or [None, None]
         if handler:
             log.debug('got handler:%s from environ', handler)
 
     if not handler:
         log.debug('request start: checking if request for `%s:%s` is of VCS type in order: %s',
                   req_method, path_url, backends)
         for vcs_type in backends:
             vcs_check, _handler = checks[vcs_type]
             if vcs_check(environ):
                 log.debug('vcs handler found %s', _handler)
                 handler = _handler
                 break
 
     return handler
 
 
 class VCSMiddleware(object):
 
     def __init__(self, app, registry, config, appenlight_client):
         self.application = app
         self.registry = registry
         self.config = config
         self.appenlight_client = appenlight_client
         self.use_gzip = True
         # order in which we check the middlewares, based on vcs.backends config
         self.check_middlewares = config['vcs.backends']
 
     def vcs_config(self, repo_name=None):
         """
         returns serialized VcsSettings
         """
         try:
             return VcsSettingsModel(
                 repo=repo_name).get_ui_settings_as_config_obj()
         except Exception:
             pass
 
     def wrap_in_gzip_if_enabled(self, app, config):
         if self.use_gzip:
             app = GunzipMiddleware(app)
         return app
 
     def _get_handler_app(self, environ):
         app = None
         log.debug('VCSMiddleware: detecting vcs type.')
         handler = detect_vcs_request(environ, self.check_middlewares)
         if handler:
             app = handler(self.config, self.registry)
 
         return app
 
     def __call__(self, environ, start_response):
         # check if we handle one of interesting protocols, optionally extract
         # specific vcsSettings and allow changes of how things are wrapped
         vcs_handler = self._get_handler_app(environ)
         if vcs_handler:
             # translate the _REPO_ID into real repo NAME for usage
             # in middleware
 
             path_info = get_path_info(environ)
             environ['PATH_INFO'] = vcs_handler._get_by_id(path_info)
 
             # Set acl, url and vcs repo names.
             vcs_handler.set_repo_names(environ)
 
             # register repo config back to the handler
             vcs_conf = self.vcs_config(vcs_handler.acl_repo_name)
             # maybe damaged/non existent settings. We still want to
             # pass that point to validate on is_valid_and_existing_repo
             # and return proper HTTP Code back to client
             if vcs_conf:
                 vcs_handler.repo_vcs_config = vcs_conf
 
             # check for type, presence in database and on filesystem
             if not vcs_handler.is_valid_and_existing_repo(
                     vcs_handler.acl_repo_name,
                     vcs_handler.base_path,
                     vcs_handler.SCM):
                 return HTTPNotFound()(environ, start_response)
 
             environ['REPO_NAME'] = vcs_handler.url_repo_name
 
             # Wrap handler in middlewares if they are enabled.
             vcs_handler = self.wrap_in_gzip_if_enabled(
                 vcs_handler, self.config)
             vcs_handler, _ = wrap_in_appenlight_if_enabled(
                 vcs_handler, self.config, self.appenlight_client)
 
             return vcs_handler(environ, start_response)
 
         return self.application(environ, start_response)

rhodecode/lib/rc_cache/backends.py

@@ -1,335 +1,342 b''
 # Copyright (C) 2015-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import errno
 import fcntl
 import functools
 import logging
 import os
 import pickle
 import time
 
 import gevent
 import msgpack
 import redis
 
 flock_org = fcntl.flock
 from typing import Union
 
 from dogpile.cache.api import Deserializer, Serializer
 from dogpile.cache.backends import file as file_backend
 from dogpile.cache.backends import memory as memory_backend
 from dogpile.cache.backends import redis as redis_backend
 from dogpile.cache.backends.file import FileLock
 from dogpile.cache.util import memoized_property
 
-from rhodecode.lib.memory_lru_dict import LRUDict, LRUDictDebug
-from rhodecode.lib.str_utils import safe_bytes, safe_str
-from rhodecode.lib.type_utils import str2bool
+from ...lib.memory_lru_dict import LRUDict, LRUDictDebug
+from ...lib.str_utils import safe_bytes, safe_str
+from ...lib.type_utils import str2bool
 
 _default_max_size = 1024
 
 log = logging.getLogger(__name__)
 
 
 class LRUMemoryBackend(memory_backend.MemoryBackend):
     key_prefix = 'lru_mem_backend'
     pickle_values = False
 
     def __init__(self, arguments):
         self.max_size = arguments.pop('max_size', _default_max_size)
 
         LRUDictClass = LRUDict
         if arguments.pop('log_key_count', None):
             LRUDictClass = LRUDictDebug
 
         arguments['cache_dict'] = LRUDictClass(self.max_size)
         super().__init__(arguments)
 
     def __repr__(self):
         return f'{self.__class__}(maxsize=`{self.max_size}`)'
 
     def __str__(self):
         return self.__repr__()
 
     def delete(self, key):
         try:
             del self._cache[key]
         except KeyError:
             # we don't care if key isn't there at deletion
             pass
 
     def list_keys(self, prefix):
         return list(self._cache.keys())
 
     def delete_multi(self, keys):
         for key in keys:
             self.delete(key)
 
     def delete_multi_by_prefix(self, prefix):
         cache_keys = self.list_keys(prefix=prefix)
         num_affected_keys = len(cache_keys)
         if num_affected_keys:
             self.delete_multi(cache_keys)
         return num_affected_keys
 
 
 class PickleSerializer:
     serializer: None | Serializer = staticmethod(  # type: ignore
         functools.partial(pickle.dumps, protocol=pickle.HIGHEST_PROTOCOL)
     )
     deserializer: None | Deserializer = staticmethod(  # type: ignore
         functools.partial(pickle.loads)
     )
 
 
 class MsgPackSerializer:
     serializer: None | Serializer = staticmethod(  # type: ignore
         msgpack.packb
     )
     deserializer: None | Deserializer = staticmethod(  # type: ignore
         functools.partial(msgpack.unpackb, use_list=False)
     )
 
 
 class CustomLockFactory(FileLock):
 
     @memoized_property
     def _module(self):
 
         def gevent_flock(fd, operation):
             """
             Gevent compatible flock
             """
             # set non-blocking, this will cause an exception if we cannot acquire a lock
             operation |= fcntl.LOCK_NB
             start_lock_time = time.time()
             timeout = 60 * 15  # 15min
             while True:
                 try:
                     flock_org(fd, operation)
                     # lock has been acquired
                     break
                 except (OSError, IOError) as e:
                     # raise on other errors than Resource temporarily unavailable
                     if e.errno != errno.EAGAIN:
                         raise
                     elif (time.time() - start_lock_time) > timeout:
                         # waited to much time on a lock, better fail than loop for ever
                         log.error('Failed to acquire lock on `%s` after waiting %ss',
                                   self.filename, timeout)
                         raise
                     wait_timeout = 0.03
                     log.debug('Failed to acquire lock on `%s`, retry in %ss',
                               self.filename, wait_timeout)
                     gevent.sleep(wait_timeout)
 
         fcntl.flock = gevent_flock
         return fcntl
 
 
 class FileNamespaceBackend(PickleSerializer, file_backend.DBMBackend):
     key_prefix = 'file_backend'
 
     def __init__(self, arguments):
         arguments['lock_factory'] = CustomLockFactory
         db_file = arguments.get('filename')
 
         log.debug('initialing cache-backend=%s db in %s', self.__class__.__name__, db_file)
         db_file_dir = os.path.dirname(db_file)
         if not os.path.isdir(db_file_dir):
             os.makedirs(db_file_dir)
 
         try:
             super().__init__(arguments)
         except Exception:
             log.exception('Failed to initialize db at: %s', db_file)
             raise
 
     def __repr__(self):
         return f'{self.__class__}(file=`{self.filename}`)'
 
     def __str__(self):
         return self.__repr__()
 
     def _get_keys_pattern(self, prefix: bytes = b''):
         return b'%b:%b' % (safe_bytes(self.key_prefix), safe_bytes(prefix))
 
     def list_keys(self, prefix: bytes = b''):
         prefix = self._get_keys_pattern(prefix)
 
         def cond(dbm_key: bytes):
             if not prefix:
                 return True
 
             if dbm_key.startswith(prefix):
                 return True
             return False
 
         with self._dbm_file(True) as dbm:
             try:
                 return list(filter(cond, dbm.keys()))
             except Exception:
                 log.error('Failed to fetch DBM keys from DB: %s', self.get_store())
                 raise
 
     def delete_multi_by_prefix(self, prefix):
         cache_keys = self.list_keys(prefix=prefix)
         num_affected_keys = len(cache_keys)
         if num_affected_keys:
             self.delete_multi(cache_keys)
         return num_affected_keys
 
     def get_store(self):
         return self.filename
 
+    def cleanup_store(self):
+        for ext in ("db", "dat", "pag", "dir"):
+            final_filename = self.filename + os.extsep + ext
+            if os.path.exists(final_filename):
+                os.remove(final_filename)
+                log.warning('Removed dbm file %s', final_filename)
+
 
 class BaseRedisBackend(redis_backend.RedisBackend):
     key_prefix = ''
 
     def __init__(self, arguments):
         self.db_conn = arguments.get('host', '') or arguments.get('url', '') or 'redis-host'
         super().__init__(arguments)
 
         self._lock_timeout = self.lock_timeout
         self._lock_auto_renewal = str2bool(arguments.pop("lock_auto_renewal", True))
 
         if self._lock_auto_renewal and not self._lock_timeout:
             # set default timeout for auto_renewal
             self._lock_timeout = 30
 
     def __repr__(self):
         return f'{self.__class__}(conn=`{self.db_conn}`)'
 
     def __str__(self):
         return self.__repr__()
 
     def _create_client(self):
         args = {}
 
         if self.url is not None:
             args.update(url=self.url)
 
         else:
             args.update(
                 host=self.host, password=self.password,
                 port=self.port, db=self.db
             )
 
         connection_pool = redis.ConnectionPool(**args)
         self.writer_client = redis.StrictRedis(
             connection_pool=connection_pool
         )
         self.reader_client = self.writer_client
 
     def _get_keys_pattern(self, prefix: bytes = b''):
         return b'%b:%b*' % (safe_bytes(self.key_prefix), safe_bytes(prefix))
 
     def list_keys(self, prefix: bytes = b''):
         prefix = self._get_keys_pattern(prefix)
         return self.reader_client.keys(prefix)
 
     def delete_multi_by_prefix(self, prefix, use_lua=False):
         if use_lua:
             # high efficient LUA script to delete ALL keys by prefix...
             lua = """local keys = redis.call('keys', ARGV[1])
                      for i=1,#keys,5000 do
                      redis.call('del', unpack(keys, i, math.min(i+(5000-1), #keys)))
                      end
                      return #keys"""
             num_affected_keys = self.writer_client.eval(
                 lua,
                 0,
                 f"{prefix}*")
         else:
             cache_keys = self.list_keys(prefix=prefix)
             num_affected_keys = len(cache_keys)
             if num_affected_keys:
                 self.delete_multi(cache_keys)
         return num_affected_keys
 
     def get_store(self):
         return self.reader_client.connection_pool
 
     def get_mutex(self, key):
         if self.distributed_lock:
             lock_key = f'_lock_{safe_str(key)}'
             return get_mutex_lock(
                 self.writer_client, lock_key,
                 self._lock_timeout,
                 auto_renewal=self._lock_auto_renewal
             )
         else:
             return None
 
 
 class RedisPickleBackend(PickleSerializer, BaseRedisBackend):
     key_prefix = 'redis_pickle_backend'
     pass
 
 
 class RedisMsgPackBackend(MsgPackSerializer, BaseRedisBackend):
     key_prefix = 'redis_msgpack_backend'
     pass
 
 
 def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
-    from rhodecode.lib._vendor import redis_lock
+    from ...lib._vendor import redis_lock
 
     class _RedisLockWrapper:
         """LockWrapper for redis_lock"""
 
         @classmethod
         def get_lock(cls):
             return redis_lock.Lock(
                 redis_client=client,
                 name=lock_key,
                 expire=lock_timeout,
                 auto_renewal=auto_renewal,
                 strict=True,
             )
 
         def __repr__(self):
             return f"{self.__class__.__name__}:{lock_key}"
 
         def __str__(self):
             return f"{self.__class__.__name__}:{lock_key}"
 
         def __init__(self):
             self.lock = self.get_lock()
             self.lock_key = lock_key
 
         def acquire(self, wait=True):
             log.debug('Trying to acquire Redis lock for key %s', self.lock_key)
             try:
                 acquired = self.lock.acquire(wait)
                 log.debug('Got lock for key %s, %s', self.lock_key, acquired)
                 return acquired
             except redis_lock.AlreadyAcquired:
                 return False
             except redis_lock.AlreadyStarted:
                 # refresh thread exists, but it also means we acquired the lock
                 return True
 
         def release(self):
             try:
                 self.lock.release()
             except redis_lock.NotAcquired:
                 pass
 
     return _RedisLockWrapper()

rhodecode/lib/rc_cache/utils.py

@@ -1,357 +1,357 b''
 # Copyright (C) 2015-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import functools
 import logging
 import os
 import threading
 import time
 
 import decorator
 from dogpile.cache import CacheRegion
 
 import rhodecode
-from rhodecode.lib.hash_utils import sha1
-from rhodecode.lib.str_utils import safe_bytes
-from rhodecode.lib.type_utils import str2bool # noqa :required by imports from .utils
+from ...lib.hash_utils import sha1
+from ...lib.str_utils import safe_bytes
+from ...lib.type_utils import str2bool # noqa :required by imports from .utils
 
 from . import region_meta
 
 log = logging.getLogger(__name__)
 
 
 def isCython(func):
     """
     Private helper that checks if a function is a cython function.
     """
     return func.__class__.__name__ == 'cython_function_or_method'
 
 
 class RhodeCodeCacheRegion(CacheRegion):
 
     def __repr__(self):
         return f'`{self.__class__.__name__}(name={self.name}, backend={self.backend.__class__})`'
 
     def conditional_cache_on_arguments(
             self, namespace=None,
             expiration_time=None,
             should_cache_fn=None,
             to_str=str,
             function_key_generator=None,
             condition=True):
         """
         Custom conditional decorator, that will not touch any dogpile internals if
         condition isn't meet. This works a bit different from should_cache_fn
         And it's faster in cases we don't ever want to compute cached values
         """
         expiration_time_is_callable = callable(expiration_time)
         if not namespace:
             namespace = getattr(self, '_default_namespace', None)
 
         if function_key_generator is None:
             function_key_generator = self.function_key_generator
 
         def get_or_create_for_user_func(func_key_generator, user_func, *arg, **kw):
 
             if not condition:
                 log.debug('Calling un-cached method:%s', user_func.__name__)
                 start = time.time()
                 result = user_func(*arg, **kw)
                 total = time.time() - start
                 log.debug('un-cached method:%s took %.4fs', user_func.__name__, total)
                 return result
 
             key = func_key_generator(*arg, **kw)
 
             timeout = expiration_time() if expiration_time_is_callable \
                 else expiration_time
 
             log.debug('Calling cached method:`%s`', user_func.__name__)
             return self.get_or_create(key, user_func, timeout, should_cache_fn, (arg, kw))
 
         def cache_decorator(user_func):
             if to_str is str:
                 # backwards compatible
                 key_generator = function_key_generator(namespace, user_func)
             else:
                 key_generator = function_key_generator(namespace, user_func, to_str=to_str)
 
             def refresh(*arg, **kw):
                 """
                 Like invalidate, but regenerates the value instead
                 """
                 key = key_generator(*arg, **kw)
                 value = user_func(*arg, **kw)
                 self.set(key, value)
                 return value
 
             def invalidate(*arg, **kw):
                 key = key_generator(*arg, **kw)
                 self.delete(key)
 
             def set_(value, *arg, **kw):
                 key = key_generator(*arg, **kw)
                 self.set(key, value)
 
             def get(*arg, **kw):
                 key = key_generator(*arg, **kw)
                 return self.get(key)
 
             user_func.set = set_
             user_func.invalidate = invalidate
             user_func.get = get
             user_func.refresh = refresh
             user_func.key_generator = key_generator
             user_func.original = user_func
 
             # Use `decorate` to preserve the signature of :param:`user_func`.
             return decorator.decorate(user_func, functools.partial(
                 get_or_create_for_user_func, key_generator))
 
         return cache_decorator
 
 
 def make_region(*arg, **kw):
     return RhodeCodeCacheRegion(*arg, **kw)
 
 
 def get_default_cache_settings(settings, prefixes=None):
     prefixes = prefixes or []
     cache_settings = {}
     for key in settings.keys():
         for prefix in prefixes:
             if key.startswith(prefix):
                 name = key.split(prefix)[1].strip()
                 val = settings[key]
                 if isinstance(val, str):
                     val = val.strip()
                 cache_settings[name] = val
     return cache_settings
 
 
 def compute_key_from_params(*args):
     """
     Helper to compute key from given params to be used in cache manager
     """
     return sha1(safe_bytes("_".join(map(str, args))))
 
 
 def custom_key_generator(backend, namespace, fn):
     func_name = fn.__name__
 
     def generate_key(*args):
         backend_pref = getattr(backend, 'key_prefix', None) or 'backend_prefix'
         namespace_pref = namespace or 'default_namespace'
         arg_key = compute_key_from_params(*args)
         final_key = f"{backend_pref}:{namespace_pref}:{func_name}_{arg_key}"
 
         return final_key
 
     return generate_key
 
 
 def backend_key_generator(backend):
     """
     Special wrapper that also sends over the backend to the key generator
     """
     def wrapper(namespace, fn):
         return custom_key_generator(backend, namespace, fn)
     return wrapper
 
 
 def get_or_create_region(region_name, region_namespace: str = None, use_async_runner=False):
     from .backends import FileNamespaceBackend
     from . import async_creation_runner
 
     region_obj = region_meta.dogpile_cache_regions.get(region_name)
     if not region_obj:
         reg_keys = list(region_meta.dogpile_cache_regions.keys())
         raise OSError(f'Region `{region_name}` not in configured: {reg_keys}.')
 
     region_uid_name = f'{region_name}:{region_namespace}'
 
     # Special case for ONLY the FileNamespaceBackend backend. We register one-file-per-region
     if isinstance(region_obj.actual_backend, FileNamespaceBackend):
         if not region_namespace:
             raise ValueError(f'{FileNamespaceBackend} used requires to specify region_namespace param')
 
         region_exist = region_meta.dogpile_cache_regions.get(region_namespace)
         if region_exist:
             log.debug('Using already configured region: %s', region_namespace)
             return region_exist
 
         expiration_time = region_obj.expiration_time
 
         cache_dir = region_meta.dogpile_config_defaults['cache_dir']
         namespace_cache_dir = cache_dir
 
         # we default the namespace_cache_dir to our default cache dir.
         # however, if this backend is configured with filename= param, we prioritize that
         # so all caches within that particular region, even those namespaced end up in the same path
         if region_obj.actual_backend.filename:
             namespace_cache_dir = os.path.dirname(region_obj.actual_backend.filename)
 
         if not os.path.isdir(namespace_cache_dir):
             os.makedirs(namespace_cache_dir)
         new_region = make_region(
             name=region_uid_name,
             function_key_generator=backend_key_generator(region_obj.actual_backend)
         )
 
         namespace_filename = os.path.join(
             namespace_cache_dir, f"{region_name}_{region_namespace}.cache_db")
         # special type that allows 1db per namespace
         new_region.configure(
             backend='dogpile.cache.rc.file_namespace',
             expiration_time=expiration_time,
             arguments={"filename": namespace_filename}
         )
 
         # create and save in region caches
         log.debug('configuring new region: %s', region_uid_name)
         region_obj = region_meta.dogpile_cache_regions[region_namespace] = new_region
 
     region_obj._default_namespace = region_namespace
     if use_async_runner:
         region_obj.async_creation_runner = async_creation_runner
     return region_obj
 
 
 def clear_cache_namespace(cache_region: str | RhodeCodeCacheRegion, cache_namespace_uid: str, method: str) -> int:
     from . import CLEAR_DELETE, CLEAR_INVALIDATE
 
     if not isinstance(cache_region, RhodeCodeCacheRegion):
         cache_region = get_or_create_region(cache_region, cache_namespace_uid)
     log.debug('clearing cache region: %s [prefix:%s] with method=%s',
               cache_region, cache_namespace_uid, method)
 
     num_affected_keys = 0
 
     if method == CLEAR_INVALIDATE:
         # NOTE: The CacheRegion.invalidate() method’s default mode of
         # operation is to set a timestamp local to this CacheRegion in this Python process only.
         # It does not impact other Python processes or regions as the timestamp is only stored locally in memory.
         cache_region.invalidate(hard=True)
 
     if method == CLEAR_DELETE:
         num_affected_keys = cache_region.backend.delete_multi_by_prefix(prefix=cache_namespace_uid)
     return num_affected_keys
 
 
 class ActiveRegionCache(object):
     def __init__(self, context, cache_data: dict):
         self.context = context
         self.cache_data = cache_data
 
     @property
     def state_uid(self) -> str:
         return self.cache_data['cache_state_uid']
 
 
 class InvalidationContext(object):
     """
     usage::
 
         from rhodecode.lib import rc_cache
 
         repo_namespace_key = 'some-cache-for-repo-id-100'
         inv_context_manager = rc_cache.InvalidationContext(key=repo_namespace_key)
 
         def cache_generator(_state_uid):
 
             @region.conditional_cache_on_arguments(namespace='some-common-namespace-100')
             def _dummy_func(*args):
                 # compute heavy function
                 return _state_uid, 'result'
 
             return _dummy_func
 
         with inv_context_manager as invalidation_context:
             cache_state_uid = invalidation_context.state_uid
             cache_func = cache_generator(cache_state_uid)
             previous_state_uid, result = cache_func(*call_args)
 
             should_invalidate = previous_state_uid != cache_state_uid
             if should_invalidate:
                 _, result = cache_func.refresh(*call_args)
 
         # To send global invalidation signal, simply run
         CacheKey.set_invalidate(repo_namespace_key)
 
     """
 
     def __repr__(self):
         return f'<InvalidationContext:{self.cache_key}>'
 
     def __init__(self, key, raise_exception=False, thread_scoped=None):
         self.cache_key = key
 
         self.raise_exception = raise_exception
         self.proc_id = rhodecode.ConfigGet().get_str('instance_id') or 'DEFAULT'
         self.thread_id = 'global'
 
         if thread_scoped is None:
             # if we set "default" we can override this via .ini settings
             thread_scoped = rhodecode.ConfigGet().get_bool('cache_thread_scoped')
 
         # Append the thread id to the cache key if this invalidation context
         # should be scoped to the current thread.
         if thread_scoped is True:
             self.thread_id = threading.current_thread().ident
 
         self.proc_key = f'proc:{self.proc_id}|thread:{self.thread_id}|key:{self.cache_key}'
         self.compute_time = 0
 
     def get_or_create_cache_obj(self):
         from rhodecode.model.db import CacheKey, Session, IntegrityError
 
         cache_obj = CacheKey.get_active_cache(self.cache_key)
         log.debug('Fetched cache obj %s using %s cache key.', cache_obj, self.cache_key)
 
         if not cache_obj:
             # generate new UID for non-existing cache object
             cache_state_uid = CacheKey.generate_new_state_uid()
             cache_obj = CacheKey(self.cache_key, cache_args=f'repo_state:{self._start_time}',
                                  cache_state_uid=cache_state_uid, cache_active=True)
             try:
                 Session().add(cache_obj)
                 Session().commit()
             except IntegrityError:
                 # if we catch integrity error, it means we inserted this object
                 # assumption is that's really an edge race-condition case and
                 # it's safe is to skip it
                 Session().rollback()
             except Exception:
                 log.exception('Failed to commit on cache key update')
                 Session().rollback()
                 if self.raise_exception:
                     raise
         return cache_obj
 
     def __enter__(self):
         log.debug('Entering cache invalidation check context: %s', self)
         self._start_time = time.time()
 
         self.cache_obj = self.get_or_create_cache_obj()
         cache_data = self.cache_obj.get_dict()
 
         return ActiveRegionCache(context=self, cache_data=cache_data)
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         # save compute time
         self.compute_time = time.time() - self._start_time

rhodecode/lib/rc_commands/add_artifact.py

@@ -1,105 +1,104 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import sys
 import logging
 
 import click
 
 from rhodecode.lib.pyramid_utils import bootstrap
 from rhodecode.model.db import Session, User, Repository
 from rhodecode.model.user import UserModel
 from rhodecode.apps.file_store import utils as store_utils
 
 log = logging.getLogger(__name__)
 
 
 @click.command()
 @click.argument('ini_path', type=click.Path(exists=True))
 @click.option(
     '--filename',
     required=True,
     help='Filename for artifact.')
 @click.option(
     '--file-path',
     required=True,
     type=click.Path(exists=True, dir_okay=False, readable=True),
     help='Path to a file to be added as artifact')
 @click.option(
     '--repo-id',
     required=True,
     type=int,
     help='ID of repository to add this artifact to.')
 @click.option(
     '--user-id',
     default=None,
     type=int,
     help='User ID for creator of artifact. '
          'Default would be first super admin.')
 @click.option(
     '--description',
     default=None,
     type=str,
     help='Add description to this artifact')
 def main(ini_path, filename, file_path, repo_id, user_id, description):
     return command(ini_path, filename, file_path, repo_id, user_id, description)
 
 
 def command(ini_path, filename, file_path, repo_id, user_id, description):
     with bootstrap(ini_path, env={'RC_CMD_SETUP_RC': '1'}) as env:
         try:
             from rc_ee.api.views.store_api import _store_file
         except ImportError:
             click.secho('ERROR: Unable to import store_api. '
                         'store_api is only available in EE edition of RhodeCode',
                         fg='red')
             sys.exit(-1)
 
         request = env['request']
 
         repo = Repository.get(repo_id)
         if not repo:
             click.secho(f'ERROR: Unable to find repository with id `{repo_id}`',
                         fg='red')
             sys.exit(-1)
 
         # if we don't give user, or it's "DEFAULT" user we pick super-admin
         if user_id is not None or user_id == 1:
             db_user = User.get(user_id)
         else:
             db_user = User.get_first_super_admin()
 
         if not db_user:
             click.secho(f'ERROR: Unable to find user with id/username `{user_id}`',
                         fg='red')
             sys.exit(-1)
 
         auth_user = db_user.AuthUser(ip_addr='127.0.0.1')
 
-        storage = store_utils.get_file_storage(request.registry.settings)
+        f_store = store_utils.get_filestore_backend(request.registry.settings)
 
         with open(file_path, 'rb') as f:
             click.secho(f'Adding new artifact from path: `{file_path}`',
                         fg='green')
 
             file_data = _store_file(
-                storage, auth_user, filename, content=None, check_acl=True,
+                f_store, auth_user, filename, content=None, check_acl=True,
                 file_obj=f, description=description,
                 scope_repo_id=repo.repo_id)
-            click.secho(f'File Data: {file_data}',
-                        fg='green')
+            click.secho(f'File Data: {file_data}', fg='green')

rhodecode/lib/rc_commands/setup_rc.py

@@ -1,124 +1,123 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import logging
 
 import click
 import pyramid.paster
 
 from rhodecode.lib.pyramid_utils import bootstrap
 from rhodecode.lib.config_utils import get_app_config
 from rhodecode.lib.db_manage import DbManage
 from rhodecode.lib.utils2 import get_encryption_key
 from rhodecode.model.db import Session
 
 
 log = logging.getLogger(__name__)
 
 
 @click.command()
 @click.argument('ini_path', type=click.Path(exists=True))
 @click.option(
     '--force-yes/--force-no', default=None,
     help="Force yes/no to every question")
 @click.option(
     '--user',
     default=None,
     help='Initial super-admin username')
 @click.option(
     '--email',
     default=None,
     help='Initial super-admin email address.')
 @click.option(
     '--password',
     default=None,
     help='Initial super-admin password. Minimum 6 chars.')
 @click.option(
     '--api-key',
     help='Initial API key for the admin user')
 @click.option(
     '--repos',
     default=None,
     help='Absolute path to storage location. This is storage for all '
          'existing and future repositories, and repository groups.')
 @click.option(
     '--public-access/--no-public-access',
     default=None,
     help='Enable public access on this installation. '
          'Default is public access enabled.')
 @click.option(
     '--skip-existing-db',
     default=False,
     is_flag=True,
     help='Do not destroy and re-initialize the database if it already exist.')
 @click.option(
     '--apply-license-key',
     default=False,
     is_flag=True,
     help='Get the license key from a license file or ENV and apply during DB creation.')
 def main(ini_path, force_yes, user, email, password, api_key, repos,
          public_access, skip_existing_db, apply_license_key):
     return command(ini_path, force_yes, user, email, password, api_key,
                    repos, public_access, skip_existing_db, apply_license_key)
 
 
 def command(ini_path, force_yes, user, email, password, api_key, repos,
             public_access, skip_existing_db, apply_license_key):
     # mapping of old parameters to new CLI from click
     options = dict(
         username=user,
         email=email,
         password=password,
         api_key=api_key,
         repos_location=repos,
         force_ask=force_yes,
         public_access=public_access
     )
     pyramid.paster.setup_logging(ini_path)
 
     config = get_app_config(ini_path)
 
     db_uri = config['sqlalchemy.db1.url']
     enc_key = get_encryption_key(config)
     dbmanage = DbManage(log_sql=True, dbconf=db_uri, root='.',
                         tests=False, cli_args=options, enc_key=enc_key)
     if skip_existing_db and dbmanage.db_exists():
         return
 
     dbmanage.create_tables(override=True)
     dbmanage.set_db_version()
     opts = dbmanage.config_prompt(None)
     dbmanage.create_settings(opts)
     dbmanage.create_default_user()
     dbmanage.create_admin_and_prompt()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     if apply_license_key:
-        try:
-            from rc_license.models import apply_trial_license_if_missing
-            apply_trial_license_if_missing(force=True)
-        except ImportError:
-            pass
+        from rhodecode.model.license import apply_license_from_file
+        license_file_path = config.get('license.import_path')
+        if license_file_path:
+            apply_license_from_file(license_file_path, force=True)
 
     Session().commit()
 
     with bootstrap(ini_path, env={'RC_CMD_SETUP_RC': '1'}) as env:
         msg = 'Successfully initialized database, schema and default data.'
         print()
         print('*' * len(msg))
         print(msg.upper())
         print('*' * len(msg))

rhodecode/lib/str_utils.py

@@ -1,183 +1,187 b''
 # Copyright (C) 2011-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import typing
 import base64
 import logging
 from unidecode import unidecode
 
 import rhodecode
 from rhodecode.lib.type_utils import aslist
 
 
 log = logging.getLogger(__name__)
 
 
 def safe_int(val, default=None) -> int:
     """
     Returns int() of val if val is not convertable to int use default
     instead
 
     :param val:
     :param default:
     """
 
     try:
         val = int(val)
     except (ValueError, TypeError):
         val = default
 
     return val
 
 
 def safe_float(val, default=None) -> float:
     """
     Returns float() of val if val is not convertable to float use default
     instead
 
     :param val:
     :param default:
     """
 
     try:
         val = float(val)
     except (ValueError, TypeError):
         val = default
 
     return val
 
 
 def base64_to_str(text: str | bytes) -> str:
     return safe_str(base64.encodebytes(safe_bytes(text))).strip()
 
 
 def get_default_encodings() -> list[str]:
     return aslist(rhodecode.CONFIG.get('default_encoding', 'utf8'), sep=',')
 
 
 DEFAULT_ENCODINGS = get_default_encodings()
 
 
 def safe_str(str_, to_encoding=None) -> str:
     """
     safe str function. Does few trick to turn unicode_ into string
 
     :param str_: str to encode
     :param to_encoding: encode to this type UTF8 default
     """
     if isinstance(str_, str):
         return str_
 
     # if it's bytes cast to str
     if not isinstance(str_, bytes):
         return str(str_)
 
     to_encoding = to_encoding or DEFAULT_ENCODINGS
     if not isinstance(to_encoding, (list, tuple)):
         to_encoding = [to_encoding]
 
     for enc in to_encoding:
         try:
             return str(str_, enc)
         except UnicodeDecodeError:
             pass
 
     return str(str_, to_encoding[0], 'replace')
 
 
 def safe_bytes(str_, from_encoding=None) -> bytes:
     """
     safe bytes function. Does few trick to turn str_ into bytes string:
 
     :param str_: string to decode
     :param from_encoding: encode from this type UTF8 default
     """
     if isinstance(str_, bytes):
         return str_
 
     if not isinstance(str_, str):
         raise ValueError(f'safe_bytes cannot convert other types than str: got: {type(str_)}')
 
     from_encoding = from_encoding or get_default_encodings()
     if not isinstance(from_encoding, (list, tuple)):
         from_encoding = [from_encoding]
 
     for enc in from_encoding:
         try:
             return str_.encode(enc)
         except UnicodeDecodeError:
             pass
 
     return str_.encode(from_encoding[0], 'replace')
 
 
 def ascii_bytes(str_, allow_bytes=False) -> bytes:
     """
     Simple conversion from str to bytes, with assumption that str_ is pure ASCII.
     Fails with UnicodeError on invalid input.
     This should be used where encoding and "safe" ambiguity should be avoided.
     Where strings already have been encoded in other ways but still are unicode
     string - for example to hex, base64, json, urlencoding, or are known to be
     identifiers.
     """
     if allow_bytes and isinstance(str_, bytes):
         return str_
 
     if not isinstance(str_, str):
         raise ValueError(f'ascii_bytes cannot convert other types than str: got: {type(str_)}')
     return str_.encode('ascii')
 
 
 def ascii_str(str_) -> str:
     """
     Simple conversion from bytes to str, with assumption that str_ is pure ASCII.
     Fails with UnicodeError on invalid input.
     This should be used where encoding and "safe" ambiguity should be avoided.
     Where strings are encoded but also in other ways are known to be ASCII, and
     where a unicode string is wanted without caring about encoding. For example
     to hex, base64, urlencoding, or are known to be identifiers.
     """
 
     if not isinstance(str_, bytes):
         raise ValueError(f'ascii_str cannot convert other types than bytes: got: {type(str_)}')
     return str_.decode('ascii')
 
 
 def convert_special_chars(str_) -> str:
     """
         trie to replace non-ascii letters to their ascii representation eg::
 
             `żołw` converts into `zolw`
     """
     value = safe_str(str_)
     converted_value = unidecode(value)
     return converted_value
 
 
 def splitnewlines(text: bytes):
     """
     like splitlines, but only split on newlines.
     """
 
     lines = [_l + b'\n' for _l in text.split(b'\n')]
     if lines:
         if lines[-1] == b'\n':
             lines.pop()
         else:
             lines[-1] = lines[-1][:-1]
     return lines
+
+
+def header_safe_str(val):
+    return safe_bytes(val).decode('latin-1', errors='replace')

rhodecode/lib/system_info.py

@@ -1,827 +1,866 b''
 # Copyright (C) 2017-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import os
 import sys
 import time
 import platform
 import collections
 import psutil
 from functools import wraps
 
 import pkg_resources
 import logging
 import resource
 
 import configparser
 
 from rc_license.models import LicenseModel
 from rhodecode.lib.str_utils import safe_str
 
 log = logging.getLogger(__name__)
 
 
 _NA = 'NOT AVAILABLE'
 _NA_FLOAT = 0.0
 
 STATE_OK = 'ok'
 STATE_ERR = 'error'
 STATE_WARN = 'warning'
 
 STATE_OK_DEFAULT = {'message': '', 'type': STATE_OK}
 
 
 registered_helpers = {}
 
 
 def register_sysinfo(func):
     """
     @register_helper
     def db_check():
         pass
 
     db_check == registered_helpers['db_check']
     """
     global registered_helpers
     registered_helpers[func.__name__] = func
 
     @wraps(func)
     def _wrapper(*args, **kwargs):
         return func(*args, **kwargs)
     return _wrapper
 
 
 # HELPERS
 def percentage(part: (int, float), whole: (int, float)):
     whole = float(whole)
     if whole > 0:
         return round(100 * float(part) / whole, 1)
     return 0.0
 
 
 def get_storage_size(storage_path):
     sizes = []
     for file_ in os.listdir(storage_path):
         storage_file = os.path.join(storage_path, file_)
         if os.path.isfile(storage_file):
             try:
                 sizes.append(os.path.getsize(storage_file))
             except OSError:
                 log.exception('Failed to get size of storage file %s', storage_file)
                 pass
 
     return sum(sizes)
 
 
 def get_resource(resource_type):
     try:
         return resource.getrlimit(resource_type)
     except Exception:
         return 'NOT_SUPPORTED'
 
 
 def get_cert_path(ini_path):
     default = '/etc/ssl/certs/ca-certificates.crt'
     control_ca_bundle = os.path.join(
         os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(ini_path)))),
         '/etc/ssl/certs/ca-certificates.crt')
     if os.path.isfile(control_ca_bundle):
         default = control_ca_bundle
 
     return default
 
 
 class SysInfoRes(object):
     def __init__(self, value, state=None, human_value=None):
         self.value = value
         self.state = state or STATE_OK_DEFAULT
         self.human_value = human_value or value
 
     def __json__(self):
         return {
             'value': self.value,
             'state': self.state,
             'human_value': self.human_value,
         }
 
     def get_value(self):
         return self.__json__()
 
     def __str__(self):
         return f'<SysInfoRes({self.__json__()})>'
 
 
 class SysInfo(object):
 
     def __init__(self, func_name, **kwargs):
         self.function_name = func_name
         self.value = _NA
         self.state = None
         self.kwargs = kwargs or {}
 
     def __call__(self):
         computed = self.compute(**self.kwargs)
         if not isinstance(computed, SysInfoRes):
             raise ValueError(
                 'computed value for {} is not instance of '
                 '{}, got {} instead'.format(
                     self.function_name, SysInfoRes, type(computed)))
         return computed.__json__()
 
     def __str__(self):
         return f'<SysInfo({self.function_name})>'
 
     def compute(self, **kwargs):
         return self.function_name(**kwargs)
 
 
 # SysInfo functions
 @register_sysinfo
 def python_info():
     value = dict(version=f'{platform.python_version()}:{platform.python_implementation()}',
                  executable=sys.executable)
     return SysInfoRes(value=value)
 
 
 @register_sysinfo
 def py_modules():
     mods = dict([(p.project_name, {'version': p.version, 'location': p.location})
                  for p in pkg_resources.working_set])
 
     value = sorted(mods.items(), key=lambda k: k[0].lower())
     return SysInfoRes(value=value)
 
 
 @register_sysinfo
 def platform_type():
     from rhodecode.lib.utils import generate_platform_uuid
 
     value = dict(
         name=safe_str(platform.platform()),
         uuid=generate_platform_uuid()
     )
     return SysInfoRes(value=value)
 
 
 @register_sysinfo
 def locale_info():
     import locale
 
     def safe_get_locale(locale_name):
         try:
             locale.getlocale(locale_name)
         except TypeError:
             return f'FAILED_LOCALE_GET:{locale_name}'
 
     value = dict(
         locale_default=locale.getlocale(),
         locale_lc_all=safe_get_locale(locale.LC_ALL),
         locale_lc_ctype=safe_get_locale(locale.LC_CTYPE),
         lang_env=os.environ.get('LANG'),
         lc_all_env=os.environ.get('LC_ALL'),
         local_archive_env=os.environ.get('LOCALE_ARCHIVE'),
     )
     human_value = \
         f"LANG: {value['lang_env']}, \
           locale LC_ALL: {value['locale_lc_all']},  \
           locale LC_CTYPE: {value['locale_lc_ctype']},  \
           Default locales: {value['locale_default']}"
 
     return SysInfoRes(value=value, human_value=human_value)
 
 
 @register_sysinfo
 def ulimit_info():
     data = collections.OrderedDict([
         ('cpu time (seconds)', get_resource(resource.RLIMIT_CPU)),
         ('file size', get_resource(resource.RLIMIT_FSIZE)),
         ('stack size', get_resource(resource.RLIMIT_STACK)),
         ('core file size', get_resource(resource.RLIMIT_CORE)),
         ('address space size', get_resource(resource.RLIMIT_AS)),
         ('locked in mem size', get_resource(resource.RLIMIT_MEMLOCK)),
         ('heap size', get_resource(resource.RLIMIT_DATA)),
         ('rss size', get_resource(resource.RLIMIT_RSS)),
         ('number of processes', get_resource(resource.RLIMIT_NPROC)),
         ('open files', get_resource(resource.RLIMIT_NOFILE)),
     ])
 
     text = ', '.join(f'{k}:{v}' for k, v in data.items())
 
     value = {
         'limits': data,
         'text': text,
     }
     return SysInfoRes(value=value)
 
 
 @register_sysinfo
 def uptime():
     from rhodecode.lib.helpers import age, time_to_datetime
     from rhodecode.translation import TranslationString
 
     value = dict(boot_time=0, uptime=0, text='')
     state = STATE_OK_DEFAULT
 
     boot_time = psutil.boot_time()
     value['boot_time'] = boot_time
     value['uptime'] = time.time() - boot_time
 
     date_or_age = age(time_to_datetime(boot_time))
     if isinstance(date_or_age, TranslationString):
         date_or_age = date_or_age.interpolate()
 
     human_value = value.copy()
     human_value['boot_time'] = time_to_datetime(boot_time)
     human_value['uptime'] = age(time_to_datetime(boot_time), show_suffix=False)
 
     human_value['text'] = f'Server started {date_or_age}'
     return SysInfoRes(value=value, human_value=human_value)
 
 
 @register_sysinfo
 def memory():
     from rhodecode.lib.helpers import format_byte_size_binary
     value = dict(available=0, used=0, used_real=0, cached=0, percent=0,
                  percent_used=0, free=0, inactive=0, active=0, shared=0,
                  total=0, buffers=0, text='')
 
     state = STATE_OK_DEFAULT
 
     value.update(dict(psutil.virtual_memory()._asdict()))
     value['used_real'] = value['total'] - value['available']
     value['percent_used'] = psutil._common.usage_percent(value['used_real'], value['total'], 1)
 
     human_value = value.copy()
     human_value['text'] = '{}/{}, {}% used'.format(
         format_byte_size_binary(value['used_real']),
         format_byte_size_binary(value['total']),
         value['percent_used'])
 
     keys = list(value.keys())[::]
     keys.pop(keys.index('percent'))
     keys.pop(keys.index('percent_used'))
     keys.pop(keys.index('text'))
     for k in keys:
         human_value[k] = format_byte_size_binary(value[k])
 
     if state['type'] == STATE_OK and value['percent_used'] > 90:
         msg = 'Critical: your available RAM memory is very low.'
         state = {'message': msg, 'type': STATE_ERR}
 
     elif state['type'] == STATE_OK and value['percent_used'] > 70:
         msg = 'Warning: your available RAM memory is running low.'
         state = {'message': msg, 'type': STATE_WARN}
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 @register_sysinfo
 def machine_load():
     value = {'1_min': _NA_FLOAT, '5_min': _NA_FLOAT, '15_min': _NA_FLOAT, 'text': ''}
     state = STATE_OK_DEFAULT
 
     # load averages
     if hasattr(psutil.os, 'getloadavg'):
         value.update(dict(
             list(zip(['1_min', '5_min', '15_min'], psutil.os.getloadavg()))
         ))
 
     human_value = value.copy()
     human_value['text'] = '1min: {}, 5min: {}, 15min: {}'.format(
         value['1_min'], value['5_min'], value['15_min'])
 
     if state['type'] == STATE_OK and value['15_min'] > 5.0:
         msg = 'Warning: your machine load is very high.'
         state = {'message': msg, 'type': STATE_WARN}
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 @register_sysinfo
 def cpu():
     value = {'cpu': 0, 'cpu_count': 0, 'cpu_usage': []}
     state = STATE_OK_DEFAULT
 
     value['cpu'] = psutil.cpu_percent(0.5)
     value['cpu_usage'] = psutil.cpu_percent(0.5, percpu=True)
     value['cpu_count'] = psutil.cpu_count()
 
     human_value = value.copy()
     human_value['text'] = f'{value["cpu_count"]} cores at {value["cpu"]} %'
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 @register_sysinfo
 def storage():
     from rhodecode.lib.helpers import format_byte_size_binary
     from rhodecode.lib.utils import get_rhodecode_repo_store_path
     path = get_rhodecode_repo_store_path()
 
     value = dict(percent=0, used=0, total=0, path=path, text='')
     state = STATE_OK_DEFAULT
 
     try:
         value.update(dict(psutil.disk_usage(path)._asdict()))
     except Exception as e:
         log.exception('Failed to fetch disk info')
         state = {'message': str(e), 'type': STATE_ERR}
 
     human_value = value.copy()
     human_value['used'] = format_byte_size_binary(value['used'])
     human_value['total'] = format_byte_size_binary(value['total'])
     human_value['text'] = "{}/{}, {}% used".format(
         format_byte_size_binary(value['used']),
         format_byte_size_binary(value['total']),
         value['percent'])
 
     if state['type'] == STATE_OK and value['percent'] > 90:
         msg = 'Critical: your disk space is very low.'
         state = {'message': msg, 'type': STATE_ERR}
 
     elif state['type'] == STATE_OK and value['percent'] > 70:
         msg = 'Warning: your disk space is running low.'
         state = {'message': msg, 'type': STATE_WARN}
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 @register_sysinfo
 def storage_inodes():
     from rhodecode.lib.utils import get_rhodecode_repo_store_path
     path = get_rhodecode_repo_store_path()
 
     value = dict(percent=0.0, free=0, used=0, total=0, path=path, text='')
     state = STATE_OK_DEFAULT
 
     try:
         i_stat = os.statvfs(path)
         value['free'] = i_stat.f_ffree
         value['used'] = i_stat.f_files-i_stat.f_favail
         value['total'] = i_stat.f_files
         value['percent'] = percentage(value['used'], value['total'])
     except Exception as e:
         log.exception('Failed to fetch disk inodes info')
         state = {'message': str(e), 'type': STATE_ERR}
 
     human_value = value.copy()
     human_value['text'] = "{}/{}, {}% used".format(
         value['used'], value['total'], value['percent'])
 
     if state['type'] == STATE_OK and value['percent'] > 90:
         msg = 'Critical: your disk free inodes are very low.'
         state = {'message': msg, 'type': STATE_ERR}
 
     elif state['type'] == STATE_OK and value['percent'] > 70:
         msg = 'Warning: your disk free inodes are running low.'
         state = {'message': msg, 'type': STATE_WARN}
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 @register_sysinfo
-def storage_archives():
+def storage_artifacts():
     import rhodecode
     from rhodecode.lib.helpers import format_byte_size_binary
     from rhodecode.lib.archive_cache import get_archival_cache_store
 
-    storage_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type')
+    backend_type = rhodecode.ConfigGet().get_str('archive_cache.backend.type')
 
-    value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=storage_type)
+    value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=backend_type)
     state = STATE_OK_DEFAULT
     try:
         d_cache = get_archival_cache_store(config=rhodecode.CONFIG)
+        backend_type = str(d_cache)
 
         total_files, total_size, _directory_stats = d_cache.get_statistics()
 
         value.update({
             'percent': 100,
             'used': total_size,
             'total': total_size,
             'items': total_files,
-            'path': d_cache.storage_path
+            'path': d_cache.storage_path,
+            'type': backend_type
         })
 
     except Exception as e:
         log.exception('failed to fetch archive cache storage')
         state = {'message': str(e), 'type': STATE_ERR}
 
     human_value = value.copy()
     human_value['used'] = format_byte_size_binary(value['used'])
     human_value['total'] = format_byte_size_binary(value['total'])
-    human_value['text'] = "{} ({} items)".format(
-        human_value['used'], value['items'])
+    human_value['text'] = f"{human_value['used']} ({value['items']} items)"
+
+    return SysInfoRes(value=value, state=state, human_value=human_value)
+
+
+@register_sysinfo
+def storage_archives():
+    import rhodecode
+    from rhodecode.lib.helpers import format_byte_size_binary
+    import rhodecode.apps.file_store.utils as store_utils
+    from rhodecode import CONFIG
+
+    backend_type = rhodecode.ConfigGet().get_str(store_utils.config_keys.backend_type)
+
+    value = dict(percent=0, used=0, total=0, items=0, path='', text='', type=backend_type)
+    state = STATE_OK_DEFAULT
+    try:
+        f_store = store_utils.get_filestore_backend(config=CONFIG)
+        backend_type = str(f_store)
+        total_files, total_size, _directory_stats = f_store.get_statistics()
+
+        value.update({
+            'percent': 100,
+            'used': total_size,
+            'total': total_size,
+            'items': total_files,
+            'path': f_store.storage_path,
+            'type': backend_type
+        })
+
+    except Exception as e:
+        log.exception('failed to fetch archive cache storage')
+        state = {'message': str(e), 'type': STATE_ERR}
+
+    human_value = value.copy()
+    human_value['used'] = format_byte_size_binary(value['used'])
+    human_value['total'] = format_byte_size_binary(value['total'])
+    human_value['text'] = f"{human_value['used']} ({value['items']} items)"
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 @register_sysinfo
 def storage_gist():
     from rhodecode.model.gist import GIST_STORE_LOC
     from rhodecode.lib.utils import safe_str, get_rhodecode_repo_store_path
     from rhodecode.lib.helpers import format_byte_size_binary, get_directory_statistics
 
     path = safe_str(os.path.join(
         get_rhodecode_repo_store_path(), GIST_STORE_LOC))
 
     # gist storage
     value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
     state = STATE_OK_DEFAULT
 
     try:
         total_files, total_size, _directory_stats = get_directory_statistics(path)
         value.update({
             'percent': 100,
             'used': total_size,
             'total': total_size,
             'items': total_files
         })
     except Exception as e:
         log.exception('failed to fetch gist storage items')
         state = {'message': str(e), 'type': STATE_ERR}
 
     human_value = value.copy()
     human_value['used'] = format_byte_size_binary(value['used'])
     human_value['total'] = format_byte_size_binary(value['total'])
     human_value['text'] = "{} ({} items)".format(
         human_value['used'], value['items'])
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 @register_sysinfo
 def storage_temp():
     import tempfile
     from rhodecode.lib.helpers import format_byte_size_binary
 
     path = tempfile.gettempdir()
     value = dict(percent=0, used=0, total=0, items=0, path=path, text='')
     state = STATE_OK_DEFAULT
 
     if not psutil:
         return SysInfoRes(value=value, state=state)
 
     try:
         value.update(dict(psutil.disk_usage(path)._asdict()))
     except Exception as e:
         log.exception('Failed to fetch temp dir info')
         state = {'message': str(e), 'type': STATE_ERR}
 
     human_value = value.copy()
     human_value['used'] = format_byte_size_binary(value['used'])
     human_value['total'] = format_byte_size_binary(value['total'])
     human_value['text'] = "{}/{}, {}% used".format(
         format_byte_size_binary(value['used']),
         format_byte_size_binary(value['total']),
         value['percent'])
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 @register_sysinfo
 def search_info():
     import rhodecode
     from rhodecode.lib.index import searcher_from_config
 
     backend = rhodecode.CONFIG.get('search.module', '')
     location = rhodecode.CONFIG.get('search.location', '')
 
     try:
         searcher = searcher_from_config(rhodecode.CONFIG)
         searcher = searcher.__class__.__name__
     except Exception:
         searcher = None
 
     value = dict(
         backend=backend, searcher=searcher, location=location, text='')
     state = STATE_OK_DEFAULT
 
     human_value = value.copy()
     human_value['text'] = "backend:`{}`".format(human_value['backend'])
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 @register_sysinfo
 def git_info():
     from rhodecode.lib.vcs.backends import git
     state = STATE_OK_DEFAULT
     value = human_value = ''
     try:
         value = git.discover_git_version(raise_on_exc=True)
         human_value = f'version reported from VCSServer: {value}'
     except Exception as e:
         state = {'message': str(e), 'type': STATE_ERR}
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 @register_sysinfo
 def hg_info():
     from rhodecode.lib.vcs.backends import hg
     state = STATE_OK_DEFAULT
     value = human_value = ''
     try:
         value = hg.discover_hg_version(raise_on_exc=True)
         human_value = f'version reported from VCSServer: {value}'
     except Exception as e:
         state = {'message': str(e), 'type': STATE_ERR}
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 @register_sysinfo
 def svn_info():
     from rhodecode.lib.vcs.backends import svn
     state = STATE_OK_DEFAULT
     value = human_value = ''
     try:
         value = svn.discover_svn_version(raise_on_exc=True)
         human_value = f'version reported from VCSServer: {value}'
     except Exception as e:
         state = {'message': str(e), 'type': STATE_ERR}
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 @register_sysinfo
 def vcs_backends():
     import rhodecode
     value = rhodecode.CONFIG.get('vcs.backends')
     human_value = 'Enabled backends in order: {}'.format(','.join(value))
     return SysInfoRes(value=value, human_value=human_value)
 
 
 @register_sysinfo
 def vcs_server():
     import rhodecode
     from rhodecode.lib.vcs.backends import get_vcsserver_service_data
 
     server_url = rhodecode.CONFIG.get('vcs.server')
     enabled = rhodecode.CONFIG.get('vcs.server.enable')
     protocol = rhodecode.CONFIG.get('vcs.server.protocol') or 'http'
     state = STATE_OK_DEFAULT
     version = None
     workers = 0
 
     try:
         data = get_vcsserver_service_data()
         if data and 'version' in data:
             version = data['version']
 
         if data and 'config' in data:
             conf = data['config']
             workers = conf.get('workers', 'NOT AVAILABLE')
 
         connection = 'connected'
     except Exception as e:
         connection = 'failed'
         state = {'message': str(e), 'type': STATE_ERR}
 
     value = dict(
         url=server_url,
         enabled=enabled,
         protocol=protocol,
         connection=connection,
         version=version,
         text='',
     )
 
     human_value = value.copy()
     human_value['text'] = \
         '{url}@ver:{ver} via {mode} mode[workers:{workers}], connection:{conn}'.format(
             url=server_url, ver=version, workers=workers, mode=protocol,
             conn=connection)
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 @register_sysinfo
 def vcs_server_config():
     from rhodecode.lib.vcs.backends import get_vcsserver_service_data
     state = STATE_OK_DEFAULT
 
     value = {}
     try:
         data = get_vcsserver_service_data()
         value = data['app_config']
     except Exception as e:
         state = {'message': str(e), 'type': STATE_ERR}
 
     human_value = value.copy()
     human_value['text'] = 'VCS Server config'
 
     return SysInfoRes(value=value, state=state, human_value=human_value)
 
 
 @register_sysinfo
 def rhodecode_app_info():
     import rhodecode
     edition = rhodecode.CONFIG.get('rhodecode.edition')
 
     value = dict(
         rhodecode_version=rhodecode.__version__,
         rhodecode_lib_path=os.path.abspath(rhodecode.__file__),
         text=''
     )
     human_value = value.copy()
     human_value['text'] = 'RhodeCode {edition}, version {ver}'.format(
         edition=edition, ver=value['rhodecode_version']
     )
     return SysInfoRes(value=value, human_value=human_value)
 
 
 @register_sysinfo
 def rhodecode_config():
     import rhodecode
     path = rhodecode.CONFIG.get('__file__')
     rhodecode_ini_safe = rhodecode.CONFIG.copy()
     cert_path = get_cert_path(path)
 
     try:
         config = configparser.ConfigParser()
         config.read(path)
         parsed_ini = config
         if parsed_ini.has_section('server:main'):
             parsed_ini = dict(parsed_ini.items('server:main'))
     except Exception:
         log.exception('Failed to read .ini file for display')
         parsed_ini = {}
 
     rhodecode_ini_safe['server:main'] = parsed_ini
 
     blacklist = [
         f'rhodecode_{LicenseModel.LICENSE_DB_KEY}',
         'routes.map',
         'sqlalchemy.db1.url',
         'channelstream.secret',
         'beaker.session.secret',
         'rhodecode.encrypted_values.secret',
         'rhodecode_auth_github_consumer_key',
         'rhodecode_auth_github_consumer_secret',
         'rhodecode_auth_google_consumer_key',
         'rhodecode_auth_google_consumer_secret',
         'rhodecode_auth_bitbucket_consumer_secret',
         'rhodecode_auth_bitbucket_consumer_key',
         'rhodecode_auth_twitter_consumer_secret',
         'rhodecode_auth_twitter_consumer_key',
 
         'rhodecode_auth_twitter_secret',
         'rhodecode_auth_github_secret',
         'rhodecode_auth_google_secret',
         'rhodecode_auth_bitbucket_secret',
 
         'appenlight.api_key',
         ('app_conf', 'sqlalchemy.db1.url')
     ]
     for k in blacklist:
         if isinstance(k, tuple):
             section, key = k
             if section in rhodecode_ini_safe:
                 rhodecode_ini_safe[section] = '**OBFUSCATED**'
         else:
             rhodecode_ini_safe.pop(k, None)
 
     # TODO: maybe put some CONFIG checks here ?
     return SysInfoRes(value={'config': rhodecode_ini_safe,
                              'path': path, 'cert_path': cert_path})
 
 
 @register_sysinfo
 def database_info():
     import rhodecode
     from sqlalchemy.engine import url as engine_url
     from rhodecode.model import meta
     from rhodecode.model.meta import Session
     from rhodecode.model.db import DbMigrateVersion
 
     state = STATE_OK_DEFAULT
 
     db_migrate = DbMigrateVersion.query().filter(
         DbMigrateVersion.repository_id == 'rhodecode_db_migrations').one()
 
     db_url_obj = engine_url.make_url(rhodecode.CONFIG['sqlalchemy.db1.url'])
 
     try:
         engine = meta.get_engine()
         db_server_info = engine.dialect._get_server_version_info(
             Session.connection(bind=engine))
         db_version = '.'.join(map(str, db_server_info))
     except Exception:
         log.exception('failed to fetch db version')
         db_version = 'UNKNOWN'
 
     db_info = dict(
         migrate_version=db_migrate.version,
         type=db_url_obj.get_backend_name(),
         version=db_version,
         url=repr(db_url_obj)
     )
     current_version = db_migrate.version
     expected_version = rhodecode.__dbversion__
     if state['type'] == STATE_OK and current_version != expected_version:
         msg = 'Critical: database schema mismatch, ' \
               'expected version {}, got {}. ' \
               'Please run migrations on your database.'.format(
             expected_version, current_version)
         state = {'message': msg, 'type': STATE_ERR}
 
     human_value = db_info.copy()
     human_value['url'] = "{} @ migration version: {}".format(
         db_info['url'], db_info['migrate_version'])
     human_value['version'] = "{} {}".format(db_info['type'], db_info['version'])
     return SysInfoRes(value=db_info, state=state, human_value=human_value)
 
 
 @register_sysinfo
 def server_info(environ):
     import rhodecode
     from rhodecode.lib.base import get_server_ip_addr, get_server_port
 
     value = {
         'server_ip': '{}:{}'.format(
             get_server_ip_addr(environ, log_errors=False),
             get_server_port(environ)
         ),
         'server_id': rhodecode.CONFIG.get('instance_id'),
     }
     return SysInfoRes(value=value)
 
 
 @register_sysinfo
 def usage_info():
     from rhodecode.model.db import User, Repository, true
     value = {
         'users': User.query().count(),
         'users_active': User.query().filter(User.active == true()).count(),
         'repositories': Repository.query().count(),
         'repository_types': {
             'hg': Repository.query().filter(
                 Repository.repo_type == 'hg').count(),
             'git': Repository.query().filter(
                 Repository.repo_type == 'git').count(),
             'svn': Repository.query().filter(
                 Repository.repo_type == 'svn').count(),
         },
     }
     return SysInfoRes(value=value)
 
 
 def get_system_info(environ):
     environ = environ or {}
     return {
         'rhodecode_app': SysInfo(rhodecode_app_info)(),
         'rhodecode_config': SysInfo(rhodecode_config)(),
         'rhodecode_usage': SysInfo(usage_info)(),
         'python': SysInfo(python_info)(),
         'py_modules': SysInfo(py_modules)(),
 
         'platform': SysInfo(platform_type)(),
         'locale': SysInfo(locale_info)(),
         'server': SysInfo(server_info, environ=environ)(),
         'database': SysInfo(database_info)(),
         'ulimit': SysInfo(ulimit_info)(),
         'storage': SysInfo(storage)(),
         'storage_inodes': SysInfo(storage_inodes)(),
         'storage_archive': SysInfo(storage_archives)(),
+        'storage_artifacts': SysInfo(storage_artifacts)(),
         'storage_gist': SysInfo(storage_gist)(),
         'storage_temp': SysInfo(storage_temp)(),
 
         'search': SysInfo(search_info)(),
 
         'uptime': SysInfo(uptime)(),
         'load': SysInfo(machine_load)(),
         'cpu': SysInfo(cpu)(),
         'memory': SysInfo(memory)(),
 
         'vcs_backends': SysInfo(vcs_backends)(),
         'vcs_server': SysInfo(vcs_server)(),
 
         'vcs_server_config': SysInfo(vcs_server_config)(),
 
         'git': SysInfo(git_info)(),
         'hg': SysInfo(hg_info)(),
         'svn': SysInfo(svn_info)(),
     }
 
 
 def load_system_info(key):
     """
     get_sys_info('vcs_server')
     get_sys_info('database')
     """
     return SysInfo(registered_helpers[key])()

rhodecode/lib/utils.py

@@ -1,824 +1,864 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Utilities library for RhodeCode
 """
 
 import datetime
 
 import decorator
 import logging
 import os
 import re
 import sys
 import shutil
 import socket
 import tempfile
 import traceback
 import tarfile
 
 from functools import wraps
 from os.path import join as jn
 
 import paste
 import pkg_resources
 from webhelpers2.text import collapse, strip_tags, convert_accented_entities, convert_misc_entities
 
 from mako import exceptions
 
+from rhodecode import ConfigGet
+from rhodecode.lib.exceptions import HTTPBranchProtected, HTTPLockedRC
 from rhodecode.lib.hash_utils import sha256_safe, md5, sha1
 from rhodecode.lib.type_utils import AttributeDict
 from rhodecode.lib.str_utils import safe_bytes, safe_str
 from rhodecode.lib.vcs.backends.base import Config
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.utils.helpers import get_scm, get_scm_backend
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.model import meta
 from rhodecode.model.db import (
     Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
 from rhodecode.model.meta import Session
 
 
 log = logging.getLogger(__name__)
 
 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
 
 # String which contains characters that are not allowed in slug names for
 # repositories or repository groups. It is properly escaped to use it in
 # regular expressions.
 SLUG_BAD_CHARS = re.escape(r'`?=[]\;\'"<>,/~!@#$%^&*()+{}|:')
 
 # Regex that matches forbidden characters in repo/group slugs.
 SLUG_BAD_CHAR_RE = re.compile(r'[{}\x00-\x08\x0b-\x0c\x0e-\x1f]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches allowed characters in repo/group slugs.
 SLUG_GOOD_CHAR_RE = re.compile(r'[^{}]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches whole repo/group slugs.
 SLUG_RE = re.compile(r'[^{}]+'.format(SLUG_BAD_CHARS))
 
 _license_cache = None
 
 
 def adopt_for_celery(func):
     """
     Decorator designed to adopt hooks (from rhodecode.lib.hooks_base)
     for further usage as a celery tasks.
     """
     @wraps(func)
     def wrapper(extras):
         extras = AttributeDict(extras)
-        # HooksResponse implements to_json method which must be used there.
-        return func(extras).to_json()
+        try:
+            # HooksResponse implements to_json method which must be used there.
+            return func(extras).to_json()
+        except HTTPBranchProtected as error:
+            # Those special cases don't need error reporting. It's a case of
+            # locked repo or protected branch
+            error_args = error.args
+            return {
+                'status': error.code,
+                'output': error.explanation,
+                'exception': type(error).__name__,
+                'exception_args': error_args,
+                'exception_traceback': '',
+            }
+        except HTTPLockedRC as error:
+            # Those special cases don't need error reporting. It's a case of
+            # locked repo or protected branch
+            error_args = error.args
+            return {
+                'status': error.code,
+                'output': error.explanation,
+                'exception': type(error).__name__,
+                'exception_args': error_args,
+                'exception_traceback': '',
+            }
+        except Exception as e:
+            return {
+                'status': 128,
+                'output': '',
+                'exception': type(e).__name__,
+                'exception_args': e.args,
+                'exception_traceback': '',
+            }
     return wrapper
 
 
 def repo_name_slug(value):
     """
     Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
 
     replacement_char = '-'
 
     slug = strip_tags(value)
     slug = convert_accented_entities(slug)
     slug = convert_misc_entities(slug)
 
     slug = SLUG_BAD_CHAR_RE.sub('', slug)
     slug = re.sub(r'[\s]+', '-', slug)
     slug = collapse(slug, replacement_char)
 
     return slug
 
 
 #==============================================================================
 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
 #==============================================================================
 def get_repo_slug(request):
     _repo = ''
 
     if hasattr(request, 'db_repo_name'):
         # if our requests has set db reference use it for name, this
         # translates the example.com/_<id> into proper repo names
         _repo = request.db_repo_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _repo = request.matchdict.get('repo_name')
 
     if _repo:
         _repo = _repo.rstrip('/')
     return _repo
 
 
 def get_repo_group_slug(request):
     _group = ''
     if hasattr(request, 'db_repo_group'):
         # if our requests has set db reference use it for name, this
         # translates the example.com/_<id> into proper repo group names
         _group = request.db_repo_group.group_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _group = request.matchdict.get('repo_group_name')
 
     if _group:
         _group = _group.rstrip('/')
     return _group
 
 
 def get_user_group_slug(request):
     _user_group = ''
 
     if hasattr(request, 'db_user_group'):
         _user_group = request.db_user_group.users_group_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _user_group = request.matchdict.get('user_group_id')
         _user_group_name = request.matchdict.get('user_group_name')
         try:
             if _user_group:
                 _user_group = UserGroup.get(_user_group)
             elif _user_group_name:
                 _user_group = UserGroup.get_by_group_name(_user_group_name)
 
             if _user_group:
                 _user_group = _user_group.users_group_name
         except Exception:
             log.exception('Failed to get user group by id and name')
             # catch all failures here
             return None
 
     return _user_group
 
 
 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
     """
     Scans given path for repos and return (name,(type,path)) tuple
 
     :param path: path to scan for repositories
     :param recursive: recursive search and return names with subdirs in front
     """
 
     # remove ending slash for better results
     path = path.rstrip(os.sep)
     log.debug('now scanning in %s location recursive:%s...', path, recursive)
 
     def _get_repos(p):
         dirpaths = get_dirpaths(p)
         if not _is_dir_writable(p):
             log.warning('repo path without write access: %s', p)
 
         for dirpath in dirpaths:
             if os.path.isfile(os.path.join(p, dirpath)):
                 continue
             cur_path = os.path.join(p, dirpath)
 
             # skip removed repos
             if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
                 continue
 
             #skip .<somethin> dirs
             if dirpath.startswith('.'):
                 continue
 
             try:
                 scm_info = get_scm(cur_path)
                 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
             except VCSError:
                 if not recursive:
                     continue
                 #check if this dir containts other repos for recursive scan
                 rec_path = os.path.join(p, dirpath)
                 if os.path.isdir(rec_path):
                     yield from _get_repos(rec_path)
 
     return _get_repos(path)
 
 
 def get_dirpaths(p: str) -> list:
     try:
         # OS-independable way of checking if we have at least read-only
         # access or not.
         dirpaths = os.listdir(p)
     except OSError:
         log.warning('ignoring repo path without read access: %s', p)
         return []
 
     # os.listpath has a tweak: If a unicode is passed into it, then it tries to
     # decode paths and suddenly returns unicode objects itself. The items it
     # cannot decode are returned as strings and cause issues.
     #
     # Those paths are ignored here until a solid solution for path handling has
     # been built.
     expected_type = type(p)
 
     def _has_correct_type(item):
         if type(item) is not expected_type:
             log.error(
                 "Ignoring path %s since it cannot be decoded into str.",
                 # Using "repr" to make sure that we see the byte value in case
                 # of support.
                 repr(item))
             return False
         return True
 
     dirpaths = [item for item in dirpaths if _has_correct_type(item)]
 
     return dirpaths
 
 
 def _is_dir_writable(path):
     """
     Probe if `path` is writable.
 
     Due to trouble on Cygwin / Windows, this is actually probing if it is
     possible to create a file inside of `path`, stat does not produce reliable
     results in this case.
     """
     try:
         with tempfile.TemporaryFile(dir=path):
             pass
     except OSError:
         return False
     return True
 
 
 def is_valid_repo(repo_name, base_path, expect_scm=None, explicit_scm=None, config=None):
     """
     Returns True if given path is a valid repository False otherwise.
     If expect_scm param is given also, compare if given scm is the same
     as expected from scm parameter. If explicit_scm is given don't try to
     detect the scm, just use the given one to check if repo is valid
 
     :param repo_name:
     :param base_path:
     :param expect_scm:
     :param explicit_scm:
     :param config:
 
     :return True: if given path is a valid repository
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
     log.debug('Checking if `%s` is a valid path for repository. '
               'Explicit type: %s', repo_name, explicit_scm)
 
     try:
         if explicit_scm:
             detected_scms = [get_scm_backend(explicit_scm)(
                 full_path, config=config).alias]
         else:
             detected_scms = get_scm(full_path)
 
         if expect_scm:
             return detected_scms[0] == expect_scm
         log.debug('path: %s is an vcs object:%s', full_path, detected_scms)
         return True
     except VCSError:
         log.debug('path: %s is not a valid repo !', full_path)
         return False
 
 
 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
     """
     Returns True if a given path is a repository group, False otherwise
 
     :param repo_group_name:
     :param base_path:
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
     log.debug('Checking if `%s` is a valid path for repository group',
               repo_group_name)
 
     # check if it's not a repo
     if is_valid_repo(repo_group_name, base_path):
         log.debug('Repo called %s exist, it is not a valid repo group', repo_group_name)
         return False
 
     try:
         # we need to check bare git repos at higher level
         # since we might match branches/hooks/info/objects or possible
         # other things inside bare git repo
         maybe_repo = os.path.dirname(full_path)
         if maybe_repo == base_path:
             # skip root level repo check; we know root location CANNOT BE a repo group
             return False
 
         scm_ = get_scm(maybe_repo)
         log.debug('path: %s is a vcs object:%s, not valid repo group', full_path, scm_)
         return False
     except VCSError:
         pass
 
     # check if it's a valid path
     if skip_path_check or os.path.isdir(full_path):
         log.debug('path: %s is a valid repo group !', full_path)
         return True
 
     log.debug('path: %s is not a valid repo group !', full_path)
     return False
 
 
 def ask_ok(prompt, retries=4, complaint='[y]es or [n]o please!'):
     while True:
         ok = input(prompt)
         if ok.lower() in ('y', 'ye', 'yes'):
             return True
         if ok.lower() in ('n', 'no', 'nop', 'nope'):
             return False
         retries = retries - 1
         if retries < 0:
             raise OSError
         print(complaint)
 
 # propagated from mercurial documentation
 ui_sections = [
     'alias', 'auth',
     'decode/encode', 'defaults',
     'diff', 'email',
     'extensions', 'format',
     'merge-patterns', 'merge-tools',
     'hooks', 'http_proxy',
     'smtp', 'patch',
     'paths', 'profiling',
     'server', 'trusted',
     'ui', 'web', ]
 
 
-def config_data_from_db(clear_session=True, repo=None):
+def prepare_config_data(clear_session=True, repo=None):
     """
-    Read the configuration data from the database and return configuration
+    Read the configuration data from the database, *.ini files and return configuration
     tuples.
     """
     from rhodecode.model.settings import VcsSettingsModel
 
-    config = []
-
     sa = meta.Session()
     settings_model = VcsSettingsModel(repo=repo, sa=sa)
 
     ui_settings = settings_model.get_ui_settings()
 
     ui_data = []
+    config = [
+        ('web', 'push_ssl', 'false'),
+    ]
     for setting in ui_settings:
+        # Todo: remove this section once transition to *.ini files will be completed
+        if setting.section in ('largefiles', 'vcs_git_lfs'):
+            if setting.key != 'enabled':
+                continue
         if setting.active:
             ui_data.append((setting.section, setting.key, setting.value))
             config.append((
                 safe_str(setting.section), safe_str(setting.key),
                 safe_str(setting.value)))
         if setting.key == 'push_ssl':
-            # force set push_ssl requirement to False, rhodecode
-            # handles that
+            # force set push_ssl requirement to False this is deprecated, and we must force it to False
             config.append((
                 safe_str(setting.section), safe_str(setting.key), False))
+    config_getter = ConfigGet()
+    config.append(('vcs_git_lfs', 'store_location', config_getter.get_str('vcs.git.lfs.storage_location')))
+    config.append(('largefiles', 'usercache', config_getter.get_str('vcs.hg.largefiles.storage_location')))
     log.debug(
         'settings ui from db@repo[%s]: %s',
         repo,
         ','.join(['[{}] {}={}'.format(*s) for s in ui_data]))
     if clear_session:
         meta.Session.remove()
 
     # TODO: mikhail: probably it makes no sense to re-read hooks information.
     # It's already there and activated/deactivated
     skip_entries = []
     enabled_hook_classes = get_enabled_hook_classes(ui_settings)
     if 'pull' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PULL))
     if 'push' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRETX_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PUSH_KEY))
 
     config = [entry for entry in config if entry[:2] not in skip_entries]
 
     return config
 
 
 def make_db_config(clear_session=True, repo=None):
     """
     Create a :class:`Config` instance based on the values in the database.
     """
     config = Config()
-    config_data = config_data_from_db(clear_session=clear_session, repo=repo)
+    config_data = prepare_config_data(clear_session=clear_session, repo=repo)
     for section, option, value in config_data:
         config.set(section, option, value)
     return config
 
 
 def get_enabled_hook_classes(ui_settings):
     """
     Return the enabled hook classes.
 
     :param ui_settings: List of ui_settings as returned
         by :meth:`VcsSettingsModel.get_ui_settings`
 
     :return: a list with the enabled hook classes. The order is not guaranteed.
     :rtype: list
     """
     enabled_hooks = []
     active_hook_keys = [
         key for section, key, value, active in ui_settings
         if section == 'hooks' and active]
 
     hook_names = {
         RhodeCodeUi.HOOK_PUSH: 'push',
         RhodeCodeUi.HOOK_PULL: 'pull',
         RhodeCodeUi.HOOK_REPO_SIZE: 'repo_size'
     }
 
     for key in active_hook_keys:
         hook = hook_names.get(key)
         if hook:
             enabled_hooks.append(hook)
 
     return enabled_hooks
 
 
 def set_rhodecode_config(config):
     """
     Updates pyramid config with new settings from database
 
     :param config:
     """
     from rhodecode.model.settings import SettingsModel
     app_settings = SettingsModel().get_all_settings()
 
     for k, v in list(app_settings.items()):
         config[k] = v
 
 
 def get_rhodecode_realm():
     """
     Return the rhodecode realm from database.
     """
     from rhodecode.model.settings import SettingsModel
     realm = SettingsModel().get_setting_by_name('realm')
     return safe_str(realm.app_settings_value)
 
 
 def get_rhodecode_repo_store_path():
     """
     Returns the base path. The base path is the filesystem path which points
     to the repository store.
     """
 
     import rhodecode
     return rhodecode.CONFIG['repo_store.path']
 
 
 def map_groups(path):
     """
     Given a full path to a repository, create all nested groups that this
     repo is inside. This function creates parent-child relationships between
     groups and creates default perms for all new groups.
 
     :param paths: full path to repository
     """
     from rhodecode.model.repo_group import RepoGroupModel
     sa = meta.Session()
     groups = path.split(Repository.NAME_SEP)
     parent = None
     group = None
 
     # last element is repo in nested groups structure
     groups = groups[:-1]
     rgm = RepoGroupModel(sa)
     owner = User.get_first_super_admin()
     for lvl, group_name in enumerate(groups):
         group_name = '/'.join(groups[:lvl] + [group_name])
         group = RepoGroup.get_by_group_name(group_name)
         desc = '%s group' % group_name
 
         # skip folders that are now removed repos
         if REMOVED_REPO_PAT.match(group_name):
             break
 
         if group is None:
             log.debug('creating group level: %s group_name: %s',
                        lvl, group_name)
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             group.user = owner
             sa.add(group)
             perm_obj = rgm._create_default_perms(group)
             sa.add(perm_obj)
             sa.flush()
 
         parent = group
     return group
 
 
 def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild=False):
     """
     maps all repos given in initial_repo_list, non existing repositories
     are created, if remove_obsolete is True it also checks for db entries
     that are not in initial_repo_list and removes them.
 
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.repo_group import RepoGroupModel
     from rhodecode.model.settings import SettingsModel
 
     sa = meta.Session()
     repo_model = RepoModel()
     user = User.get_first_super_admin()
     added = []
 
     # creation defaults
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_locking = defs.get('repo_enable_locking')
     enable_downloads = defs.get('repo_enable_downloads')
     private = defs.get('repo_private')
 
     for name, repo in list(initial_repo_list.items()):
         group = map_groups(name)
         str_name = safe_str(name)
         db_repo = repo_model.get_by_repo_name(str_name)
 
         # found repo that is on filesystem not in RhodeCode database
         if not db_repo:
             log.info('repository `%s` not found in the database, creating now', name)
             added.append(name)
             desc = (repo.description
                     if repo.description != 'unknown'
                     else '%s repository' % name)
 
             db_repo = repo_model._create_repo(
                 repo_name=name,
                 repo_type=repo.alias,
                 description=desc,
                 repo_group=getattr(group, 'group_id', None),
                 owner=user,
                 enable_locking=enable_locking,
                 enable_downloads=enable_downloads,
                 enable_statistics=enable_statistics,
                 private=private,
                 state=Repository.STATE_CREATED
             )
             sa.commit()
             # we added that repo just now, and make sure we updated server info
             if db_repo.repo_type == 'git':
                 git_repo = db_repo.scm_instance()
                 # update repository server-info
                 log.debug('Running update server info')
                 git_repo._update_server_info(force=True)
 
-            db_repo.update_commit_cache()
+            db_repo.update_commit_cache(recursive=False)
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         repo = db_repo.scm_instance(config=config)
         repo.install_hooks(force=force_hooks_rebuild)
 
     removed = []
     if remove_obsolete:
         # remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in list(initial_repo_list.keys()):
                 log.debug("Removing non-existing repository found in db `%s`",
                           repo.repo_name)
                 try:
                     RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
                     sa.commit()
                     removed.append(repo.repo_name)
                 except Exception:
                     # don't hold further removals on error
                     log.error(traceback.format_exc())
                     sa.rollback()
 
         def splitter(full_repo_name):
             _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
             gr_name = None
             if len(_parts) == 2:
                 gr_name = _parts[0]
             return gr_name
 
         initial_repo_group_list = [splitter(x) for x in
                                    list(initial_repo_list.keys()) if splitter(x)]
 
         # remove from database those repository groups that are not in the
         # filesystem due to parent child relationships we need to delete them
         # in a specific order of most nested first
         all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
         def nested_sort(gr):
             return len(gr.split('/'))
         for group_name in sorted(all_groups, key=nested_sort, reverse=True):
             if group_name not in initial_repo_group_list:
                 repo_group = RepoGroup.get_by_group_name(group_name)
                 if (repo_group.children.all() or
                     not RepoGroupModel().check_exist_filesystem(
                         group_name=group_name, exc_on_failure=False)):
                     continue
 
                 log.info(
                     'Removing non-existing repository group found in db `%s`',
                     group_name)
                 try:
                     RepoGroupModel(sa).delete(group_name, fs_remove=False)
                     sa.commit()
                     removed.append(group_name)
                 except Exception:
                     # don't hold further removals on error
                     log.exception(
                         'Unable to remove repository group `%s`',
                         group_name)
                     sa.rollback()
                     raise
 
     return added, removed
 
 
 def load_rcextensions(root_path):
     import rhodecode
     from rhodecode.config import conf
 
     path = os.path.join(root_path)
     sys.path.append(path)
 
     try:
         rcextensions = __import__('rcextensions')
     except ImportError:
         if os.path.isdir(os.path.join(path, 'rcextensions')):
             log.warning('Unable to load rcextensions from %s', path)
         rcextensions = None
 
     if rcextensions:
         log.info('Loaded rcextensions from %s...', rcextensions)
         rhodecode.EXTENSIONS = rcextensions
 
         # Additional mappings that are not present in the pygments lexers
         conf.LANGUAGES_EXTENSIONS_MAP.update(
             getattr(rhodecode.EXTENSIONS, 'EXTRA_MAPPINGS', {}))
 
 
 def get_custom_lexer(extension):
     """
     returns a custom lexer if it is defined in rcextensions module, or None
     if there's no custom lexer defined
     """
     import rhodecode
     from pygments import lexers
 
     # custom override made by RhodeCode
     if extension in ['mako']:
         return lexers.get_lexer_by_name('html+mako')
 
     # check if we didn't define this extension as other lexer
     extensions = rhodecode.EXTENSIONS and getattr(rhodecode.EXTENSIONS, 'EXTRA_LEXERS', None)
     if extensions and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
         _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
         return lexers.get_lexer_by_name(_lexer_name)
 
 
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, config):
     """
     Makes default test index.
     """
     try:
         import rc_testdata
     except ImportError:
         raise ImportError('Failed to import rc_testdata, '
                           'please make sure this package is installed from requirements_test.txt')
     rc_testdata.extract_search_index(
         'vcs_search_index', os.path.dirname(config['search.location']))
 
 
 def create_test_directory(test_path):
     """
     Create test directory if it doesn't exist.
     """
     if not os.path.isdir(test_path):
         log.debug('Creating testdir %s', test_path)
         os.makedirs(test_path)
 
 
 def create_test_database(test_path, config):
     """
     Makes a fresh database.
     """
     from rhodecode.lib.db_manage import DbManage
     from rhodecode.lib.utils2 import get_encryption_key
 
     # PART ONE create db
     dbconf = config['sqlalchemy.db1.url']
     enc_key = get_encryption_key(config)
 
     log.debug('making test db %s', dbconf)
 
     dbmanage = DbManage(log_sql=False, dbconf=dbconf, root=config['here'],
                         tests=True, cli_args={'force_ask': True}, enc_key=enc_key)
     dbmanage.create_tables(override=True)
     dbmanage.set_db_version()
     # for tests dynamically set new root paths based on generated content
     dbmanage.create_settings(dbmanage.config_prompt(test_path))
     dbmanage.create_default_user()
     dbmanage.create_test_admin_and_users()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     Session().commit()
 
 
 def create_test_repositories(test_path, config):
     """
     Creates test repositories in the temporary directory. Repositories are
     extracted from archives within the rc_testdata package.
     """
     import rc_testdata
     from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
 
     log.debug('making test vcs repositories')
 
     idx_path = config['search.location']
     data_path = config['cache_dir']
 
     # clean index and data
     if idx_path and os.path.exists(idx_path):
         log.debug('remove %s', idx_path)
         shutil.rmtree(idx_path)
 
     if data_path and os.path.exists(data_path):
         log.debug('remove %s', data_path)
         shutil.rmtree(data_path)
 
     rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
     rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))
 
     # Note: Subversion is in the process of being integrated with the system,
     # until we have a properly packed version of the test svn repository, this
     # tries to copy over the repo from a package "rc_testdata"
     svn_repo_path = rc_testdata.get_svn_repo_archive()
     with tarfile.open(svn_repo_path) as tar:
         tar.extractall(jn(test_path, SVN_REPO))
 
 
 def password_changed(auth_user, session):
     # Never report password change in case of default user or anonymous user.
     if auth_user.username == User.DEFAULT_USER or auth_user.user_id is None:
         return False
 
     password_hash = md5(safe_bytes(auth_user.password)) if auth_user.password else None
     rhodecode_user = session.get('rhodecode_user', {})
     session_password_hash = rhodecode_user.get('password', '')
     return password_hash != session_password_hash
 
 
 def read_opensource_licenses():
     global _license_cache
 
     if not _license_cache:
         licenses = pkg_resources.resource_string(
             'rhodecode', 'config/licenses.json')
         _license_cache = json.loads(licenses)
 
     return _license_cache
 
 
 def generate_platform_uuid():
     """
     Generates platform UUID based on it's name
     """
     import platform
 
     try:
         uuid_list = [platform.platform()]
         return sha256_safe(':'.join(uuid_list))
     except Exception as e:
         log.error('Failed to generate host uuid: %s', e)
         return 'UNDEFINED'
 
 
 def send_test_email(recipients, email_body='TEST EMAIL'):
     """
     Simple code for generating test emails.
     Usage::
 
         from rhodecode.lib import utils
         utils.send_test_email()
     """
     from rhodecode.lib.celerylib import tasks, run_task
 
     email_body = email_body_plaintext = email_body
     subject = f'SUBJECT FROM: {socket.gethostname()}'
     tasks.send_email(recipients, subject, email_body_plaintext, email_body)

rhodecode/model/db.py

@@ -1,6046 +1,6055 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Database Models for RhodeCode Enterprise
 """
 
 import re
 import os
 import time
 import string
 import logging
 import datetime
 import uuid
 import warnings
 import ipaddress
 import functools
 import traceback
 import collections
 
 import pyotp
 from sqlalchemy import (
     or_, and_, not_, func, cast, TypeDecorator, event, select,
     true, false, null, union_all,
     Index, Sequence, UniqueConstraint, ForeignKey, CheckConstraint, Column,
     Boolean, String, Unicode, UnicodeText, DateTime, Integer, LargeBinary,
     Text, Float, PickleType, BigInteger)
 from sqlalchemy.sql.expression import case
 from sqlalchemy.sql.functions import coalesce, count  # pragma: no cover
 from sqlalchemy.orm import (
     relationship, lazyload, joinedload, class_mapper, validates, aliased, load_only)
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.exc import IntegrityError  # pragma: no cover
 from sqlalchemy.dialects.mysql import LONGTEXT
 from zope.cachedescriptors.property import Lazy as LazyProperty
 from pyramid.threadlocal import get_current_request
 from webhelpers2.text import remove_formatting
 
 from rhodecode import ConfigGet
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.translation import _
 from rhodecode.lib.vcs import get_vcs_instance, VCSError
 from rhodecode.lib.vcs.backends.base import (
     EmptyCommit, Reference, unicode_to_reference, reference_to_unicode)
 from rhodecode.lib.utils2 import (
     str2bool, safe_str, get_commit_safe, sha1_safe,
     time_to_datetime, aslist, Optional, safe_int, get_clone_url, AttributeDict,
     glob2re, StrictAttributeDict, cleaned_uri, datetime_to_time)
 from rhodecode.lib.jsonalchemy import (
     MutationObj, MutationList, JsonType, JsonRaw)
 from rhodecode.lib.hash_utils import sha1
 from rhodecode.lib import ext_json
 from rhodecode.lib import enc_utils
 from rhodecode.lib.ext_json import json, str_json
 from rhodecode.lib.caching_query import FromCache
 from rhodecode.lib.exceptions import (
     ArtifactMetadataDuplicate, ArtifactMetadataBadValueType)
 from rhodecode.model.meta import Base, Session
 
 URL_SEP = '/'
 log = logging.getLogger(__name__)
 
 # =============================================================================
 # BASE CLASSES
 # =============================================================================
 
 # this is propagated from .ini file rhodecode.encrypted_values.secret or
 # beaker.session.secret if first is not set.
 # and initialized at environment.py
 ENCRYPTION_KEY: bytes = b''
 
 # used to sort permissions by types, '#' used here is not allowed to be in
 # usernames, and it's very early in sorted string.printable table.
 PERMISSION_TYPE_SORT = {
     'admin': '####',
     'write': '###',
     'read':  '##',
     'none':  '#',
 }
 
 
 def display_user_sort(obj):
     """
     Sort function used to sort permissions in .permissions() function of
     Repository, RepoGroup, UserGroup. Also it put the default user in front
     of all other resources
     """
 
     if obj.username == User.DEFAULT_USER:
         return '#####'
     prefix = PERMISSION_TYPE_SORT.get(obj.permission.split('.')[-1], '')
     extra_sort_num = '1'  # default
 
     # NOTE(dan): inactive duplicates goes last
     if getattr(obj, 'duplicate_perm', None):
         extra_sort_num = '9'
     return prefix + extra_sort_num + obj.username
 
 
 def display_user_group_sort(obj):
     """
     Sort function used to sort permissions in .permissions() function of
     Repository, RepoGroup, UserGroup. Also it put the default user in front
     of all other resources
     """
 
     prefix = PERMISSION_TYPE_SORT.get(obj.permission.split('.')[-1], '')
     return prefix + obj.users_group_name
 
 
 def _hash_key(k):
     return sha1_safe(k)
 
 
 def description_escaper(desc):
     from rhodecode.lib import helpers as h
     return h.escape(desc)
 
 
 def in_filter_generator(qry, items, limit=500):
     """
     Splits IN() into multiple with OR
     e.g.::
     cnt = Repository.query().filter(
         or_(
             *in_filter_generator(Repository.repo_id, range(100000))
         )).count()
     """
     if not items:
         # empty list will cause empty query which might cause security issues
         # this can lead to hidden unpleasant results
         items = [-1]
 
     parts = []
     for chunk in range(0, len(items), limit):
         parts.append(
             qry.in_(items[chunk: chunk + limit])
         )
 
     return parts
 
 
 base_table_args = {
     'extend_existing': True,
     'mysql_engine': 'InnoDB',
     'mysql_charset': 'utf8',
     'sqlite_autoincrement': True
 }
 
 
 class EncryptedTextValue(TypeDecorator):
     """
     Special column for encrypted long text data, use like::
 
         value = Column("encrypted_value", EncryptedValue(), nullable=False)
 
     This column is intelligent so if value is in unencrypted form it return
     unencrypted form, but on save it always encrypts
     """
     cache_ok = True
     impl = Text
 
     def process_bind_param(self, value, dialect):
         """
         Setter for storing value
         """
         import rhodecode
         if not value:
             return value
 
         # protect against double encrypting if values is already encrypted
         if value.startswith('enc$aes$') \
                 or value.startswith('enc$aes_hmac$') \
                 or value.startswith('enc2$'):
             raise ValueError('value needs to be in unencrypted format, '
                              'ie. not starting with enc$ or enc2$')
 
         algo = rhodecode.CONFIG.get('rhodecode.encrypted_values.algorithm') or 'aes'
         bytes_val = enc_utils.encrypt_value(value, enc_key=ENCRYPTION_KEY, algo=algo)
         return safe_str(bytes_val)
 
     def process_result_value(self, value, dialect):
         """
         Getter for retrieving value
         """
 
         import rhodecode
         if not value:
             return value
 
         bytes_val = enc_utils.decrypt_value(value, enc_key=ENCRYPTION_KEY)
 
         return safe_str(bytes_val)
 
 
 class BaseModel(object):
     """
     Base Model for all classes
     """
 
     @classmethod
     def _get_keys(cls):
         """return column names for this model """
         return class_mapper(cls).c.keys()
 
     def get_dict(self):
         """
         return dict with keys and values corresponding
         to this model data """
 
         d = {}
         for k in self._get_keys():
             d[k] = getattr(self, k)
 
         # also use __json__() if present to get additional fields
         _json_attr = getattr(self, '__json__', None)
         if _json_attr:
             # update with attributes from __json__
             if callable(_json_attr):
                 _json_attr = _json_attr()
             for k, val in _json_attr.items():
                 d[k] = val
         return d
 
     def get_appstruct(self):
         """return list with keys and values tuples corresponding
         to this model data """
 
         lst = []
         for k in self._get_keys():
             lst.append((k, getattr(self, k),))
         return lst
 
     def populate_obj(self, populate_dict):
         """populate model with data from given populate_dict"""
 
         for k in self._get_keys():
             if k in populate_dict:
                 setattr(self, k, populate_dict[k])
 
     @classmethod
     def query(cls):
         return Session().query(cls)
 
     @classmethod
     def select(cls, custom_cls=None):
         """
         stmt = cls.select().where(cls.user_id==1)
         # optionally
         stmt = cls.select(User.user_id).where(cls.user_id==1)
         result = cls.execute(stmt) | cls.scalars(stmt)
         """
 
         if custom_cls:
             stmt = select(custom_cls)
         else:
             stmt = select(cls)
         return stmt
 
     @classmethod
     def execute(cls, stmt):
         return Session().execute(stmt)
 
     @classmethod
     def scalars(cls, stmt):
         return Session().scalars(stmt)
 
     @classmethod
     def get(cls, id_):
         if id_:
             return cls.query().get(id_)
 
     @classmethod
     def get_or_404(cls, id_):
         from pyramid.httpexceptions import HTTPNotFound
 
         try:
             id_ = int(id_)
         except (TypeError, ValueError):
             raise HTTPNotFound()
 
         res = cls.query().get(id_)
         if not res:
             raise HTTPNotFound()
         return res
 
     @classmethod
     def getAll(cls):
         # deprecated and left for backward compatibility
         return cls.get_all()
 
     @classmethod
     def get_all(cls):
         return cls.query().all()
 
     @classmethod
     def delete(cls, id_):
         obj = cls.query().get(id_)
         Session().delete(obj)
 
     @classmethod
     def identity_cache(cls, session, attr_name, value):
         exist_in_session = []
         for (item_cls, pkey), instance in session.identity_map.items():
             if cls == item_cls and getattr(instance, attr_name) == value:
                 exist_in_session.append(instance)
         if exist_in_session:
             if len(exist_in_session) == 1:
                 return exist_in_session[0]
             log.exception(
                 'multiple objects with attr %s and '
                 'value %s found with same name: %r',
                 attr_name, value, exist_in_session)
 
     @property
     def cls_name(self):
         return self.__class__.__name__
 
     def __repr__(self):
         return f'<DB:{self.cls_name}>'
 
 
 class RhodeCodeSetting(Base, BaseModel):
     __tablename__ = 'rhodecode_settings'
     __table_args__ = (
         UniqueConstraint('app_settings_name'),
         base_table_args
     )
 
     SETTINGS_TYPES = {
         'str': safe_str,
         'int': safe_int,
         'unicode': safe_str,
         'bool': str2bool,
         'list': functools.partial(aslist, sep=',')
     }
     DEFAULT_UPDATE_URL = 'https://rhodecode.com/api/v1/info/versions'
     GLOBAL_CONF_KEY = 'app_settings'
 
     app_settings_id = Column("app_settings_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     app_settings_name = Column("app_settings_name", String(255), nullable=True, unique=None, default=None)
     _app_settings_value = Column("app_settings_value", String(4096), nullable=True, unique=None, default=None)
     _app_settings_type = Column("app_settings_type", String(255), nullable=True, unique=None, default=None)
 
     def __init__(self, key='', val='', type='unicode'):
         self.app_settings_name = key
         self.app_settings_type = type
         self.app_settings_value = val
 
     @validates('_app_settings_value')
     def validate_settings_value(self, key, val):
         assert type(val) == str
         return val
 
     @hybrid_property
     def app_settings_value(self):
         v = self._app_settings_value
         _type = self.app_settings_type
         if _type:
             _type = self.app_settings_type.split('.')[0]
             # decode the encrypted value
             if 'encrypted' in self.app_settings_type:
                 cipher = EncryptedTextValue()
                 v = safe_str(cipher.process_result_value(v, None))
 
         converter = self.SETTINGS_TYPES.get(_type) or \
             self.SETTINGS_TYPES['unicode']
         return converter(v)
 
     @app_settings_value.setter
     def app_settings_value(self, val):
         """
         Setter that will always make sure we use unicode in app_settings_value
 
         :param val:
         """
         val = safe_str(val)
         # encode the encrypted value
         if 'encrypted' in self.app_settings_type:
             cipher = EncryptedTextValue()
             val = safe_str(cipher.process_bind_param(val, None))
         self._app_settings_value = val
 
     @hybrid_property
     def app_settings_type(self):
         return self._app_settings_type
 
     @app_settings_type.setter
     def app_settings_type(self, val):
         if val.split('.')[0] not in self.SETTINGS_TYPES:
             raise Exception('type must be one of %s got %s'
                             % (self.SETTINGS_TYPES.keys(), val))
         self._app_settings_type = val
 
     @classmethod
     def get_by_prefix(cls, prefix):
         return RhodeCodeSetting.query()\
             .filter(RhodeCodeSetting.app_settings_name.startswith(prefix))\
             .all()
 
     def __repr__(self):
         return "<%s('%s:%s[%s]')>" % (
             self.cls_name,
             self.app_settings_name, self.app_settings_value,
             self.app_settings_type
         )
 
 
 class RhodeCodeUi(Base, BaseModel):
     __tablename__ = 'rhodecode_ui'
     __table_args__ = (
         UniqueConstraint('ui_key'),
         base_table_args
     )
     # Sync those values with vcsserver.config.hooks
 
     HOOK_REPO_SIZE = 'changegroup.repo_size'
     # HG
     HOOK_PRE_PULL = 'preoutgoing.pre_pull'
     HOOK_PULL = 'outgoing.pull_logger'
     HOOK_PRE_PUSH = 'prechangegroup.pre_push'
     HOOK_PRETX_PUSH = 'pretxnchangegroup.pre_push'
     HOOK_PUSH = 'changegroup.push_logger'
     HOOK_PUSH_KEY = 'pushkey.key_push'
 
     HOOKS_BUILTIN = [
         HOOK_PRE_PULL,
         HOOK_PULL,
         HOOK_PRE_PUSH,
         HOOK_PRETX_PUSH,
         HOOK_PUSH,
         HOOK_PUSH_KEY,
     ]
 
     # TODO: johbo: Unify way how hooks are configured for git and hg,
     # git part is currently hardcoded.
 
     # SVN PATTERNS
     SVN_BRANCH_ID = 'vcs_svn_branch'
     SVN_TAG_ID = 'vcs_svn_tag'
 
     ui_id = Column(
         "ui_id", Integer(), nullable=False, unique=True, default=None,
         primary_key=True)
     ui_section = Column(
         "ui_section", String(255), nullable=True, unique=None, default=None)
     ui_key = Column(
         "ui_key", String(255), nullable=True, unique=None, default=None)
     ui_value = Column(
         "ui_value", String(255), nullable=True, unique=None, default=None)
     ui_active = Column(
         "ui_active", Boolean(), nullable=True, unique=None, default=True)
 
     def __repr__(self):
         return '<%s[%s]%s=>%s]>' % (self.cls_name, self.ui_section,
                                     self.ui_key, self.ui_value)
 
 
 class RepoRhodeCodeSetting(Base, BaseModel):
     __tablename__ = 'repo_rhodecode_settings'
     __table_args__ = (
         UniqueConstraint(
             'app_settings_name', 'repository_id',
             name='uq_repo_rhodecode_setting_name_repo_id'),
         base_table_args
     )
 
     repository_id = Column(
         "repository_id", Integer(), ForeignKey('repositories.repo_id'),
         nullable=False)
     app_settings_id = Column(
         "app_settings_id", Integer(), nullable=False, unique=True,
         default=None, primary_key=True)
     app_settings_name = Column(
         "app_settings_name", String(255), nullable=True, unique=None,
         default=None)
     _app_settings_value = Column(
         "app_settings_value", String(4096), nullable=True, unique=None,
         default=None)
     _app_settings_type = Column(
         "app_settings_type", String(255), nullable=True, unique=None,
         default=None)
 
     repository = relationship('Repository', viewonly=True)
 
     def __init__(self, repository_id, key='', val='', type='unicode'):
         self.repository_id = repository_id
         self.app_settings_name = key
         self.app_settings_type = type
         self.app_settings_value = val
 
     @validates('_app_settings_value')
     def validate_settings_value(self, key, val):
         assert type(val) == str
         return val
 
     @hybrid_property
     def app_settings_value(self):
         v = self._app_settings_value
         type_ = self.app_settings_type
         SETTINGS_TYPES = RhodeCodeSetting.SETTINGS_TYPES
         converter = SETTINGS_TYPES.get(type_) or SETTINGS_TYPES['unicode']
         return converter(v)
 
     @app_settings_value.setter
     def app_settings_value(self, val):
         """
         Setter that will always make sure we use unicode in app_settings_value
 
         :param val:
         """
         self._app_settings_value = safe_str(val)
 
     @hybrid_property
     def app_settings_type(self):
         return self._app_settings_type
 
     @app_settings_type.setter
     def app_settings_type(self, val):
         SETTINGS_TYPES = RhodeCodeSetting.SETTINGS_TYPES
         if val not in SETTINGS_TYPES:
             raise Exception('type must be one of %s got %s'
                             % (SETTINGS_TYPES.keys(), val))
         self._app_settings_type = val
 
     def __repr__(self):
         return "<%s('%s:%s:%s[%s]')>" % (
             self.cls_name, self.repository.repo_name,
             self.app_settings_name, self.app_settings_value,
             self.app_settings_type
         )
 
 
 class RepoRhodeCodeUi(Base, BaseModel):
     __tablename__ = 'repo_rhodecode_ui'
     __table_args__ = (
         UniqueConstraint(
             'repository_id', 'ui_section', 'ui_key',
             name='uq_repo_rhodecode_ui_repository_id_section_key'),
         base_table_args
     )
 
     repository_id = Column(
         "repository_id", Integer(), ForeignKey('repositories.repo_id'),
         nullable=False)
     ui_id = Column(
         "ui_id", Integer(), nullable=False, unique=True, default=None,
         primary_key=True)
     ui_section = Column(
         "ui_section", String(255), nullable=True, unique=None, default=None)
     ui_key = Column(
         "ui_key", String(255), nullable=True, unique=None, default=None)
     ui_value = Column(
         "ui_value", String(255), nullable=True, unique=None, default=None)
     ui_active = Column(
         "ui_active", Boolean(), nullable=True, unique=None, default=True)
 
     repository = relationship('Repository', viewonly=True)
 
     def __repr__(self):
         return '<%s[%s:%s]%s=>%s]>' % (
             self.cls_name, self.repository.repo_name,
             self.ui_section, self.ui_key, self.ui_value)
 
 
 class User(Base, BaseModel):
     __tablename__ = 'users'
     __table_args__ = (
         UniqueConstraint('username'), UniqueConstraint('email'),
         Index('u_username_idx', 'username'),
         Index('u_email_idx', 'email'),
         base_table_args
     )
 
     DEFAULT_USER = 'default'
     DEFAULT_USER_EMAIL = '[e-mail]'
     DEFAULT_GRAVATAR_URL = 'https://secure.gravatar.com/avatar/{md5email}?d=identicon&s={size}'
     RECOVERY_CODES_COUNT = 10
 
     user_id = Column("user_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     username = Column("username", String(255), nullable=True, unique=None, default=None)
     password = Column("password", String(255), nullable=True, unique=None, default=None)
     active = Column("active", Boolean(), nullable=True, unique=None, default=True)
     admin = Column("admin", Boolean(), nullable=True, unique=None, default=False)
     name = Column("firstname", String(255), nullable=True, unique=None, default=None)
     lastname = Column("lastname", String(255), nullable=True, unique=None, default=None)
     _email = Column("email", String(255), nullable=True, unique=None, default=None)
     last_login = Column("last_login", DateTime(timezone=False), nullable=True, unique=None, default=None)
     last_activity = Column('last_activity', DateTime(timezone=False), nullable=True, unique=None, default=None)
     description = Column('description', UnicodeText().with_variant(UnicodeText(1024), 'mysql'))
 
     extern_type = Column("extern_type", String(255), nullable=True, unique=None, default=None)
     extern_name = Column("extern_name", String(255), nullable=True, unique=None, default=None)
     _api_key = Column("api_key", String(255), nullable=True, unique=None, default=None)
     inherit_default_permissions = Column("inherit_default_permissions", Boolean(), nullable=False, unique=None, default=True)
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     _user_data = Column("user_data", LargeBinary(), nullable=True)  # JSON data
 
     user_log = relationship('UserLog', back_populates='user')
     user_perms = relationship('UserToPerm', primaryjoin="User.user_id==UserToPerm.user_id", cascade='all, delete-orphan')
 
     repositories = relationship('Repository', back_populates='user')
     repository_groups = relationship('RepoGroup', back_populates='user')
     user_groups = relationship('UserGroup', back_populates='user')
 
     user_followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_user_id==User.user_id', cascade='all', back_populates='follows_user')
     followings = relationship('UserFollowing', primaryjoin='UserFollowing.user_id==User.user_id', cascade='all', back_populates='user')
 
     repo_to_perm = relationship('UserRepoToPerm', primaryjoin='UserRepoToPerm.user_id==User.user_id', cascade='all, delete-orphan')
     repo_group_to_perm = relationship('UserRepoGroupToPerm', primaryjoin='UserRepoGroupToPerm.user_id==User.user_id', cascade='all, delete-orphan', back_populates='user')
     user_group_to_perm = relationship('UserUserGroupToPerm', primaryjoin='UserUserGroupToPerm.user_id==User.user_id', cascade='all, delete-orphan', back_populates='user')
 
     group_member = relationship('UserGroupMember', cascade='all', back_populates='user')
 
     notifications = relationship('UserNotification', cascade='all', back_populates='user')
     # notifications assigned to this user
     user_created_notifications = relationship('Notification', cascade='all', back_populates='created_by_user')
     # comments created by this user
     user_comments = relationship('ChangesetComment', cascade='all', back_populates='author')
     # user profile extra info
     user_emails = relationship('UserEmailMap', cascade='all', back_populates='user')
     user_ip_map = relationship('UserIpMap', cascade='all', back_populates='user')
     user_auth_tokens = relationship('UserApiKeys', cascade='all', back_populates='user')
     user_ssh_keys = relationship('UserSshKeys', cascade='all', back_populates='user')
 
     # gists
     user_gists = relationship('Gist', cascade='all', back_populates='owner')
     # user pull requests
     user_pull_requests = relationship('PullRequest', cascade='all', back_populates='author')
 
     # external identities
     external_identities = relationship('ExternalIdentity', primaryjoin="User.user_id==ExternalIdentity.local_user_id", cascade='all')
     # review rules
     user_review_rules = relationship('RepoReviewRuleUser', cascade='all', back_populates='user')
 
     # artifacts owned
     artifacts = relationship('FileStore', primaryjoin='FileStore.user_id==User.user_id', back_populates='upload_user')
 
     # no cascade, set NULL
     scope_artifacts = relationship('FileStore', primaryjoin='FileStore.scope_user_id==User.user_id', cascade='', back_populates='user')
 
     def __repr__(self):
         return f"<{self.cls_name}('id={self.user_id}, username={self.username}')>"
 
     @hybrid_property
     def email(self):
         return self._email
 
     @email.setter
     def email(self, val):
         self._email = val.lower() if val else None
 
     @hybrid_property
     def first_name(self):
         if self.name:
             return description_escaper(self.name)
         return self.name
 
     @hybrid_property
     def last_name(self):
         if self.lastname:
             return description_escaper(self.lastname)
         return self.lastname
 
     @hybrid_property
     def api_key(self):
         """
         Fetch if exist an auth-token with role ALL connected to this user
         """
         user_auth_token = UserApiKeys.query()\
             .filter(UserApiKeys.user_id == self.user_id)\
             .filter(or_(UserApiKeys.expires == -1,
                             UserApiKeys.expires >= time.time()))\
             .filter(UserApiKeys.role == UserApiKeys.ROLE_ALL).first()
         if user_auth_token:
             user_auth_token = user_auth_token.api_key
 
         return user_auth_token
 
     @api_key.setter
     def api_key(self, val):
         # don't allow to set API key this is deprecated for now
         self._api_key = None
 
     @property
     def reviewer_pull_requests(self):
         return PullRequestReviewers.query() \
             .options(joinedload(PullRequestReviewers.pull_request)) \
             .filter(PullRequestReviewers.user_id == self.user_id) \
             .all()
 
     @property
     def firstname(self):
         # alias for future
         return self.name
 
     @property
     def emails(self):
         other = UserEmailMap.query()\
             .filter(UserEmailMap.user == self) \
             .order_by(UserEmailMap.email_id.asc()) \
             .all()
         return [self.email] + [x.email for x in other]
 
     def emails_cached(self):
         emails = []
         if self.user_id != self.get_default_user_id():
             emails = UserEmailMap.query()\
                 .filter(UserEmailMap.user == self) \
                 .order_by(UserEmailMap.email_id.asc())
 
             emails = emails.options(
                 FromCache("sql_cache_short", f"get_user_{self.user_id}_emails")
             )
 
         return [self.email] + [x.email for x in emails]
 
     @property
     def auth_tokens(self):
         auth_tokens = self.get_auth_tokens()
         return [x.api_key for x in auth_tokens]
 
     def get_auth_tokens(self):
         return UserApiKeys.query()\
             .filter(UserApiKeys.user == self)\
             .order_by(UserApiKeys.user_api_key_id.asc())\
             .all()
 
     @LazyProperty
     def feed_token(self):
         return self.get_feed_token()
 
     def get_feed_token(self, cache=True):
         feed_tokens = UserApiKeys.query()\
             .filter(UserApiKeys.user == self)\
             .filter(UserApiKeys.role == UserApiKeys.ROLE_FEED)
         if cache:
             feed_tokens = feed_tokens.options(
                 FromCache("sql_cache_short", f"get_user_feed_token_{self.user_id}"))
 
         feed_tokens = feed_tokens.all()
         if feed_tokens:
             return feed_tokens[0].api_key
         return 'NO_FEED_TOKEN_AVAILABLE'
 
     @LazyProperty
     def artifact_token(self):
         return self.get_artifact_token()
 
     def get_artifact_token(self, cache=True):
         artifacts_tokens = UserApiKeys.query()\
             .filter(UserApiKeys.user == self) \
             .filter(or_(UserApiKeys.expires == -1,
                         UserApiKeys.expires >= time.time())) \
             .filter(UserApiKeys.role == UserApiKeys.ROLE_ARTIFACT_DOWNLOAD)
 
         if cache:
             artifacts_tokens = artifacts_tokens.options(
                 FromCache("sql_cache_short", f"get_user_artifact_token_{self.user_id}"))
 
         artifacts_tokens = artifacts_tokens.all()
         if artifacts_tokens:
             return artifacts_tokens[0].api_key
         return 'NO_ARTIFACT_TOKEN_AVAILABLE'
 
     def get_or_create_artifact_token(self):
         artifacts_tokens = UserApiKeys.query()\
             .filter(UserApiKeys.user == self) \
             .filter(or_(UserApiKeys.expires == -1,
                         UserApiKeys.expires >= time.time())) \
             .filter(UserApiKeys.role == UserApiKeys.ROLE_ARTIFACT_DOWNLOAD)
 
         artifacts_tokens = artifacts_tokens.all()
         if artifacts_tokens:
             return artifacts_tokens[0].api_key
         else:
             from rhodecode.model.auth_token import AuthTokenModel
             artifact_token = AuthTokenModel().create(
                 self, 'auto-generated-artifact-token',
                 lifetime=-1, role=UserApiKeys.ROLE_ARTIFACT_DOWNLOAD)
             Session.commit()
             return artifact_token.api_key
 
     def is_totp_valid(self, received_code, secret):
         totp = pyotp.TOTP(secret)
         return totp.verify(received_code)
 
     def is_2fa_recovery_code_valid(self, received_code, secret):
         encrypted_recovery_codes = self.user_data.get('recovery_codes_2fa', [])
         recovery_codes = self.get_2fa_recovery_codes()
         if received_code in recovery_codes:
             encrypted_recovery_codes.pop(recovery_codes.index(received_code))
             self.update_userdata(recovery_codes_2fa=encrypted_recovery_codes)
             return True
         return False
 
     @hybrid_property
     def has_forced_2fa(self):
         """
         Checks if 2fa was forced for current user
         """
         from rhodecode.model.settings import SettingsModel
         if value := SettingsModel().get_setting_by_name(f'auth_{self.extern_type}_global_2fa'):
             return value.app_settings_value
         return False
 
     @hybrid_property
     def has_enabled_2fa(self):
         """
         Checks if user enabled 2fa
         """
         if value := self.has_forced_2fa:
             return value
         return self.user_data.get('enabled_2fa', False)
 
     @has_enabled_2fa.setter
     def has_enabled_2fa(self, val):
         val = str2bool(val)
         self.update_userdata(enabled_2fa=val)
         if not val:
             # NOTE: setting to false we clear the user_data to not store any 2fa artifacts
             self.update_userdata(secret_2fa=None, recovery_codes_2fa=[], check_2fa=False)
         Session().commit()
 
     @hybrid_property
     def check_2fa_required(self):
         """
         Check if check 2fa flag is set for this user
         """
         value = self.user_data.get('check_2fa', False)
         return value
 
     @check_2fa_required.setter
     def check_2fa_required(self, val):
         val = str2bool(val)
         self.update_userdata(check_2fa=val)
         Session().commit()
 
     @hybrid_property
     def has_seen_2fa_codes(self):
         """
         get the flag about if user has seen 2fa recovery codes
         """
         value = self.user_data.get('recovery_codes_2fa_seen', False)
         return value
 
     @has_seen_2fa_codes.setter
     def has_seen_2fa_codes(self, val):
         val = str2bool(val)
         self.update_userdata(recovery_codes_2fa_seen=val)
         Session().commit()
 
     @hybrid_property
     def needs_2fa_configure(self):
         """
         Determines if setup2fa has completed for this user. Means he has all needed data for 2fa to work.
 
         Currently this is 2fa enabled and secret exists
         """
         if self.has_enabled_2fa:
             return not self.user_data.get('secret_2fa')
         return False
 
     def init_2fa_recovery_codes(self, persist=True, force=False):
         """
         Creates 2fa recovery codes
         """
         recovery_codes = self.user_data.get('recovery_codes_2fa', [])
         encrypted_codes = []
         if not recovery_codes or force:
             for _ in range(self.RECOVERY_CODES_COUNT):
                 recovery_code = pyotp.random_base32()
                 recovery_codes.append(recovery_code)
                 encrypted_code = enc_utils.encrypt_value(safe_bytes(recovery_code), enc_key=ENCRYPTION_KEY)
                 encrypted_codes.append(safe_str(encrypted_code))
             if persist:
                 self.update_userdata(recovery_codes_2fa=encrypted_codes, recovery_codes_2fa_seen=False)
             return recovery_codes
         # User should not check the same recovery codes more than once
         return []
 
     def get_2fa_recovery_codes(self):
         encrypted_recovery_codes = self.user_data.get('recovery_codes_2fa', [])
 
         recovery_codes = list(map(
             lambda val: safe_str(
                 enc_utils.decrypt_value(
                     val,
                     enc_key=ENCRYPTION_KEY
                 )),
             encrypted_recovery_codes))
         return recovery_codes
 
     def init_secret_2fa(self, persist=True, force=False):
         secret_2fa = self.user_data.get('secret_2fa')
         if not secret_2fa or force:
             secret = pyotp.random_base32()
             if persist:
                 self.update_userdata(secret_2fa=safe_str(enc_utils.encrypt_value(safe_bytes(secret), enc_key=ENCRYPTION_KEY)))
             return secret
         return ''
 
     @hybrid_property
     def secret_2fa(self) -> str:
         """
         get stored secret for 2fa
         """
         secret_2fa = self.user_data.get('secret_2fa')
         if secret_2fa:
             return safe_str(
                 enc_utils.decrypt_value(secret_2fa, enc_key=ENCRYPTION_KEY))
         return ''
 
     @secret_2fa.setter
     def secret_2fa(self, value: str) -> None:
         encrypted_value = enc_utils.encrypt_value(safe_bytes(value), enc_key=ENCRYPTION_KEY)
         self.update_userdata(secret_2fa=safe_str(encrypted_value))
 
     def regenerate_2fa_recovery_codes(self):
         """
         Regenerates 2fa recovery codes upon request
         """
         new_recovery_codes = self.init_2fa_recovery_codes(force=True)
         Session().commit()
         return new_recovery_codes
 
     @classmethod
     def extra_valid_auth_tokens(cls, user, role=None):
         tokens = UserApiKeys.query().filter(UserApiKeys.user == user)\
                 .filter(or_(UserApiKeys.expires == -1,
                             UserApiKeys.expires >= time.time()))
         if role:
             tokens = tokens.filter(or_(UserApiKeys.role == role,
                                        UserApiKeys.role == UserApiKeys.ROLE_ALL))
         return tokens.all()
 
     def authenticate_by_token(self, auth_token, roles=None, scope_repo_id=None):
         from rhodecode.lib import auth
 
         log.debug('Trying to authenticate user: %s via auth-token, '
                   'and roles: %s', self, roles)
 
         if not auth_token:
             return False
 
         roles = (roles or []) + [UserApiKeys.ROLE_ALL]
         tokens_q = UserApiKeys.query()\
             .filter(UserApiKeys.user_id == self.user_id)\
             .filter(or_(UserApiKeys.expires == -1,
                         UserApiKeys.expires >= time.time()))
 
         tokens_q = tokens_q.filter(UserApiKeys.role.in_(roles))
 
         crypto_backend = auth.crypto_backend()
         enc_token_map = {}
         plain_token_map = {}
         for token in tokens_q:
             if token.api_key.startswith(crypto_backend.ENC_PREF):
                 enc_token_map[token.api_key] = token
             else:
                 plain_token_map[token.api_key] = token
         log.debug(
             'Found %s plain and %s encrypted tokens to check for authentication for this user',
             len(plain_token_map), len(enc_token_map))
 
         # plain token match comes first
         match = plain_token_map.get(auth_token)
 
         # check encrypted tokens now
         if not match:
             for token_hash, token in enc_token_map.items():
                 # NOTE(marcink): this is expensive to calculate, but most secure
                 if crypto_backend.hash_check(auth_token, token_hash):
                     match = token
                     break
 
         if match:
             log.debug('Found matching token %s', match)
             if match.repo_id:
                 log.debug('Found scope, checking for scope match of token %s', match)
                 if match.repo_id == scope_repo_id:
                     return True
                 else:
                     log.debug(
                         'AUTH_TOKEN: scope mismatch, token has a set repo scope: %s, '
                         'and calling scope is:%s, skipping further checks',
                          match.repo, scope_repo_id)
                     return False
             else:
                 return True
 
         return False
 
     @property
     def ip_addresses(self):
         ret = UserIpMap.query().filter(UserIpMap.user == self).all()
         return [x.ip_addr for x in ret]
 
     @property
     def username_and_name(self):
         return f'{self.username} ({self.first_name} {self.last_name})'
 
     @property
     def username_or_name_or_email(self):
         full_name = self.full_name if self.full_name != ' ' else None
         return self.username or full_name or self.email
 
     @property
     def full_name(self):
         return f'{self.first_name} {self.last_name}'
 
     @property
     def full_name_or_username(self):
         return (f'{self.first_name} {self.last_name}'
                 if (self.first_name and self.last_name) else self.username)
 
     @property
     def full_contact(self):
         return f'{self.first_name} {self.last_name} <{self.email}>'
 
     @property
     def short_contact(self):
         return f'{self.first_name} {self.last_name}'
 
     @property
     def is_admin(self):
         return self.admin
 
     @property
     def language(self):
         return self.user_data.get('language')
 
     def AuthUser(self, **kwargs):
         """
         Returns instance of AuthUser for this user
         """
         from rhodecode.lib.auth import AuthUser
         return AuthUser(user_id=self.user_id, username=self.username, **kwargs)
 
     @hybrid_property
     def user_data(self):
         if not self._user_data:
             return {}
 
         try:
             return json.loads(self._user_data) or {}
         except TypeError:
             return {}
 
     @user_data.setter
     def user_data(self, val):
         if not isinstance(val, dict):
             raise Exception(f'user_data must be dict, got {type(val)}')
         try:
             self._user_data = safe_bytes(json.dumps(val))
         except Exception:
             log.error(traceback.format_exc())
 
     @classmethod
     def get(cls, user_id, cache=False):
         if not user_id:
             return
 
         user = cls.query()
         if cache:
             user = user.options(
                 FromCache("sql_cache_short", f"get_users_{user_id}"))
         return user.get(user_id)
 
     @classmethod
     def get_by_username(cls, username, case_insensitive=False,
                         cache=False):
 
         if case_insensitive:
             q = cls.select().where(
                 func.lower(cls.username) == func.lower(username))
         else:
             q = cls.select().where(cls.username == username)
 
         if cache:
             hash_key = _hash_key(username)
             q = q.options(
                 FromCache("sql_cache_short", f"get_user_by_name_{hash_key}"))
 
         return cls.execute(q).scalar_one_or_none()
 
     @classmethod
     def get_by_username_or_primary_email(cls, user_identifier):
         qs = union_all(cls.select().where(func.lower(cls.username) == func.lower(user_identifier)),
                        cls.select().where(func.lower(cls.email) == func.lower(user_identifier)))
         return cls.execute(cls.select(User).from_statement(qs)).scalar_one_or_none()
 
     @classmethod
     def get_by_auth_token(cls, auth_token, cache=False):
 
         q = cls.select(User)\
             .join(UserApiKeys)\
             .where(UserApiKeys.api_key == auth_token)\
             .where(or_(UserApiKeys.expires == -1,
                        UserApiKeys.expires >= time.time()))
 
         if cache:
             q = q.options(
                 FromCache("sql_cache_short", f"get_auth_token_{auth_token}"))
 
         matched_user = cls.execute(q).scalar_one_or_none()
 
         return matched_user
 
     @classmethod
     def get_by_email(cls, email, case_insensitive=False, cache=False):
 
         if case_insensitive:
             q = cls.select().where(func.lower(cls.email) == func.lower(email))
         else:
             q = cls.select().where(cls.email == email)
 
         if cache:
             email_key = _hash_key(email)
             q = q.options(
                 FromCache("sql_cache_short", f"get_email_key_{email_key}"))
 
         ret = cls.execute(q).scalar_one_or_none()
 
         if ret is None:
             q = cls.select(UserEmailMap)
             # try fetching in alternate email map
             if case_insensitive:
                 q = q.where(func.lower(UserEmailMap.email) == func.lower(email))
             else:
                 q = q.where(UserEmailMap.email == email)
             q = q.options(joinedload(UserEmailMap.user))
             if cache:
                 q = q.options(
                     FromCache("sql_cache_short", f"get_email_map_key_{email_key}"))
 
             result = cls.execute(q).scalar_one_or_none()
             ret = getattr(result, 'user', None)
 
         return ret
 
     @classmethod
     def get_from_cs_author(cls, author):
         """
         Tries to get User objects out of commit author string
 
         :param author:
         """
         from rhodecode.lib.helpers import email, author_name
         # Valid email in the attribute passed, see if they're in the system
         _email = email(author)
         if _email:
             user = cls.get_by_email(_email, case_insensitive=True)
             if user:
                 return user
         # Maybe we can match by username?
         _author = author_name(author)
         user = cls.get_by_username(_author, case_insensitive=True)
         if user:
             return user
 
     def update_userdata(self, **kwargs):
         usr = self
         old = usr.user_data
         old.update(**kwargs)
         usr.user_data = old
         Session().add(usr)
         log.debug('updated userdata with %s', kwargs)
 
     def update_lastlogin(self):
         """Update user lastlogin"""
         self.last_login = datetime.datetime.now()
         Session().add(self)
         log.debug('updated user %s lastlogin', self.username)
 
     def update_password(self, new_password):
         from rhodecode.lib.auth import get_crypt_password
 
         self.password = get_crypt_password(new_password)
         Session().add(self)
 
     @classmethod
     def get_first_super_admin(cls):
         stmt = cls.select().where(User.admin == true()).order_by(User.user_id.asc())
         user = cls.scalars(stmt).first()
 
         if user is None:
             raise Exception('FATAL: Missing administrative account!')
         return user
 
     @classmethod
     def get_all_super_admins(cls, only_active=False):
         """
         Returns all admin accounts sorted by username
         """
         qry = User.query().filter(User.admin == true()).order_by(User.username.asc())
         if only_active:
             qry = qry.filter(User.active == true())
         return qry.all()
 
     @classmethod
     def get_all_user_ids(cls, only_active=True):
         """
         Returns all users IDs
         """
         qry = Session().query(User.user_id)
 
         if only_active:
             qry = qry.filter(User.active == true())
         return [x.user_id for x in qry]
 
     @classmethod
     def get_default_user(cls, cache=False, refresh=False):
         user = User.get_by_username(User.DEFAULT_USER, cache=cache)
         if user is None:
             raise Exception('FATAL: Missing default account!')
         if refresh:
             # The default user might be based on outdated state which
             # has been loaded from the cache.
             # A call to refresh() ensures that the
             # latest state from the database is used.
             Session().refresh(user)
 
         return user
 
     @classmethod
     def get_default_user_id(cls):
         import rhodecode
         return rhodecode.CONFIG['default_user_id']
 
     def _get_default_perms(self, user, suffix=''):
         from rhodecode.model.permission import PermissionModel
         return PermissionModel().get_default_perms(user.user_perms, suffix)
 
     def get_default_perms(self, suffix=''):
         return self._get_default_perms(self, suffix)
 
     def get_api_data(self, include_secrets=False, details='full'):
         """
         Common function for generating user related data for API
 
         :param include_secrets: By default secrets in the API data will be replaced
            by a placeholder value to prevent exposing this data by accident. In case
            this data shall be exposed, set this flag to ``True``.
 
         :param details: details can be 'basic|full' basic gives only a subset of
            the available user information that includes user_id, name and emails.
         """
         user = self
         user_data = self.user_data
         data = {
             'user_id': user.user_id,
             'username': user.username,
             'firstname': user.name,
             'lastname': user.lastname,
             'description': user.description,
             'email': user.email,
             'emails': user.emails,
         }
         if details == 'basic':
             return data
 
         auth_token_length = 40
         auth_token_replacement = '*' * auth_token_length
 
         extras = {
             'auth_tokens': [auth_token_replacement],
             'active': user.active,
             'admin': user.admin,
             'extern_type': user.extern_type,
             'extern_name': user.extern_name,
             'last_login': user.last_login,
             'last_activity': user.last_activity,
             'ip_addresses': user.ip_addresses,
             'language': user_data.get('language')
         }
         data.update(extras)
 
         if include_secrets:
             data['auth_tokens'] = user.auth_tokens
         return data
 
     def __json__(self):
         data = {
             'full_name': self.full_name,
             'full_name_or_username': self.full_name_or_username,
             'short_contact': self.short_contact,
             'full_contact': self.full_contact,
         }
         data.update(self.get_api_data())
         return data
 
 
 class UserApiKeys(Base, BaseModel):
     __tablename__ = 'user_api_keys'
     __table_args__ = (
         Index('uak_api_key_idx', 'api_key'),
         Index('uak_api_key_expires_idx', 'api_key', 'expires'),
         base_table_args
     )
 
     # ApiKey role
     ROLE_ALL = 'token_role_all'
     ROLE_VCS = 'token_role_vcs'
     ROLE_API = 'token_role_api'
     ROLE_HTTP = 'token_role_http'
     ROLE_FEED = 'token_role_feed'
     ROLE_ARTIFACT_DOWNLOAD = 'role_artifact_download'
     # The last one is ignored in the list as we only
     # use it for one action, and cannot be created by users
     ROLE_PASSWORD_RESET = 'token_password_reset'
 
     ROLES = [ROLE_ALL, ROLE_VCS, ROLE_API, ROLE_HTTP, ROLE_FEED, ROLE_ARTIFACT_DOWNLOAD]
 
     user_api_key_id = Column("user_api_key_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
     api_key = Column("api_key", String(255), nullable=False, unique=True)
     description = Column('description', UnicodeText().with_variant(UnicodeText(1024), 'mysql'))
     expires = Column('expires', Float(53), nullable=False)
     role = Column('role', String(255), nullable=True)
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 
     # scope columns
     repo_id = Column(
         'repo_id', Integer(), ForeignKey('repositories.repo_id'),
         nullable=True, unique=None, default=None)
     repo = relationship('Repository', lazy='joined', back_populates='scoped_tokens')
 
     repo_group_id = Column(
         'repo_group_id', Integer(), ForeignKey('groups.group_id'),
         nullable=True, unique=None, default=None)
     repo_group = relationship('RepoGroup', lazy='joined')
 
     user = relationship('User', lazy='joined', back_populates='user_auth_tokens')
 
     def __repr__(self):
         return f"<{self.cls_name}('{self.role}')>"
 
     def __json__(self):
         data = {
             'auth_token': self.api_key,
             'role': self.role,
             'scope': self.scope_humanized,
             'expired': self.expired
         }
         return data
 
     def get_api_data(self, include_secrets=False):
         data = self.__json__()
         if include_secrets:
             return data
         else:
             data['auth_token'] = self.token_obfuscated
             return data
 
     @hybrid_property
     def description_safe(self):
         return description_escaper(self.description)
 
     @property
     def expired(self):
         if self.expires == -1:
             return False
         return time.time() > self.expires
 
     @classmethod
     def _get_role_name(cls, role):
         return {
             cls.ROLE_ALL: _('all'),
             cls.ROLE_HTTP: _('http/web interface'),
             cls.ROLE_VCS: _('vcs (git/hg/svn protocol)'),
             cls.ROLE_API: _('api calls'),
             cls.ROLE_FEED: _('feed access'),
             cls.ROLE_ARTIFACT_DOWNLOAD: _('artifacts downloads'),
         }.get(role, role)
 
     @classmethod
     def _get_role_description(cls, role):
         return {
             cls.ROLE_ALL: _('Token for all actions.'),
             cls.ROLE_HTTP: _('Token to access RhodeCode pages via web interface without '
                              'login using `api_access_controllers_whitelist` functionality.'),
             cls.ROLE_VCS: _('Token to interact over git/hg/svn protocols. '
                             'Requires auth_token authentication plugin to be active. <br/>'
                             'Such Token should be used then instead of a password to '
                             'interact with a repository, and additionally can be '
                             'limited to single repository using repo scope.'),
             cls.ROLE_API: _('Token limited to api calls.'),
             cls.ROLE_FEED: _('Token to read RSS/ATOM feed.'),
             cls.ROLE_ARTIFACT_DOWNLOAD: _('Token for artifacts downloads.'),
         }.get(role, role)
 
     @property
     def role_humanized(self):
         return self._get_role_name(self.role)
 
     def _get_scope(self):
         if self.repo:
             return 'Repository: {}'.format(self.repo.repo_name)
         if self.repo_group:
             return 'RepositoryGroup: {} (recursive)'.format(self.repo_group.group_name)
         return 'Global'
 
     @property
     def scope_humanized(self):
         return self._get_scope()
 
     @property
     def token_obfuscated(self):
         if self.api_key:
             return self.api_key[:4] + "****"
 
 
 class UserEmailMap(Base, BaseModel):
     __tablename__ = 'user_email_map'
     __table_args__ = (
         Index('uem_email_idx', 'email'),
         Index('uem_user_id_idx', 'user_id'),
         UniqueConstraint('email'),
         base_table_args
     )
 
     email_id = Column("email_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
     _email = Column("email", String(255), nullable=True, unique=False, default=None)
     user = relationship('User', lazy='joined', back_populates='user_emails')
 
     @validates('_email')
     def validate_email(self, key, email):
         # check if this email is not main one
         main_email = Session().query(User).filter(User.email == email).scalar()
         if main_email is not None:
             raise AttributeError('email %s is present is user table' % email)
         return email
 
     @hybrid_property
     def email(self):
         return self._email
 
     @email.setter
     def email(self, val):
         self._email = val.lower() if val else None
 
 
 class UserIpMap(Base, BaseModel):
     __tablename__ = 'user_ip_map'
     __table_args__ = (
         UniqueConstraint('user_id', 'ip_addr'),
         base_table_args
     )
 
     ip_id = Column("ip_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
     ip_addr = Column("ip_addr", String(255), nullable=True, unique=False, default=None)
     active = Column("active", Boolean(), nullable=True, unique=None, default=True)
     description = Column("description", String(10000), nullable=True, unique=None, default=None)
     user = relationship('User', lazy='joined', back_populates='user_ip_map')
 
     @hybrid_property
     def description_safe(self):
         return description_escaper(self.description)
 
     @classmethod
     def _get_ip_range(cls, ip_addr):
         net = ipaddress.ip_network(safe_str(ip_addr), strict=False)
         return [str(net.network_address), str(net.broadcast_address)]
 
     def __json__(self):
         return {
           'ip_addr': self.ip_addr,
           'ip_range': self._get_ip_range(self.ip_addr),
         }
 
     def __repr__(self):
         return f"<{self.cls_name}('user_id={self.user_id} => ip={self.ip_addr}')>"
 
 
 class UserSshKeys(Base, BaseModel):
     __tablename__ = 'user_ssh_keys'
     __table_args__ = (
         Index('usk_ssh_key_fingerprint_idx', 'ssh_key_fingerprint'),
 
         UniqueConstraint('ssh_key_fingerprint'),
 
         base_table_args
     )
 
     ssh_key_id = Column('ssh_key_id', Integer(), nullable=False, unique=True, default=None, primary_key=True)
     ssh_key_data = Column('ssh_key_data', String(10240), nullable=False, unique=None, default=None)
     ssh_key_fingerprint = Column('ssh_key_fingerprint', String(255), nullable=False, unique=None, default=None)
 
     description = Column('description', UnicodeText().with_variant(UnicodeText(1024), 'mysql'))
 
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     accessed_on = Column('accessed_on', DateTime(timezone=False), nullable=True, default=None)
     user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
 
     user = relationship('User', lazy='joined', back_populates='user_ssh_keys')
 
     def __json__(self):
         data = {
             'ssh_fingerprint': self.ssh_key_fingerprint,
             'description': self.description,
             'created_on': self.created_on
         }
         return data
 
     def get_api_data(self):
         data = self.__json__()
         return data
 
 
 class UserLog(Base, BaseModel):
     __tablename__ = 'user_logs'
     __table_args__ = (
         base_table_args,
     )
 
     VERSION_1 = 'v1'
     VERSION_2 = 'v2'
     VERSIONS = [VERSION_1, VERSION_2]
 
     user_log_id = Column("user_log_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id',ondelete='SET NULL'), nullable=True, unique=None, default=None)
     username = Column("username", String(255), nullable=True, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id', ondelete='SET NULL'), nullable=True, unique=None, default=None)
     repository_name = Column("repository_name", String(255), nullable=True, unique=None, default=None)
     user_ip = Column("user_ip", String(255), nullable=True, unique=None, default=None)
     action = Column("action", Text().with_variant(Text(1200000), 'mysql'), nullable=True, unique=None, default=None)
     action_date = Column("action_date", DateTime(timezone=False), nullable=True, unique=None, default=None)
 
     version = Column("version", String(255), nullable=True, default=VERSION_1)
     user_data = Column('user_data_json', MutationObj.as_mutable(JsonType(dialect_map=dict(mysql=LONGTEXT()))))
     action_data = Column('action_data_json', MutationObj.as_mutable(JsonType(dialect_map=dict(mysql=LONGTEXT()))))
     user = relationship('User', cascade='', back_populates='user_log')
     repository = relationship('Repository', cascade='', back_populates='logs')
 
     def __repr__(self):
         return f"<{self.cls_name}('id:{self.repository_name}:{self.action}')>"
 
     def __json__(self):
         return {
             'user_id': self.user_id,
             'username': self.username,
             'repository_id': self.repository_id,
             'repository_name': self.repository_name,
             'user_ip': self.user_ip,
             'action_date': self.action_date,
             'action': self.action,
         }
 
     @hybrid_property
     def entry_id(self):
         return self.user_log_id
 
     @property
     def action_as_day(self):
         return datetime.date(*self.action_date.timetuple()[:3])
 
 
 class UserGroup(Base, BaseModel):
     __tablename__ = 'users_groups'
     __table_args__ = (
         base_table_args,
     )
 
     users_group_id = Column("users_group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_name = Column("users_group_name", String(255), nullable=False, unique=True, default=None)
     user_group_description = Column("user_group_description", String(10000), nullable=True, unique=None, default=None)
     users_group_active = Column("users_group_active", Boolean(), nullable=True, unique=None, default=None)
     inherit_default_permissions = Column("users_group_inherit_default_permissions", Boolean(), nullable=False, unique=None, default=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None)
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     _group_data = Column("group_data", LargeBinary(), nullable=True)  # JSON data
 
     members = relationship('UserGroupMember', cascade="all, delete-orphan", lazy="joined", back_populates='users_group')
     users_group_to_perm = relationship('UserGroupToPerm', cascade='all', back_populates='users_group')
     users_group_repo_to_perm = relationship('UserGroupRepoToPerm', cascade='all', back_populates='users_group')
     users_group_repo_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all', back_populates='users_group')
     user_user_group_to_perm = relationship('UserUserGroupToPerm', cascade='all', back_populates='user_group')
 
     user_group_user_group_to_perm = relationship('UserGroupUserGroupToPerm', primaryjoin="UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id", cascade='all', back_populates='target_user_group')
 
     user_group_review_rules = relationship('RepoReviewRuleUserGroup', cascade='all', back_populates='users_group')
     user = relationship('User', primaryjoin="User.user_id==UserGroup.user_id", back_populates='user_groups')
 
     @classmethod
     def _load_group_data(cls, column):
         if not column:
             return {}
 
         try:
             return json.loads(column) or {}
         except TypeError:
             return {}
 
     @hybrid_property
     def description_safe(self):
         return description_escaper(self.user_group_description)
 
     @hybrid_property
     def group_data(self):
         return self._load_group_data(self._group_data)
 
     @group_data.expression
     def group_data(self, **kwargs):
         return self._group_data
 
     @group_data.setter
     def group_data(self, val):
         try:
             self._group_data = json.dumps(val)
         except Exception:
             log.error(traceback.format_exc())
 
     @classmethod
     def _load_sync(cls, group_data):
         if group_data:
             return group_data.get('extern_type')
 
     @property
     def sync(self):
         return self._load_sync(self.group_data)
 
     def __repr__(self):
         return f"<{self.cls_name}('id:{self.users_group_id}:{self.users_group_name}')>"
 
     @classmethod
     def get_by_group_name(cls, group_name, cache=False,
                           case_insensitive=False):
         if case_insensitive:
             q = cls.query().filter(func.lower(cls.users_group_name) ==
                                    func.lower(group_name))
 
         else:
             q = cls.query().filter(cls.users_group_name == group_name)
         if cache:
             name_key = _hash_key(group_name)
             q = q.options(
                 FromCache("sql_cache_short", f"get_group_{name_key}"))
         return q.scalar()
 
     @classmethod
     def get(cls, user_group_id, cache=False):
         if not user_group_id:
             return
 
         user_group = cls.query()
         if cache:
             user_group = user_group.options(
                 FromCache("sql_cache_short", f"get_users_group_{user_group_id}"))
         return user_group.get(user_group_id)
 
     def permissions(self, with_admins=True, with_owner=True,
                     expand_from_user_groups=False):
         """
         Permissions for user groups
         """
         _admin_perm = 'usergroup.admin'
 
         owner_row = []
         if with_owner:
             usr = AttributeDict(self.user.get_dict())
             usr.owner_row = True
             usr.permission = _admin_perm
             owner_row.append(usr)
 
         super_admin_ids = []
         super_admin_rows = []
         if with_admins:
             for usr in User.get_all_super_admins():
                 super_admin_ids.append(usr.user_id)
                 # if this admin is also owner, don't double the record
                 if usr.user_id == owner_row[0].user_id:
                     owner_row[0].admin_row = True
                 else:
                     usr = AttributeDict(usr.get_dict())
                     usr.admin_row = True
                     usr.permission = _admin_perm
                     super_admin_rows.append(usr)
 
         q = UserUserGroupToPerm.query().filter(UserUserGroupToPerm.user_group == self)
         q = q.options(joinedload(UserUserGroupToPerm.user_group),
                       joinedload(UserUserGroupToPerm.user),
                       joinedload(UserUserGroupToPerm.permission),)
 
         # get owners and admins and permissions. We do a trick of re-writing
         # objects from sqlalchemy to named-tuples due to sqlalchemy session
         # has a global reference and changing one object propagates to all
         # others. This means if admin is also an owner admin_row that change
         # would propagate to both objects
         perm_rows = []
         for _usr in q.all():
             usr = AttributeDict(_usr.user.get_dict())
             # if this user is also owner/admin, mark as duplicate record
             if usr.user_id == owner_row[0].user_id or usr.user_id in super_admin_ids:
                 usr.duplicate_perm = True
             usr.permission = _usr.permission.permission_name
             perm_rows.append(usr)
 
         # filter the perm rows by 'default' first and then sort them by
         # admin,write,read,none permissions sorted again alphabetically in
         # each group
         perm_rows = sorted(perm_rows, key=display_user_sort)
 
         user_groups_rows = []
         if expand_from_user_groups:
             for ug in self.permission_user_groups(with_members=True):
                 for user_data in ug.members:
                     user_groups_rows.append(user_data)
 
         return super_admin_rows + owner_row + perm_rows + user_groups_rows
 
     def permission_user_groups(self, with_members=False):
         q = UserGroupUserGroupToPerm.query()\
             .filter(UserGroupUserGroupToPerm.target_user_group == self)
         q = q.options(joinedload(UserGroupUserGroupToPerm.user_group),
                       joinedload(UserGroupUserGroupToPerm.target_user_group),
                       joinedload(UserGroupUserGroupToPerm.permission),)
 
         perm_rows = []
         for _user_group in q.all():
             entry = AttributeDict(_user_group.user_group.get_dict())
             entry.permission = _user_group.permission.permission_name
             if with_members:
                 entry.members = [x.user.get_dict()
                                  for x in _user_group.user_group.members]
             perm_rows.append(entry)
 
         perm_rows = sorted(perm_rows, key=display_user_group_sort)
         return perm_rows
 
     def _get_default_perms(self, user_group, suffix=''):
         from rhodecode.model.permission import PermissionModel
         return PermissionModel().get_default_perms(user_group.users_group_to_perm, suffix)
 
     def get_default_perms(self, suffix=''):
         return self._get_default_perms(self, suffix)
 
     def get_api_data(self, with_group_members=True, include_secrets=False):
         """
         :param include_secrets: See :meth:`User.get_api_data`, this parameter is
            basically forwarded.
 
         """
         user_group = self
         data = {
             'users_group_id': user_group.users_group_id,
             'group_name': user_group.users_group_name,
             'group_description': user_group.user_group_description,
             'active': user_group.users_group_active,
             'owner': user_group.user.username,
             'sync': user_group.sync,
             'owner_email': user_group.user.email,
         }
 
         if with_group_members:
             users = []
             for user in user_group.members:
                 user = user.user
                 users.append(user.get_api_data(include_secrets=include_secrets))
             data['users'] = users
 
         return data
 
 
 class UserGroupMember(Base, BaseModel):
     __tablename__ = 'users_groups_members'
     __table_args__ = (
         base_table_args,
     )
 
     users_group_member_id = Column("users_group_member_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User', lazy='joined', back_populates='group_member')
     users_group = relationship('UserGroup', back_populates='members')
 
     def __init__(self, gr_id='', u_id=''):
         self.users_group_id = gr_id
         self.user_id = u_id
 
 
 class RepositoryField(Base, BaseModel):
     __tablename__ = 'repositories_fields'
     __table_args__ = (
         UniqueConstraint('repository_id', 'field_key'),  # no-multi field
         base_table_args,
     )
 
     PREFIX = 'ex_'  # prefix used in form to not conflict with already existing fields
 
     repo_field_id = Column("repo_field_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
     field_key = Column("field_key", String(250))
     field_label = Column("field_label", String(1024), nullable=False)
     field_value = Column("field_value", String(10000), nullable=False)
     field_desc = Column("field_desc", String(1024), nullable=False)
     field_type = Column("field_type", String(255), nullable=False, unique=None)
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 
     repository = relationship('Repository', back_populates='extra_fields')
 
     @property
     def field_key_prefixed(self):
         return 'ex_%s' % self.field_key
 
     @classmethod
     def un_prefix_key(cls, key):
         if key.startswith(cls.PREFIX):
             return key[len(cls.PREFIX):]
         return key
 
     @classmethod
     def get_by_key_name(cls, key, repo):
         row = cls.query()\
                 .filter(cls.repository == repo)\
                 .filter(cls.field_key == key).scalar()
         return row
 
 
 class Repository(Base, BaseModel):
     __tablename__ = 'repositories'
     __table_args__ = (
         Index('r_repo_name_idx', 'repo_name', mysql_length=255),
         base_table_args,
     )
     DEFAULT_CLONE_URI = '{scheme}://{user}@{netloc}/{repo}'
     DEFAULT_CLONE_URI_ID = '{scheme}://{user}@{netloc}/_{repoid}'
     DEFAULT_CLONE_URI_SSH = 'ssh://{sys_user}@{hostname}/{repo}'
 
     STATE_CREATED = 'repo_state_created'
     STATE_PENDING = 'repo_state_pending'
     STATE_ERROR = 'repo_state_error'
 
     LOCK_AUTOMATIC = 'lock_auto'
     LOCK_API = 'lock_api'
     LOCK_WEB = 'lock_web'
     LOCK_PULL = 'lock_pull'
 
     NAME_SEP = URL_SEP
 
     repo_id = Column(
         "repo_id", Integer(), nullable=False, unique=True, default=None,
         primary_key=True)
     _repo_name = Column(
         "repo_name", Text(), nullable=False, default=None)
     repo_name_hash = Column(
         "repo_name_hash", String(255), nullable=False, unique=True)
     repo_state = Column("repo_state", String(255), nullable=True)
 
     clone_uri = Column(
         "clone_uri", EncryptedTextValue(), nullable=True, unique=False,
         default=None)
     push_uri = Column(
         "push_uri", EncryptedTextValue(), nullable=True, unique=False,
         default=None)
     repo_type = Column(
         "repo_type", String(255), nullable=False, unique=False, default=None)
     user_id = Column(
         "user_id", Integer(), ForeignKey('users.user_id'), nullable=False,
         unique=False, default=None)
     private = Column(
         "private", Boolean(), nullable=True, unique=None, default=None)
     archived = Column(
         "archived", Boolean(), nullable=True, unique=None, default=None)
     enable_statistics = Column(
         "statistics", Boolean(), nullable=True, unique=None, default=True)
     enable_downloads = Column(
         "downloads", Boolean(), nullable=True, unique=None, default=True)
     description = Column(
         "description", String(10000), nullable=True, unique=None, default=None)
     created_on = Column(
         'created_on', DateTime(timezone=False), nullable=True, unique=None,
         default=datetime.datetime.now)
     updated_on = Column(
         'updated_on', DateTime(timezone=False), nullable=True, unique=None,
         default=datetime.datetime.now)
     _landing_revision = Column(
         "landing_revision", String(255), nullable=False, unique=False,
         default=None)
     enable_locking = Column(
         "enable_locking", Boolean(), nullable=False, unique=None,
         default=False)
     _locked = Column(
         "locked", String(255), nullable=True, unique=False, default=None)
     _changeset_cache = Column(
         "changeset_cache", LargeBinary(), nullable=True)  # JSON data
 
     fork_id = Column(
         "fork_id", Integer(), ForeignKey('repositories.repo_id'),
         nullable=True, unique=False, default=None)
     group_id = Column(
         "group_id", Integer(), ForeignKey('groups.group_id'), nullable=True,
         unique=False, default=None)
 
     user = relationship('User', lazy='joined', back_populates='repositories')
     fork = relationship('Repository', remote_side=repo_id, lazy='joined')
     group = relationship('RepoGroup', lazy='joined')
     repo_to_perm = relationship('UserRepoToPerm', cascade='all', order_by='UserRepoToPerm.repo_to_perm_id')
     users_group_to_perm = relationship('UserGroupRepoToPerm', cascade='all', back_populates='repository')
     stats = relationship('Statistics', cascade='all', uselist=False)
 
     followers = relationship('UserFollowing', primaryjoin='UserFollowing.follows_repo_id==Repository.repo_id', cascade='all', back_populates='follows_repository')
     extra_fields = relationship('RepositoryField', cascade="all, delete-orphan", back_populates='repository')
 
     logs = relationship('UserLog', back_populates='repository')
 
     comments = relationship('ChangesetComment', cascade="all, delete-orphan", back_populates='repo')
 
     pull_requests_source = relationship(
         'PullRequest',
         primaryjoin='PullRequest.source_repo_id==Repository.repo_id',
         cascade="all, delete-orphan",
         overlaps="source_repo"
     )
     pull_requests_target = relationship(
         'PullRequest',
         primaryjoin='PullRequest.target_repo_id==Repository.repo_id',
         cascade="all, delete-orphan",
         overlaps="target_repo"
     )
 
     ui = relationship('RepoRhodeCodeUi', cascade="all")
     settings = relationship('RepoRhodeCodeSetting', cascade="all")
     integrations = relationship('Integration', cascade="all, delete-orphan", back_populates='repo')
 
     scoped_tokens = relationship('UserApiKeys', cascade="all", back_populates='repo')
 
     # no cascade, set NULL
     artifacts = relationship('FileStore', primaryjoin='FileStore.scope_repo_id==Repository.repo_id', viewonly=True)
 
     review_rules = relationship('RepoReviewRule')
     user_branch_perms = relationship('UserToRepoBranchPermission')
     user_group_branch_perms = relationship('UserGroupToRepoBranchPermission')
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (self.cls_name, self.repo_id, self.repo_name)
 
     @hybrid_property
     def description_safe(self):
         return description_escaper(self.description)
 
     @hybrid_property
     def landing_rev(self):
         # always should return [rev_type, rev], e.g ['branch', 'master']
         if self._landing_revision:
             _rev_info = self._landing_revision.split(':')
             if len(_rev_info) < 2:
                 _rev_info.insert(0, 'rev')
             return [_rev_info[0], _rev_info[1]]
         return [None, None]
 
     @property
     def landing_ref_type(self):
         return self.landing_rev[0]
 
     @property
     def landing_ref_name(self):
         return self.landing_rev[1]
 
     @landing_rev.setter
     def landing_rev(self, val):
         if ':' not in val:
             raise ValueError('value must be delimited with `:` and consist '
                              'of <rev_type>:<rev>, got %s instead' % val)
         self._landing_revision = val
 
     @hybrid_property
     def locked(self):
         if self._locked:
             user_id, timelocked, reason = self._locked.split(':')
             lock_values = int(user_id), timelocked, reason
         else:
             lock_values = [None, None, None]
         return lock_values
 
     @locked.setter
     def locked(self, val):
         if val and isinstance(val, (list, tuple)):
             self._locked = ':'.join(map(str, val))
         else:
             self._locked = None
 
     @classmethod
     def _load_changeset_cache(cls, repo_id, changeset_cache_raw):
         from rhodecode.lib.vcs.backends.base import EmptyCommit
         dummy = EmptyCommit().__json__()
         if not changeset_cache_raw:
             dummy['source_repo_id'] = repo_id
             return json.loads(json.dumps(dummy))
 
         try:
             return json.loads(changeset_cache_raw)
         except TypeError:
             return dummy
         except Exception:
             log.error(traceback.format_exc())
             return dummy
 
     @hybrid_property
     def changeset_cache(self):
         return self._load_changeset_cache(self.repo_id, self._changeset_cache)
 
     @changeset_cache.setter
     def changeset_cache(self, val):
         try:
             self._changeset_cache = json.dumps(val)
         except Exception:
             log.error(traceback.format_exc())
 
     @hybrid_property
     def repo_name(self):
         return self._repo_name
 
     @repo_name.setter
     def repo_name(self, value):
         self._repo_name = value
         self.repo_name_hash = sha1(safe_bytes(value))
 
     @classmethod
     def normalize_repo_name(cls, repo_name):
         """
         Normalizes os specific repo_name to the format internally stored inside
         database using URL_SEP
 
         :param cls:
         :param repo_name:
         """
         return cls.NAME_SEP.join(repo_name.split(os.sep))
 
     @classmethod
     def get_by_repo_name(cls, repo_name, cache=False, identity_cache=False):
         session = Session()
         q = session.query(cls).filter(cls.repo_name == repo_name)
 
         if cache:
             if identity_cache:
                 val = cls.identity_cache(session, 'repo_name', repo_name)
                 if val:
                     return val
             else:
                 cache_key = f"get_repo_by_name_{_hash_key(repo_name)}"
                 q = q.options(
                     FromCache("sql_cache_short", cache_key))
 
         return q.scalar()
 
     @classmethod
     def get_by_id_or_repo_name(cls, repoid):
         if isinstance(repoid, int):
             try:
                 repo = cls.get(repoid)
             except ValueError:
                 repo = None
         else:
             repo = cls.get_by_repo_name(repoid)
         return repo
 
     @classmethod
     def get_by_full_path(cls, repo_full_path):
         repo_name = repo_full_path.split(cls.base_path(), 1)[-1]
         repo_name = cls.normalize_repo_name(repo_name)
         return cls.get_by_repo_name(repo_name.strip(URL_SEP))
 
     @classmethod
     def get_repo_forks(cls, repo_id):
         return cls.query().filter(Repository.fork_id == repo_id)
 
     @classmethod
     def base_path(cls):
         """
         Returns base path when all repos are stored
 
         :param cls:
         """
         from rhodecode.lib.utils import get_rhodecode_repo_store_path
         return get_rhodecode_repo_store_path()
 
     @classmethod
     def get_all_repos(cls, user_id=Optional(None), group_id=Optional(None),
                       case_insensitive=True, archived=False):
         q = Repository.query()
 
         if not archived:
             q = q.filter(Repository.archived.isnot(true()))
 
         if not isinstance(user_id, Optional):
             q = q.filter(Repository.user_id == user_id)
 
         if not isinstance(group_id, Optional):
             q = q.filter(Repository.group_id == group_id)
 
         if case_insensitive:
             q = q.order_by(func.lower(Repository.repo_name))
         else:
             q = q.order_by(Repository.repo_name)
 
         return q.all()
 
     @property
     def repo_uid(self):
         return '_{}'.format(self.repo_id)
 
     @property
     def forks(self):
         """
         Return forks of this repo
         """
         return Repository.get_repo_forks(self.repo_id)
 
     @property
     def parent(self):
         """
         Returns fork parent
         """
         return self.fork
 
     @property
     def just_name(self):
         return self.repo_name.split(self.NAME_SEP)[-1]
 
     @property
     def groups_with_parents(self):
         groups = []
         if self.group is None:
             return groups
 
         cur_gr = self.group
         groups.insert(0, cur_gr)
         while 1:
             gr = getattr(cur_gr, 'parent_group', None)
             cur_gr = cur_gr.parent_group
             if gr is None:
                 break
             groups.insert(0, gr)
 
         return groups
 
     @property
     def groups_and_repo(self):
         return self.groups_with_parents, self
 
     @property
     def repo_path(self):
         """
         Returns base full path for that repository means where it actually
         exists on a filesystem
         """
         return self.base_path()
 
     @property
     def repo_full_path(self):
         p = [self.repo_path]
         # we need to split the name by / since this is how we store the
         # names in the database, but that eventually needs to be converted
         # into a valid system path
         p += self.repo_name.split(self.NAME_SEP)
         return os.path.join(*map(safe_str, p))
 
     @property
     def cache_keys(self):
         """
         Returns associated cache keys for that repo
         """
         repo_namespace_key = CacheKey.REPO_INVALIDATION_NAMESPACE.format(repo_id=self.repo_id)
         return CacheKey.query()\
             .filter(CacheKey.cache_key == repo_namespace_key)\
             .order_by(CacheKey.cache_key)\
             .all()
 
     @property
     def cached_diffs_relative_dir(self):
         """
         Return a relative to the repository store path of cached diffs
         used for safe display for users, who shouldn't know the absolute store
         path
         """
         return os.path.join(
             os.path.dirname(self.repo_name),
             self.cached_diffs_dir.split(os.path.sep)[-1])
 
     @property
     def cached_diffs_dir(self):
         path = self.repo_full_path
         return os.path.join(
             os.path.dirname(path),
             f'.__shadow_diff_cache_repo_{self.repo_id}')
 
     def cached_diffs(self):
         diff_cache_dir = self.cached_diffs_dir
         if os.path.isdir(diff_cache_dir):
             return os.listdir(diff_cache_dir)
         return []
 
     def shadow_repos(self):
         shadow_repos_pattern = f'.__shadow_repo_{self.repo_id}'
         return [
             x for x in os.listdir(os.path.dirname(self.repo_full_path))
             if x.startswith(shadow_repos_pattern)
         ]
 
     def get_new_name(self, repo_name):
         """
         returns new full repository name based on assigned group and new new
 
         :param repo_name:
         """
         path_prefix = self.group.full_path_splitted if self.group else []
         return self.NAME_SEP.join(path_prefix + [repo_name])
 
     @property
     def _config(self):
         """
         Returns db based config object.
         """
         from rhodecode.lib.utils import make_db_config
         return make_db_config(clear_session=False, repo=self)
 
     def permissions(self, with_admins=True, with_owner=True,
                     expand_from_user_groups=False):
         """
         Permissions for repositories
         """
         _admin_perm = 'repository.admin'
 
         owner_row = []
         if with_owner:
             usr = AttributeDict(self.user.get_dict())
             usr.owner_row = True
             usr.permission = _admin_perm
             usr.permission_id = None
             owner_row.append(usr)
 
         super_admin_ids = []
         super_admin_rows = []
         if with_admins:
             for usr in User.get_all_super_admins():
                 super_admin_ids.append(usr.user_id)
                 # if this admin is also owner, don't double the record
                 if usr.user_id == owner_row[0].user_id:
                     owner_row[0].admin_row = True
                 else:
                     usr = AttributeDict(usr.get_dict())
                     usr.admin_row = True
                     usr.permission = _admin_perm
                     usr.permission_id = None
                     super_admin_rows.append(usr)
 
         q = UserRepoToPerm.query().filter(UserRepoToPerm.repository == self)
         q = q.options(joinedload(UserRepoToPerm.repository),
                       joinedload(UserRepoToPerm.user),
                       joinedload(UserRepoToPerm.permission),)
 
         # get owners and admins and permissions. We do a trick of re-writing
         # objects from sqlalchemy to named-tuples due to sqlalchemy session
         # has a global reference and changing one object propagates to all
         # others. This means if admin is also an owner admin_row that change
         # would propagate to both objects
         perm_rows = []
         for _usr in q.all():
             usr = AttributeDict(_usr.user.get_dict())
             # if this user is also owner/admin, mark as duplicate record
             if usr.user_id == owner_row[0].user_id or usr.user_id in super_admin_ids:
                 usr.duplicate_perm = True
             # also check if this permission is maybe used by branch_permissions
             if _usr.branch_perm_entry:
                 usr.branch_rules = [x.branch_rule_id for x in _usr.branch_perm_entry]
 
             usr.permission = _usr.permission.permission_name
             usr.permission_id = _usr.repo_to_perm_id
             perm_rows.append(usr)
 
         # filter the perm rows by 'default' first and then sort them by
         # admin,write,read,none permissions sorted again alphabetically in
         # each group
         perm_rows = sorted(perm_rows, key=display_user_sort)
 
         user_groups_rows = []
         if expand_from_user_groups:
             for ug in self.permission_user_groups(with_members=True):
                 for user_data in ug.members:
                     user_groups_rows.append(user_data)
 
         return super_admin_rows + owner_row + perm_rows + user_groups_rows
 
     def permission_user_groups(self, with_members=True):
         q = UserGroupRepoToPerm.query()\
             .filter(UserGroupRepoToPerm.repository == self)
         q = q.options(joinedload(UserGroupRepoToPerm.repository),
                       joinedload(UserGroupRepoToPerm.users_group),
                       joinedload(UserGroupRepoToPerm.permission),)
 
         perm_rows = []
         for _user_group in q.all():
             entry = AttributeDict(_user_group.users_group.get_dict())
             entry.permission = _user_group.permission.permission_name
             if with_members:
                 entry.members = [x.user.get_dict()
                                  for x in _user_group.users_group.members]
             perm_rows.append(entry)
 
         perm_rows = sorted(perm_rows, key=display_user_group_sort)
         return perm_rows
 
     def get_api_data(self, include_secrets=False):
         """
         Common function for generating repo api data
 
         :param include_secrets: See :meth:`User.get_api_data`.
 
         """
         # TODO: mikhail: Here there is an anti-pattern, we probably need to
         # move this methods on models level.
         from rhodecode.model.settings import SettingsModel
         from rhodecode.model.repo import RepoModel
 
         repo = self
         _user_id, _time, _reason = self.locked
 
         data = {
             'repo_id': repo.repo_id,
             'repo_name': repo.repo_name,
             'repo_type': repo.repo_type,
             'clone_uri': repo.clone_uri or '',
             'push_uri': repo.push_uri or '',
             'url': RepoModel().get_url(self),
             'private': repo.private,
             'created_on': repo.created_on,
             'description': repo.description_safe,
             'landing_rev': repo.landing_rev,
             'owner': repo.user.username,
             'fork_of': repo.fork.repo_name if repo.fork else None,
             'fork_of_id': repo.fork.repo_id if repo.fork else None,
             'enable_statistics': repo.enable_statistics,
             'enable_locking': repo.enable_locking,
             'enable_downloads': repo.enable_downloads,
             'last_changeset': repo.changeset_cache,
             'locked_by': User.get(_user_id).get_api_data(
                 include_secrets=include_secrets) if _user_id else None,
             'locked_date': time_to_datetime(_time) if _time else None,
             'lock_reason': _reason if _reason else None,
         }
 
         # TODO: mikhail: should be per-repo settings here
         rc_config = SettingsModel().get_all_settings()
         repository_fields = str2bool(
             rc_config.get('rhodecode_repository_fields'))
         if repository_fields:
             for f in self.extra_fields:
                 data[f.field_key_prefixed] = f.field_value
 
         return data
 
     @classmethod
     def lock(cls, repo, user_id, lock_time=None, lock_reason=None):
         if not lock_time:
             lock_time = time.time()
         if not lock_reason:
             lock_reason = cls.LOCK_AUTOMATIC
         repo.locked = [user_id, lock_time, lock_reason]
         Session().add(repo)
         Session().commit()
 
     @classmethod
     def unlock(cls, repo):
         repo.locked = None
         Session().add(repo)
         Session().commit()
 
     @classmethod
     def getlock(cls, repo):
         return repo.locked
 
     def get_locking_state(self, action, user_id, only_when_enabled=True):
         """
         Checks locking on this repository, if locking is enabled and lock is
         present returns a tuple of make_lock, locked, locked_by.
         make_lock can have 3 states None (do nothing) True, make lock
         False release lock, This value is later propagated to hooks, which
         do the locking. Think about this as signals passed to hooks what to do.
 
         """
         # TODO: johbo: This is part of the business logic and should be moved
         # into the RepositoryModel.
 
         if action not in ('push', 'pull'):
             raise ValueError("Invalid action value: %s" % repr(action))
 
         # defines if locked error should be thrown to user
         currently_locked = False
         # defines if new lock should be made, tri-state
         make_lock = None
         repo = self
         user = User.get(user_id)
 
         lock_info = repo.locked
 
         if repo and (repo.enable_locking or not only_when_enabled):
             if action == 'push':
                 # check if it's already locked !, if it is compare users
                 locked_by_user_id = lock_info[0]
                 if user.user_id == locked_by_user_id:
                     log.debug(
                         'Got `push` action from user %s, now unlocking', user)
                     # unlock if we have push from user who locked
                     make_lock = False
                 else:
                     # we're not the same user who locked, ban with
                     # code defined in settings (default is 423 HTTP Locked) !
                     log.debug('Repo %s is currently locked by %s', repo, user)
                     currently_locked = True
             elif action == 'pull':
                 # [0] user [1] date
                 if lock_info[0] and lock_info[1]:
                     log.debug('Repo %s is currently locked by %s', repo, user)
                     currently_locked = True
                 else:
                     log.debug('Setting lock on repo %s by %s', repo, user)
                     make_lock = True
 
         else:
             log.debug('Repository %s do not have locking enabled', repo)
 
         log.debug('FINAL locking values make_lock:%s,locked:%s,locked_by:%s',
                   make_lock, currently_locked, lock_info)
 
         from rhodecode.lib.auth import HasRepoPermissionAny
         perm_check = HasRepoPermissionAny('repository.write', 'repository.admin')
         if make_lock and not perm_check(repo_name=repo.repo_name, user=user):
             # if we don't have at least write permission we cannot make a lock
             log.debug('lock state reset back to FALSE due to lack '
                       'of at least read permission')
             make_lock = False
 
         return make_lock, currently_locked, lock_info
 
     @property
     def last_commit_cache_update_diff(self):
         return time.time() - (safe_int(self.changeset_cache.get('updated_on')) or 0)
 
     @classmethod
     def _load_commit_change(cls, last_commit_cache):
         from rhodecode.lib.vcs.utils.helpers import parse_datetime
         empty_date = datetime.datetime.fromtimestamp(0)
         date_latest = last_commit_cache.get('date', empty_date)
         try:
             return parse_datetime(date_latest)
         except Exception:
             return empty_date
 
     @property
     def last_commit_change(self):
         return self._load_commit_change(self.changeset_cache)
 
     @property
     def last_db_change(self):
         return self.updated_on
 
     @property
     def clone_uri_hidden(self):
         clone_uri = self.clone_uri
         if clone_uri:
             import urlobject
             url_obj = urlobject.URLObject(cleaned_uri(clone_uri))
             if url_obj.password:
                 clone_uri = url_obj.with_password('*****')
         return clone_uri
 
     @property
     def push_uri_hidden(self):
         push_uri = self.push_uri
         if push_uri:
             import urlobject
             url_obj = urlobject.URLObject(cleaned_uri(push_uri))
             if url_obj.password:
                 push_uri = url_obj.with_password('*****')
         return push_uri
 
     def clone_url(self, **override):
         from rhodecode.model.settings import SettingsModel
 
         uri_tmpl = None
         if 'with_id' in override:
             uri_tmpl = self.DEFAULT_CLONE_URI_ID
             del override['with_id']
 
         if 'uri_tmpl' in override:
             uri_tmpl = override['uri_tmpl']
             del override['uri_tmpl']
 
         ssh = False
         if 'ssh' in override:
             ssh = True
             del override['ssh']
 
         # we didn't override our tmpl from **overrides
         request = get_current_request()
         if not uri_tmpl:
             if hasattr(request, 'call_context') and hasattr(request.call_context, 'rc_config'):
                 rc_config = request.call_context.rc_config
             else:
                 rc_config = SettingsModel().get_all_settings(cache=True)
 
             if ssh:
                 uri_tmpl = rc_config.get(
                     'rhodecode_clone_uri_ssh_tmpl') or self.DEFAULT_CLONE_URI_SSH
 
             else:
                 uri_tmpl = rc_config.get(
                     'rhodecode_clone_uri_tmpl') or self.DEFAULT_CLONE_URI
 
         return get_clone_url(request=request,
                              uri_tmpl=uri_tmpl,
                              repo_name=self.repo_name,
                              repo_id=self.repo_id,
                              repo_type=self.repo_type,
                              **override)
 
     def set_state(self, state):
         self.repo_state = state
         Session().add(self)
     #==========================================================================
     # SCM PROPERTIES
     #==========================================================================
 
     def get_commit(self, commit_id=None, commit_idx=None, pre_load=None, maybe_unreachable=False, reference_obj=None):
         return get_commit_safe(
             self.scm_instance(), commit_id, commit_idx, pre_load=pre_load,
             maybe_unreachable=maybe_unreachable, reference_obj=reference_obj)
 
     def get_changeset(self, rev=None, pre_load=None):
         warnings.warn("Use get_commit", DeprecationWarning)
         commit_id = None
         commit_idx = None
         if isinstance(rev, str):
             commit_id = rev
         else:
             commit_idx = rev
         return self.get_commit(commit_id=commit_id, commit_idx=commit_idx,
                                pre_load=pre_load)
 
     def get_landing_commit(self):
         """
         Returns landing commit, or if that doesn't exist returns the tip
         """
         _rev_type, _rev = self.landing_rev
         commit = self.get_commit(_rev)
         if isinstance(commit, EmptyCommit):
             return self.get_commit()
         return commit
 
     def flush_commit_cache(self):
-        self.update_commit_cache(cs_cache={'raw_id':'0'})
+        self.update_commit_cache(cs_cache={'raw_id': '0'})
         self.update_commit_cache()
 
-    def update_commit_cache(self, cs_cache=None, config=None):
+    def update_commit_cache(self, cs_cache=None, config=None, recursive=True):
         """
         Update cache of last commit for repository
         cache_keys should be::
 
             source_repo_id
             short_id
             raw_id
             revision
             parents
             message
             date
             author
             updated_on
 
         """
         from rhodecode.lib.vcs.backends.base import BaseCommit
         from rhodecode.lib.vcs.utils.helpers import parse_datetime
         empty_date = datetime.datetime.fromtimestamp(0)
         repo_commit_count = 0
 
         if cs_cache is None:
             # use no-cache version here
             try:
                 scm_repo = self.scm_instance(cache=False, config=config)
             except VCSError:
                 scm_repo = None
             empty = scm_repo is None or scm_repo.is_empty()
 
             if not empty:
                 cs_cache = scm_repo.get_commit(
                     pre_load=["author", "date", "message", "parents", "branch"])
                 repo_commit_count = scm_repo.count()
             else:
                 cs_cache = EmptyCommit()
 
         if isinstance(cs_cache, BaseCommit):
             cs_cache = cs_cache.__json__()
 
+        def maybe_update_recursive(instance, _config, _recursive, _cs_cache, _last_change):
+            if _recursive:
+                repo_id = instance.repo_id
+                _cs_cache['source_repo_id'] = repo_id
+                for gr in instance.groups_with_parents:
+                    gr.changeset_cache = _cs_cache
+                    gr.updated_on = _last_change
+
         def is_outdated(new_cs_cache):
             if (new_cs_cache['raw_id'] != self.changeset_cache['raw_id'] or
                 new_cs_cache['revision'] != self.changeset_cache['revision']):
                 return True
             return False
 
         # check if we have maybe already latest cached revision
         if is_outdated(cs_cache) or not self.changeset_cache:
             _current_datetime = datetime.datetime.utcnow()
             last_change = cs_cache.get('date') or _current_datetime
             # we check if last update is newer than the new value
             # if yes, we use the current timestamp instead. Imagine you get
             # old commit pushed 1y ago, we'd set last update 1y to ago.
             last_change_timestamp = datetime_to_time(last_change)
             current_timestamp = datetime_to_time(last_change)
             if last_change_timestamp > current_timestamp and not empty:
                 cs_cache['date'] = _current_datetime
 
             # also store size of repo
             cs_cache['repo_commit_count'] = repo_commit_count
 
             _date_latest = parse_datetime(cs_cache.get('date') or empty_date)
             cs_cache['updated_on'] = time.time()
             self.changeset_cache = cs_cache
             self.updated_on = last_change
             Session().add(self)
+            maybe_update_recursive(self, config, recursive, cs_cache, last_change)
             Session().commit()
 
         else:
             if empty:
                 cs_cache = EmptyCommit().__json__()
             else:
                 cs_cache = self.changeset_cache
 
             _date_latest = parse_datetime(cs_cache.get('date') or empty_date)
 
             cs_cache['updated_on'] = time.time()
             self.changeset_cache = cs_cache
             self.updated_on = _date_latest
             Session().add(self)
+            maybe_update_recursive(self, config, recursive, cs_cache, _date_latest)
             Session().commit()
 
         log.debug('updated repo `%s` with new commit cache %s, and last update_date: %s',
                   self.repo_name, cs_cache, _date_latest)
 
     @property
     def tip(self):
         return self.get_commit('tip')
 
     @property
     def author(self):
         return self.tip.author
 
     @property
     def last_change(self):
         return self.scm_instance().last_change
 
     def get_comments(self, revisions=None):
         """
         Returns comments for this repository grouped by revisions
 
         :param revisions: filter query by revisions only
         """
         cmts = ChangesetComment.query()\
             .filter(ChangesetComment.repo == self)
         if revisions:
             cmts = cmts.filter(ChangesetComment.revision.in_(revisions))
         grouped = collections.defaultdict(list)
         for cmt in cmts.all():
             grouped[cmt.revision].append(cmt)
         return grouped
 
     def statuses(self, revisions=None):
         """
         Returns statuses for this repository
 
         :param revisions: list of revisions to get statuses for
         """
         statuses = ChangesetStatus.query()\
             .filter(ChangesetStatus.repo == self)\
             .filter(ChangesetStatus.version == 0)
 
         if revisions:
             # Try doing the filtering in chunks to avoid hitting limits
             size = 500
             status_results = []
             for chunk in range(0, len(revisions), size):
                 status_results += statuses.filter(
                     ChangesetStatus.revision.in_(
                         revisions[chunk: chunk+size])
                 ).all()
         else:
             status_results = statuses.all()
 
         grouped = {}
 
         # maybe we have open new pullrequest without a status?
         stat = ChangesetStatus.STATUS_UNDER_REVIEW
         status_lbl = ChangesetStatus.get_status_lbl(stat)
         for pr in PullRequest.query().filter(PullRequest.source_repo == self).all():
             for rev in pr.revisions:
                 pr_id = pr.pull_request_id
                 pr_repo = pr.target_repo.repo_name
                 grouped[rev] = [stat, status_lbl, pr_id, pr_repo]
 
         for stat in status_results:
             pr_id = pr_repo = None
             if stat.pull_request:
                 pr_id = stat.pull_request.pull_request_id
                 pr_repo = stat.pull_request.target_repo.repo_name
             grouped[stat.revision] = [str(stat.status), stat.status_lbl,
                                       pr_id, pr_repo]
         return grouped
 
     # ==========================================================================
     # SCM CACHE INSTANCE
     # ==========================================================================
 
     def scm_instance(self, **kwargs):
         import rhodecode
 
         # Passing a config will not hit the cache currently only used
         # for repo2dbmapper
         config = kwargs.pop('config', None)
         cache = kwargs.pop('cache', None)
         vcs_full_cache = kwargs.pop('vcs_full_cache', None)
         if vcs_full_cache is not None:
             # allows override global config
             full_cache = vcs_full_cache
         else:
             full_cache = rhodecode.ConfigGet().get_bool('vcs_full_cache')
         # if cache is NOT defined use default global, else we have a full
         # control over cache behaviour
         if cache is None and full_cache and not config:
             log.debug('Initializing pure cached instance for %s', self.repo_path)
             return self._get_instance_cached()
 
         # cache here is sent to the "vcs server"
         return self._get_instance(cache=bool(cache), config=config)
 
     def _get_instance_cached(self):
         from rhodecode.lib import rc_cache
 
         cache_namespace_uid = f'repo_instance.{self.repo_id}'
         region = rc_cache.get_or_create_region('cache_repo_longterm', cache_namespace_uid)
 
         # we must use thread scoped cache here,
         # because each thread of gevent needs it's own not shared connection and cache
         # we also alter `args` so the cache key is individual for every green thread.
         repo_namespace_key = CacheKey.REPO_INVALIDATION_NAMESPACE.format(repo_id=self.repo_id)
         inv_context_manager = rc_cache.InvalidationContext(key=repo_namespace_key, thread_scoped=True)
 
         # our wrapped caching function that takes state_uid to save the previous state in
         def cache_generator(_state_uid):
 
             @region.conditional_cache_on_arguments(namespace=cache_namespace_uid)
             def get_instance_cached(_repo_id, _process_context_id):
                 # we save in cached func the generation state so we can detect a change and invalidate caches
                 return _state_uid, self._get_instance(repo_state_uid=_state_uid)
 
             return get_instance_cached
 
         with inv_context_manager as invalidation_context:
             cache_state_uid = invalidation_context.state_uid
             cache_func = cache_generator(cache_state_uid)
 
             args = self.repo_id, inv_context_manager.proc_key
 
             previous_state_uid, instance = cache_func(*args)
 
             # now compare keys, the "cache" state vs expected state.
             if previous_state_uid != cache_state_uid:
                 log.warning('Cached state uid %s is different than current state uid %s',
                             previous_state_uid, cache_state_uid)
                 _, instance = cache_func.refresh(*args)
 
             log.debug('Repo instance fetched in %.4fs', inv_context_manager.compute_time)
             return instance
 
     def _get_instance(self, cache=True, config=None, repo_state_uid=None):
         log.debug('Initializing %s instance `%s` with cache flag set to: %s',
                   self.repo_type, self.repo_path, cache)
         config = config or self._config
         custom_wire = {
             'cache': cache,  # controls the vcs.remote cache
             'repo_state_uid': repo_state_uid
         }
 
         repo = get_vcs_instance(
             repo_path=safe_str(self.repo_full_path),
             config=config,
             with_wire=custom_wire,
             create=False,
             _vcs_alias=self.repo_type)
         if repo is not None:
             repo.count()  # cache rebuild
 
         return repo
 
     def get_shadow_repository_path(self, workspace_id):
         from rhodecode.lib.vcs.backends.base import BaseRepository
         shadow_repo_path = BaseRepository._get_shadow_repository_path(
             self.repo_full_path, self.repo_id, workspace_id)
         return shadow_repo_path
 
     def __json__(self):
         return {'landing_rev': self.landing_rev}
 
     def get_dict(self):
 
         # Since we transformed `repo_name` to a hybrid property, we need to
         # keep compatibility with the code which uses `repo_name` field.
 
         result = super(Repository, self).get_dict()
         result['repo_name'] = result.pop('_repo_name', None)
         result.pop('_changeset_cache', '')
         return result
 
 
 class RepoGroup(Base, BaseModel):
     __tablename__ = 'groups'
     __table_args__ = (
         UniqueConstraint('group_name', 'group_parent_id'),
         base_table_args,
     )
 
     CHOICES_SEPARATOR = '/'  # used to generate select2 choices for nested groups
 
     group_id = Column("group_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     _group_name = Column("group_name", String(255), nullable=False, unique=True, default=None)
     group_name_hash = Column("repo_group_name_hash", String(1024), nullable=False, unique=False)
     group_parent_id = Column("group_parent_id", Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None)
     group_description = Column("group_description", String(10000), nullable=True, unique=None, default=None)
     enable_locking = Column("enable_locking", Boolean(), nullable=False, unique=None, default=False)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=False, default=None)
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     updated_on = Column('updated_on', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
     personal = Column('personal', Boolean(), nullable=True, unique=None, default=None)
     _changeset_cache = Column("changeset_cache", LargeBinary(), nullable=True)  # JSON data
 
     repo_group_to_perm = relationship('UserRepoGroupToPerm', cascade='all', order_by='UserRepoGroupToPerm.group_to_perm_id', back_populates='group')
     users_group_to_perm = relationship('UserGroupRepoGroupToPerm', cascade='all', back_populates='group')
     parent_group = relationship('RepoGroup', remote_side=group_id)
     user = relationship('User', back_populates='repository_groups')
     integrations = relationship('Integration', cascade="all, delete-orphan", back_populates='repo_group')
 
     # no cascade, set NULL
     scope_artifacts = relationship('FileStore', primaryjoin='FileStore.scope_repo_group_id==RepoGroup.group_id', viewonly=True)
 
     def __init__(self, group_name='', parent_group=None):
         self.group_name = group_name
         self.parent_group = parent_group
 
     def __repr__(self):
         return f"<{self.cls_name}('id:{self.group_id}:{self.group_name}')>"
 
     @hybrid_property
     def group_name(self):
         return self._group_name
 
     @group_name.setter
     def group_name(self, value):
         self._group_name = value
         self.group_name_hash = self.hash_repo_group_name(value)
 
     @classmethod
     def _load_changeset_cache(cls, repo_id, changeset_cache_raw):
         from rhodecode.lib.vcs.backends.base import EmptyCommit
         dummy = EmptyCommit().__json__()
         if not changeset_cache_raw:
             dummy['source_repo_id'] = repo_id
             return json.loads(json.dumps(dummy))
 
         try:
             return json.loads(changeset_cache_raw)
         except TypeError:
             return dummy
         except Exception:
             log.error(traceback.format_exc())
             return dummy
 
     @hybrid_property
     def changeset_cache(self):
         return self._load_changeset_cache('', self._changeset_cache)
 
     @changeset_cache.setter
     def changeset_cache(self, val):
         try:
             self._changeset_cache = json.dumps(val)
         except Exception:
             log.error(traceback.format_exc())
 
     @validates('group_parent_id')
     def validate_group_parent_id(self, key, val):
         """
         Check cycle references for a parent group to self
         """
         if self.group_id and val:
             assert val != self.group_id
 
         return val
 
     @hybrid_property
     def description_safe(self):
         return description_escaper(self.group_description)
 
     @classmethod
     def hash_repo_group_name(cls, repo_group_name):
         val = remove_formatting(repo_group_name)
         val = safe_str(val).lower()
         chars = []
         for c in val:
             if c not in string.ascii_letters:
                 c = str(ord(c))
             chars.append(c)
 
         return ''.join(chars)
 
     @classmethod
     def _generate_choice(cls, repo_group):
         from webhelpers2.html import literal as _literal
 
         def _name(k):
             return _literal(cls.CHOICES_SEPARATOR.join(k))
 
         return repo_group.group_id, _name(repo_group.full_path_splitted)
 
     @classmethod
     def groups_choices(cls, groups=None, show_empty_group=True):
         if not groups:
             groups = cls.query().all()
 
         repo_groups = []
         if show_empty_group:
             repo_groups = [(-1, '-- %s --' % _('No parent'))]
 
         repo_groups.extend([cls._generate_choice(x) for x in groups])
 
         repo_groups = sorted(
             repo_groups, key=lambda t: t[1].split(cls.CHOICES_SEPARATOR)[0])
         return repo_groups
 
     @classmethod
     def url_sep(cls):
         return URL_SEP
 
     @classmethod
     def get_by_group_name(cls, group_name, cache=False, case_insensitive=False):
         if case_insensitive:
             gr = cls.query().filter(func.lower(cls.group_name)
                                     == func.lower(group_name))
         else:
             gr = cls.query().filter(cls.group_name == group_name)
         if cache:
             name_key = _hash_key(group_name)
             gr = gr.options(
                 FromCache("sql_cache_short", f"get_group_{name_key}"))
         return gr.scalar()
 
     @classmethod
     def get_user_personal_repo_group(cls, user_id):
         user = User.get(user_id)
         if user.username == User.DEFAULT_USER:
             return None
 
         return cls.query()\
             .filter(cls.personal == true()) \
             .filter(cls.user == user) \
             .order_by(cls.group_id.asc()) \
             .first()
 
     @classmethod
     def get_all_repo_groups(cls, user_id=Optional(None), group_id=Optional(None),
                             case_insensitive=True):
         q = RepoGroup.query()
 
         if not isinstance(user_id, Optional):
             q = q.filter(RepoGroup.user_id == user_id)
 
         if not isinstance(group_id, Optional):
             q = q.filter(RepoGroup.group_parent_id == group_id)
 
         if case_insensitive:
             q = q.order_by(func.lower(RepoGroup.group_name))
         else:
             q = q.order_by(RepoGroup.group_name)
         return q.all()
 
     @property
     def parents(self, parents_recursion_limit=10):
         groups = []
         if self.parent_group is None:
             return groups
         cur_gr = self.parent_group
         groups.insert(0, cur_gr)
         cnt = 0
         while 1:
             cnt += 1
             gr = getattr(cur_gr, 'parent_group', None)
             cur_gr = cur_gr.parent_group
             if gr is None:
                 break
             if cnt == parents_recursion_limit:
                 # this will prevent accidental infinit loops
                 log.error('more than %s parents found for group %s, stopping '
                           'recursive parent fetching', parents_recursion_limit, self)
                 break
 
             groups.insert(0, gr)
         return groups
 
     @property
     def last_commit_cache_update_diff(self):
         return time.time() - (safe_int(self.changeset_cache.get('updated_on')) or 0)
 
     @classmethod
     def _load_commit_change(cls, last_commit_cache):
         from rhodecode.lib.vcs.utils.helpers import parse_datetime
         empty_date = datetime.datetime.fromtimestamp(0)
         date_latest = last_commit_cache.get('date', empty_date)
         try:
             return parse_datetime(date_latest)
         except Exception:
             return empty_date
 
     @property
     def last_commit_change(self):
         return self._load_commit_change(self.changeset_cache)
 
     @property
     def last_db_change(self):
         return self.updated_on
 
     @property
     def children(self):
         return RepoGroup.query().filter(RepoGroup.parent_group == self)
 
     @property
     def name(self):
         return self.group_name.split(RepoGroup.url_sep())[-1]
 
     @property
     def full_path(self):
         return self.group_name
 
     @property
     def full_path_splitted(self):
         return self.group_name.split(RepoGroup.url_sep())
 
     @property
     def repositories(self):
         return Repository.query()\
                 .filter(Repository.group == self)\
                 .order_by(Repository.repo_name)
 
     @property
     def repositories_recursive_count(self):
         cnt = self.repositories.count()
 
         def children_count(group):
             cnt = 0
             for child in group.children:
                 cnt += child.repositories.count()
                 cnt += children_count(child)
             return cnt
 
         return cnt + children_count(self)
 
     def _recursive_objects(self, include_repos=True, include_groups=True):
         all_ = []
 
         def _get_members(root_gr):
             if include_repos:
                 for r in root_gr.repositories:
                     all_.append(r)
             childs = root_gr.children.all()
             if childs:
                 for gr in childs:
                     if include_groups:
                         all_.append(gr)
                     _get_members(gr)
 
         root_group = []
         if include_groups:
             root_group = [self]
 
         _get_members(self)
         return root_group + all_
 
     def recursive_groups_and_repos(self):
         """
         Recursive return all groups, with repositories in those groups
         """
         return self._recursive_objects()
 
     def recursive_groups(self):
         """
         Returns all children groups for this group including children of children
         """
         return self._recursive_objects(include_repos=False)
 
     def recursive_repos(self):
         """
         Returns all children repositories for this group
         """
         return self._recursive_objects(include_groups=False)
 
     def get_new_name(self, group_name):
         """
         returns new full group name based on parent and new name
 
         :param group_name:
         """
         path_prefix = (self.parent_group.full_path_splitted if
                        self.parent_group else [])
         return RepoGroup.url_sep().join(path_prefix + [group_name])
 
     def update_commit_cache(self, config=None):
         """
         Update cache of last commit for newest repository inside this repository group.
         cache_keys should be::
 
             source_repo_id
             short_id
             raw_id
             revision
             parents
             message
             date
             author
 
         """
         from rhodecode.lib.vcs.utils.helpers import parse_datetime
         empty_date = datetime.datetime.fromtimestamp(0)
 
         def repo_groups_and_repos(root_gr):
             for _repo in root_gr.repositories:
                 yield _repo
             for child_group in root_gr.children.all():
                 yield child_group
 
         latest_repo_cs_cache = {}
         for obj in repo_groups_and_repos(self):
             repo_cs_cache = obj.changeset_cache
             date_latest = latest_repo_cs_cache.get('date', empty_date)
             date_current = repo_cs_cache.get('date', empty_date)
             current_timestamp = datetime_to_time(parse_datetime(date_latest))
             if current_timestamp < datetime_to_time(parse_datetime(date_current)):
                 latest_repo_cs_cache = repo_cs_cache
                 if hasattr(obj, 'repo_id'):
                     latest_repo_cs_cache['source_repo_id'] = obj.repo_id
                 else:
                     latest_repo_cs_cache['source_repo_id'] = repo_cs_cache.get('source_repo_id')
 
         _date_latest = parse_datetime(latest_repo_cs_cache.get('date') or empty_date)
 
         latest_repo_cs_cache['updated_on'] = time.time()
         self.changeset_cache = latest_repo_cs_cache
         self.updated_on = _date_latest
         Session().add(self)
         Session().commit()
 
         log.debug('updated repo group `%s` with new commit cache %s, and last update_date: %s',
                   self.group_name, latest_repo_cs_cache, _date_latest)
 
     def permissions(self, with_admins=True, with_owner=True,
                     expand_from_user_groups=False):
         """
         Permissions for repository groups
         """
         _admin_perm = 'group.admin'
 
         owner_row = []
         if with_owner:
             usr = AttributeDict(self.user.get_dict())
             usr.owner_row = True
             usr.permission = _admin_perm
             owner_row.append(usr)
 
         super_admin_ids = []
         super_admin_rows = []
         if with_admins:
             for usr in User.get_all_super_admins():
                 super_admin_ids.append(usr.user_id)
                 # if this admin is also owner, don't double the record
                 if usr.user_id == owner_row[0].user_id:
                     owner_row[0].admin_row = True
                 else:
                     usr = AttributeDict(usr.get_dict())
                     usr.admin_row = True
                     usr.permission = _admin_perm
                     super_admin_rows.append(usr)
 
         q = UserRepoGroupToPerm.query().filter(UserRepoGroupToPerm.group == self)
         q = q.options(joinedload(UserRepoGroupToPerm.group),
                       joinedload(UserRepoGroupToPerm.user),
                       joinedload(UserRepoGroupToPerm.permission),)
 
         # get owners and admins and permissions. We do a trick of re-writing
         # objects from sqlalchemy to named-tuples due to sqlalchemy session
         # has a global reference and changing one object propagates to all
         # others. This means if admin is also an owner admin_row that change
         # would propagate to both objects
         perm_rows = []
         for _usr in q.all():
             usr = AttributeDict(_usr.user.get_dict())
             # if this user is also owner/admin, mark as duplicate record
             if usr.user_id == owner_row[0].user_id or usr.user_id in super_admin_ids:
                 usr.duplicate_perm = True
             usr.permission = _usr.permission.permission_name
             perm_rows.append(usr)
 
         # filter the perm rows by 'default' first and then sort them by
         # admin,write,read,none permissions sorted again alphabetically in
         # each group
         perm_rows = sorted(perm_rows, key=display_user_sort)
 
         user_groups_rows = []
         if expand_from_user_groups:
             for ug in self.permission_user_groups(with_members=True):
                 for user_data in ug.members:
                     user_groups_rows.append(user_data)
 
         return super_admin_rows + owner_row + perm_rows + user_groups_rows
 
     def permission_user_groups(self, with_members=False):
         q = UserGroupRepoGroupToPerm.query()\
             .filter(UserGroupRepoGroupToPerm.group == self)
         q = q.options(joinedload(UserGroupRepoGroupToPerm.group),
                       joinedload(UserGroupRepoGroupToPerm.users_group),
                       joinedload(UserGroupRepoGroupToPerm.permission),)
 
         perm_rows = []
         for _user_group in q.all():
             entry = AttributeDict(_user_group.users_group.get_dict())
             entry.permission = _user_group.permission.permission_name
             if with_members:
                 entry.members = [x.user.get_dict()
                                  for x in _user_group.users_group.members]
             perm_rows.append(entry)
 
         perm_rows = sorted(perm_rows, key=display_user_group_sort)
         return perm_rows
 
     def get_api_data(self):
         """
         Common function for generating api data
 
         """
         group = self
         data = {
             'group_id': group.group_id,
             'group_name': group.group_name,
             'group_description': group.description_safe,
             'parent_group': group.parent_group.group_name if group.parent_group else None,
             'repositories': [x.repo_name for x in group.repositories],
             'owner': group.user.username,
         }
         return data
 
     def get_dict(self):
         # Since we transformed `group_name` to a hybrid property, we need to
         # keep compatibility with the code which uses `group_name` field.
         result = super(RepoGroup, self).get_dict()
         result['group_name'] = result.pop('_group_name', None)
         result.pop('_changeset_cache', '')
         return result
 
 
 class Permission(Base, BaseModel):
     __tablename__ = 'permissions'
     __table_args__ = (
         Index('p_perm_name_idx', 'permission_name'),
         base_table_args,
     )
 
     PERMS = [
         ('hg.admin', _('RhodeCode Super Administrator')),
 
         ('repository.none', _('Repository no access')),
         ('repository.read', _('Repository read access')),
         ('repository.write', _('Repository write access')),
         ('repository.admin', _('Repository admin access')),
 
         ('group.none', _('Repository group no access')),
         ('group.read', _('Repository group read access')),
         ('group.write', _('Repository group write access')),
         ('group.admin', _('Repository group admin access')),
 
         ('usergroup.none', _('User group no access')),
         ('usergroup.read', _('User group read access')),
         ('usergroup.write', _('User group write access')),
         ('usergroup.admin', _('User group admin access')),
 
         ('branch.none', _('Branch no permissions')),
         ('branch.merge', _('Branch access by web merge')),
         ('branch.push', _('Branch access by push')),
         ('branch.push_force', _('Branch access by push with force')),
 
         ('hg.repogroup.create.false', _('Repository Group creation disabled')),
         ('hg.repogroup.create.true', _('Repository Group creation enabled')),
 
         ('hg.usergroup.create.false', _('User Group creation disabled')),
         ('hg.usergroup.create.true', _('User Group creation enabled')),
 
         ('hg.create.none', _('Repository creation disabled')),
         ('hg.create.repository', _('Repository creation enabled')),
         ('hg.create.write_on_repogroup.true', _('Repository creation enabled with write permission to a repository group')),
         ('hg.create.write_on_repogroup.false', _('Repository creation disabled with write permission to a repository group')),
 
         ('hg.fork.none', _('Repository forking disabled')),
         ('hg.fork.repository', _('Repository forking enabled')),
 
         ('hg.register.none', _('Registration disabled')),
         ('hg.register.manual_activate', _('User Registration with manual account activation')),
         ('hg.register.auto_activate', _('User Registration with automatic account activation')),
 
         ('hg.password_reset.enabled', _('Password reset enabled')),
         ('hg.password_reset.hidden', _('Password reset hidden')),
         ('hg.password_reset.disabled', _('Password reset disabled')),
 
         ('hg.extern_activate.manual', _('Manual activation of external account')),
         ('hg.extern_activate.auto', _('Automatic activation of external account')),
 
         ('hg.inherit_default_perms.false', _('Inherit object permissions from default user disabled')),
         ('hg.inherit_default_perms.true', _('Inherit object permissions from default user enabled')),
     ]
 
     # definition of system default permissions for DEFAULT user, created on
     # system setup
     DEFAULT_USER_PERMISSIONS = [
         # object perms
         'repository.read',
         'group.read',
         'usergroup.read',
         # branch, for backward compat we need same value as before so forced pushed
         'branch.push_force',
         # global
         'hg.create.repository',
         'hg.repogroup.create.false',
         'hg.usergroup.create.false',
         'hg.create.write_on_repogroup.true',
         'hg.fork.repository',
         'hg.register.manual_activate',
         'hg.password_reset.enabled',
         'hg.extern_activate.auto',
         'hg.inherit_default_perms.true',
     ]
 
     # defines which permissions are more important higher the more important
     # Weight defines which permissions are more important.
     # The higher number the more important.
     PERM_WEIGHTS = {
         'repository.none': 0,
         'repository.read': 1,
         'repository.write': 3,
         'repository.admin': 4,
 
         'group.none': 0,
         'group.read': 1,
         'group.write': 3,
         'group.admin': 4,
 
         'usergroup.none': 0,
         'usergroup.read': 1,
         'usergroup.write': 3,
         'usergroup.admin': 4,
 
         'branch.none': 0,
         'branch.merge': 1,
         'branch.push': 3,
         'branch.push_force': 4,
 
         'hg.repogroup.create.false': 0,
         'hg.repogroup.create.true': 1,
 
         'hg.usergroup.create.false': 0,
         'hg.usergroup.create.true': 1,
 
         'hg.fork.none': 0,
         'hg.fork.repository': 1,
         'hg.create.none': 0,
         'hg.create.repository': 1
     }
 
     permission_id = Column("permission_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     permission_name = Column("permission_name", String(255), nullable=True, unique=None, default=None)
     permission_longname = Column("permission_longname", String(255), nullable=True, unique=None, default=None)
 
     def __repr__(self):
         return "<%s('%s:%s')>" % (
             self.cls_name, self.permission_id, self.permission_name
         )
 
     @classmethod
     def get_by_key(cls, key):
         return cls.query().filter(cls.permission_name == key).scalar()
 
     @classmethod
     def get_default_repo_perms(cls, user_id, repo_id=None):
         q = Session().query(UserRepoToPerm, Repository, Permission)\
             .join((Permission, UserRepoToPerm.permission_id == Permission.permission_id))\
             .join((Repository, UserRepoToPerm.repository_id == Repository.repo_id))\
             .filter(UserRepoToPerm.user_id == user_id)
         if repo_id:
             q = q.filter(UserRepoToPerm.repository_id == repo_id)
         return q.all()
 
     @classmethod
     def get_default_repo_branch_perms(cls, user_id, repo_id=None):
         q = Session().query(UserToRepoBranchPermission, UserRepoToPerm, Permission) \
             .join(
                 Permission,
                 UserToRepoBranchPermission.permission_id == Permission.permission_id) \
             .join(
                 UserRepoToPerm,
                 UserToRepoBranchPermission.rule_to_perm_id == UserRepoToPerm.repo_to_perm_id) \
             .filter(UserRepoToPerm.user_id == user_id)
 
         if repo_id:
             q = q.filter(UserToRepoBranchPermission.repository_id == repo_id)
         return q.order_by(UserToRepoBranchPermission.rule_order).all()
 
     @classmethod
     def get_default_repo_perms_from_user_group(cls, user_id, repo_id=None):
         q = Session().query(UserGroupRepoToPerm, Repository, Permission)\
             .join(
                 Permission,
                 UserGroupRepoToPerm.permission_id == Permission.permission_id)\
             .join(
                 Repository,
                 UserGroupRepoToPerm.repository_id == Repository.repo_id)\
             .join(
                 UserGroup,
                 UserGroupRepoToPerm.users_group_id ==
                 UserGroup.users_group_id)\
             .join(
                 UserGroupMember,
                 UserGroupRepoToPerm.users_group_id ==
                 UserGroupMember.users_group_id)\
             .filter(
                 UserGroupMember.user_id == user_id,
                 UserGroup.users_group_active == true())
         if repo_id:
             q = q.filter(UserGroupRepoToPerm.repository_id == repo_id)
         return q.all()
 
     @classmethod
     def get_default_repo_branch_perms_from_user_group(cls, user_id, repo_id=None):
         q = Session().query(UserGroupToRepoBranchPermission, UserGroupRepoToPerm, Permission) \
             .join(
                 Permission,
                 UserGroupToRepoBranchPermission.permission_id == Permission.permission_id) \
             .join(
                 UserGroupRepoToPerm,
                 UserGroupToRepoBranchPermission.rule_to_perm_id == UserGroupRepoToPerm.users_group_to_perm_id) \
             .join(
                 UserGroup,
                 UserGroupRepoToPerm.users_group_id == UserGroup.users_group_id) \
             .join(
                 UserGroupMember,
                 UserGroupRepoToPerm.users_group_id == UserGroupMember.users_group_id) \
             .filter(
                 UserGroupMember.user_id == user_id,
                 UserGroup.users_group_active == true())
 
         if repo_id:
             q = q.filter(UserGroupToRepoBranchPermission.repository_id == repo_id)
         return q.order_by(UserGroupToRepoBranchPermission.rule_order).all()
 
     @classmethod
     def get_default_group_perms(cls, user_id, repo_group_id=None):
         q = Session().query(UserRepoGroupToPerm, RepoGroup, Permission)\
             .join(
                 Permission,
                 UserRepoGroupToPerm.permission_id == Permission.permission_id)\
             .join(
                 RepoGroup,
                 UserRepoGroupToPerm.group_id == RepoGroup.group_id)\
             .filter(UserRepoGroupToPerm.user_id == user_id)
         if repo_group_id:
             q = q.filter(UserRepoGroupToPerm.group_id == repo_group_id)
         return q.all()
 
     @classmethod
     def get_default_group_perms_from_user_group(
             cls, user_id, repo_group_id=None):
         q = Session().query(UserGroupRepoGroupToPerm, RepoGroup, Permission)\
             .join(
                 Permission,
                 UserGroupRepoGroupToPerm.permission_id ==
                 Permission.permission_id)\
             .join(
                 RepoGroup,
                 UserGroupRepoGroupToPerm.group_id == RepoGroup.group_id)\
             .join(
                 UserGroup,
                 UserGroupRepoGroupToPerm.users_group_id ==
                 UserGroup.users_group_id)\
             .join(
                 UserGroupMember,
                 UserGroupRepoGroupToPerm.users_group_id ==
                 UserGroupMember.users_group_id)\
             .filter(
                 UserGroupMember.user_id == user_id,
                 UserGroup.users_group_active == true())
         if repo_group_id:
             q = q.filter(UserGroupRepoGroupToPerm.group_id == repo_group_id)
         return q.all()
 
     @classmethod
     def get_default_user_group_perms(cls, user_id, user_group_id=None):
         q = Session().query(UserUserGroupToPerm, UserGroup, Permission)\
             .join((Permission, UserUserGroupToPerm.permission_id == Permission.permission_id))\
             .join((UserGroup, UserUserGroupToPerm.user_group_id == UserGroup.users_group_id))\
             .filter(UserUserGroupToPerm.user_id == user_id)
         if user_group_id:
             q = q.filter(UserUserGroupToPerm.user_group_id == user_group_id)
         return q.all()
 
     @classmethod
     def get_default_user_group_perms_from_user_group(
             cls, user_id, user_group_id=None):
         TargetUserGroup = aliased(UserGroup, name='target_user_group')
         q = Session().query(UserGroupUserGroupToPerm, UserGroup, Permission)\
             .join(
                 Permission,
                 UserGroupUserGroupToPerm.permission_id ==
                 Permission.permission_id)\
             .join(
                 TargetUserGroup,
                 UserGroupUserGroupToPerm.target_user_group_id ==
                 TargetUserGroup.users_group_id)\
             .join(
                 UserGroup,
                 UserGroupUserGroupToPerm.user_group_id ==
                 UserGroup.users_group_id)\
             .join(
                 UserGroupMember,
                 UserGroupUserGroupToPerm.user_group_id ==
                 UserGroupMember.users_group_id)\
             .filter(
                 UserGroupMember.user_id == user_id,
                 UserGroup.users_group_active == true())
         if user_group_id:
             q = q.filter(
                 UserGroupUserGroupToPerm.user_group_id == user_group_id)
 
         return q.all()
 
 
 class UserRepoToPerm(Base, BaseModel):
     __tablename__ = 'repo_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'repository_id', 'permission_id'),
         base_table_args
     )
 
     repo_to_perm_id = Column("repo_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User', back_populates="repo_to_perm")
     repository = relationship('Repository', back_populates="repo_to_perm")
     permission = relationship('Permission')
 
     branch_perm_entry = relationship('UserToRepoBranchPermission', cascade="all, delete-orphan", lazy='joined', back_populates='user_repo_to_perm')
 
     @classmethod
     def create(cls, user, repository, permission):
         n = cls()
         n.user = user
         n.repository = repository
         n.permission = permission
         Session().add(n)
         return n
 
     def __repr__(self):
         return f'<{self.user} => {self.repository} >'
 
 
 class UserUserGroupToPerm(Base, BaseModel):
     __tablename__ = 'user_user_group_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'user_group_id', 'permission_id'),
         base_table_args
     )
 
     user_user_group_to_perm_id = Column("user_user_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     user_group_id = Column("user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User', back_populates='user_group_to_perm')
     user_group = relationship('UserGroup', back_populates='user_user_group_to_perm')
     permission = relationship('Permission')
 
     @classmethod
     def create(cls, user, user_group, permission):
         n = cls()
         n.user = user
         n.user_group = user_group
         n.permission = permission
         Session().add(n)
         return n
 
     def __repr__(self):
         return f'<{self.user} => {self.user_group} >'
 
 
 class UserToPerm(Base, BaseModel):
     __tablename__ = 'user_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'permission_id'),
         base_table_args
     )
 
     user_to_perm_id = Column("user_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User', back_populates='user_perms')
     permission = relationship('Permission', lazy='joined')
 
     def __repr__(self):
         return f'<{self.user} => {self.permission} >'
 
 
 class UserGroupRepoToPerm(Base, BaseModel):
     __tablename__ = 'users_group_repo_to_perm'
     __table_args__ = (
         UniqueConstraint('repository_id', 'users_group_id', 'permission_id'),
         base_table_args
     )
 
     users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
 
     users_group = relationship('UserGroup', back_populates='users_group_repo_to_perm')
     permission = relationship('Permission')
     repository = relationship('Repository', back_populates='users_group_to_perm')
     user_group_branch_perms = relationship('UserGroupToRepoBranchPermission', cascade='all', back_populates='user_group_repo_to_perm')
 
     @classmethod
     def create(cls, users_group, repository, permission):
         n = cls()
         n.users_group = users_group
         n.repository = repository
         n.permission = permission
         Session().add(n)
         return n
 
     def __repr__(self):
         return f'<UserGroupRepoToPerm:{self.users_group} => {self.repository} >'
 
 
 class UserGroupUserGroupToPerm(Base, BaseModel):
     __tablename__ = 'user_group_user_group_to_perm'
     __table_args__ = (
         UniqueConstraint('target_user_group_id', 'user_group_id', 'permission_id'),
         CheckConstraint('target_user_group_id != user_group_id'),
         base_table_args
     )
 
     user_group_user_group_to_perm_id = Column("user_group_user_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     target_user_group_id = Column("target_user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     user_group_id = Column("user_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
 
     target_user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.target_user_group_id==UserGroup.users_group_id', back_populates='user_group_user_group_to_perm')
     user_group = relationship('UserGroup', primaryjoin='UserGroupUserGroupToPerm.user_group_id==UserGroup.users_group_id')
     permission = relationship('Permission')
 
     @classmethod
     def create(cls, target_user_group, user_group, permission):
         n = cls()
         n.target_user_group = target_user_group
         n.user_group = user_group
         n.permission = permission
         Session().add(n)
         return n
 
     def __repr__(self):
         return f'<UserGroupUserGroup:{self.target_user_group} => {self.user_group} >'
 
 
 class UserGroupToPerm(Base, BaseModel):
     __tablename__ = 'users_group_to_perm'
     __table_args__ = (
         UniqueConstraint('users_group_id', 'permission_id',),
         base_table_args
     )
 
     users_group_to_perm_id = Column("users_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     users_group = relationship('UserGroup', back_populates='users_group_to_perm')
     permission = relationship('Permission')
 
 
 class UserRepoGroupToPerm(Base, BaseModel):
     __tablename__ = 'user_repo_group_to_perm'
     __table_args__ = (
         UniqueConstraint('user_id', 'group_id', 'permission_id'),
         base_table_args
     )
 
     group_to_perm_id = Column("group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     user = relationship('User', back_populates='repo_group_to_perm')
     group = relationship('RepoGroup', back_populates='repo_group_to_perm')
     permission = relationship('Permission')
 
     @classmethod
     def create(cls, user, repository_group, permission):
         n = cls()
         n.user = user
         n.group = repository_group
         n.permission = permission
         Session().add(n)
         return n
 
 
 class UserGroupRepoGroupToPerm(Base, BaseModel):
     __tablename__ = 'users_group_repo_group_to_perm'
     __table_args__ = (
         UniqueConstraint('users_group_id', 'group_id'),
         base_table_args
     )
 
     users_group_repo_group_to_perm_id = Column("users_group_repo_group_to_perm_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False, unique=None, default=None)
     group_id = Column("group_id", Integer(), ForeignKey('groups.group_id'), nullable=False, unique=None, default=None)
     permission_id = Column("permission_id", Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
 
     users_group = relationship('UserGroup', back_populates='users_group_repo_group_to_perm')
     permission = relationship('Permission')
     group = relationship('RepoGroup', back_populates='users_group_to_perm')
 
     @classmethod
     def create(cls, user_group, repository_group, permission):
         n = cls()
         n.users_group = user_group
         n.group = repository_group
         n.permission = permission
         Session().add(n)
         return n
 
     def __repr__(self):
         return '<UserGroupRepoGroupToPerm:%s => %s >' % (self.users_group, self.group)
 
 
 class Statistics(Base, BaseModel):
     __tablename__ = 'statistics'
     __table_args__ = (
          base_table_args
     )
 
     stat_id = Column("stat_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     repository_id = Column("repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=True, default=None)
     stat_on_revision = Column("stat_on_revision", Integer(), nullable=False)
     commit_activity = Column("commit_activity", LargeBinary(1000000), nullable=False)  #JSON data
     commit_activity_combined = Column("commit_activity_combined", LargeBinary(), nullable=False)  #JSON data
     languages = Column("languages", LargeBinary(1000000), nullable=False)  #JSON data
 
     repository = relationship('Repository', single_parent=True, viewonly=True)
 
 
 class UserFollowing(Base, BaseModel):
     __tablename__ = 'user_followings'
     __table_args__ = (
         UniqueConstraint('user_id', 'follows_repository_id'),
         UniqueConstraint('user_id', 'follows_user_id'),
         base_table_args
     )
 
     user_following_id = Column("user_following_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     follows_repo_id = Column("follows_repository_id", Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=None, default=None)
     follows_user_id = Column("follows_user_id", Integer(), ForeignKey('users.user_id'), nullable=True, unique=None, default=None)
     follows_from = Column('follows_from', DateTime(timezone=False), nullable=True, unique=None, default=datetime.datetime.now)
 
     user = relationship('User', primaryjoin='User.user_id==UserFollowing.user_id', back_populates='followings')
 
     follows_user = relationship('User', primaryjoin='User.user_id==UserFollowing.follows_user_id')
     follows_repository = relationship('Repository', order_by='Repository.repo_name', back_populates='followers')
 
     @classmethod
     def get_repo_followers(cls, repo_id):
         return cls.query().filter(cls.follows_repo_id == repo_id)
 
 
 class CacheKey(Base, BaseModel):
     __tablename__ = 'cache_invalidation'
     __table_args__ = (
         UniqueConstraint('cache_key'),
         Index('key_idx', 'cache_key'),
         Index('cache_args_idx', 'cache_args'),
         base_table_args,
     )
 
     CACHE_TYPE_FEED = 'FEED'
 
     # namespaces used to register process/thread aware caches
     REPO_INVALIDATION_NAMESPACE = 'repo_cache.v1:{repo_id}'
 
     cache_id = Column("cache_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     cache_key = Column("cache_key", String(255), nullable=True, unique=None, default=None)
     cache_args = Column("cache_args", String(255), nullable=True, unique=None, default=None)
     cache_state_uid = Column("cache_state_uid", String(255), nullable=True, unique=None, default=None)
     cache_active = Column("cache_active", Boolean(), nullable=True, unique=None, default=False)
 
     def __init__(self, cache_key, cache_args='', cache_state_uid=None, cache_active=False):
         self.cache_key = cache_key
         self.cache_args = cache_args
         self.cache_active = cache_active
         # first key should be same for all entries, since all workers should share it
         self.cache_state_uid = cache_state_uid or self.generate_new_state_uid()
 
     def __repr__(self):
         return "<%s('%s:%s[%s]')>" % (
             self.cls_name,
             self.cache_id, self.cache_key, self.cache_active)
 
     def _cache_key_partition(self):
         prefix, repo_name, suffix = self.cache_key.partition(self.cache_args)
         return prefix, repo_name, suffix
 
     def get_prefix(self):
         """
         Try to extract prefix from existing cache key. The key could consist
         of prefix, repo_name, suffix
         """
         # this returns prefix, repo_name, suffix
         return self._cache_key_partition()[0]
 
     def get_suffix(self):
         """
         get suffix that might have been used in _get_cache_key to
         generate self.cache_key. Only used for informational purposes
         in repo_edit.mako.
         """
         # prefix, repo_name, suffix
         return self._cache_key_partition()[2]
 
     @classmethod
     def generate_new_state_uid(cls, based_on=None):
         if based_on:
             return str(uuid.uuid5(uuid.NAMESPACE_URL, safe_str(based_on)))
         else:
             return str(uuid.uuid4())
 
     @classmethod
     def delete_all_cache(cls):
         """
         Delete all cache keys from database.
         Should only be run when all instances are down and all entries
         thus stale.
         """
         cls.query().delete()
         Session().commit()
 
     @classmethod
     def set_invalidate(cls, cache_uid, delete=False):
         """
         Mark all caches of a repo as invalid in the database.
         """
         try:
             qry = Session().query(cls).filter(cls.cache_key == cache_uid)
             if delete:
                 qry.delete()
                 log.debug('cache objects deleted for cache args %s',
                           safe_str(cache_uid))
             else:
                 new_uid = cls.generate_new_state_uid()
                 qry.update({"cache_state_uid": new_uid,
                             "cache_args": f"repo_state:{time.time()}"})
                 log.debug('cache object %s set new UID %s',
                           safe_str(cache_uid), new_uid)
 
             Session().commit()
         except Exception:
             log.exception(
                 'Cache key invalidation failed for cache args %s',
                 safe_str(cache_uid))
             Session().rollback()
 
     @classmethod
     def get_active_cache(cls, cache_key):
         inv_obj = cls.query().filter(cls.cache_key == cache_key).scalar()
         if inv_obj:
             return inv_obj
         return None
 
     @classmethod
     def get_namespace_map(cls, namespace):
         return {
             x.cache_key: x
             for x in cls.query().filter(cls.cache_args == namespace)}
 
 
 class ChangesetComment(Base, BaseModel):
     __tablename__ = 'changeset_comments'
     __table_args__ = (
         Index('cc_revision_idx', 'revision'),
         base_table_args,
     )
 
     COMMENT_OUTDATED = 'comment_outdated'
     COMMENT_TYPE_NOTE = 'note'
     COMMENT_TYPE_TODO = 'todo'
     COMMENT_TYPES = [COMMENT_TYPE_NOTE, COMMENT_TYPE_TODO]
 
     OP_IMMUTABLE = 'immutable'
     OP_CHANGEABLE = 'changeable'
 
     comment_id = Column('comment_id', Integer(), nullable=False, primary_key=True)
     repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     revision = Column('revision', String(40), nullable=True)
     pull_request_id = Column("pull_request_id", Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)
     pull_request_version_id = Column("pull_request_version_id", Integer(), ForeignKey('pull_request_versions.pull_request_version_id'), nullable=True)
     line_no = Column('line_no', Unicode(10), nullable=True)
     hl_lines = Column('hl_lines', Unicode(512), nullable=True)
     f_path = Column('f_path', Unicode(1000), nullable=True)
     user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     text = Column('text', UnicodeText().with_variant(UnicodeText(25000), 'mysql'), nullable=False)
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     modified_at = Column('modified_at', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     renderer = Column('renderer', Unicode(64), nullable=True)
     display_state = Column('display_state',  Unicode(128), nullable=True)
     immutable_state = Column('immutable_state', Unicode(128), nullable=True, default=OP_CHANGEABLE)
     draft = Column('draft', Boolean(), nullable=True, default=False)
 
     comment_type = Column('comment_type',  Unicode(128), nullable=True, default=COMMENT_TYPE_NOTE)
     resolved_comment_id = Column('resolved_comment_id', Integer(), ForeignKey('changeset_comments.comment_id'), nullable=True)
 
     resolved_comment = relationship('ChangesetComment', remote_side=comment_id, back_populates='resolved_by')
     resolved_by = relationship('ChangesetComment', back_populates='resolved_comment')
 
     author = relationship('User', lazy='select', back_populates='user_comments')
     repo = relationship('Repository', back_populates='comments')
     status_change = relationship('ChangesetStatus', cascade="all, delete-orphan", lazy='select', back_populates='comment')
     pull_request = relationship('PullRequest', lazy='select', back_populates='comments')
     pull_request_version = relationship('PullRequestVersion', lazy='select')
     history = relationship('ChangesetCommentHistory', cascade='all, delete-orphan', lazy='select', order_by='ChangesetCommentHistory.version', back_populates="comment")
 
     @classmethod
     def get_users(cls, revision=None, pull_request_id=None):
         """
         Returns user associated with this ChangesetComment. ie those
         who actually commented
 
         :param cls:
         :param revision:
         """
         q = Session().query(User).join(ChangesetComment.author)
         if revision:
             q = q.filter(cls.revision == revision)
         elif pull_request_id:
             q = q.filter(cls.pull_request_id == pull_request_id)
         return q.all()
 
     @classmethod
     def get_index_from_version(cls, pr_version, versions=None, num_versions=None) -> int:
         if pr_version is None:
             return 0
 
         if versions is not None:
             num_versions = [x.pull_request_version_id for x in versions]
 
         num_versions = num_versions or []
         try:
             return num_versions.index(pr_version) + 1
         except (IndexError, ValueError):
             return 0
 
     @property
     def outdated(self):
         return self.display_state == self.COMMENT_OUTDATED
 
     @property
     def outdated_js(self):
         return str_json(self.display_state == self.COMMENT_OUTDATED)
 
     @property
     def immutable(self):
         return self.immutable_state == self.OP_IMMUTABLE
 
     def outdated_at_version(self, version: int) -> bool:
         """
         Checks if comment is outdated for given pull request version
         """
 
         def version_check():
             return self.pull_request_version_id and self.pull_request_version_id != version
 
         if self.is_inline:
             return self.outdated and version_check()
         else:
             # general comments don't have .outdated set, also latest don't have a version
             return version_check()
 
     def outdated_at_version_js(self, version):
         """
         Checks if comment is outdated for given pull request version
         """
         return str_json(self.outdated_at_version(version))
 
     def older_than_version(self, version: int) -> bool:
         """
         Checks if comment is made from a previous version than given.
         Assumes self.pull_request_version.pull_request_version_id is an integer if not None.
         """
 
         # If version is None, return False as the current version cannot be less than None
         if version is None:
             return False
 
         # Ensure that the version is an integer to prevent TypeError on comparison
         if not isinstance(version, int):
             raise ValueError("The provided version must be an integer.")
 
         # Initialize current version to 0 or pull_request_version_id if it's available
         cur_ver = 0
         if self.pull_request_version and self.pull_request_version.pull_request_version_id is not None:
             cur_ver = self.pull_request_version.pull_request_version_id
 
         # Return True if the current version is less than the given version
         return cur_ver < version
 
     def older_than_version_js(self, version):
         """
         Checks if comment is made from previous version than given
         """
         return str_json(self.older_than_version(version))
 
     @property
     def commit_id(self):
         """New style naming to stop using .revision"""
         return self.revision
 
     @property
     def resolved(self):
         return self.resolved_by[0] if self.resolved_by else None
 
     @property
     def is_todo(self):
         return self.comment_type == self.COMMENT_TYPE_TODO
 
     @property
     def is_inline(self):
         if self.line_no and self.f_path:
             return True
         return False
 
     @property
     def last_version(self):
         version = 0
         if self.history:
             version = self.history[-1].version
         return version
 
     def get_index_version(self, versions):
         return self.get_index_from_version(
             self.pull_request_version_id, versions)
 
     @property
     def review_status(self):
         if self.status_change:
             return self.status_change[0].status
 
     @property
     def review_status_lbl(self):
         if self.status_change:
             return self.status_change[0].status_lbl
 
     def __repr__(self):
         if self.comment_id:
             return f'<DB:Comment #{self.comment_id}>'
         else:
             return f'<DB:Comment at {id(self)!r}>'
 
     def get_api_data(self):
         comment = self
 
         data = {
             'comment_id': comment.comment_id,
             'comment_type': comment.comment_type,
             'comment_text': comment.text,
             'comment_status': comment.status_change,
             'comment_f_path': comment.f_path,
             'comment_lineno': comment.line_no,
             'comment_author': comment.author,
             'comment_created_on': comment.created_on,
             'comment_resolved_by': self.resolved,
             'comment_commit_id': comment.revision,
             'comment_pull_request_id': comment.pull_request_id,
             'comment_last_version': self.last_version
         }
         return data
 
     def __json__(self):
         data = dict()
         data.update(self.get_api_data())
         return data
 
 
 class ChangesetCommentHistory(Base, BaseModel):
     __tablename__ = 'changeset_comments_history'
     __table_args__ = (
         Index('cch_comment_id_idx', 'comment_id'),
         base_table_args,
     )
 
     comment_history_id = Column('comment_history_id', Integer(), nullable=False, primary_key=True)
     comment_id = Column('comment_id', Integer(), ForeignKey('changeset_comments.comment_id'), nullable=False)
     version = Column("version", Integer(), nullable=False, default=0)
     created_by_user_id = Column('created_by_user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     text = Column('text', UnicodeText().with_variant(UnicodeText(25000), 'mysql'), nullable=False)
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     deleted = Column('deleted', Boolean(), default=False)
 
     author = relationship('User', lazy='joined')
     comment = relationship('ChangesetComment', cascade="all, delete", back_populates="history")
 
     @classmethod
     def get_version(cls, comment_id):
         q = Session().query(ChangesetCommentHistory).filter(
             ChangesetCommentHistory.comment_id == comment_id).order_by(ChangesetCommentHistory.version.desc())
         if q.count() == 0:
             return 1
         elif q.count() >= q[0].version:
             return q.count() + 1
         else:
             return q[0].version + 1
 
 
 class ChangesetStatus(Base, BaseModel):
     __tablename__ = 'changeset_statuses'
     __table_args__ = (
         Index('cs_revision_idx', 'revision'),
         Index('cs_version_idx', 'version'),
         UniqueConstraint('repo_id', 'revision', 'version'),
         base_table_args
     )
 
     STATUS_NOT_REVIEWED = DEFAULT = 'not_reviewed'
     STATUS_APPROVED = 'approved'
     STATUS_REJECTED = 'rejected'
     STATUS_UNDER_REVIEW = 'under_review'
 
     STATUSES = [
         (STATUS_NOT_REVIEWED, _("Not Reviewed")),  # (no icon) and default
         (STATUS_APPROVED, _("Approved")),
         (STATUS_REJECTED, _("Rejected")),
         (STATUS_UNDER_REVIEW, _("Under Review")),
     ]
 
     changeset_status_id = Column('changeset_status_id', Integer(), nullable=False, primary_key=True)
     repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None)
     revision = Column('revision', String(40), nullable=False)
     status = Column('status', String(128), nullable=False, default=DEFAULT)
     changeset_comment_id = Column('changeset_comment_id', Integer(), ForeignKey('changeset_comments.comment_id'))
     modified_at = Column('modified_at', DateTime(), nullable=False, default=datetime.datetime.now)
     version = Column('version', Integer(), nullable=False, default=0)
     pull_request_id = Column("pull_request_id", Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=True)
 
     author = relationship('User', lazy='select')
     repo = relationship('Repository', lazy='select')
     comment = relationship('ChangesetComment', lazy='select', back_populates='status_change')
     pull_request = relationship('PullRequest', lazy='select', back_populates='statuses')
 
     def __repr__(self):
         return f"<{self.cls_name}('{self.status}[v{self.version}]:{self.author}')>"
 
     @classmethod
     def get_status_lbl(cls, value):
         return dict(cls.STATUSES).get(value)
 
     @property
     def status_lbl(self):
         return ChangesetStatus.get_status_lbl(self.status)
 
     def get_api_data(self):
         status = self
         data = {
             'status_id': status.changeset_status_id,
             'status': status.status,
         }
         return data
 
     def __json__(self):
         data = dict()
         data.update(self.get_api_data())
         return data
 
 
 class _SetState(object):
     """
     Context processor allowing changing state for sensitive operation such as
     pull request update or merge
     """
 
     def __init__(self, pull_request, pr_state, back_state=None):
         self._pr = pull_request
         self._org_state = back_state or pull_request.pull_request_state
         self._pr_state = pr_state
         self._current_state = None
 
     def __enter__(self):
         log.debug('StateLock: entering set state context of pr %s, setting state to: `%s`',
                   self._pr, self._pr_state)
         self.set_pr_state(self._pr_state)
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         if exc_val is not None or exc_type is not None:
             log.error(traceback.format_tb(exc_tb))
             return None
 
         self.set_pr_state(self._org_state)
         log.debug('StateLock: exiting set state context of pr %s, setting state to: `%s`',
                   self._pr, self._org_state)
 
     @property
     def state(self):
         return self._current_state
 
     def set_pr_state(self, pr_state):
         try:
             self._pr.pull_request_state = pr_state
             Session().add(self._pr)
             Session().commit()
             self._current_state = pr_state
         except Exception:
             log.exception('Failed to set PullRequest %s state to %s', self._pr, pr_state)
             raise
 
 
 class _PullRequestBase(BaseModel):
     """
     Common attributes of pull request and version entries.
     """
 
     # .status values
     STATUS_NEW = 'new'
     STATUS_OPEN = 'open'
     STATUS_CLOSED = 'closed'
 
     # available states
     STATE_CREATING = 'creating'
     STATE_UPDATING = 'updating'
     STATE_MERGING = 'merging'
     STATE_CREATED = 'created'
 
     title = Column('title', Unicode(255), nullable=True)
     description = Column(
         'description', UnicodeText().with_variant(UnicodeText(10240), 'mysql'),
         nullable=True)
     description_renderer = Column('description_renderer', Unicode(64), nullable=True)
 
     # new/open/closed status of pull request (not approve/reject/etc)
     status = Column('status', Unicode(255), nullable=False, default=STATUS_NEW)
     created_on = Column(
         'created_on', DateTime(timezone=False), nullable=False,
         default=datetime.datetime.now)
     updated_on = Column(
         'updated_on', DateTime(timezone=False), nullable=False,
         default=datetime.datetime.now)
 
     pull_request_state = Column("pull_request_state", String(255), nullable=True)
 
     @declared_attr
     def user_id(cls):
         return Column(
             "user_id", Integer(), ForeignKey('users.user_id'), nullable=False,
             unique=None)
 
     # 500 revisions max
     _revisions = Column(
         'revisions', UnicodeText().with_variant(UnicodeText(20500), 'mysql'))
 
     common_ancestor_id = Column('common_ancestor_id', Unicode(255), nullable=True)
 
     @declared_attr
     def source_repo_id(cls):
         # TODO: dan: rename column to source_repo_id
         return Column(
             'org_repo_id', Integer(), ForeignKey('repositories.repo_id'),
             nullable=False)
 
     @declared_attr
     def pr_source(cls):
         return relationship(
             'Repository',
             primaryjoin=f'{cls.__name__}.source_repo_id==Repository.repo_id',
             overlaps="pull_requests_source"
         )
 
     _source_ref = Column('org_ref', Unicode(255), nullable=False)
 
     @hybrid_property
     def source_ref(self):
         return self._source_ref
 
     @source_ref.setter
     def source_ref(self, val):
         parts = (val or '').split(':')
         if len(parts) != 3:
             raise ValueError(
                 'Invalid reference format given: {}, expected X:Y:Z'.format(val))
         self._source_ref = safe_str(val)
 
     _target_ref = Column('other_ref', Unicode(255), nullable=False)
 
     @hybrid_property
     def target_ref(self):
         return self._target_ref
 
     @target_ref.setter
     def target_ref(self, val):
         parts = (val or '').split(':')
         if len(parts) != 3:
             raise ValueError(
                 'Invalid reference format given: {}, expected X:Y:Z'.format(val))
         self._target_ref = safe_str(val)
 
     @declared_attr
     def target_repo_id(cls):
         # TODO: dan: rename column to target_repo_id
         return Column(
             'other_repo_id', Integer(), ForeignKey('repositories.repo_id'),
             nullable=False)
 
     @declared_attr
     def pr_target(cls):
         return relationship(
             'Repository',
             primaryjoin=f'{cls.__name__}.target_repo_id==Repository.repo_id',
             overlaps="pull_requests_target"
         )
 
     _shadow_merge_ref = Column('shadow_merge_ref', Unicode(255), nullable=True)
 
     # TODO: dan: rename column to last_merge_source_rev
     _last_merge_source_rev = Column(
         'last_merge_org_rev', String(40), nullable=True)
     # TODO: dan: rename column to last_merge_target_rev
     _last_merge_target_rev = Column(
         'last_merge_other_rev', String(40), nullable=True)
     _last_merge_status = Column('merge_status', Integer(), nullable=True)
     last_merge_metadata = Column(
         'last_merge_metadata', MutationObj.as_mutable(
             JsonType(dialect_map=dict(mysql=UnicodeText(16384)))))
 
     merge_rev = Column('merge_rev', String(40), nullable=True)
 
     reviewer_data = Column(
         'reviewer_data_json', MutationObj.as_mutable(
             JsonType(dialect_map=dict(mysql=UnicodeText(16384)))))
 
     @property
     def reviewer_data_json(self):
         return str_json(self.reviewer_data)
 
     @property
     def last_merge_metadata_parsed(self):
         metadata = {}
         if not self.last_merge_metadata:
             return metadata
 
         if hasattr(self.last_merge_metadata, 'de_coerce'):
             for k, v in self.last_merge_metadata.de_coerce().items():
                 if k in ['target_ref', 'source_ref']:
                     metadata[k] = Reference(v['type'], v['name'], v['commit_id'])
                 else:
                     if hasattr(v, 'de_coerce'):
                         metadata[k] = v.de_coerce()
                     else:
                         metadata[k] = v
         return metadata
 
     @property
     def work_in_progress(self):
         """checks if pull request is work in progress by checking the title"""
         title = self.title.upper()
         if re.match(r'^(\[WIP\]\s*|WIP:\s*|WIP\s+)', title):
             return True
         return False
 
     @property
     def title_safe(self):
         return self.title\
             .replace('{', '{{')\
             .replace('}', '}}')
 
     @hybrid_property
     def description_safe(self):
         return description_escaper(self.description)
 
     @hybrid_property
     def revisions(self):
         return self._revisions.split(':') if self._revisions else []
 
     @revisions.setter
     def revisions(self, val):
         self._revisions = ':'.join(val)
 
     @hybrid_property
     def last_merge_status(self):
         return safe_int(self._last_merge_status)
 
     @last_merge_status.setter
     def last_merge_status(self, val):
         self._last_merge_status = val
 
     @declared_attr
     def author(cls):
         return relationship(
             'User', lazy='joined',
             #TODO, problem that is somehow :?
             #back_populates='user_pull_requests'
             )
 
     @declared_attr
     def source_repo(cls):
         return relationship(
             'Repository',
             primaryjoin=f'{cls.__name__}.source_repo_id==Repository.repo_id',
             overlaps="pr_source"
         )
 
     @property
     def source_ref_parts(self):
         return self.unicode_to_reference(self.source_ref)
 
     @declared_attr
     def target_repo(cls):
         return relationship(
             'Repository',
             primaryjoin=f'{cls.__name__}.target_repo_id==Repository.repo_id',
             overlaps="pr_target"
         )
 
     @property
     def target_ref_parts(self):
         return self.unicode_to_reference(self.target_ref)
 
     @property
     def shadow_merge_ref(self):
         return self.unicode_to_reference(self._shadow_merge_ref)
 
     @shadow_merge_ref.setter
     def shadow_merge_ref(self, ref):
         self._shadow_merge_ref = self.reference_to_unicode(ref)
 
     @staticmethod
     def unicode_to_reference(raw):
         return unicode_to_reference(raw)
 
     @staticmethod
     def reference_to_unicode(ref):
         return reference_to_unicode(ref)
 
     def get_api_data(self, with_merge_state=True):
         from rhodecode.model.pull_request import PullRequestModel
 
         pull_request = self
         if with_merge_state:
             merge_response, merge_status, msg = \
                 PullRequestModel().merge_status(pull_request)
             merge_state = {
                 'status': merge_status,
                 'message': safe_str(msg),
             }
         else:
             merge_state = {'status': 'not_available',
                            'message': 'not_available'}
 
         merge_data = {
             'clone_url': PullRequestModel().get_shadow_clone_url(pull_request),
             'reference': (
                 pull_request.shadow_merge_ref.asdict()
                 if pull_request.shadow_merge_ref else None),
         }
 
         data = {
             'pull_request_id': pull_request.pull_request_id,
             'url': PullRequestModel().get_url(pull_request),
             'title': pull_request.title,
             'description': pull_request.description,
             'status': pull_request.status,
             'state': pull_request.pull_request_state,
             'created_on': pull_request.created_on,
             'updated_on': pull_request.updated_on,
             'commit_ids': pull_request.revisions,
             'review_status': pull_request.calculated_review_status(),
             'mergeable': merge_state,
             'source': {
                 'clone_url': pull_request.source_repo.clone_url(),
                 'repository': pull_request.source_repo.repo_name,
                 'reference': {
                     'name': pull_request.source_ref_parts.name,
                     'type': pull_request.source_ref_parts.type,
                     'commit_id': pull_request.source_ref_parts.commit_id,
                 },
             },
             'target': {
                 'clone_url': pull_request.target_repo.clone_url(),
                 'repository': pull_request.target_repo.repo_name,
                 'reference': {
                     'name': pull_request.target_ref_parts.name,
                     'type': pull_request.target_ref_parts.type,
                     'commit_id': pull_request.target_ref_parts.commit_id,
                 },
             },
             'merge': merge_data,
             'author': pull_request.author.get_api_data(include_secrets=False,
                                                        details='basic'),
             'reviewers': [
                 {
                     'user': reviewer.get_api_data(include_secrets=False,
                                                   details='basic'),
                     'reasons': reasons,
                     'review_status': st[0][1].status if st else 'not_reviewed',
                 }
                 for obj, reviewer, reasons, mandatory, st in
                 pull_request.reviewers_statuses()
             ]
         }
 
         return data
 
     def set_state(self, pull_request_state, final_state=None):
         """
         # goes from initial state to updating to initial state.
         # initial state can be changed by specifying back_state=
         with pull_request_obj.set_state(PullRequest.STATE_UPDATING):
            pull_request.merge()
 
         :param pull_request_state:
         :param final_state:
 
         """
 
         return _SetState(self, pull_request_state, back_state=final_state)
 
 
 class PullRequest(Base, _PullRequestBase):
     __tablename__ = 'pull_requests'
     __table_args__ = (
         base_table_args,
     )
     LATEST_VER = 'latest'
 
     pull_request_id = Column(
         'pull_request_id', Integer(), nullable=False, primary_key=True)
 
     def __repr__(self):
         if self.pull_request_id:
             return f'<DB:PullRequest #{self.pull_request_id}>'
         else:
             return f'<DB:PullRequest at {id(self)!r}>'
 
     def __str__(self):
         if self.pull_request_id:
             return f'#{self.pull_request_id}'
         else:
             return f'#{id(self)!r}'
 
     reviewers = relationship('PullRequestReviewers', cascade="all, delete-orphan", back_populates='pull_request')
     statuses = relationship('ChangesetStatus', cascade="all, delete-orphan", back_populates='pull_request')
     comments = relationship('ChangesetComment', cascade="all, delete-orphan", back_populates='pull_request')
     versions = relationship('PullRequestVersion', cascade="all, delete-orphan", lazy='dynamic', back_populates='pull_request')
 
     @classmethod
     def get_pr_display_object(cls, pull_request_obj, org_pull_request_obj,
                               internal_methods=None):
 
         class PullRequestDisplay(object):
             """
             Special object wrapper for showing PullRequest data via Versions
             It mimics PR object as close as possible. This is read only object
             just for display
             """
 
             def __init__(self, attrs, internal=None):
                 self.attrs = attrs
                 # internal have priority over the given ones via attrs
                 self.internal = internal or ['versions']
 
             def __getattr__(self, item):
                 if item in self.internal:
                     return getattr(self, item)
                 try:
                     return self.attrs[item]
                 except KeyError:
                     raise AttributeError(
                         '%s object has no attribute %s' % (self, item))
 
             def __repr__(self):
                 pr_id = self.attrs.get('pull_request_id')
                 return f'<DB:PullRequestDisplay #{pr_id}>'
 
             def versions(self):
                 return pull_request_obj.versions.order_by(
                     PullRequestVersion.pull_request_version_id).all()
 
             def is_closed(self):
                 return pull_request_obj.is_closed()
 
             def is_state_changing(self):
                 return pull_request_obj.is_state_changing()
 
             @property
             def pull_request_version_id(self):
                 return getattr(pull_request_obj, 'pull_request_version_id', None)
 
             @property
             def pull_request_last_version(self):
                 return pull_request_obj.pull_request_last_version
 
         attrs = StrictAttributeDict(pull_request_obj.get_api_data(with_merge_state=False))
 
         attrs.author = StrictAttributeDict(
             pull_request_obj.author.get_api_data())
         if pull_request_obj.target_repo:
             attrs.target_repo = StrictAttributeDict(
                 pull_request_obj.target_repo.get_api_data())
             attrs.target_repo.clone_url = pull_request_obj.target_repo.clone_url
 
         if pull_request_obj.source_repo:
             attrs.source_repo = StrictAttributeDict(
                 pull_request_obj.source_repo.get_api_data())
             attrs.source_repo.clone_url = pull_request_obj.source_repo.clone_url
 
         attrs.source_ref_parts = pull_request_obj.source_ref_parts
         attrs.target_ref_parts = pull_request_obj.target_ref_parts
         attrs.revisions = pull_request_obj.revisions
         attrs.common_ancestor_id = pull_request_obj.common_ancestor_id
         attrs.shadow_merge_ref = org_pull_request_obj.shadow_merge_ref
         attrs.reviewer_data = org_pull_request_obj.reviewer_data
         attrs.reviewer_data_json = org_pull_request_obj.reviewer_data_json
 
         return PullRequestDisplay(attrs, internal=internal_methods)
 
     def is_closed(self):
         return self.status == self.STATUS_CLOSED
 
     def is_state_changing(self):
         return self.pull_request_state != PullRequest.STATE_CREATED
 
     def __json__(self):
         return {
             'revisions': self.revisions,
             'versions': self.versions_count
         }
 
     def calculated_review_status(self):
         from rhodecode.model.changeset_status import ChangesetStatusModel
         return ChangesetStatusModel().calculated_review_status(self)
 
     def reviewers_statuses(self, user=None):
         from rhodecode.model.changeset_status import ChangesetStatusModel
         return ChangesetStatusModel().reviewers_statuses(self, user=user)
 
     def get_pull_request_reviewers(self, role=None):
         qry = PullRequestReviewers.query()\
             .filter(PullRequestReviewers.pull_request_id == self.pull_request_id)
         if role:
             qry = qry.filter(PullRequestReviewers.role == role)
 
         return qry.all()
 
     @property
     def reviewers_count(self):
         qry = PullRequestReviewers.query()\
             .filter(PullRequestReviewers.pull_request_id == self.pull_request_id)\
             .filter(PullRequestReviewers.role == PullRequestReviewers.ROLE_REVIEWER)
         return qry.count()
 
     @property
     def observers_count(self):
         qry = PullRequestReviewers.query()\
             .filter(PullRequestReviewers.pull_request_id == self.pull_request_id)\
             .filter(PullRequestReviewers.role == PullRequestReviewers.ROLE_OBSERVER)
         return qry.count()
 
     def observers(self):
         qry = PullRequestReviewers.query()\
             .filter(PullRequestReviewers.pull_request_id == self.pull_request_id)\
             .filter(PullRequestReviewers.role == PullRequestReviewers.ROLE_OBSERVER)\
             .all()
 
         for entry in qry:
             yield entry, entry.user
 
     @property
     def workspace_id(self):
         from rhodecode.model.pull_request import PullRequestModel
         return PullRequestModel()._workspace_id(self)
 
     def get_shadow_repo(self):
         workspace_id = self.workspace_id
         shadow_repository_path = self.target_repo.get_shadow_repository_path(workspace_id)
         if os.path.isdir(shadow_repository_path):
             vcs_obj = self.target_repo.scm_instance()
             return vcs_obj.get_shadow_instance(shadow_repository_path)
 
     @property
     def versions_count(self):
         """
         return number of versions this PR have, e.g a PR that once been
         updated will have 2 versions
         """
         return self.versions.count() + 1
 
     @property
     def pull_request_last_version(self):
         return self.versions_count
 
 
 class PullRequestVersion(Base, _PullRequestBase):
     __tablename__ = 'pull_request_versions'
     __table_args__ = (
         base_table_args,
     )
 
     pull_request_version_id = Column('pull_request_version_id', Integer(), nullable=False, primary_key=True)
     pull_request_id = Column('pull_request_id', Integer(), ForeignKey('pull_requests.pull_request_id'), nullable=False)
     pull_request = relationship('PullRequest',  back_populates='versions')
 
     def __repr__(self):
         if self.pull_request_version_id:
             return f'<DB:PullRequestVersion #{self.pull_request_version_id}>'
         else:
             return f'<DB:PullRequestVersion at {id(self)!r}>'
 
     @property
     def reviewers(self):
         return self.pull_request.reviewers
 
     @property
     def versions(self):
         return self.pull_request.versions
 
     def is_closed(self):
         # calculate from original
         return self.pull_request.status == self.STATUS_CLOSED
 
     def is_state_changing(self):
         return self.pull_request.pull_request_state != PullRequest.STATE_CREATED
 
     def calculated_review_status(self):
         return self.pull_request.calculated_review_status()
 
     def reviewers_statuses(self):
         return self.pull_request.reviewers_statuses()
 
     def observers(self):
         return self.pull_request.observers()
 
 
 class PullRequestReviewers(Base, BaseModel):
     __tablename__ = 'pull_request_reviewers'
     __table_args__ = (
         base_table_args,
     )
     ROLE_REVIEWER = 'reviewer'
     ROLE_OBSERVER = 'observer'
     ROLES = [ROLE_REVIEWER, ROLE_OBSERVER]
 
     @hybrid_property
     def reasons(self):
         if not self._reasons:
             return []
         return self._reasons
 
     @reasons.setter
     def reasons(self, val):
         val = val or []
         if any(not isinstance(x, str) for x in val):
             raise Exception('invalid reasons type, must be list of strings')
         self._reasons = val
 
     pull_requests_reviewers_id = Column(
         'pull_requests_reviewers_id', Integer(), nullable=False,
         primary_key=True)
     pull_request_id = Column(
         "pull_request_id", Integer(),
         ForeignKey('pull_requests.pull_request_id'), nullable=False)
     user_id = Column(
         "user_id", Integer(), ForeignKey('users.user_id'), nullable=True)
     _reasons = Column(
         'reason', MutationList.as_mutable(
             JsonType('list', dialect_map=dict(mysql=UnicodeText(16384)))))
 
     mandatory = Column("mandatory", Boolean(), nullable=False, default=False)
     role = Column('role', Unicode(255), nullable=True, default=ROLE_REVIEWER)
 
     user = relationship('User')
     pull_request = relationship('PullRequest', back_populates='reviewers')
 
     rule_data = Column(
         'rule_data_json',
         JsonType(dialect_map=dict(mysql=UnicodeText(16384))))
 
     def rule_user_group_data(self):
         """
         Returns the voting user group rule data for this reviewer
         """
 
         if self.rule_data and 'vote_rule' in self.rule_data:
             user_group_data = {}
             if 'rule_user_group_entry_id' in self.rule_data:
                 # means a group with voting rules !
                 user_group_data['id'] = self.rule_data['rule_user_group_entry_id']
                 user_group_data['name'] = self.rule_data['rule_name']
                 user_group_data['vote_rule'] = self.rule_data['vote_rule']
 
             return user_group_data
 
     @classmethod
     def get_pull_request_reviewers(cls, pull_request_id, role=None):
         qry = PullRequestReviewers.query()\
             .filter(PullRequestReviewers.pull_request_id == pull_request_id)
         if role:
             qry = qry.filter(PullRequestReviewers.role == role)
 
         return qry.all()
 
     def __repr__(self):
         return f"<{self.cls_name}('id:{self.pull_requests_reviewers_id}')>"
 
 
 class Notification(Base, BaseModel):
     __tablename__ = 'notifications'
     __table_args__ = (
         Index('notification_type_idx', 'type'),
         base_table_args,
     )
 
     TYPE_CHANGESET_COMMENT = 'cs_comment'
     TYPE_MESSAGE = 'message'
     TYPE_MENTION = 'mention'
     TYPE_REGISTRATION = 'registration'
     TYPE_PULL_REQUEST = 'pull_request'
     TYPE_PULL_REQUEST_COMMENT = 'pull_request_comment'
     TYPE_PULL_REQUEST_UPDATE = 'pull_request_update'
 
     notification_id = Column('notification_id', Integer(), nullable=False, primary_key=True)
     subject = Column('subject', Unicode(512), nullable=True)
     body = Column('body', UnicodeText().with_variant(UnicodeText(50000), 'mysql'), nullable=True)
     created_by = Column("created_by", Integer(), ForeignKey('users.user_id'), nullable=True)
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     type_ = Column('type', Unicode(255))
 
     created_by_user = relationship('User', back_populates='user_created_notifications')
     notifications_to_users = relationship('UserNotification', lazy='joined', cascade="all, delete-orphan", back_populates='notification')
 
     @property
     def recipients(self):
         return [x.user for x in UserNotification.query()\
                 .filter(UserNotification.notification == self)\
                 .order_by(UserNotification.user_id.asc()).all()]
 
     @classmethod
     def create(cls, created_by, subject, body, recipients, type_=None):
         if type_ is None:
             type_ = Notification.TYPE_MESSAGE
 
         notification = cls()
         notification.created_by_user = created_by
         notification.subject = subject
         notification.body = body
         notification.type_ = type_
         notification.created_on = datetime.datetime.now()
 
         # For each recipient link the created notification to his account
         for u in recipients:
             assoc = UserNotification()
             assoc.user_id = u.user_id
             assoc.notification = notification
 
             # if created_by is inside recipients mark his notification
             # as read
             if u.user_id == created_by.user_id:
                 assoc.read = True
             Session().add(assoc)
 
         Session().add(notification)
 
         return notification
 
 
 class UserNotification(Base, BaseModel):
     __tablename__ = 'user_to_notification'
     __table_args__ = (
         UniqueConstraint('user_id', 'notification_id'),
         base_table_args
     )
 
     user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), primary_key=True)
     notification_id = Column("notification_id", Integer(), ForeignKey('notifications.notification_id'), primary_key=True)
     read = Column('read', Boolean, default=False)
     sent_on = Column('sent_on', DateTime(timezone=False), nullable=True, unique=None)
 
     user = relationship('User', lazy="joined", back_populates='notifications')
     notification = relationship('Notification', lazy="joined", order_by=lambda: Notification.created_on.desc(), back_populates='notifications_to_users')
 
     def mark_as_read(self):
         self.read = True
         Session().add(self)
 
 
 class UserNotice(Base, BaseModel):
     __tablename__ = 'user_notices'
     __table_args__ = (
         base_table_args
     )
 
     NOTIFICATION_TYPE_MESSAGE = 'message'
     NOTIFICATION_TYPE_NOTICE = 'notice'
 
     NOTIFICATION_LEVEL_INFO = 'info'
     NOTIFICATION_LEVEL_WARNING = 'warning'
     NOTIFICATION_LEVEL_ERROR = 'error'
 
     user_notice_id = Column('gist_id', Integer(), primary_key=True)
 
     notice_subject = Column('notice_subject', Unicode(512), nullable=True)
     notice_body = Column('notice_body', UnicodeText().with_variant(UnicodeText(50000), 'mysql'), nullable=True)
 
     notice_read = Column('notice_read', Boolean, default=False)
 
     notification_level = Column('notification_level', String(1024), default=NOTIFICATION_LEVEL_INFO)
     notification_type = Column('notification_type', String(1024), default=NOTIFICATION_TYPE_NOTICE)
 
     notice_created_by = Column('notice_created_by', Integer(), ForeignKey('users.user_id'), nullable=True)
     notice_created_on = Column('notice_created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 
     user_id = Column('user_id', Integer(), ForeignKey('users.user_id'))
     user = relationship('User', lazy="joined", primaryjoin='User.user_id==UserNotice.user_id')
 
     @classmethod
     def create_for_user(cls, user, subject, body, notice_level=NOTIFICATION_LEVEL_INFO, allow_duplicate=False):
 
         if notice_level not in [cls.NOTIFICATION_LEVEL_ERROR,
                                 cls.NOTIFICATION_LEVEL_WARNING,
                                 cls.NOTIFICATION_LEVEL_INFO]:
             return
 
         from rhodecode.model.user import UserModel
         user = UserModel().get_user(user)
 
         new_notice = UserNotice()
         if not allow_duplicate:
             existing_msg = UserNotice().query() \
                 .filter(UserNotice.user == user) \
                 .filter(UserNotice.notice_body == body) \
                 .filter(UserNotice.notice_read == false()) \
                 .scalar()
             if existing_msg:
                 log.warning('Ignoring duplicate notice for user %s', user)
                 return
 
         new_notice.user = user
         new_notice.notice_subject = subject
         new_notice.notice_body = body
         new_notice.notification_level = notice_level
         Session().add(new_notice)
         Session().commit()
 
 
 class Gist(Base, BaseModel):
     __tablename__ = 'gists'
     __table_args__ = (
         Index('g_gist_access_id_idx', 'gist_access_id'),
         Index('g_created_on_idx', 'created_on'),
         base_table_args
     )
 
     GIST_PUBLIC = 'public'
     GIST_PRIVATE = 'private'
     DEFAULT_FILENAME = 'gistfile1.txt'
 
     ACL_LEVEL_PUBLIC = 'acl_public'
     ACL_LEVEL_PRIVATE = 'acl_private'
 
     gist_id = Column('gist_id', Integer(), primary_key=True)
     gist_access_id = Column('gist_access_id', Unicode(250))
     gist_description = Column('gist_description', UnicodeText().with_variant(UnicodeText(1024), 'mysql'))
     gist_owner = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=True)
     gist_expires = Column('gist_expires', Float(53), nullable=False)
     gist_type = Column('gist_type', Unicode(128), nullable=False)
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     modified_at = Column('modified_at', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     acl_level = Column('acl_level', Unicode(128), nullable=True)
 
     owner = relationship('User', back_populates='user_gists')
 
     def __repr__(self):
         return f'<Gist:[{self.gist_type}]{self.gist_access_id}>'
 
     @hybrid_property
     def description_safe(self):
         return description_escaper(self.gist_description)
 
     @classmethod
     def get_or_404(cls, id_):
         from pyramid.httpexceptions import HTTPNotFound
 
         res = cls.query().filter(cls.gist_access_id == id_).scalar()
         if not res:
             log.debug('WARN: No DB entry with id %s', id_)
             raise HTTPNotFound()
         return res
 
     @classmethod
     def get_by_access_id(cls, gist_access_id):
         return cls.query().filter(cls.gist_access_id == gist_access_id).scalar()
 
     def gist_url(self):
         from rhodecode.model.gist import GistModel
         return GistModel().get_url(self)
 
     @classmethod
     def base_path(cls):
         """
         Returns base path when all gists are stored
 
         :param cls:
         """
         from rhodecode.model.gist import GIST_STORE_LOC
         from rhodecode.lib.utils import get_rhodecode_repo_store_path
         repo_store_path = get_rhodecode_repo_store_path()
         return os.path.join(repo_store_path, GIST_STORE_LOC)
 
     def get_api_data(self):
         """
         Common function for generating gist related data for API
         """
         gist = self
         data = {
             'gist_id': gist.gist_id,
             'type': gist.gist_type,
             'access_id': gist.gist_access_id,
             'description': gist.gist_description,
             'url': gist.gist_url(),
             'expires': gist.gist_expires,
             'created_on': gist.created_on,
             'modified_at': gist.modified_at,
             'content': None,
             'acl_level': gist.acl_level,
         }
         return data
 
     def __json__(self):
         data = dict()
         data.update(self.get_api_data())
         return data
     # SCM functions
 
     def scm_instance(self, **kwargs):
         """
         Get an instance of VCS Repository
 
         :param kwargs:
         """
         from rhodecode.model.gist import GistModel
         full_repo_path = os.path.join(self.base_path(), self.gist_access_id)
         return get_vcs_instance(
             repo_path=safe_str(full_repo_path), create=False,
             _vcs_alias=GistModel.vcs_backend)
 
 
 class ExternalIdentity(Base, BaseModel):
     __tablename__ = 'external_identities'
     __table_args__ = (
         Index('local_user_id_idx', 'local_user_id'),
         Index('external_id_idx', 'external_id'),
         base_table_args
     )
 
     external_id = Column('external_id', Unicode(255), default='', primary_key=True)
     external_username = Column('external_username', Unicode(1024), default='')
     local_user_id = Column('local_user_id', Integer(), ForeignKey('users.user_id'), primary_key=True)
     provider_name = Column('provider_name', Unicode(255), default='', primary_key=True)
     access_token = Column('access_token', String(1024), default='')
     alt_token = Column('alt_token', String(1024), default='')
     token_secret = Column('token_secret', String(1024), default='')
 
     @classmethod
     def by_external_id_and_provider(cls, external_id, provider_name, local_user_id=None):
         """
         Returns ExternalIdentity instance based on search params
 
         :param external_id:
         :param provider_name:
         :return: ExternalIdentity
         """
         query = cls.query()
         query = query.filter(cls.external_id == external_id)
         query = query.filter(cls.provider_name == provider_name)
         if local_user_id:
             query = query.filter(cls.local_user_id == local_user_id)
         return query.first()
 
     @classmethod
     def user_by_external_id_and_provider(cls, external_id, provider_name):
         """
         Returns User instance based on search params
 
         :param external_id:
         :param provider_name:
         :return: User
         """
         query = User.query()
         query = query.filter(cls.external_id == external_id)
         query = query.filter(cls.provider_name == provider_name)
         query = query.filter(User.user_id == cls.local_user_id)
         return query.first()
 
     @classmethod
     def by_local_user_id(cls, local_user_id):
         """
         Returns all tokens for user
 
         :param local_user_id:
         :return: ExternalIdentity
         """
         query = cls.query()
         query = query.filter(cls.local_user_id == local_user_id)
         return query
 
     @classmethod
     def load_provider_plugin(cls, plugin_id):
         from rhodecode.authentication.base import loadplugin
         _plugin_id = 'egg:rhodecode-enterprise-ee#{}'.format(plugin_id)
         auth_plugin = loadplugin(_plugin_id)
         return auth_plugin
 
 
 class Integration(Base, BaseModel):
     __tablename__ = 'integrations'
     __table_args__ = (
         base_table_args
     )
 
     integration_id = Column('integration_id', Integer(), primary_key=True)
     integration_type = Column('integration_type', String(255))
     enabled = Column('enabled', Boolean(), nullable=False)
     name = Column('name', String(255), nullable=False)
     child_repos_only = Column('child_repos_only', Boolean(), nullable=False, default=False)
 
     settings = Column(
         'settings_json', MutationObj.as_mutable(
             JsonType(dialect_map=dict(mysql=UnicodeText(16384)))))
     repo_id = Column('repo_id', Integer(), ForeignKey('repositories.repo_id'), nullable=True, unique=None, default=None)
     repo = relationship('Repository', lazy='joined', back_populates='integrations')
 
     repo_group_id = Column('repo_group_id', Integer(), ForeignKey('groups.group_id'), nullable=True, unique=None, default=None)
     repo_group = relationship('RepoGroup', lazy='joined', back_populates='integrations')
 
     @property
     def scope(self):
         if self.repo:
             return repr(self.repo)
         if self.repo_group:
             if self.child_repos_only:
                 return repr(self.repo_group) + ' (child repos only)'
             else:
                 return repr(self.repo_group) + ' (recursive)'
         if self.child_repos_only:
             return 'root_repos'
         return 'global'
 
     def __repr__(self):
         return '<Integration(%r, %r)>' % (self.integration_type, self.scope)
 
 
 class RepoReviewRuleUser(Base, BaseModel):
     __tablename__ = 'repo_review_rules_users'
     __table_args__ = (
         base_table_args
     )
     ROLE_REVIEWER = 'reviewer'
     ROLE_OBSERVER = 'observer'
     ROLES = [ROLE_REVIEWER, ROLE_OBSERVER]
 
     repo_review_rule_user_id = Column('repo_review_rule_user_id', Integer(), primary_key=True)
     repo_review_rule_id = Column("repo_review_rule_id", Integer(), ForeignKey('repo_review_rules.repo_review_rule_id'))
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False)
     mandatory = Column("mandatory", Boolean(), nullable=False, default=False)
     role = Column('role', Unicode(255), nullable=True, default=ROLE_REVIEWER)
     user = relationship('User', back_populates='user_review_rules')
 
     def rule_data(self):
         return {
             'mandatory': self.mandatory,
             'role': self.role,
         }
 
 
 class RepoReviewRuleUserGroup(Base, BaseModel):
     __tablename__ = 'repo_review_rules_users_groups'
     __table_args__ = (
         base_table_args
     )
 
     VOTE_RULE_ALL = -1
     ROLE_REVIEWER = 'reviewer'
     ROLE_OBSERVER = 'observer'
     ROLES = [ROLE_REVIEWER, ROLE_OBSERVER]
 
     repo_review_rule_users_group_id = Column('repo_review_rule_users_group_id', Integer(), primary_key=True)
     repo_review_rule_id = Column("repo_review_rule_id", Integer(), ForeignKey('repo_review_rules.repo_review_rule_id'))
     users_group_id = Column("users_group_id", Integer(), ForeignKey('users_groups.users_group_id'), nullable=False)
     mandatory = Column("mandatory", Boolean(), nullable=False, default=False)
     role = Column('role', Unicode(255), nullable=True, default=ROLE_REVIEWER)
     vote_rule = Column("vote_rule", Integer(), nullable=True, default=VOTE_RULE_ALL)
     users_group = relationship('UserGroup')
 
     def rule_data(self):
         return {
             'mandatory': self.mandatory,
             'role': self.role,
             'vote_rule': self.vote_rule
         }
 
     @property
     def vote_rule_label(self):
         if not self.vote_rule or self.vote_rule == self.VOTE_RULE_ALL:
             return 'all must vote'
         else:
             return 'min. vote {}'.format(self.vote_rule)
 
 
 class RepoReviewRule(Base, BaseModel):
     __tablename__ = 'repo_review_rules'
     __table_args__ = (
         base_table_args
     )
 
     repo_review_rule_id = Column(
         'repo_review_rule_id', Integer(), primary_key=True)
     repo_id = Column(
         "repo_id", Integer(), ForeignKey('repositories.repo_id'))
     repo = relationship('Repository', back_populates='review_rules')
 
     review_rule_name = Column('review_rule_name', String(255))
     _branch_pattern = Column("branch_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default='*')  # glob
     _target_branch_pattern = Column("target_branch_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default='*')  # glob
     _file_pattern = Column("file_pattern", UnicodeText().with_variant(UnicodeText(255), 'mysql'), default='*')  # glob
 
     use_authors_for_review = Column("use_authors_for_review", Boolean(), nullable=False, default=False)
 
     # Legacy fields, just for backward compat
     _forbid_author_to_review = Column("forbid_author_to_review", Boolean(), nullable=False, default=False)
     _forbid_commit_author_to_review = Column("forbid_commit_author_to_review", Boolean(), nullable=False, default=False)
 
     pr_author = Column("pr_author", UnicodeText().with_variant(UnicodeText(255), 'mysql'), nullable=True)
     commit_author = Column("commit_author", UnicodeText().with_variant(UnicodeText(255), 'mysql'), nullable=True)
 
     forbid_adding_reviewers = Column("forbid_adding_reviewers", Boolean(), nullable=False, default=False)
 
     rule_users = relationship('RepoReviewRuleUser')
     rule_user_groups = relationship('RepoReviewRuleUserGroup')
 
     def _validate_pattern(self, value):
         re.compile('^' + glob2re(value) + '$')
 
     @hybrid_property
     def source_branch_pattern(self):
         return self._branch_pattern or '*'
 
     @source_branch_pattern.setter
     def source_branch_pattern(self, value):
         self._validate_pattern(value)
         self._branch_pattern = value or '*'
 
     @hybrid_property
     def target_branch_pattern(self):
         return self._target_branch_pattern or '*'
 
     @target_branch_pattern.setter
     def target_branch_pattern(self, value):
         self._validate_pattern(value)
         self._target_branch_pattern = value or '*'
 
     @hybrid_property
     def file_pattern(self):
         return self._file_pattern or '*'
 
     @file_pattern.setter
     def file_pattern(self, value):
         self._validate_pattern(value)
         self._file_pattern = value or '*'
 
     @hybrid_property
     def forbid_pr_author_to_review(self):
         return self.pr_author == 'forbid_pr_author'
 
     @hybrid_property
     def include_pr_author_to_review(self):
         return self.pr_author == 'include_pr_author'
 
     @hybrid_property
     def forbid_commit_author_to_review(self):
         return self.commit_author == 'forbid_commit_author'
 
     @hybrid_property
     def include_commit_author_to_review(self):
         return self.commit_author == 'include_commit_author'
 
     def matches(self, source_branch, target_branch, files_changed):
         """
         Check if this review rule matches a branch/files in a pull request
 
         :param source_branch: source branch name for the commit
         :param target_branch: target branch name for the commit
         :param files_changed: list of file paths changed in the pull request
         """
 
         source_branch = source_branch or ''
         target_branch = target_branch or ''
         files_changed = files_changed or []
 
         branch_matches = True
         if source_branch or target_branch:
             if self.source_branch_pattern == '*':
                 source_branch_match = True
             else:
                 if self.source_branch_pattern.startswith('re:'):
                     source_pattern = self.source_branch_pattern[3:]
                 else:
                     source_pattern = '^' + glob2re(self.source_branch_pattern) + '$'
                 source_branch_regex = re.compile(source_pattern)
                 source_branch_match = bool(source_branch_regex.search(source_branch))
             if self.target_branch_pattern == '*':
                 target_branch_match = True
             else:
                 if self.target_branch_pattern.startswith('re:'):
                     target_pattern = self.target_branch_pattern[3:]
                 else:
                     target_pattern = '^' + glob2re(self.target_branch_pattern) + '$'
                 target_branch_regex = re.compile(target_pattern)
                 target_branch_match = bool(target_branch_regex.search(target_branch))
 
             branch_matches = source_branch_match and target_branch_match
 
         files_matches = True
         if self.file_pattern != '*':
             files_matches = False
             if self.file_pattern.startswith('re:'):
                 file_pattern = self.file_pattern[3:]
             else:
                 file_pattern = glob2re(self.file_pattern)
             file_regex = re.compile(file_pattern)
             for file_data in files_changed:
                 filename = file_data.get('filename')
 
                 if file_regex.search(filename):
                     files_matches = True
                     break
 
         return branch_matches and files_matches
 
     @property
     def review_users(self):
         """ Returns the users which this rule applies to """
 
         users = collections.OrderedDict()
 
         for rule_user in self.rule_users:
             if rule_user.user.active:
                 if rule_user.user not in users:
                     users[rule_user.user.username] = {
                         'user': rule_user.user,
                         'source': 'user',
                         'source_data': {},
                         'data': rule_user.rule_data()
                     }
 
         for rule_user_group in self.rule_user_groups:
             source_data = {
                 'user_group_id': rule_user_group.users_group.users_group_id,
                 'name': rule_user_group.users_group.users_group_name,
                 'members': len(rule_user_group.users_group.members)
             }
             for member in rule_user_group.users_group.members:
                 if member.user.active:
                     key = member.user.username
                     if key in users:
                         # skip this member as we have him already
                         # this prevents from override the "first" matched
                         # users with duplicates in multiple groups
                         continue
 
                     users[key] = {
                         'user': member.user,
                         'source': 'user_group',
                         'source_data': source_data,
                         'data': rule_user_group.rule_data()
                     }
 
         return users
 
     def user_group_vote_rule(self, user_id):
 
         rules = []
         if not self.rule_user_groups:
             return rules
 
         for user_group in self.rule_user_groups:
             user_group_members = [x.user_id for x in user_group.users_group.members]
             if user_id in user_group_members:
                 rules.append(user_group)
         return rules
 
     def __repr__(self):
         return f'<RepoReviewerRule(id={self.repo_review_rule_id}, repo={self.repo!r})>'
 
 
 class ScheduleEntry(Base, BaseModel):
     __tablename__ = 'schedule_entries'
     __table_args__ = (
         UniqueConstraint('schedule_name', name='s_schedule_name_idx'),
         UniqueConstraint('task_uid', name='s_task_uid_idx'),
         base_table_args,
     )
     SCHEDULE_TYPE_INTEGER = "integer"
     SCHEDULE_TYPE_CRONTAB = "crontab"
 
     schedule_types = [SCHEDULE_TYPE_CRONTAB, SCHEDULE_TYPE_INTEGER]
     schedule_entry_id = Column('schedule_entry_id', Integer(), primary_key=True)
 
     schedule_name = Column("schedule_name", String(255), nullable=False, unique=None, default=None)
     schedule_description = Column("schedule_description", String(10000), nullable=True, unique=None, default=None)
     schedule_enabled = Column("schedule_enabled", Boolean(), nullable=False, unique=None, default=True)
 
     _schedule_type = Column("schedule_type", String(255), nullable=False, unique=None, default=None)
     schedule_definition = Column('schedule_definition_json', MutationObj.as_mutable(JsonType(default=lambda: "", dialect_map=dict(mysql=LONGTEXT()))))
 
     schedule_last_run = Column('schedule_last_run', DateTime(timezone=False), nullable=True, unique=None, default=None)
     schedule_total_run_count = Column('schedule_total_run_count', Integer(), nullable=True, unique=None, default=0)
 
     # task
     task_uid = Column("task_uid", String(255), nullable=False, unique=None, default=None)
     task_dot_notation = Column("task_dot_notation", String(4096), nullable=False, unique=None, default=None)
     task_args = Column('task_args_json', MutationObj.as_mutable(JsonType(default=list, dialect_map=dict(mysql=LONGTEXT()))))
     task_kwargs = Column('task_kwargs_json', MutationObj.as_mutable(JsonType(default=dict, dialect_map=dict(mysql=LONGTEXT()))))
 
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     updated_on = Column('updated_on', DateTime(timezone=False), nullable=True, unique=None, default=None)
 
     @hybrid_property
     def schedule_type(self):
         return self._schedule_type
 
     @schedule_type.setter
     def schedule_type(self, val):
         if val not in self.schedule_types:
             raise ValueError(f'Value must be on of `{val}` and got `{self.schedule_type}`')
 
         self._schedule_type = val
 
     @classmethod
     def get_uid(cls, obj):
         args = obj.task_args
         kwargs = obj.task_kwargs
 
         if isinstance(args, JsonRaw):
             try:
                 args = json.loads(str(args))
             except ValueError:
                 log.exception('json.loads of args failed...')
                 args = tuple()
 
         if isinstance(kwargs, JsonRaw):
             try:
                 kwargs = json.loads(str(kwargs))
             except ValueError:
                 log.exception('json.loads of kwargs failed...')
                 kwargs = dict()
 
         dot_notation = obj.task_dot_notation
         val = '.'.join(map(safe_str, [dot_notation, args, sorted(kwargs.items())]))
         log.debug('calculating task uid using id:`%s`', val)
 
         return sha1(safe_bytes(val))
 
     @classmethod
     def get_by_schedule_name(cls, schedule_name):
         return cls.query().filter(cls.schedule_name == schedule_name).scalar()
 
     @classmethod
     def get_by_schedule_id(cls, schedule_id):
         return cls.query().filter(cls.schedule_entry_id == schedule_id).scalar()
 
     @classmethod
     def get_by_task_uid(cls, task_uid):
         return cls.query().filter(cls.task_uid == task_uid).scalar()
 
     @property
     def task(self):
         return self.task_dot_notation
 
     @property
     def schedule(self):
         from rhodecode.lib.celerylib.utils import raw_2_schedule
         schedule = raw_2_schedule(self.schedule_definition, self.schedule_type)
         return schedule
 
     @property
     def args(self):
         try:
             return list(self.task_args or [])
         except ValueError:
             return list()
 
     @property
     def kwargs(self):
         try:
             return dict(self.task_kwargs or {})
         except ValueError:
             return dict()
 
     def _as_raw(self, val, indent=False):
         if hasattr(val, 'de_coerce'):
             val = val.de_coerce()
             if val:
                 if indent:
                     val = ext_json.formatted_str_json(val)
                 else:
                     val = ext_json.str_json(val)
 
         return val
 
     @property
     def schedule_definition_raw(self):
         return self._as_raw(self.schedule_definition)
 
     def args_raw(self, indent=False):
         return self._as_raw(self.task_args, indent)
 
     def kwargs_raw(self, indent=False):
         return self._as_raw(self.task_kwargs, indent)
 
     def __repr__(self):
         return f'<DB:ScheduleEntry({self.schedule_entry_id}:{self.schedule_name})>'
 
 
 @event.listens_for(ScheduleEntry, 'before_update')
 def update_task_uid(mapper, connection, target):
     target.task_uid = ScheduleEntry.get_uid(target)
 
 
 @event.listens_for(ScheduleEntry, 'before_insert')
 def set_task_uid(mapper, connection, target):
     target.task_uid = ScheduleEntry.get_uid(target)
 
 
 class _BaseBranchPerms(BaseModel):
     @classmethod
     def compute_hash(cls, value):
         return sha1_safe(value)
 
     @hybrid_property
     def branch_pattern(self):
         return self._branch_pattern or '*'
 
     @hybrid_property
     def branch_hash(self):
         return self._branch_hash
 
     def _validate_glob(self, value):
         re.compile('^' + glob2re(value) + '$')
 
     @branch_pattern.setter
     def branch_pattern(self, value):
         self._validate_glob(value)
         self._branch_pattern = value or '*'
         # set the Hash when setting the branch pattern
         self._branch_hash = self.compute_hash(self._branch_pattern)
 
     def matches(self, branch):
         """
         Check if this the branch matches entry
 
         :param branch: branch name for the commit
         """
 
         branch = branch or ''
 
         branch_matches = True
         if branch:
             branch_regex = re.compile('^' + glob2re(self.branch_pattern) + '$')
             branch_matches = bool(branch_regex.search(branch))
 
         return branch_matches
 
 
 class UserToRepoBranchPermission(Base, _BaseBranchPerms):
     __tablename__ = 'user_to_repo_branch_permissions'
     __table_args__ = (
         base_table_args
     )
 
     branch_rule_id = Column('branch_rule_id', Integer(), primary_key=True)
 
     repository_id = Column('repository_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
     repo = relationship('Repository', back_populates='user_branch_perms')
 
     permission_id = Column('permission_id', Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     permission = relationship('Permission')
 
     rule_to_perm_id = Column('rule_to_perm_id', Integer(), ForeignKey('repo_to_perm.repo_to_perm_id'), nullable=False, unique=None, default=None)
     user_repo_to_perm = relationship('UserRepoToPerm', back_populates='branch_perm_entry')
 
     rule_order = Column('rule_order', Integer(), nullable=False)
     _branch_pattern = Column('branch_pattern', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), default='*')  # glob
     _branch_hash = Column('branch_hash', UnicodeText().with_variant(UnicodeText(2048), 'mysql'))
 
     def __repr__(self):
         return f'<UserBranchPermission({self.user_repo_to_perm} => {self.branch_pattern!r})>'
 
 
 class UserGroupToRepoBranchPermission(Base, _BaseBranchPerms):
     __tablename__ = 'user_group_to_repo_branch_permissions'
     __table_args__ = (
         base_table_args
     )
 
     branch_rule_id = Column('branch_rule_id', Integer(), primary_key=True)
 
     repository_id = Column('repository_id', Integer(), ForeignKey('repositories.repo_id'), nullable=False, unique=None, default=None)
     repo = relationship('Repository', back_populates='user_group_branch_perms')
 
     permission_id = Column('permission_id', Integer(), ForeignKey('permissions.permission_id'), nullable=False, unique=None, default=None)
     permission = relationship('Permission')
 
     rule_to_perm_id = Column('rule_to_perm_id', Integer(), ForeignKey('users_group_repo_to_perm.users_group_to_perm_id'), nullable=False, unique=None, default=None)
     user_group_repo_to_perm = relationship('UserGroupRepoToPerm', back_populates='user_group_branch_perms')
 
     rule_order = Column('rule_order', Integer(), nullable=False)
     _branch_pattern = Column('branch_pattern', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), default='*')  # glob
     _branch_hash = Column('branch_hash', UnicodeText().with_variant(UnicodeText(2048), 'mysql'))
 
     def __repr__(self):
         return f'<UserBranchPermission({self.user_group_repo_to_perm} => {self.branch_pattern!r})>'
 
 
 class UserBookmark(Base, BaseModel):
     __tablename__ = 'user_bookmarks'
     __table_args__ = (
         UniqueConstraint('user_id', 'bookmark_repo_id'),
         UniqueConstraint('user_id', 'bookmark_repo_group_id'),
         UniqueConstraint('user_id', 'bookmark_position'),
         base_table_args
     )
 
     user_bookmark_id = Column("user_bookmark_id", Integer(), nullable=False, unique=True, default=None, primary_key=True)
     user_id = Column("user_id", Integer(), ForeignKey('users.user_id'), nullable=False, unique=None, default=None)
     position = Column("bookmark_position", Integer(), nullable=False)
     title = Column("bookmark_title", String(255), nullable=True, unique=None, default=None)
     redirect_url = Column("bookmark_redirect_url", String(10240), nullable=True, unique=None, default=None)
     created_on = Column("created_on", DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
 
     bookmark_repo_id = Column("bookmark_repo_id", Integer(), ForeignKey("repositories.repo_id"), nullable=True, unique=None, default=None)
     bookmark_repo_group_id = Column("bookmark_repo_group_id", Integer(), ForeignKey("groups.group_id"), nullable=True, unique=None, default=None)
 
     user = relationship("User")
 
     repository = relationship("Repository")
     repository_group = relationship("RepoGroup")
 
     @classmethod
     def get_by_position_for_user(cls, position, user_id):
         return cls.query() \
             .filter(UserBookmark.user_id == user_id) \
             .filter(UserBookmark.position == position).scalar()
 
     @classmethod
     def get_bookmarks_for_user(cls, user_id, cache=True):
         bookmarks = select(
             UserBookmark.title,
             UserBookmark.position,
             ) \
             .add_columns(Repository.repo_id, Repository.repo_type, Repository.repo_name) \
             .add_columns(RepoGroup.group_id, RepoGroup.group_name) \
             .where(UserBookmark.user_id == user_id) \
             .outerjoin(Repository, Repository.repo_id == UserBookmark.bookmark_repo_id) \
             .outerjoin(RepoGroup, RepoGroup.group_id == UserBookmark.bookmark_repo_group_id) \
             .order_by(UserBookmark.position.asc())
 
         if cache:
             bookmarks = bookmarks.options(
                 FromCache("sql_cache_short", f"get_user_{user_id}_bookmarks")
             )
 
         return Session().execute(bookmarks).all()
 
     def __repr__(self):
         return f'<UserBookmark({self.position} @ {self.redirect_url!r})>'
 
 
 class FileStore(Base, BaseModel):
     __tablename__ = 'file_store'
     __table_args__ = (
         base_table_args
     )
 
     file_store_id = Column('file_store_id', Integer(), primary_key=True)
     file_uid = Column('file_uid', String(1024), nullable=False)
     file_display_name = Column('file_display_name', UnicodeText().with_variant(UnicodeText(2048), 'mysql'), nullable=True)
     file_description = Column('file_description', UnicodeText().with_variant(UnicodeText(10240), 'mysql'), nullable=True)
     file_org_name = Column('file_org_name', UnicodeText().with_variant(UnicodeText(10240), 'mysql'), nullable=False)
 
     # sha256 hash
     file_hash = Column('file_hash', String(512), nullable=False)
     file_size = Column('file_size', BigInteger(), nullable=False)
 
     created_on = Column('created_on', DateTime(timezone=False), nullable=False, default=datetime.datetime.now)
     accessed_on = Column('accessed_on', DateTime(timezone=False), nullable=True)
     accessed_count = Column('accessed_count', Integer(), default=0)
 
     enabled = Column('enabled', Boolean(), nullable=False, default=True)
 
     # if repo/repo_group reference is set, check for permissions
     check_acl = Column('check_acl', Boolean(), nullable=False, default=True)
 
     # hidden defines an attachment that should be hidden from showing in artifact listing
     hidden = Column('hidden', Boolean(), nullable=False, default=False)
 
     user_id = Column('user_id', Integer(), ForeignKey('users.user_id'), nullable=False)
     upload_user = relationship('User', lazy='joined', primaryjoin='User.user_id==FileStore.user_id', back_populates='artifacts')
 
     file_metadata = relationship('FileStoreMetadata', lazy='joined')
 
     # scope limited to user, which requester have access to
     scope_user_id = Column(
         'scope_user_id', Integer(), ForeignKey('users.user_id'),
         nullable=True, unique=None, default=None)
     user = relationship('User', lazy='joined', primaryjoin='User.user_id==FileStore.scope_user_id', back_populates='scope_artifacts')
 
     # scope limited to user group, which requester have access to
     scope_user_group_id = Column(
         'scope_user_group_id', Integer(), ForeignKey('users_groups.users_group_id'),
         nullable=True, unique=None, default=None)
     user_group = relationship('UserGroup', lazy='joined')
 
     # scope limited to repo, which requester have access to
     scope_repo_id = Column(
         'scope_repo_id', Integer(), ForeignKey('repositories.repo_id'),
         nullable=True, unique=None, default=None)
     repo = relationship('Repository', lazy='joined')
 
     # scope limited to repo group, which requester have access to
     scope_repo_group_id = Column(
         'scope_repo_group_id', Integer(), ForeignKey('groups.group_id'),
         nullable=True, unique=None, default=None)
     repo_group = relationship('RepoGroup', lazy='joined')
 
     @classmethod
     def get_scope(cls, scope_type, scope_id):
         if scope_type == 'repo':
             return f'repo:{scope_id}'
         elif scope_type == 'repo-group':
             return f'repo-group:{scope_id}'
         elif scope_type == 'user':
             return f'user:{scope_id}'
         elif scope_type == 'user-group':
             return f'user-group:{scope_id}'
         else:
             return scope_type
 
     @classmethod
     def get_by_store_uid(cls, file_store_uid, safe=False):
         if safe:
             return FileStore.query().filter(FileStore.file_uid == file_store_uid).first()
         else:
             return FileStore.query().filter(FileStore.file_uid == file_store_uid).scalar()
 
     @classmethod
     def create(cls, file_uid, filename, file_hash, file_size, file_display_name='',
                file_description='', enabled=True, hidden=False, check_acl=True,
                user_id=None, scope_user_id=None, scope_repo_id=None, scope_repo_group_id=None):
 
         store_entry = FileStore()
         store_entry.file_uid = file_uid
         store_entry.file_display_name = file_display_name
         store_entry.file_org_name = filename
         store_entry.file_size = file_size
         store_entry.file_hash = file_hash
         store_entry.file_description = file_description
 
         store_entry.check_acl = check_acl
         store_entry.enabled = enabled
         store_entry.hidden = hidden
 
         store_entry.user_id = user_id
         store_entry.scope_user_id = scope_user_id
         store_entry.scope_repo_id = scope_repo_id
         store_entry.scope_repo_group_id = scope_repo_group_id
 
         return store_entry
 
     @classmethod
     def store_metadata(cls, file_store_id, args, commit=True):
         file_store = FileStore.get(file_store_id)
         if file_store is None:
             return
 
         for section, key, value, value_type in args:
             has_key = FileStoreMetadata().query() \
                 .filter(FileStoreMetadata.file_store_id == file_store.file_store_id) \
                 .filter(FileStoreMetadata.file_store_meta_section == section) \
                 .filter(FileStoreMetadata.file_store_meta_key == key) \
                 .scalar()
             if has_key:
-                msg = 'key `{}` already defined under section `{}` for this file.'\
-                    .format(key, section)
+                msg = f'key `{key}` already defined under section `{section}` for this file.'
                 raise ArtifactMetadataDuplicate(msg, err_section=section, err_key=key)
 
             # NOTE(marcink): raises ArtifactMetadataBadValueType
             FileStoreMetadata.valid_value_type(value_type)
 
             meta_entry = FileStoreMetadata()
             meta_entry.file_store = file_store
             meta_entry.file_store_meta_section = section
             meta_entry.file_store_meta_key = key
             meta_entry.file_store_meta_value_type = value_type
             meta_entry.file_store_meta_value = value
 
             Session().add(meta_entry)
 
         try:
             if commit:
                 Session().commit()
         except IntegrityError:
             Session().rollback()
             raise ArtifactMetadataDuplicate('Duplicate section/key found for this file.')
 
     @classmethod
     def bump_access_counter(cls, file_uid, commit=True):
         FileStore().query()\
             .filter(FileStore.file_uid == file_uid)\
             .update({FileStore.accessed_count: (FileStore.accessed_count + 1),
                      FileStore.accessed_on: datetime.datetime.now()})
         if commit:
             Session().commit()
 
     def __json__(self):
         data = {
             'filename': self.file_display_name,
             'filename_org': self.file_org_name,
             'file_uid': self.file_uid,
             'description': self.file_description,
             'hidden': self.hidden,
             'size': self.file_size,
             'created_on': self.created_on,
             'uploaded_by': self.upload_user.get_api_data(details='basic'),
             'downloaded_times': self.accessed_count,
             'sha256': self.file_hash,
             'metadata': self.file_metadata,
         }
 
         return data
 
     def __repr__(self):
         return f'<FileStore({self.file_store_id})>'
 
 
 class FileStoreMetadata(Base, BaseModel):
     __tablename__ = 'file_store_metadata'
     __table_args__ = (
         UniqueConstraint('file_store_id', 'file_store_meta_section_hash', 'file_store_meta_key_hash'),
         Index('file_store_meta_section_idx', 'file_store_meta_section', mysql_length=255),
         Index('file_store_meta_key_idx', 'file_store_meta_key', mysql_length=255),
         base_table_args
     )
     SETTINGS_TYPES = {
         'str': safe_str,
         'int': safe_int,
         'unicode': safe_str,
         'bool': str2bool,
         'list': functools.partial(aslist, sep=',')
     }
 
     file_store_meta_id = Column(
         "file_store_meta_id", Integer(), nullable=False, unique=True, default=None,
         primary_key=True)
     _file_store_meta_section = Column(
         "file_store_meta_section", UnicodeText().with_variant(UnicodeText(1024), 'mysql'),
         nullable=True, unique=None, default=None)
     _file_store_meta_section_hash = Column(
         "file_store_meta_section_hash", String(255),
         nullable=True, unique=None, default=None)
     _file_store_meta_key = Column(
         "file_store_meta_key", UnicodeText().with_variant(UnicodeText(1024), 'mysql'),
         nullable=True, unique=None, default=None)
     _file_store_meta_key_hash = Column(
         "file_store_meta_key_hash", String(255), nullable=True, unique=None, default=None)
     _file_store_meta_value = Column(
         "file_store_meta_value", UnicodeText().with_variant(UnicodeText(20480), 'mysql'),
         nullable=True, unique=None, default=None)
     _file_store_meta_value_type = Column(
         "file_store_meta_value_type", String(255), nullable=True, unique=None,
         default='unicode')
 
     file_store_id = Column(
         'file_store_id', Integer(), ForeignKey('file_store.file_store_id'),
         nullable=True, unique=None, default=None)
 
     file_store = relationship('FileStore', lazy='joined', viewonly=True)
 
     @classmethod
     def valid_value_type(cls, value):
         if value.split('.')[0] not in cls.SETTINGS_TYPES:
             raise ArtifactMetadataBadValueType(
-                'value_type must be one of %s got %s' % (cls.SETTINGS_TYPES.keys(), value))
+                f'value_type must be one of {cls.SETTINGS_TYPES.keys()} got {value}')
 
     @hybrid_property
     def file_store_meta_section(self):
         return self._file_store_meta_section
 
     @file_store_meta_section.setter
     def file_store_meta_section(self, value):
         self._file_store_meta_section = value
         self._file_store_meta_section_hash = _hash_key(value)
 
     @hybrid_property
     def file_store_meta_key(self):
         return self._file_store_meta_key
 
     @file_store_meta_key.setter
     def file_store_meta_key(self, value):
         self._file_store_meta_key = value
         self._file_store_meta_key_hash = _hash_key(value)
 
     @hybrid_property
     def file_store_meta_value(self):
         val = self._file_store_meta_value
 
         if self._file_store_meta_value_type:
             # e.g unicode.encrypted == unicode
             _type = self._file_store_meta_value_type.split('.')[0]
             # decode the encrypted value if it's encrypted field type
             if '.encrypted' in self._file_store_meta_value_type:
                 cipher = EncryptedTextValue()
                 val = safe_str(cipher.process_result_value(val, None))
             # do final type conversion
             converter = self.SETTINGS_TYPES.get(_type) or self.SETTINGS_TYPES['unicode']
             val = converter(val)
 
         return val
 
     @file_store_meta_value.setter
     def file_store_meta_value(self, val):
         val = safe_str(val)
         # encode the encrypted value
         if '.encrypted' in self.file_store_meta_value_type:
             cipher = EncryptedTextValue()
             val = safe_str(cipher.process_bind_param(val, None))
         self._file_store_meta_value = val
 
     @hybrid_property
     def file_store_meta_value_type(self):
         return self._file_store_meta_value_type
 
     @file_store_meta_value_type.setter
     def file_store_meta_value_type(self, val):
         # e.g unicode.encrypted
         self.valid_value_type(val)
         self._file_store_meta_value_type = val
 
     def __json__(self):
         data = {
             'artifact': self.file_store.file_uid,
             'section': self.file_store_meta_section,
             'key': self.file_store_meta_key,
             'value': self.file_store_meta_value,
         }
 
         return data
 
     def __repr__(self):
         return '<%s[%s]%s=>%s]>' % (self.cls_name, self.file_store_meta_section,
                                     self.file_store_meta_key, self.file_store_meta_value)
 
 
 class DbMigrateVersion(Base, BaseModel):
     __tablename__ = 'db_migrate_version'
     __table_args__ = (
         base_table_args,
     )
 
     repository_id = Column('repository_id', String(250), primary_key=True)
     repository_path = Column('repository_path', Text)
     version = Column('version', Integer)
 
     @classmethod
     def set_version(cls, version):
         """
         Helper for forcing a different version, usually for debugging purposes via ishell.
         """
         ver = DbMigrateVersion.query().first()
         ver.version = version
         Session().commit()
 
 
 class DbSession(Base, BaseModel):
     __tablename__ = 'db_session'
     __table_args__ = (
         base_table_args,
     )
 
     def __repr__(self):
         return f'<DB:DbSession({self.id})>'
 
     id = Column('id', Integer())
     namespace = Column('namespace', String(255), primary_key=True)
     accessed = Column('accessed', DateTime, nullable=False)
     created = Column('created', DateTime, nullable=False)
     data = Column('data', PickleType, nullable=False)

rhodecode/model/forms.py

@@ -1,655 +1,662 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 this is forms validation classes
 http://formencode.org/module-formencode.validators.html
 for list off all availible validators
 
 we can create our own validators
 
 The table below outlines the options which can be used in a schema in addition to the validators themselves
 pre_validators          []     These validators will be applied before the schema
 chained_validators      []     These validators will be applied after the schema
 allow_extra_fields      False     If True, then it is not an error when keys that aren't associated with a validator are present
 filter_extra_fields     False     If True, then keys that aren't associated with a validator are removed
 if_key_missing          NoDefault If this is given, then any keys that aren't available but are expected will be replaced with this value (and then validated). This does not override a present .if_missing attribute on validators. NoDefault is a special FormEncode class to mean that no default values has been specified and therefore missing keys shouldn't take a default value.
 ignore_key_missing      False     If True, then missing keys will be missing in the result, if the validator doesn't have .if_missing on it already
 
 
 <name> = formencode.validators.<name of validator>
 <name> must equal form name
 list=[1,2,3,4,5]
 for SELECT use formencode.All(OneOf(list), Int())
 
 """
 
 import deform
 import logging
 import formencode
 
 from pkg_resources import resource_filename
 from formencode import All, Pipe
 
 from pyramid.threadlocal import get_current_request
 
 from rhodecode import BACKENDS
 from rhodecode.lib import helpers
 from rhodecode.model import validators as v
 
 log = logging.getLogger(__name__)
 
 
 deform_templates = resource_filename('deform', 'templates')
 rhodecode_templates = resource_filename('rhodecode', 'templates/forms')
 search_path = (rhodecode_templates, deform_templates)
 
 
 class RhodecodeFormZPTRendererFactory(deform.ZPTRendererFactory):
     """ Subclass of ZPTRendererFactory to add rhodecode context variables """
     def __call__(self, template_name, **kw):
         kw['h'] = helpers
         kw['request'] = get_current_request()
         return self.load(template_name)(**kw)
 
 
 form_renderer = RhodecodeFormZPTRendererFactory(search_path)
 deform.Form.set_default_renderer(form_renderer)
 
 
 def LoginForm(localizer):
     _ = localizer
 
     class _LoginForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         username = v.UnicodeString(
             strip=True,
             min=1,
             not_empty=True,
             messages={
                 'empty': _('Please enter a login'),
                 'tooShort': _('Enter a value %(min)i characters long or more')
             }
         )
 
         password = v.UnicodeString(
             strip=False,
             min=3,
             max=72,
             not_empty=True,
             messages={
                 'empty': _('Please enter a password'),
                 'tooShort': _('Enter %(min)i characters or more')}
         )
 
         remember = v.StringBoolean(if_missing=False)
 
         chained_validators = [v.ValidAuth(localizer)]
     return _LoginForm
 
 
 def TOTPForm(localizer, user, allow_recovery_code_use=False):
     _ = localizer
 
     class _TOTPForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         totp = v.Regex(r'^(?:\d{6}|[A-Z0-9]{32})$')
         secret_totp = v.String()
 
         def to_python(self, value, state=None):
             validation_checks = [user.is_totp_valid]
             if allow_recovery_code_use:
                 validation_checks.append(user.is_2fa_recovery_code_valid)
             form_data = super().to_python(value, state)
             received_code = form_data['totp']
             secret = form_data.get('secret_totp')
 
             if not any(map(lambda func: func(received_code, secret), validation_checks)):
                 error_msg = _('Code is invalid. Try again!')
                 raise formencode.Invalid(error_msg, v, state, error_dict={'totp': error_msg})
             return form_data
 
     return _TOTPForm
 
 
+def WhitelistedVcsClientsForm(localizer):
+    _ = localizer
+
+    class _WhitelistedVcsClientsForm(formencode.Schema):
+        regexp = r'^(?:\s*[<>=~^!]*\s*\d{1,2}\.\d{1,2}(?:\.\d{1,2})?\s*|\*)\s*(?:,\s*[<>=~^!]*\s*\d{1,2}\.\d{1,2}(?:\.\d{1,2})?\s*|\s*\*\s*)*$'
+        allow_extra_fields = True
+        filter_extra_fields = True
+        git = v.Regex(regexp)
+        hg = v.Regex(regexp)
+        svn = v.Regex(regexp)
+
+    return _WhitelistedVcsClientsForm
+
+
 def UserForm(localizer, edit=False, available_languages=None, old_data=None):
     old_data = old_data or {}
     available_languages = available_languages or []
     _ = localizer
 
     class _UserForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         username = All(v.UnicodeString(strip=True, min=1, not_empty=True),
                        v.ValidUsername(localizer, edit, old_data))
         if edit:
             new_password = All(
                 v.ValidPassword(localizer),
                 v.UnicodeString(strip=False, min=6, max=72, not_empty=False)
             )
             password_confirmation = All(
                 v.ValidPassword(localizer),
                 v.UnicodeString(strip=False, min=6, max=72, not_empty=False),
             )
             admin = v.StringBoolean(if_missing=False)
         else:
             password = All(
                 v.ValidPassword(localizer),
                 v.UnicodeString(strip=False, min=6, max=72, not_empty=True)
             )
             password_confirmation = All(
                 v.ValidPassword(localizer),
                 v.UnicodeString(strip=False, min=6, max=72, not_empty=False)
             )
 
         password_change = v.StringBoolean(if_missing=False)
         create_repo_group = v.StringBoolean(if_missing=False)
 
         active = v.StringBoolean(if_missing=False)
         firstname = v.UnicodeString(strip=True, min=1, not_empty=False)
         lastname = v.UnicodeString(strip=True, min=1, not_empty=False)
         email = All(v.UniqSystemEmail(localizer, old_data), v.Email(not_empty=True))
         description = v.UnicodeString(strip=True, min=1, max=250, not_empty=False,
                                       if_missing='')
         extern_name = v.UnicodeString(strip=True)
         extern_type = v.UnicodeString(strip=True)
         language = v.OneOf(available_languages, hideList=False,
                            testValueList=True, if_missing=None)
         chained_validators = [v.ValidPasswordsMatch(localizer)]
     return _UserForm
 
 
 def UserGroupForm(localizer, edit=False, old_data=None, allow_disabled=False):
     old_data = old_data or {}
     _ = localizer
 
     class _UserGroupForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
 
         users_group_name = All(
             v.UnicodeString(strip=True, min=1, not_empty=True),
             v.ValidUserGroup(localizer, edit, old_data)
         )
         user_group_description = v.UnicodeString(strip=True, min=1,
                                                  not_empty=False)
 
         users_group_active = v.StringBoolean(if_missing=False)
 
         if edit:
             # this is user group owner
             user = All(
                 v.UnicodeString(not_empty=True),
                 v.ValidRepoUser(localizer, allow_disabled))
     return _UserGroupForm
 
 
 def RepoGroupForm(localizer, edit=False, old_data=None, available_groups=None,
                   can_create_in_root=False, allow_disabled=False):
     _ = localizer
     old_data = old_data or {}
     available_groups = available_groups or []
 
     class _RepoGroupForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
 
         group_name = All(v.UnicodeString(strip=True, min=1, not_empty=True),
                          v.SlugifyName(localizer),)
         group_description = v.UnicodeString(strip=True, min=1,
                                             not_empty=False)
         group_copy_permissions = v.StringBoolean(if_missing=False)
 
         group_parent_id = v.OneOf(available_groups, hideList=False,
                                   testValueList=True, not_empty=True)
         enable_locking = v.StringBoolean(if_missing=False)
         chained_validators = [
             v.ValidRepoGroup(localizer, edit, old_data, can_create_in_root)]
 
         if edit:
             # this is repo group owner
             user = All(
                 v.UnicodeString(not_empty=True),
                 v.ValidRepoUser(localizer, allow_disabled))
     return _RepoGroupForm
 
 
 def RegisterForm(localizer, edit=False, old_data=None):
     _ = localizer
     old_data = old_data or {}
 
     class _RegisterForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         username = All(
             v.ValidUsername(localizer, edit, old_data),
             v.UnicodeString(strip=True, min=1, not_empty=True)
         )
         password = All(
             v.ValidPassword(localizer),
             v.UnicodeString(strip=False, min=6, max=72, not_empty=True)
         )
         password_confirmation = All(
             v.ValidPassword(localizer),
             v.UnicodeString(strip=False, min=6, max=72, not_empty=True)
         )
         active = v.StringBoolean(if_missing=False)
         firstname = v.UnicodeString(strip=True, min=1, not_empty=False)
         lastname = v.UnicodeString(strip=True, min=1, not_empty=False)
         email = All(v.UniqSystemEmail(localizer, old_data), v.Email(not_empty=True))
 
         chained_validators = [v.ValidPasswordsMatch(localizer)]
     return _RegisterForm
 
 
 def PasswordResetForm(localizer):
     _ = localizer
 
     class _PasswordResetForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         email = All(v.ValidSystemEmail(localizer), v.Email(not_empty=True))
     return _PasswordResetForm
 
 
 def RepoForm(localizer, edit=False, old_data=None, repo_groups=None, allow_disabled=False):
     _ = localizer
     old_data = old_data or {}
     repo_groups = repo_groups or []
     supported_backends = BACKENDS.keys()
 
     class _RepoForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         repo_name = All(v.UnicodeString(strip=True, min=1, not_empty=True),
                         v.SlugifyName(localizer), v.CannotHaveGitSuffix(localizer))
         repo_group = All(v.CanWriteGroup(localizer, old_data),
                          v.OneOf(repo_groups, hideList=True))
         repo_type = v.OneOf(supported_backends, required=False,
                             if_missing=old_data.get('repo_type'))
         repo_description = v.UnicodeString(strip=True, min=1, not_empty=False)
         repo_private = v.StringBoolean(if_missing=False)
         repo_copy_permissions = v.StringBoolean(if_missing=False)
         clone_uri = All(v.UnicodeString(strip=True, min=1, not_empty=False))
 
         repo_enable_statistics = v.StringBoolean(if_missing=False)
         repo_enable_downloads = v.StringBoolean(if_missing=False)
         repo_enable_locking = v.StringBoolean(if_missing=False)
 
         if edit:
             # this is repo owner
             user = All(
                 v.UnicodeString(not_empty=True),
                 v.ValidRepoUser(localizer, allow_disabled))
             clone_uri_change = v.UnicodeString(
                 not_empty=False, if_missing=v.Missing)
 
         chained_validators = [v.ValidCloneUri(localizer),
                               v.ValidRepoName(localizer, edit, old_data)]
     return _RepoForm
 
 
 def RepoPermsForm(localizer):
     _ = localizer
 
     class _RepoPermsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         chained_validators = [v.ValidPerms(localizer, type_='repo')]
     return _RepoPermsForm
 
 
 def RepoGroupPermsForm(localizer, valid_recursive_choices):
     _ = localizer
 
     class _RepoGroupPermsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         recursive = v.OneOf(valid_recursive_choices)
         chained_validators = [v.ValidPerms(localizer, type_='repo_group')]
     return _RepoGroupPermsForm
 
 
 def UserGroupPermsForm(localizer):
     _ = localizer
 
     class _UserPermsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         chained_validators = [v.ValidPerms(localizer, type_='user_group')]
     return _UserPermsForm
 
 
 def RepoFieldForm(localizer):
     _ = localizer
 
     class _RepoFieldForm(formencode.Schema):
         filter_extra_fields = True
         allow_extra_fields = True
 
         new_field_key = All(v.FieldKey(localizer),
                             v.UnicodeString(strip=True, min=3, not_empty=True))
         new_field_value = v.UnicodeString(not_empty=False, if_missing='')
         new_field_type = v.OneOf(['str', 'unicode', 'list', 'tuple'],
                                  if_missing='str')
         new_field_label = v.UnicodeString(not_empty=False)
         new_field_desc = v.UnicodeString(not_empty=False)
     return _RepoFieldForm
 
 
 def RepoForkForm(localizer, edit=False, old_data=None,
                  supported_backends=BACKENDS.keys(), repo_groups=None):
     _ = localizer
     old_data = old_data or {}
     repo_groups = repo_groups or []
 
     class _RepoForkForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         repo_name = All(v.UnicodeString(strip=True, min=1, not_empty=True),
                         v.SlugifyName(localizer))
         repo_group = All(v.CanWriteGroup(localizer, ),
                          v.OneOf(repo_groups, hideList=True))
         repo_type = All(v.ValidForkType(localizer, old_data), v.OneOf(supported_backends))
         description = v.UnicodeString(strip=True, min=1, not_empty=True)
         private = v.StringBoolean(if_missing=False)
         copy_permissions = v.StringBoolean(if_missing=False)
         fork_parent_id = v.UnicodeString()
         chained_validators = [v.ValidForkName(localizer, edit, old_data)]
     return _RepoForkForm
 
 
 def ApplicationSettingsForm(localizer):
     _ = localizer
 
     class _ApplicationSettingsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         rhodecode_title = v.UnicodeString(strip=True, max=40, not_empty=False)
         rhodecode_realm = v.UnicodeString(strip=True, min=1, not_empty=True)
         rhodecode_pre_code = v.UnicodeString(strip=True, min=1, not_empty=False)
         rhodecode_post_code = v.UnicodeString(strip=True, min=1, not_empty=False)
         rhodecode_captcha_public_key = v.UnicodeString(strip=True, min=1, not_empty=False)
         rhodecode_captcha_private_key = v.UnicodeString(strip=True, min=1, not_empty=False)
         rhodecode_create_personal_repo_group = v.StringBoolean(if_missing=False)
         rhodecode_personal_repo_group_pattern = v.UnicodeString(strip=True, min=1, not_empty=False)
     return _ApplicationSettingsForm
 
 
 def ApplicationVisualisationForm(localizer):
     from rhodecode.model.db import Repository
     _ = localizer
 
     class _ApplicationVisualisationForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         rhodecode_show_public_icon = v.StringBoolean(if_missing=False)
         rhodecode_show_private_icon = v.StringBoolean(if_missing=False)
         rhodecode_stylify_metatags = v.StringBoolean(if_missing=False)
 
         rhodecode_repository_fields = v.StringBoolean(if_missing=False)
         rhodecode_lightweight_journal = v.StringBoolean(if_missing=False)
         rhodecode_dashboard_items = v.Int(min=5, not_empty=True)
         rhodecode_admin_grid_items = v.Int(min=5, not_empty=True)
         rhodecode_show_version = v.StringBoolean(if_missing=False)
         rhodecode_use_gravatar = v.StringBoolean(if_missing=False)
         rhodecode_markup_renderer = v.OneOf(['markdown', 'rst'])
         rhodecode_gravatar_url = v.UnicodeString(min=3)
         rhodecode_clone_uri_tmpl = v.UnicodeString(not_empty=False, if_empty=Repository.DEFAULT_CLONE_URI)
         rhodecode_clone_uri_id_tmpl = v.UnicodeString(not_empty=False, if_empty=Repository.DEFAULT_CLONE_URI_ID)
         rhodecode_clone_uri_ssh_tmpl = v.UnicodeString(not_empty=False, if_empty=Repository.DEFAULT_CLONE_URI_SSH)
         rhodecode_support_url = v.UnicodeString()
         rhodecode_show_revision_number = v.StringBoolean(if_missing=False)
         rhodecode_show_sha_length = v.Int(min=4, not_empty=True)
     return _ApplicationVisualisationForm
 
 
 class _BaseVcsSettingsForm(formencode.Schema):
 
     allow_extra_fields = True
     filter_extra_fields = False
     hooks_changegroup_repo_size = v.StringBoolean(if_missing=False)
     hooks_changegroup_push_logger = v.StringBoolean(if_missing=False)
     hooks_outgoing_pull_logger = v.StringBoolean(if_missing=False)
 
     # PR/Code-review
     rhodecode_pr_merge_enabled = v.StringBoolean(if_missing=False)
     rhodecode_use_outdated_comments = v.StringBoolean(if_missing=False)
 
     # hg
     extensions_largefiles = v.StringBoolean(if_missing=False)
     extensions_evolve = v.StringBoolean(if_missing=False)
     phases_publish = v.StringBoolean(if_missing=False)
 
     rhodecode_hg_use_rebase_for_merging = v.StringBoolean(if_missing=False)
     rhodecode_hg_close_branch_before_merging = v.StringBoolean(if_missing=False)
 
     # git
     vcs_git_lfs_enabled = v.StringBoolean(if_missing=False)
     rhodecode_git_use_rebase_for_merging = v.StringBoolean(if_missing=False)
     rhodecode_git_close_branch_before_merging = v.StringBoolean(if_missing=False)
 
     # cache
     rhodecode_diff_cache = v.StringBoolean(if_missing=False)
 
 
 def ApplicationUiSettingsForm(localizer):
     _ = localizer
 
     class _ApplicationUiSettingsForm(_BaseVcsSettingsForm):
-        web_push_ssl = v.StringBoolean(if_missing=False)
-        largefiles_usercache = All(
-            v.ValidPath(localizer),
-            v.UnicodeString(strip=True, min=2, not_empty=True))
-        vcs_git_lfs_store_location = All(
-            v.ValidPath(localizer),
-            v.UnicodeString(strip=True, min=2, not_empty=True))
         extensions_hggit = v.StringBoolean(if_missing=False)
         new_svn_branch = v.ValidSvnPattern(localizer, section='vcs_svn_branch')
         new_svn_tag = v.ValidSvnPattern(localizer, section='vcs_svn_tag')
     return _ApplicationUiSettingsForm
 
 
 def RepoVcsSettingsForm(localizer, repo_name):
     _ = localizer
 
     class _RepoVcsSettingsForm(_BaseVcsSettingsForm):
         inherit_global_settings = v.StringBoolean(if_missing=False)
         new_svn_branch = v.ValidSvnPattern(localizer,
             section='vcs_svn_branch', repo_name=repo_name)
         new_svn_tag = v.ValidSvnPattern(localizer,
             section='vcs_svn_tag', repo_name=repo_name)
     return _RepoVcsSettingsForm
 
 
 def LabsSettingsForm(localizer):
     _ = localizer
 
     class _LabSettingsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
     return _LabSettingsForm
 
 
 def ApplicationPermissionsForm(
         localizer, register_choices, password_reset_choices,
         extern_activate_choices):
     _ = localizer
 
     class _DefaultPermissionsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
 
         anonymous = v.StringBoolean(if_missing=False)
         default_register = v.OneOf(register_choices)
         default_register_message = v.UnicodeString()
         default_password_reset = v.OneOf(password_reset_choices)
         default_extern_activate = v.OneOf(extern_activate_choices)
     return _DefaultPermissionsForm
 
 
 def ObjectPermissionsForm(localizer, repo_perms_choices, group_perms_choices,
                           user_group_perms_choices):
     _ = localizer
 
     class _ObjectPermissionsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         overwrite_default_repo = v.StringBoolean(if_missing=False)
         overwrite_default_group = v.StringBoolean(if_missing=False)
         overwrite_default_user_group = v.StringBoolean(if_missing=False)
 
         default_repo_perm = v.OneOf(repo_perms_choices)
         default_group_perm = v.OneOf(group_perms_choices)
         default_user_group_perm = v.OneOf(user_group_perms_choices)
 
     return _ObjectPermissionsForm
 
 
 def BranchPermissionsForm(localizer, branch_perms_choices):
     _ = localizer
 
     class _BranchPermissionsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         overwrite_default_branch = v.StringBoolean(if_missing=False)
         default_branch_perm = v.OneOf(branch_perms_choices)
 
     return _BranchPermissionsForm
 
 
 def UserPermissionsForm(localizer, create_choices, create_on_write_choices,
                         repo_group_create_choices, user_group_create_choices,
                         fork_choices, inherit_default_permissions_choices):
     _ = localizer
 
     class _DefaultPermissionsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
 
         anonymous = v.StringBoolean(if_missing=False)
 
         default_repo_create = v.OneOf(create_choices)
         default_repo_create_on_write = v.OneOf(create_on_write_choices)
         default_user_group_create = v.OneOf(user_group_create_choices)
         default_repo_group_create = v.OneOf(repo_group_create_choices)
         default_fork_create = v.OneOf(fork_choices)
         default_inherit_default_permissions = v.OneOf(inherit_default_permissions_choices)
     return _DefaultPermissionsForm
 
 
 def UserIndividualPermissionsForm(localizer):
     _ = localizer
 
     class _DefaultPermissionsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
 
         inherit_default_permissions = v.StringBoolean(if_missing=False)
     return _DefaultPermissionsForm
 
 
 def DefaultsForm(localizer, edit=False, old_data=None, supported_backends=BACKENDS.keys()):
     _ = localizer
     old_data = old_data or {}
 
     class _DefaultsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         default_repo_type = v.OneOf(supported_backends)
         default_repo_private = v.StringBoolean(if_missing=False)
         default_repo_enable_statistics = v.StringBoolean(if_missing=False)
         default_repo_enable_downloads = v.StringBoolean(if_missing=False)
         default_repo_enable_locking = v.StringBoolean(if_missing=False)
     return _DefaultsForm
 
 
 def AuthSettingsForm(localizer):
     _ = localizer
 
     class _AuthSettingsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
         auth_plugins = All(v.ValidAuthPlugins(localizer),
                            v.UniqueListFromString(localizer)(not_empty=True))
     return _AuthSettingsForm
 
 
 def UserExtraEmailForm(localizer):
     _ = localizer
 
     class _UserExtraEmailForm(formencode.Schema):
         email = All(v.UniqSystemEmail(localizer), v.Email(not_empty=True))
     return _UserExtraEmailForm
 
 
 def UserExtraIpForm(localizer):
     _ = localizer
 
     class _UserExtraIpForm(formencode.Schema):
         ip = v.ValidIp(localizer)(not_empty=True)
     return _UserExtraIpForm
 
 
 def PullRequestForm(localizer, repo_id):
     _ = localizer
 
     class ReviewerForm(formencode.Schema):
         user_id = v.Int(not_empty=True)
         reasons = All()
         rules = All(v.UniqueList(localizer, convert=int)())
         mandatory = v.StringBoolean()
         role = v.String(if_missing='reviewer')
 
     class ObserverForm(formencode.Schema):
         user_id = v.Int(not_empty=True)
         reasons = All()
         rules = All(v.UniqueList(localizer, convert=int)())
         mandatory = v.StringBoolean()
         role = v.String(if_missing='observer')
 
     class _PullRequestForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = True
 
         common_ancestor = v.UnicodeString(strip=True, required=True)
         source_repo = v.UnicodeString(strip=True, required=True)
         source_ref = v.UnicodeString(strip=True, required=True)
         target_repo = v.UnicodeString(strip=True, required=True)
         target_ref = v.UnicodeString(strip=True, required=True)
         revisions = All(#v.NotReviewedRevisions(localizer, repo_id)(),
                         v.UniqueList(localizer)(not_empty=True))
         review_members = formencode.ForEach(ReviewerForm())
         observer_members = formencode.ForEach(ObserverForm())
         pullrequest_title = v.UnicodeString(strip=True, required=True, min=1, max=255)
         pullrequest_desc = v.UnicodeString(strip=True, required=False)
         description_renderer = v.UnicodeString(strip=True, required=False)
 
     return _PullRequestForm
 
 
 def IssueTrackerPatternsForm(localizer):
     _ = localizer
 
     class _IssueTrackerPatternsForm(formencode.Schema):
         allow_extra_fields = True
         filter_extra_fields = False
         chained_validators = [v.ValidPattern(localizer)]
     return _IssueTrackerPatternsForm

rhodecode/model/notification.py

@@ -1,456 +1,458 b''
 # Copyright (C) 2011-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 """
 Model for notifications
 """
 
 import logging
 import traceback
 
 import premailer
 from pyramid.threadlocal import get_current_request
 from sqlalchemy.sql.expression import false, true
 
 import rhodecode
 from rhodecode.lib import helpers as h
 from rhodecode.model import BaseModel
 from rhodecode.model.db import Notification, User, UserNotification
 from rhodecode.model.meta import Session
 from rhodecode.translation import TranslationString
 
 log = logging.getLogger(__name__)
 
 
 class NotificationModel(BaseModel):
 
     cls = Notification
 
     def __get_notification(self, notification):
         if isinstance(notification, Notification):
             return notification
         elif isinstance(notification, int):
             return Notification.get(notification)
         else:
             if notification:
                 raise Exception('notification must be int or Instance'
                                 ' of Notification got %s' % type(notification))
 
     def create(
             self, created_by, notification_subject='', notification_body='',
             notification_type=Notification.TYPE_MESSAGE, recipients=None,
             mention_recipients=None, with_email=True, email_kwargs=None):
         """
 
         Creates notification of given type
 
         :param created_by: int, str or User instance. User who created this
             notification
         :param notification_subject: subject of notification itself,
             it will be generated automatically from notification_type if not specified
         :param notification_body: body of notification text
             it will be generated automatically from notification_type if not specified
         :param notification_type: type of notification, based on that we
             pick templates
         :param recipients: list of int, str or User objects, when None
             is given send to all admins
         :param mention_recipients: list of int, str or User objects,
             that were mentioned
         :param with_email: send email with this notification
         :param email_kwargs: dict with arguments to generate email
         """
 
         from rhodecode.lib.celerylib import tasks, run_task
 
         if recipients and not getattr(recipients, '__iter__', False):
             raise Exception('recipients must be an iterable object')
 
         if not (notification_subject and notification_body) and not notification_type:
             raise ValueError('notification_subject, and notification_body '
                              'cannot be empty when notification_type is not specified')
 
         created_by_obj = self._get_user(created_by)
 
         if not created_by_obj:
             raise Exception('unknown user %s' % created_by)
 
         # default MAIN body if not given
         email_kwargs = email_kwargs or {'body': notification_body}
         mention_recipients = mention_recipients or set()
 
         if recipients is None:
             # recipients is None means to all admins
             recipients_objs = User.query().filter(User.admin == true()).all()
             log.debug('sending notifications %s to admins: %s',
                       notification_type, recipients_objs)
         else:
             recipients_objs = set()
             for u in recipients:
                 obj = self._get_user(u)
                 if obj:
                     recipients_objs.add(obj)
                 else:  # we didn't find this user, log the error and carry on
                     log.error('cannot notify unknown user %r', u)
 
             if not recipients_objs:
                 raise Exception('no valid recipients specified')
 
             log.debug('sending notifications %s to %s',
                       notification_type, recipients_objs)
 
         # add mentioned users into recipients
         final_recipients = set(recipients_objs).union(mention_recipients)
 
-        (subject, email_body, email_body_plaintext) = \
-            EmailNotificationModel().render_email(notification_type, **email_kwargs)
+        # No need to render email if we are sending just notification
+        if with_email:
+            (subject, email_body, email_body_plaintext) = \
+                EmailNotificationModel().render_email(notification_type, **email_kwargs)
 
-        if not notification_subject:
-            notification_subject = subject
+            if not notification_subject:
+                notification_subject = subject
 
-        if not notification_body:
-            notification_body = email_body_plaintext
+            if not notification_body:
+                notification_body = email_body_plaintext
 
         notification = Notification.create(
             created_by=created_by_obj, subject=notification_subject,
             body=notification_body, recipients=final_recipients,
             type_=notification_type
         )
 
         if not with_email:  # skip sending email, and just create notification
             return notification
 
         # don't send email to person who created this comment
         rec_objs = set(recipients_objs).difference({created_by_obj})
 
         # now notify all recipients in question
 
         for recipient in rec_objs.union(mention_recipients):
             # inject current recipient
             email_kwargs['recipient'] = recipient
             email_kwargs['mention'] = recipient in mention_recipients
             (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
                 notification_type, **email_kwargs)
 
             extra_headers = None
             if 'thread_ids' in email_kwargs:
                 extra_headers = {'thread_ids': email_kwargs.pop('thread_ids')}
 
             log.debug('Creating notification email task for user:`%s`', recipient)
             task = run_task(tasks.send_email, recipient.email, subject,
                             email_body_plaintext, email_body, extra_headers=extra_headers)
             log.debug('Created email task: %s', task)
 
         return notification
 
     def delete(self, user, notification):
         # we don't want to remove actual notification just the assignment
         try:
             notification = self.__get_notification(notification)
             user = self._get_user(user)
             if notification and user:
                 obj = UserNotification.query()\
                     .filter(UserNotification.user == user)\
                     .filter(UserNotification.notification == notification)\
                     .one()
                 Session().delete(obj)
                 return True
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def get_for_user(self, user, filter_=None):
         """
         Get mentions for given user, filter them if filter dict is given
         """
         user = self._get_user(user)
 
         q = UserNotification.query()\
             .filter(UserNotification.user == user)\
             .join((
                 Notification, UserNotification.notification_id ==
                 Notification.notification_id))
         if filter_ == ['all']:
             q = q  # no filter
         elif filter_ == ['unread']:
             q = q.filter(UserNotification.read == false())
         elif filter_:
             q = q.filter(Notification.type_.in_(filter_))
 
         q = q.order_by(Notification.created_on.desc())
         return q
 
     def mark_read(self, user, notification):
         try:
             notification = self.__get_notification(notification)
             user = self._get_user(user)
             if notification and user:
                 obj = UserNotification.query()\
                     .filter(UserNotification.user == user)\
                     .filter(UserNotification.notification == notification)\
                     .one()
                 obj.read = True
                 Session().add(obj)
                 return True
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def mark_all_read_for_user(self, user, filter_=None):
         user = self._get_user(user)
         q = UserNotification.query()\
             .filter(UserNotification.user == user)\
             .filter(UserNotification.read == false())\
             .join((
                 Notification, UserNotification.notification_id ==
                 Notification.notification_id))
         if filter_ == ['unread']:
             q = q.filter(UserNotification.read == false())
         elif filter_:
             q = q.filter(Notification.type_.in_(filter_))
 
         # this is a little inefficient but sqlalchemy doesn't support
         # update on joined tables :(
         for obj in q.all():
             obj.read = True
             Session().add(obj)
 
     def get_unread_cnt_for_user(self, user):
         user = self._get_user(user)
         return UserNotification.query()\
             .filter(UserNotification.read == false())\
             .filter(UserNotification.user == user).count()
 
     def get_unread_for_user(self, user):
         user = self._get_user(user)
         return [x.notification for x in UserNotification.query()
                 .filter(UserNotification.read == false())
                 .filter(UserNotification.user == user).all()]
 
     def get_user_notification(self, user, notification):
         user = self._get_user(user)
         notification = self.__get_notification(notification)
 
         return UserNotification.query()\
             .filter(UserNotification.notification == notification)\
             .filter(UserNotification.user == user).scalar()
 
     def make_description(self, notification, translate, show_age=True):
         """
         Creates a human readable description based on properties
         of notification object
         """
         _ = translate
         _map = {
             notification.TYPE_CHANGESET_COMMENT: [
                 _('%(user)s commented on commit %(date_or_age)s'),
                 _('%(user)s commented on commit at %(date_or_age)s'),
                 ],
             notification.TYPE_MESSAGE: [
                 _('%(user)s sent message %(date_or_age)s'),
                 _('%(user)s sent message at %(date_or_age)s'),
                 ],
             notification.TYPE_MENTION: [
                 _('%(user)s mentioned you %(date_or_age)s'),
                 _('%(user)s mentioned you at %(date_or_age)s'),
                 ],
             notification.TYPE_REGISTRATION: [
                 _('%(user)s registered in RhodeCode %(date_or_age)s'),
                 _('%(user)s registered in RhodeCode at %(date_or_age)s'),
                 ],
             notification.TYPE_PULL_REQUEST: [
                 _('%(user)s opened new pull request %(date_or_age)s'),
                 _('%(user)s opened new pull request at %(date_or_age)s'),
                 ],
             notification.TYPE_PULL_REQUEST_UPDATE: [
                 _('%(user)s updated pull request %(date_or_age)s'),
                 _('%(user)s updated pull request at %(date_or_age)s'),
             ],
             notification.TYPE_PULL_REQUEST_COMMENT: [
                 _('%(user)s commented on pull request %(date_or_age)s'),
                 _('%(user)s commented on pull request at %(date_or_age)s'),
                 ],
         }
 
         templates = _map[notification.type_]
 
         if show_age:
             template = templates[0]
             date_or_age = h.age(notification.created_on)
             if translate:
                 date_or_age = translate(date_or_age)
 
             if isinstance(date_or_age, TranslationString):
                 date_or_age = date_or_age.interpolate()
 
         else:
             template = templates[1]
             date_or_age = h.format_date(notification.created_on)
 
         return template % {
             'user': notification.created_by_user.username,
             'date_or_age': date_or_age,
         }
 
 
 # Templates for Titles, that could be overwritten by rcextensions
 # Title of email for pull-request update
 EMAIL_PR_UPDATE_SUBJECT_TEMPLATE = ''
 # Title of email for request for pull request review
 EMAIL_PR_REVIEW_SUBJECT_TEMPLATE = ''
 
 # Title of email for general comment on pull request
 EMAIL_PR_COMMENT_SUBJECT_TEMPLATE = ''
 # Title of email for general comment which includes status change on pull request
 EMAIL_PR_COMMENT_STATUS_CHANGE_SUBJECT_TEMPLATE = ''
 # Title of email for inline comment on a file in pull request
 EMAIL_PR_COMMENT_FILE_SUBJECT_TEMPLATE = ''
 
 # Title of email for general comment on commit
 EMAIL_COMMENT_SUBJECT_TEMPLATE = ''
 # Title of email for general comment which includes status change on commit
 EMAIL_COMMENT_STATUS_CHANGE_SUBJECT_TEMPLATE = ''
 # Title of email for inline comment on a file in commit
 EMAIL_COMMENT_FILE_SUBJECT_TEMPLATE = ''
 
 import cssutils
 # hijack css utils logger and replace with ours
 log = logging.getLogger('rhodecode.cssutils.premailer')
 log.setLevel(logging.INFO)
 cssutils.log.setLog(log)
 
 
 class EmailNotificationModel(BaseModel):
     TYPE_COMMIT_COMMENT = Notification.TYPE_CHANGESET_COMMENT
     TYPE_REGISTRATION = Notification.TYPE_REGISTRATION
     TYPE_PULL_REQUEST = Notification.TYPE_PULL_REQUEST
     TYPE_PULL_REQUEST_COMMENT = Notification.TYPE_PULL_REQUEST_COMMENT
     TYPE_PULL_REQUEST_UPDATE = Notification.TYPE_PULL_REQUEST_UPDATE
     TYPE_MAIN = Notification.TYPE_MESSAGE
 
     TYPE_PASSWORD_RESET = 'password_reset'
     TYPE_PASSWORD_RESET_CONFIRMATION = 'password_reset_confirmation'
     TYPE_EMAIL_TEST = 'email_test'
     TYPE_EMAIL_EXCEPTION = 'exception'
     TYPE_UPDATE_AVAILABLE = 'update_available'
     TYPE_TEST = 'test'
 
     email_types = {
         TYPE_MAIN:
             'rhodecode:templates/email_templates/main.mako',
         TYPE_TEST:
             'rhodecode:templates/email_templates/test.mako',
         TYPE_EMAIL_EXCEPTION:
             'rhodecode:templates/email_templates/exception_tracker.mako',
         TYPE_UPDATE_AVAILABLE:
             'rhodecode:templates/email_templates/update_available.mako',
         TYPE_EMAIL_TEST:
             'rhodecode:templates/email_templates/email_test.mako',
         TYPE_REGISTRATION:
             'rhodecode:templates/email_templates/user_registration.mako',
         TYPE_PASSWORD_RESET:
             'rhodecode:templates/email_templates/password_reset.mako',
         TYPE_PASSWORD_RESET_CONFIRMATION:
             'rhodecode:templates/email_templates/password_reset_confirmation.mako',
         TYPE_COMMIT_COMMENT:
             'rhodecode:templates/email_templates/commit_comment.mako',
         TYPE_PULL_REQUEST:
             'rhodecode:templates/email_templates/pull_request_review.mako',
         TYPE_PULL_REQUEST_COMMENT:
             'rhodecode:templates/email_templates/pull_request_comment.mako',
         TYPE_PULL_REQUEST_UPDATE:
             'rhodecode:templates/email_templates/pull_request_update.mako',
     }
 
     premailer_instance = premailer.Premailer(
         #cssutils_logging_handler=log.handlers[0],
         #cssutils_logging_level=logging.INFO
     )
 
     def __init__(self):
         """
         Example usage::
 
             (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
                 EmailNotificationModel.TYPE_TEST, **email_kwargs)
 
         """
         super().__init__()
         self.rhodecode_instance_name = rhodecode.CONFIG.get('rhodecode_title')
 
     def _update_kwargs_for_render(self, kwargs):
         """
         Inject params required for Mako rendering
 
         :param kwargs:
         """
 
         kwargs['rhodecode_instance_name'] = self.rhodecode_instance_name
         kwargs['rhodecode_version'] = rhodecode.__version__
         instance_url = h.route_url('home')
         _kwargs = {
             'instance_url': instance_url,
             'whitespace_filter': self.whitespace_filter,
             'email_pr_update_subject_template': EMAIL_PR_UPDATE_SUBJECT_TEMPLATE,
             'email_pr_review_subject_template': EMAIL_PR_REVIEW_SUBJECT_TEMPLATE,
             'email_pr_comment_subject_template': EMAIL_PR_COMMENT_SUBJECT_TEMPLATE,
             'email_pr_comment_status_change_subject_template': EMAIL_PR_COMMENT_STATUS_CHANGE_SUBJECT_TEMPLATE,
             'email_pr_comment_file_subject_template': EMAIL_PR_COMMENT_FILE_SUBJECT_TEMPLATE,
             'email_comment_subject_template': EMAIL_COMMENT_SUBJECT_TEMPLATE,
             'email_comment_status_change_subject_template': EMAIL_COMMENT_STATUS_CHANGE_SUBJECT_TEMPLATE,
             'email_comment_file_subject_template': EMAIL_COMMENT_FILE_SUBJECT_TEMPLATE,
         }
         _kwargs.update(kwargs)
         return _kwargs
 
     def whitespace_filter(self, text):
         return text.replace('\n', '').replace('\t', '')
 
     def get_renderer(self, type_, request):
         template_name = self.email_types[type_]
         return request.get_partial_renderer(template_name)
 
     def render_email(self, type_, **kwargs):
         """
         renders template for email, and returns a tuple of
         (subject, email_headers, email_html_body, email_plaintext_body)
         """
         request = get_current_request()
 
         # translator and helpers inject
         _kwargs = self._update_kwargs_for_render(kwargs)
         email_template = self.get_renderer(type_, request=request)
         subject = email_template.render('subject', **_kwargs)
 
         try:
             body_plaintext = email_template.render('body_plaintext', **_kwargs)
         except AttributeError:
             # it's not defined in template, ok we can skip it
             body_plaintext = ''
 
         # render WHOLE template
         body = email_template.render(None, **_kwargs)
 
         try:
             # Inline CSS styles and conversion
             body = self.premailer_instance.transform(body)
         except Exception:
             log.exception('Failed to parse body with premailer')
             pass
 
         return subject, body, body_plaintext

rhodecode/model/repo.py

@@ -1,1195 +1,1203 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import re
 import shutil
 import time
 import logging
 import traceback
 import datetime
 
 from pyramid.threadlocal import get_current_request
 from sqlalchemy.orm import aliased
 from zope.cachedescriptors.property import Lazy as LazyProperty
 
 from rhodecode import events
 from rhodecode.lib.auth import HasUserGroupPermissionAny
 from rhodecode.lib.caching_query import FromCache
-from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError
+from rhodecode.lib.exceptions import AttachedForksError, AttachedPullRequestsError, AttachedArtifactsError
 from rhodecode.lib import hooks_base
 from rhodecode.lib.user_log_filter import user_log_filter
 from rhodecode.lib.utils import make_db_config
 from rhodecode.lib.utils2 import (
     safe_str, remove_prefix, obfuscate_url_pw,
     get_current_rhodecode_user, safe_int, action_logger_generic)
 from rhodecode.lib.vcs.backends import get_backend
 from rhodecode.lib.vcs.nodes import NodeKind
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     _hash_key, func, case, joinedload, or_, in_filter_generator,
     Session, Repository, UserRepoToPerm, UserGroupRepoToPerm,
     UserRepoGroupToPerm, UserGroupRepoGroupToPerm, User, Permission,
     Statistics, UserGroup, RepoGroup, RepositoryField, UserLog)
 from rhodecode.model.permission import PermissionModel
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 class RepoModel(BaseModel):
 
     cls = Repository
 
     def _get_user_group(self, users_group):
         return self._get_instance(UserGroup, users_group,
                                   callback=UserGroup.get_by_group_name)
 
     def _get_repo_group(self, repo_group):
         return self._get_instance(RepoGroup, repo_group,
                                   callback=RepoGroup.get_by_group_name)
 
     def _create_default_perms(self, repository, private):
         # create default permission
         default = 'repository.read'
         def_user = User.get_default_user()
         for p in def_user.user_perms:
             if p.permission.permission_name.startswith('repository.'):
                 default = p.permission.permission_name
                 break
 
         default_perm = 'repository.none' if private else default
 
         repo_to_perm = UserRepoToPerm()
         repo_to_perm.permission = Permission.get_by_key(default_perm)
 
         repo_to_perm.repository = repository
         repo_to_perm.user = def_user
 
         return repo_to_perm
 
     def get(self, repo_id):
         repo = self.sa.query(Repository) \
             .filter(Repository.repo_id == repo_id)
 
         return repo.scalar()
 
     def get_repo(self, repository):
         return self._get_repo(repository)
 
     def get_by_repo_name(self, repo_name, cache=False):
         repo = self.sa.query(Repository) \
             .filter(Repository.repo_name == repo_name)
 
         if cache:
             name_key = _hash_key(repo_name)
             repo = repo.options(
                 FromCache("sql_cache_short", f"get_repo_{name_key}"))
         return repo.scalar()
 
     def _extract_id_from_repo_name(self, repo_name):
         if repo_name.startswith('/'):
             repo_name = repo_name.lstrip('/')
         by_id_match = re.match(r'^_(\d+)', repo_name)
         if by_id_match:
             return by_id_match.groups()[0]
 
     def get_repo_by_id(self, repo_name):
         """
         Extracts repo_name by id from special urls.
         Example url is _11/repo_name
 
         :param repo_name:
         :return: repo object if matched else None
         """
         _repo_id = None
         try:
             _repo_id = self._extract_id_from_repo_name(repo_name)
             if _repo_id:
                 return self.get(_repo_id)
         except Exception:
             log.exception('Failed to extract repo_name from URL')
             if _repo_id:
                 Session().rollback()
 
         return None
 
     def get_repos_for_root(self, root, traverse=False):
         if traverse:
             like_expression = u'{}%'.format(safe_str(root))
             repos = Repository.query().filter(
                 Repository.repo_name.like(like_expression)).all()
         else:
             if root and not isinstance(root, RepoGroup):
                 raise ValueError(
                     'Root must be an instance '
                     'of RepoGroup, got:{} instead'.format(type(root)))
             repos = Repository.query().filter(Repository.group == root).all()
         return repos
 
     def get_url(self, repo, request=None, permalink=False):
         if not request:
             request = get_current_request()
 
         if not request:
             return
 
         if permalink:
             return request.route_url(
                 'repo_summary', repo_name='_{}'.format(safe_str(repo.repo_id)))
         else:
             return request.route_url(
                 'repo_summary', repo_name=safe_str(repo.repo_name))
 
     def get_commit_url(self, repo, commit_id, request=None, permalink=False):
         if not request:
             request = get_current_request()
 
         if not request:
             return
 
         if permalink:
             return request.route_url(
                 'repo_commit', repo_name=safe_str(repo.repo_id),
                 commit_id=commit_id)
 
         else:
             return request.route_url(
                 'repo_commit', repo_name=safe_str(repo.repo_name),
                 commit_id=commit_id)
 
     def get_repo_log(self, repo, filter_term):
         repo_log = UserLog.query()\
             .filter(or_(UserLog.repository_id == repo.repo_id,
                         UserLog.repository_name == repo.repo_name))\
             .options(joinedload(UserLog.user))\
             .options(joinedload(UserLog.repository))\
             .order_by(UserLog.action_date.desc())
 
         repo_log = user_log_filter(repo_log, filter_term)
         return repo_log
 
     @classmethod
     def update_commit_cache(cls, repositories=None):
         if not repositories:
             repositories = Repository.getAll()
         for repo in repositories:
             repo.update_commit_cache()
 
     def get_repos_as_dict(self, repo_list=None, admin=False,
                           super_user_actions=False, short_name=None):
 
         _render = get_current_request().get_partial_renderer(
             'rhodecode:templates/data_table/_dt_elements.mako')
         c = _render.get_call_context()
         h = _render.get_helpers()
 
         def quick_menu(repo_name):
             return _render('quick_menu', repo_name)
 
         def repo_lnk(name, rtype, rstate, private, archived, fork_repo_name):
             if short_name is not None:
                 short_name_var = short_name
             else:
                 short_name_var = not admin
             return _render('repo_name', name, rtype, rstate, private, archived, fork_repo_name,
                            short_name=short_name_var, admin=False)
 
         def last_change(last_change):
             if admin and isinstance(last_change, datetime.datetime) and not last_change.tzinfo:
                 ts = time.time()
                 utc_offset = (datetime.datetime.fromtimestamp(ts)
                               - datetime.datetime.utcfromtimestamp(ts)).total_seconds()
                 last_change = last_change + datetime.timedelta(seconds=utc_offset)
 
             return _render("last_change", last_change)
 
         def rss_lnk(repo_name):
             return _render("rss", repo_name)
 
         def atom_lnk(repo_name):
             return _render("atom", repo_name)
 
         def last_rev(repo_name, cs_cache):
             return _render('revision', repo_name, cs_cache.get('revision'),
                            cs_cache.get('raw_id'), cs_cache.get('author'),
                            cs_cache.get('message'), cs_cache.get('date'))
 
         def desc(desc):
             return _render('repo_desc', desc, c.visual.stylify_metatags)
 
         def state(repo_state):
             return _render("repo_state", repo_state)
 
         def repo_actions(repo_name):
             return _render('repo_actions', repo_name, super_user_actions)
 
         def user_profile(username):
             return _render('user_profile', username)
 
         repos_data = []
         for repo in repo_list:
             # NOTE(marcink): because we use only raw column we need to load it like that
             changeset_cache = Repository._load_changeset_cache(
                 repo.repo_id, repo._changeset_cache)
 
             row = {
                 "menu": quick_menu(repo.repo_name),
 
                 "name": repo_lnk(repo.repo_name, repo.repo_type, repo.repo_state,
                                  repo.private, repo.archived, repo.fork_repo_name),
 
                 "desc": desc(h.escape(repo.description)),
 
                 "last_change": last_change(repo.updated_on),
 
                 "last_changeset": last_rev(repo.repo_name, changeset_cache),
                 "last_changeset_raw": changeset_cache.get('revision'),
 
                 "owner": user_profile(repo.owner_username),
 
                 "state": state(repo.repo_state),
                 "rss": rss_lnk(repo.repo_name),
                 "atom": atom_lnk(repo.repo_name),
             }
             if admin:
                 row.update({
                     "action": repo_actions(repo.repo_name),
                 })
             repos_data.append(row)
 
         return repos_data
 
     def get_repos_data_table(
             self, draw, start, limit,
             search_q, order_by, order_dir,
             auth_user, repo_group_id):
         from rhodecode.model.scm import RepoList
 
         _perms = ['repository.read', 'repository.write', 'repository.admin']
 
         repos = Repository.query() \
             .filter(Repository.group_id == repo_group_id) \
             .all()
         auth_repo_list = RepoList(
             repos, perm_set=_perms,
             extra_kwargs=dict(user=auth_user))
 
         allowed_ids = [-1]
         for repo in auth_repo_list:
             allowed_ids.append(repo.repo_id)
 
         repos_data_total_count = Repository.query() \
             .filter(Repository.group_id == repo_group_id) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(Repository.repo_id, allowed_ids))
             ) \
             .count()
 
         RepoFork = aliased(Repository)
         OwnerUser = aliased(User)
         base_q = Session.query(
             Repository.repo_id,
             Repository.repo_name,
             Repository.description,
             Repository.repo_type,
             Repository.repo_state,
             Repository.private,
             Repository.archived,
             Repository.updated_on,
             Repository._changeset_cache,
             RepoFork.repo_name.label('fork_repo_name'),
             OwnerUser.username.label('owner_username'),
             ) \
             .filter(Repository.group_id == repo_group_id) \
             .filter(or_(
                 # generate multiple IN to fix limitation problems
                 *in_filter_generator(Repository.repo_id, allowed_ids))
             ) \
             .outerjoin(RepoFork, Repository.fork_id == RepoFork.repo_id) \
             .join(OwnerUser, Repository.user_id == OwnerUser.user_id)
 
         repos_data_total_filtered_count = base_q.count()
 
         sort_defined = False
         if order_by == 'repo_name':
             sort_col = func.lower(Repository.repo_name)
             sort_defined = True
         elif order_by == 'user_username':
             sort_col = User.username
         else:
             sort_col = getattr(Repository, order_by, None)
 
         if sort_defined or sort_col:
             if order_dir == 'asc':
                 sort_col = sort_col.asc()
             else:
                 sort_col = sort_col.desc()
 
         base_q = base_q.order_by(sort_col)
         base_q = base_q.offset(start).limit(limit)
 
         repos_list = base_q.all()
 
         repos_data = RepoModel().get_repos_as_dict(
             repo_list=repos_list, admin=False)
 
         data = ({
             'draw': draw,
             'data': repos_data,
             'recordsTotal': repos_data_total_count,
             'recordsFiltered': repos_data_total_filtered_count,
         })
         return data
 
     def _get_defaults(self, repo_name):
         """
         Gets information about repository, and returns a dict for
         usage in forms
 
         :param repo_name:
         """
 
         repo_info = Repository.get_by_repo_name(repo_name)
 
         if repo_info is None:
             return None
 
         defaults = repo_info.get_dict()
         defaults['repo_name'] = repo_info.just_name
 
         groups = repo_info.groups_with_parents
         parent_group = groups[-1] if groups else None
 
         # we use -1 as this is how in HTML, we mark an empty group
         defaults['repo_group'] = getattr(parent_group, 'group_id', -1)
 
         keys_to_process = (
             {'k': 'repo_type', 'strip': False},
             {'k': 'repo_enable_downloads', 'strip': True},
             {'k': 'repo_description', 'strip': True},
             {'k': 'repo_enable_locking', 'strip': True},
             {'k': 'repo_landing_rev', 'strip': True},
             {'k': 'clone_uri', 'strip': False},
             {'k': 'push_uri', 'strip': False},
             {'k': 'repo_private', 'strip': True},
             {'k': 'repo_enable_statistics', 'strip': True}
         )
 
         for item in keys_to_process:
             attr = item['k']
             if item['strip']:
                 attr = remove_prefix(item['k'], 'repo_')
 
             val = defaults[attr]
             if item['k'] == 'repo_landing_rev':
                 val = ':'.join(defaults[attr])
             defaults[item['k']] = val
             if item['k'] == 'clone_uri':
                 defaults['clone_uri_hidden'] = repo_info.clone_uri_hidden
             if item['k'] == 'push_uri':
                 defaults['push_uri_hidden'] = repo_info.push_uri_hidden
 
         # fill owner
         if repo_info.user:
             defaults.update({'user': repo_info.user.username})
         else:
             replacement_user = User.get_first_super_admin().username
             defaults.update({'user': replacement_user})
 
         return defaults
 
     def update(self, repo, **kwargs):
         try:
             cur_repo = self._get_repo(repo)
             source_repo_name = cur_repo.repo_name
 
             affected_user_ids = []
             if 'user' in kwargs:
                 old_owner_id = cur_repo.user.user_id
                 new_owner = User.get_by_username(kwargs['user'])
                 cur_repo.user = new_owner
 
                 if old_owner_id != new_owner.user_id:
                     affected_user_ids = [new_owner.user_id, old_owner_id]
 
             if 'repo_group' in kwargs:
                 cur_repo.group = RepoGroup.get(kwargs['repo_group'])
             log.debug('Updating repo %s with params:%s', cur_repo, kwargs)
 
             update_keys = [
                 (1, 'repo_description'),
                 (1, 'repo_landing_rev'),
                 (1, 'repo_private'),
                 (1, 'repo_enable_downloads'),
                 (1, 'repo_enable_locking'),
                 (1, 'repo_enable_statistics'),
                 (0, 'clone_uri'),
                 (0, 'push_uri'),
                 (0, 'fork_id')
             ]
             for strip, k in update_keys:
                 if k in kwargs:
                     val = kwargs[k]
                     if strip:
                         k = remove_prefix(k, 'repo_')
 
                     setattr(cur_repo, k, val)
 
             new_name = cur_repo.get_new_name(kwargs['repo_name'])
             cur_repo.repo_name = new_name
 
             # if private flag is set, reset default permission to NONE
             if kwargs.get('repo_private'):
                 EMPTY_PERM = 'repository.none'
                 RepoModel().grant_user_permission(
                     repo=cur_repo, user=User.DEFAULT_USER, perm=EMPTY_PERM
                 )
             if kwargs.get('repo_landing_rev'):
                 landing_rev_val = kwargs['repo_landing_rev']
                 RepoModel().set_landing_rev(cur_repo, landing_rev_val)
 
             # handle extra fields
             for field in filter(lambda k: k.startswith(RepositoryField.PREFIX), kwargs):
                 k = RepositoryField.un_prefix_key(field)
                 ex_field = RepositoryField.get_by_key_name(
                     key=k, repo=cur_repo)
                 if ex_field:
                     ex_field.field_value = kwargs[field]
                     self.sa.add(ex_field)
 
             self.sa.add(cur_repo)
 
             if source_repo_name != new_name:
                 # rename repository
                 self._rename_filesystem_repo(
                     old=source_repo_name, new=new_name)
 
             if affected_user_ids:
                 PermissionModel().trigger_permission_flush(affected_user_ids)
 
             return cur_repo
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def _create_repo(self, repo_name, repo_type, description, owner,
                      private=False, clone_uri=None, repo_group=None,
                      landing_rev=None, fork_of=None,
                      copy_fork_permissions=False, enable_statistics=False,
                      enable_locking=False, enable_downloads=False,
                      copy_group_permissions=False,
                      state=Repository.STATE_PENDING):
         """
         Create repository inside database with PENDING state, this should be
         only executed by create() repo. With exception of importing existing
         repos
         """
         from rhodecode.model.scm import ScmModel
 
         owner = self._get_user(owner)
         fork_of = self._get_repo(fork_of)
         repo_group = self._get_repo_group(safe_int(repo_group))
         default_landing_ref, _lbl = ScmModel.backend_landing_ref(repo_type)
         landing_rev = landing_rev or default_landing_ref
 
         try:
             repo_name = safe_str(repo_name)
             description = safe_str(description)
             # repo name is just a name of repository
             # while repo_name_full is a full qualified name that is combined
             # with name and path of group
             repo_name_full = repo_name
             repo_name = repo_name.split(Repository.NAME_SEP)[-1]
 
             new_repo = Repository()
             new_repo.repo_state = state
             new_repo.enable_statistics = False
             new_repo.repo_name = repo_name_full
             new_repo.repo_type = repo_type
             new_repo.user = owner
             new_repo.group = repo_group
             new_repo.description = description or repo_name
             new_repo.private = private
             new_repo.archived = False
             new_repo.clone_uri = clone_uri
             new_repo.landing_rev = landing_rev
 
             new_repo.enable_statistics = enable_statistics
             new_repo.enable_locking = enable_locking
             new_repo.enable_downloads = enable_downloads
 
             if repo_group:
                 new_repo.enable_locking = repo_group.enable_locking
 
             if fork_of:
                 parent_repo = fork_of
                 new_repo.fork = parent_repo
 
             events.trigger(events.RepoPreCreateEvent(new_repo))
 
             self.sa.add(new_repo)
 
             EMPTY_PERM = 'repository.none'
             if fork_of and copy_fork_permissions:
                 repo = fork_of
                 user_perms = UserRepoToPerm.query() \
                     .filter(UserRepoToPerm.repository == repo).all()
                 group_perms = UserGroupRepoToPerm.query() \
                     .filter(UserGroupRepoToPerm.repository == repo).all()
 
                 for perm in user_perms:
                     UserRepoToPerm.create(
                         perm.user, new_repo, perm.permission)
 
                 for perm in group_perms:
                     UserGroupRepoToPerm.create(
                         perm.users_group, new_repo, perm.permission)
                 # in case we copy permissions and also set this repo to private
                 # override the default user permission to make it a private repo
                 if private:
                     RepoModel(self.sa).grant_user_permission(
                         repo=new_repo, user=User.DEFAULT_USER, perm=EMPTY_PERM)
 
             elif repo_group and copy_group_permissions:
                 user_perms = UserRepoGroupToPerm.query() \
                     .filter(UserRepoGroupToPerm.group == repo_group).all()
 
                 group_perms = UserGroupRepoGroupToPerm.query() \
                     .filter(UserGroupRepoGroupToPerm.group == repo_group).all()
 
                 for perm in user_perms:
                     perm_name = perm.permission.permission_name.replace(
                         'group.', 'repository.')
                     perm_obj = Permission.get_by_key(perm_name)
                     UserRepoToPerm.create(perm.user, new_repo, perm_obj)
 
                 for perm in group_perms:
                     perm_name = perm.permission.permission_name.replace(
                         'group.', 'repository.')
                     perm_obj = Permission.get_by_key(perm_name)
                     UserGroupRepoToPerm.create(perm.users_group, new_repo, perm_obj)
 
                 if private:
                     RepoModel(self.sa).grant_user_permission(
                         repo=new_repo, user=User.DEFAULT_USER, perm=EMPTY_PERM)
 
             else:
                 perm_obj = self._create_default_perms(new_repo, private)
                 self.sa.add(perm_obj)
 
             # now automatically start following this repository as owner
             ScmModel(self.sa).toggle_following_repo(new_repo.repo_id, owner.user_id)
 
             # we need to flush here, in order to check if database won't
             # throw any exceptions, create filesystem dirs at the very end
             self.sa.flush()
             events.trigger(events.RepoCreateEvent(new_repo, actor=owner))
             return new_repo
 
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def create(self, form_data, cur_user):
         """
         Create repository using celery tasks
 
         :param form_data:
         :param cur_user:
         """
         from rhodecode.lib.celerylib import tasks, run_task
         return run_task(tasks.create_repo, form_data, cur_user)
 
     def update_permissions(self, repo, perm_additions=None, perm_updates=None,
                            perm_deletions=None, check_perms=True,
                            cur_user=None):
         if not perm_additions:
             perm_additions = []
         if not perm_updates:
             perm_updates = []
         if not perm_deletions:
             perm_deletions = []
 
         req_perms = ('usergroup.read', 'usergroup.write', 'usergroup.admin')
 
         changes = {
             'added': [],
             'updated': [],
             'deleted': [],
             'default_user_changed': None
         }
 
         repo = self._get_repo(repo)
 
         # update permissions
         for member_id, perm, member_type in perm_updates:
             member_id = int(member_id)
             if member_type == 'user':
                 member_name = User.get(member_id).username
                 if member_name == User.DEFAULT_USER:
                     # NOTE(dan): detect if we changed permissions for default user
                     perm_obj = self.sa.query(UserRepoToPerm) \
                         .filter(UserRepoToPerm.user_id == member_id) \
                         .filter(UserRepoToPerm.repository == repo) \
                         .scalar()
                     if perm_obj and perm_obj.permission.permission_name != perm:
                         changes['default_user_changed'] = True
 
                 # this updates also current one if found
                 self.grant_user_permission(
                     repo=repo, user=member_id, perm=perm)
             elif member_type == 'user_group':
                 # check if we have permissions to alter this usergroup
                 member_name = UserGroup.get(member_id).users_group_name
                 if not check_perms or HasUserGroupPermissionAny(
                         *req_perms)(member_name, user=cur_user):
                     self.grant_user_group_permission(
                         repo=repo, group_name=member_id, perm=perm)
             else:
                 raise ValueError("member_type must be 'user' or 'user_group' "
                                  "got {} instead".format(member_type))
             changes['updated'].append({'type': member_type, 'id': member_id,
                                        'name': member_name, 'new_perm': perm})
 
         # set new permissions
         for member_id, perm, member_type in perm_additions:
             member_id = int(member_id)
             if member_type == 'user':
                 member_name = User.get(member_id).username
                 self.grant_user_permission(
                     repo=repo, user=member_id, perm=perm)
             elif member_type == 'user_group':
                 # check if we have permissions to alter this usergroup
                 member_name = UserGroup.get(member_id).users_group_name
                 if not check_perms or HasUserGroupPermissionAny(
                         *req_perms)(member_name, user=cur_user):
                     self.grant_user_group_permission(
                         repo=repo, group_name=member_id, perm=perm)
             else:
                 raise ValueError("member_type must be 'user' or 'user_group' "
                                  "got {} instead".format(member_type))
 
             changes['added'].append({'type': member_type, 'id': member_id,
                                      'name': member_name, 'new_perm': perm})
         # delete permissions
         for member_id, perm, member_type in perm_deletions:
             member_id = int(member_id)
             if member_type == 'user':
                 member_name = User.get(member_id).username
                 self.revoke_user_permission(repo=repo, user=member_id)
             elif member_type == 'user_group':
                 # check if we have permissions to alter this usergroup
                 member_name = UserGroup.get(member_id).users_group_name
                 if not check_perms or HasUserGroupPermissionAny(
                         *req_perms)(member_name, user=cur_user):
                     self.revoke_user_group_permission(
                         repo=repo, group_name=member_id)
             else:
                 raise ValueError("member_type must be 'user' or 'user_group' "
                                  "got {} instead".format(member_type))
 
             changes['deleted'].append({'type': member_type, 'id': member_id,
                                        'name': member_name, 'new_perm': perm})
         return changes
 
     def create_fork(self, form_data, cur_user):
         """
         Simple wrapper into executing celery task for fork creation
 
         :param form_data:
         :param cur_user:
         """
         from rhodecode.lib.celerylib import tasks, run_task
         return run_task(tasks.create_repo_fork, form_data, cur_user)
 
     def archive(self, repo):
         """
         Archive given repository. Set archive flag.
 
         :param repo:
         """
         repo = self._get_repo(repo)
         if repo:
 
             try:
                 repo.archived = True
                 self.sa.add(repo)
                 self.sa.commit()
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
-    def delete(self, repo, forks=None, pull_requests=None, fs_remove=True, cur_user=None):
+    def delete(self, repo, forks=None, pull_requests=None, artifacts=None, fs_remove=True, cur_user=None):
         """
         Delete given repository, forks parameter defines what do do with
         attached forks. Throws AttachedForksError if deleted repo has attached
         forks
 
         :param repo:
         :param forks: str 'delete' or 'detach'
         :param pull_requests: str 'delete' or None
+        :param artifacts: str 'delete' or None
         :param fs_remove: remove(archive) repo from filesystem
         """
         if not cur_user:
             cur_user = getattr(get_current_rhodecode_user(), 'username', None)
         repo = self._get_repo(repo)
         if repo:
             if forks == 'detach':
                 for r in repo.forks:
                     r.fork = None
                     self.sa.add(r)
             elif forks == 'delete':
                 for r in repo.forks:
                     self.delete(r, forks='delete')
             elif [f for f in repo.forks]:
                 raise AttachedForksError()
 
             # check for pull requests
             pr_sources = repo.pull_requests_source
             pr_targets = repo.pull_requests_target
             if pull_requests != 'delete' and (pr_sources or pr_targets):
                 raise AttachedPullRequestsError()
 
+            artifacts_objs = repo.artifacts
+            if artifacts == 'delete':
+                for a in artifacts_objs:
+                    self.sa.delete(a)
+            elif [a for a in artifacts_objs]:
+                raise AttachedArtifactsError()
+
             old_repo_dict = repo.get_dict()
             events.trigger(events.RepoPreDeleteEvent(repo))
             try:
                 self.sa.delete(repo)
                 if fs_remove:
                     self._delete_filesystem_repo(repo)
                 else:
                     log.debug('skipping removal from filesystem')
                 old_repo_dict.update({
                     'deleted_by': cur_user,
                     'deleted_on': time.time(),
                 })
                 hooks_base.delete_repository(**old_repo_dict)
                 events.trigger(events.RepoDeleteEvent(repo))
             except Exception:
                 log.error(traceback.format_exc())
                 raise
 
     def grant_user_permission(self, repo, user, perm):
         """
         Grant permission for user on given repository, or update existing one
         if found
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         :param perm: Instance of Permission, or permission_name
         """
         user = self._get_user(user)
         repo = self._get_repo(repo)
         permission = self._get_perm(perm)
 
         # check if we have that permission already
         obj = self.sa.query(UserRepoToPerm) \
             .filter(UserRepoToPerm.user == user) \
             .filter(UserRepoToPerm.repository == repo) \
             .scalar()
         if obj is None:
             # create new !
             obj = UserRepoToPerm()
         obj.repository = repo
         obj.user = user
         obj.permission = permission
         self.sa.add(obj)
         log.debug('Granted perm %s to %s on %s', perm, user, repo)
         action_logger_generic(
             'granted permission: {} to user: {} on repo: {}'.format(
                 perm, user, repo), namespace='security.repo')
         return obj
 
     def revoke_user_permission(self, repo, user):
         """
         Revoke permission for user on given repository
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param user: Instance of User, user_id or username
         """
 
         user = self._get_user(user)
         repo = self._get_repo(repo)
 
         obj = self.sa.query(UserRepoToPerm) \
             .filter(UserRepoToPerm.repository == repo) \
             .filter(UserRepoToPerm.user == user) \
             .scalar()
         if obj:
             self.sa.delete(obj)
             log.debug('Revoked perm on %s on %s', repo, user)
             action_logger_generic(
                 'revoked permission from user: {} on repo: {}'.format(
                     user, repo), namespace='security.repo')
 
     def grant_user_group_permission(self, repo, group_name, perm):
         """
         Grant permission for user group on given repository, or update
         existing one if found
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         :param perm: Instance of Permission, or permission_name
         """
         repo = self._get_repo(repo)
         group_name = self._get_user_group(group_name)
         permission = self._get_perm(perm)
 
         # check if we have that permission already
         obj = self.sa.query(UserGroupRepoToPerm) \
             .filter(UserGroupRepoToPerm.users_group == group_name) \
             .filter(UserGroupRepoToPerm.repository == repo) \
             .scalar()
 
         if obj is None:
             # create new
             obj = UserGroupRepoToPerm()
 
         obj.repository = repo
         obj.users_group = group_name
         obj.permission = permission
         self.sa.add(obj)
         log.debug('Granted perm %s to %s on %s', perm, group_name, repo)
         action_logger_generic(
             'granted permission: {} to usergroup: {} on repo: {}'.format(
                 perm, group_name, repo), namespace='security.repo')
 
         return obj
 
     def revoke_user_group_permission(self, repo, group_name):
         """
         Revoke permission for user group on given repository
 
         :param repo: Instance of Repository, repository_id, or repository name
         :param group_name: Instance of UserGroup, users_group_id,
             or user group name
         """
         repo = self._get_repo(repo)
         group_name = self._get_user_group(group_name)
 
         obj = self.sa.query(UserGroupRepoToPerm) \
             .filter(UserGroupRepoToPerm.repository == repo) \
             .filter(UserGroupRepoToPerm.users_group == group_name) \
             .scalar()
         if obj:
             self.sa.delete(obj)
             log.debug('Revoked perm to %s on %s', repo, group_name)
             action_logger_generic(
                 'revoked permission from usergroup: {} on repo: {}'.format(
                     group_name, repo), namespace='security.repo')
 
     def delete_stats(self, repo_name):
         """
         removes stats for given repo
 
         :param repo_name:
         """
         repo = self._get_repo(repo_name)
         try:
             obj = self.sa.query(Statistics) \
                 .filter(Statistics.repository == repo).scalar()
             if obj:
                 self.sa.delete(obj)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def add_repo_field(self, repo_name, field_key, field_label, field_value='',
                        field_type='str', field_desc=''):
 
         repo = self._get_repo(repo_name)
 
         new_field = RepositoryField()
         new_field.repository = repo
         new_field.field_key = field_key
         new_field.field_type = field_type  # python type
         new_field.field_value = field_value
         new_field.field_desc = field_desc
         new_field.field_label = field_label
         self.sa.add(new_field)
         return new_field
 
     def delete_repo_field(self, repo_name, field_key):
         repo = self._get_repo(repo_name)
         field = RepositoryField.get_by_key_name(field_key, repo)
         if field:
             self.sa.delete(field)
 
     def set_landing_rev(self, repo, landing_rev_name):
         if landing_rev_name.startswith('branch:'):
             landing_rev_name = landing_rev_name.split('branch:')[-1]
         scm_instance = repo.scm_instance()
         if scm_instance:
             return scm_instance._remote.set_head_ref(landing_rev_name)
 
     def _create_filesystem_repo(self, repo_name, repo_type, repo_group,
                                 clone_uri=None, repo_store_location=None,
                                 use_global_config=False, install_hooks=True):
         """
         makes repository on filesystem. It's group aware means it'll create
         a repository within a group, and alter the paths accordingly of
         group location
 
         :param repo_name:
         :param alias:
         :param parent:
         :param clone_uri:
         :param repo_store_location:
         """
         from rhodecode.lib.utils import is_valid_repo, is_valid_repo_group
         from rhodecode.model.scm import ScmModel
 
         if Repository.NAME_SEP in repo_name:
             raise ValueError(
                 'repo_name must not contain groups got `%s`' % repo_name)
 
         if isinstance(repo_group, RepoGroup):
             new_parent_path = os.sep.join(repo_group.full_path_splitted)
         else:
             new_parent_path = repo_group or ''
 
         if repo_store_location:
             _paths = [repo_store_location]
         else:
             _paths = [self.repos_path, new_parent_path, repo_name]
             # we need to make it str for mercurial
         repo_path = os.path.join(*map(lambda x: safe_str(x), _paths))
 
         # check if this path is not a repository
         if is_valid_repo(repo_path, self.repos_path):
             raise Exception(f'This path {repo_path} is a valid repository')
 
         # check if this path is a group
         if is_valid_repo_group(repo_path, self.repos_path):
             raise Exception(f'This path {repo_path} is a valid group')
 
         log.info('creating repo %s in %s from url: `%s`',
                  repo_name, safe_str(repo_path),
                  obfuscate_url_pw(clone_uri))
 
         backend = get_backend(repo_type)
 
         config_repo = None if use_global_config else repo_name
         if config_repo and new_parent_path:
             config_repo = Repository.NAME_SEP.join(
                 (new_parent_path, config_repo))
         config = make_db_config(clear_session=False, repo=config_repo)
         config.set('extensions', 'largefiles', '')
 
         # patch and reset hooks section of UI config to not run any
         # hooks on creating remote repo
         config.clear_section('hooks')
 
         # TODO: johbo: Unify this, hardcoded "bare=True" does not look nice
         if repo_type == 'git':
             repo = backend(
                 repo_path, config=config, create=True, src_url=clone_uri, bare=True,
                 with_wire={"cache": False})
         else:
             repo = backend(
                 repo_path, config=config, create=True, src_url=clone_uri,
                 with_wire={"cache": False})
 
         if install_hooks:
             repo.install_hooks()
 
         log.debug('Created repo %s with %s backend',
                   safe_str(repo_name), safe_str(repo_type))
         return repo
 
     def _rename_filesystem_repo(self, old, new):
         """
         renames repository on filesystem
 
         :param old: old name
         :param new: new name
         """
         log.info('renaming repo from %s to %s', old, new)
 
         old_path = os.path.join(self.repos_path, old)
         new_path = os.path.join(self.repos_path, new)
         if os.path.isdir(new_path):
             raise Exception(
                 'Was trying to rename to already existing dir %s' % new_path
             )
         shutil.move(old_path, new_path)
 
     def _delete_filesystem_repo(self, repo):
         """
         removes repo from filesystem, the removal is actually made by
         added rm__ prefix into dir, and rename internal .hg/.git dirs so this
         repository is no longer valid for rhodecode, can be undeleted later on
         by reverting the renames on this repository
 
         :param repo: repo object
         """
         rm_path = os.path.join(self.repos_path, repo.repo_name)
         repo_group = repo.group
         log.info("delete_filesystem_repo: removing repository %s", rm_path)
         # disable hg/git internal that it doesn't get detected as repo
         alias = repo.repo_type
 
         config = make_db_config(clear_session=False)
         config.set('extensions', 'largefiles', '')
         bare = getattr(repo.scm_instance(config=config), 'bare', False)
 
         # skip this for bare git repos
         if not bare:
             # disable VCS repo
             vcs_path = os.path.join(rm_path, '.%s' % alias)
             if os.path.exists(vcs_path):
                 shutil.move(vcs_path, os.path.join(rm_path, 'rm__.%s' % alias))
 
         _now = datetime.datetime.now()
         _ms = str(_now.microsecond).rjust(6, '0')
         _d = 'rm__{}__{}'.format(_now.strftime('%Y%m%d_%H%M%S_' + _ms),
                              repo.just_name)
         if repo_group:
             # if repository is in group, prefix the removal path with the group
             args = repo_group.full_path_splitted + [_d]
             _d = os.path.join(*args)
 
         if os.path.isdir(rm_path):
             shutil.move(rm_path, os.path.join(self.repos_path, _d))
 
         # finally cleanup diff-cache if it exists
         cached_diffs_dir = repo.cached_diffs_dir
         if os.path.isdir(cached_diffs_dir):
             shutil.rmtree(cached_diffs_dir)
 
 
 class ReadmeFinder:
     """
     Utility which knows how to find a readme for a specific commit.
 
     The main idea is that this is a configurable algorithm. When creating an
     instance you can define parameters, currently only the `default_renderer`.
     Based on this configuration the method :meth:`search` behaves slightly
     different.
     """
 
     readme_re = re.compile(r'^readme(\.[^\.]+)?$', re.IGNORECASE)
     path_re = re.compile(r'^docs?', re.IGNORECASE)
 
     default_priorities = {
         None:        0,
         '.rst':      1,
         '.md':       1,
         '.rest':     2,
         '.mkdn':     2,
         '.text':     2,
         '.txt':      3,
         '.mdown':    3,
         '.markdown': 4,
     }
 
     path_priority = {
         'doc':  0,
         'docs': 1,
     }
 
     FALLBACK_PRIORITY = 99
 
     RENDERER_TO_EXTENSION = {
         'rst': ['.rst', '.rest'],
         'markdown': ['.md', 'mkdn', '.mdown', '.markdown'],
     }
 
     def __init__(self, default_renderer=None):
         self._default_renderer = default_renderer
         self._renderer_extensions = self.RENDERER_TO_EXTENSION.get(
             default_renderer, [])
 
     def search(self, commit, path='/'):
         """
         Find a readme in the given `commit`.
         """
         # firstly, check the PATH type if it is actually a DIR
         if commit.get_node(path).kind != NodeKind.DIR:
             return None
 
         nodes = commit.get_nodes(path)
         matches = self._match_readmes(nodes)
         matches = self._sort_according_to_priority(matches)
         if matches:
             return matches[0].node
 
         paths = self._match_paths(nodes)
         paths = self._sort_paths_according_to_priority(paths)
         for path in paths:
             match = self.search(commit, path=path)
             if match:
                 return match
 
         return None
 
     def _match_readmes(self, nodes):
         for node in nodes:
             if not node.is_file():
                 continue
             path = node.path.rsplit('/', 1)[-1]
             match = self.readme_re.match(path)
             if match:
                 extension = match.group(1)
                 yield ReadmeMatch(node, match, self._priority(extension))
 
     def _match_paths(self, nodes):
         for node in nodes:
             if not node.is_dir():
                 continue
             match = self.path_re.match(node.path)
             if match:
                 yield node.path
 
     def _priority(self, extension):
         renderer_priority = (
             0 if extension in self._renderer_extensions else 1)
         extension_priority = self.default_priorities.get(
             extension, self.FALLBACK_PRIORITY)
         return (renderer_priority, extension_priority)
 
     def _sort_according_to_priority(self, matches):
 
         def priority_and_path(match):
             return (match.priority, match.path)
 
         return sorted(matches, key=priority_and_path)
 
     def _sort_paths_according_to_priority(self, paths):
 
         def priority_and_path(path):
             return (self.path_priority.get(path, self.FALLBACK_PRIORITY), path)
 
         return sorted(paths, key=priority_and_path)
 
 
 class ReadmeMatch:
 
     def __init__(self, node, match, priority):
         self.node = node
         self._match = match
         self.priority = priority
 
     @property
     def path(self):
         return self.node.path
 
     def __repr__(self):
         return f'<ReadmeMatch {self.path} priority={self.priority}'

rhodecode/model/settings.py

@@ -1,902 +1,888 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import re
 import logging
 import time
 import functools
 from collections import namedtuple
 
 from pyramid.threadlocal import get_current_request
 
 from rhodecode.lib import rc_cache
 from rhodecode.lib.hash_utils import sha1_safe
 from rhodecode.lib.html_filters import sanitize_html
 from rhodecode.lib.utils2 import (
     Optional, AttributeDict, safe_str, remove_prefix, str2bool)
 from rhodecode.lib.vcs.backends import base
 from rhodecode.lib.statsd_client import StatsdClient
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     RepoRhodeCodeUi, RepoRhodeCodeSetting, RhodeCodeUi, RhodeCodeSetting)
 from rhodecode.model.meta import Session
 
 
 log = logging.getLogger(__name__)
 
 
 UiSetting = namedtuple(
     'UiSetting', ['section', 'key', 'value', 'active'])
 
 SOCIAL_PLUGINS_LIST = ['github', 'bitbucket', 'twitter', 'google']
 
 
 class SettingNotFound(Exception):
     def __init__(self, setting_id):
         msg = f'Setting `{setting_id}` is not found'
         super().__init__(msg)
 
 
 class SettingsModel(BaseModel):
     BUILTIN_HOOKS = (
         RhodeCodeUi.HOOK_REPO_SIZE, RhodeCodeUi.HOOK_PUSH,
         RhodeCodeUi.HOOK_PRE_PUSH, RhodeCodeUi.HOOK_PRETX_PUSH,
         RhodeCodeUi.HOOK_PULL, RhodeCodeUi.HOOK_PRE_PULL,
         RhodeCodeUi.HOOK_PUSH_KEY,)
     HOOKS_SECTION = 'hooks'
 
     def __init__(self, sa=None, repo=None):
         self.repo = repo
         self.UiDbModel = RepoRhodeCodeUi if repo else RhodeCodeUi
         self.SettingsDbModel = (
             RepoRhodeCodeSetting if repo else RhodeCodeSetting)
         super().__init__(sa)
 
     def get_keyname(self, key_name, prefix='rhodecode_'):
         return f'{prefix}{key_name}'
 
     def get_ui_by_key(self, key):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_key == key)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.scalar()
 
     def get_ui_by_section(self, section):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_section == section)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.all()
 
     def get_ui_by_section_and_key(self, section, key):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_section == section)
         q = q.filter(self.UiDbModel.ui_key == key)
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
         return q.scalar()
 
     def get_ui(self, section=None, key=None):
         q = self.UiDbModel.query()
         q = self._filter_by_repo(RepoRhodeCodeUi, q)
 
         if section:
             q = q.filter(self.UiDbModel.ui_section == section)
         if key:
             q = q.filter(self.UiDbModel.ui_key == key)
 
         # TODO: mikhail: add caching
         result = [
             UiSetting(
                 section=safe_str(r.ui_section), key=safe_str(r.ui_key),
                 value=safe_str(r.ui_value), active=r.ui_active
             )
             for r in q.all()
         ]
         return result
 
     def get_builtin_hooks(self):
         q = self.UiDbModel.query()
         q = q.filter(self.UiDbModel.ui_key.in_(self.BUILTIN_HOOKS))
         return self._get_hooks(q)
 
     def get_custom_hooks(self):
         q = self.UiDbModel.query()
         q = q.filter(~self.UiDbModel.ui_key.in_(self.BUILTIN_HOOKS))
         return self._get_hooks(q)
 
     def create_ui_section_value(self, section, val, key=None, active=True):
         new_ui = self.UiDbModel()
         new_ui.ui_section = section
         new_ui.ui_value = val
         new_ui.ui_active = active
 
         repository_id = ''
         if self.repo:
             repo = self._get_repo(self.repo)
             repository_id = repo.repo_id
             new_ui.repository_id = repository_id
 
         if not key:
             # keys are unique so they need appended info
             if self.repo:
                 key = sha1_safe(f'{section}{val}{repository_id}')
             else:
                 key = sha1_safe(f'{section}{val}')
 
         new_ui.ui_key = key
 
         Session().add(new_ui)
         return new_ui
 
     def create_or_update_hook(self, key, value):
         ui = (
             self.get_ui_by_section_and_key(self.HOOKS_SECTION, key) or
             self.UiDbModel())
         ui.ui_section = self.HOOKS_SECTION
         ui.ui_active = True
         ui.ui_key = key
         ui.ui_value = value
 
         if self.repo:
             repo = self._get_repo(self.repo)
             repository_id = repo.repo_id
             ui.repository_id = repository_id
 
         Session().add(ui)
         return ui
 
     def delete_ui(self, id_):
         ui = self.UiDbModel.get(id_)
         if not ui:
             raise SettingNotFound(id_)
         Session().delete(ui)
 
     def get_setting_by_name(self, name):
         q = self._get_settings_query()
         q = q.filter(self.SettingsDbModel.app_settings_name == name)
         return q.scalar()
 
     def create_or_update_setting(
             self, name, val: Optional | str = Optional(''), type_: Optional | str = Optional('unicode')):
         """
         Creates or updates RhodeCode setting. If updates are triggered, it will
         only update parameters that are explicitly set Optional instance will
         be skipped
 
         :param name:
         :param val:
         :param type_:
         :return:
         """
 
         res = self.get_setting_by_name(name)
         repo = self._get_repo(self.repo) if self.repo else None
 
         if not res:
             val = Optional.extract(val)
             type_ = Optional.extract(type_)
 
             args = (
                 (repo.repo_id, name, val, type_)
                 if repo else (name, val, type_))
             res = self.SettingsDbModel(*args)
 
         else:
             if self.repo:
                 res.repository_id = repo.repo_id
 
             res.app_settings_name = name
             if not isinstance(type_, Optional):
                 # update if set
                 res.app_settings_type = type_
             if not isinstance(val, Optional):
                 # update if set
                 res.app_settings_value = val
 
         Session().add(res)
         return res
 
     def get_cache_region(self):
         repo = self._get_repo(self.repo) if self.repo else None
         cache_key = f"repo.v1.{repo.repo_id}" if repo else "repo.v1.ALL"
         cache_namespace_uid = f'cache_settings.{cache_key}'
         region = rc_cache.get_or_create_region('cache_general', cache_namespace_uid)
         return region, cache_namespace_uid
 
     def invalidate_settings_cache(self, hard=False):
         region, namespace_key = self.get_cache_region()
         log.debug('Invalidation cache [%s] region %s for cache_key: %s',
                   'invalidate_settings_cache', region, namespace_key)
 
         # we use hard cleanup if invalidation is sent
         rc_cache.clear_cache_namespace(region, namespace_key, method=rc_cache.CLEAR_DELETE)
 
     def get_cache_call_method(self, cache=True):
         region, cache_key = self.get_cache_region()
 
         @region.conditional_cache_on_arguments(condition=cache)
         def _get_all_settings(name, key):
             q = self._get_settings_query()
             if not q:
                 raise Exception('Could not get application settings !')
 
             settings = {
                 self.get_keyname(res.app_settings_name): res.app_settings_value
                 for res in q
             }
             return settings
         return _get_all_settings
 
     def get_all_settings(self, cache=False, from_request=True):
         # defines if we use GLOBAL, or PER_REPO
         repo = self._get_repo(self.repo) if self.repo else None
 
         # initially try the request context; this is the fastest
         # we only fetch global config, NOT for repo-specific
         if from_request and not repo:
             request = get_current_request()
 
             if request and hasattr(request, 'call_context') and hasattr(request.call_context, 'rc_config'):
                 rc_config = request.call_context.rc_config
                 if rc_config:
                     return rc_config
 
         _region, cache_key = self.get_cache_region()
         _get_all_settings = self.get_cache_call_method(cache=cache)
 
         start = time.time()
         result = _get_all_settings('rhodecode_settings', cache_key)
         compute_time = time.time() - start
         log.debug('cached method:%s took %.4fs', _get_all_settings.__name__, compute_time)
 
         statsd = StatsdClient.statsd
         if statsd:
             elapsed_time_ms = round(1000.0 * compute_time)  # use ms only
             statsd.timing("rhodecode_settings_timing.histogram", elapsed_time_ms,
                           use_decimals=False)
 
         log.debug('Fetching app settings for key: %s took: %.4fs: cache: %s', cache_key, compute_time, cache)
 
         return result
 
     def get_auth_settings(self):
         q = self._get_settings_query()
         q = q.filter(
             self.SettingsDbModel.app_settings_name.startswith('auth_'))
         rows = q.all()
         auth_settings = {
             row.app_settings_name: row.app_settings_value for row in rows}
         return auth_settings
 
     def get_auth_plugins(self):
         auth_plugins = self.get_setting_by_name("auth_plugins")
         return auth_plugins.app_settings_value
 
     def get_default_repo_settings(self, strip_prefix=False):
         q = self._get_settings_query()
         q = q.filter(
             self.SettingsDbModel.app_settings_name.startswith('default_'))
         rows = q.all()
 
         result = {}
         for row in rows:
             key = row.app_settings_name
             if strip_prefix:
                 key = remove_prefix(key, prefix='default_')
             result.update({key: row.app_settings_value})
         return result
 
     def get_repo(self):
         repo = self._get_repo(self.repo)
         if not repo:
             raise Exception(
                 f'Repository `{self.repo}` cannot be found inside the database')
         return repo
 
     def _filter_by_repo(self, model, query):
         if self.repo:
             repo = self.get_repo()
             query = query.filter(model.repository_id == repo.repo_id)
         return query
 
     def _get_hooks(self, query):
         query = query.filter(self.UiDbModel.ui_section == self.HOOKS_SECTION)
         query = self._filter_by_repo(RepoRhodeCodeUi, query)
         return query.all()
 
     def _get_settings_query(self):
         q = self.SettingsDbModel.query()
         return self._filter_by_repo(RepoRhodeCodeSetting, q)
 
     def list_enabled_social_plugins(self, settings):
         enabled = []
         for plug in SOCIAL_PLUGINS_LIST:
             if str2bool(settings.get(f'rhodecode_auth_{plug}_enabled')):
                 enabled.append(plug)
         return enabled
 
 
 def assert_repo_settings(func):
     @functools.wraps(func)
     def _wrapper(self, *args, **kwargs):
         if not self.repo_settings:
             raise Exception('Repository is not specified')
         return func(self, *args, **kwargs)
     return _wrapper
 
 
 class IssueTrackerSettingsModel(object):
     INHERIT_SETTINGS = 'inherit_issue_tracker_settings'
     SETTINGS_PREFIX = 'issuetracker_'
 
     def __init__(self, sa=None, repo=None):
         self.global_settings = SettingsModel(sa=sa)
         self.repo_settings = SettingsModel(sa=sa, repo=repo) if repo else None
 
     @property
     def inherit_global_settings(self):
         if not self.repo_settings:
             return True
         setting = self.repo_settings.get_setting_by_name(self.INHERIT_SETTINGS)
         return setting.app_settings_value if setting else True
 
     @inherit_global_settings.setter
     def inherit_global_settings(self, value):
         if self.repo_settings:
             settings = self.repo_settings.create_or_update_setting(
                 self.INHERIT_SETTINGS, value, type_='bool')
             Session().add(settings)
 
     def _get_keyname(self, key, uid, prefix='rhodecode_'):
         return f'{prefix}{self.SETTINGS_PREFIX}{key}_{uid}'
 
     def _make_dict_for_settings(self, qs):
         prefix_match = self._get_keyname('pat', '',)
 
         issuetracker_entries = {}
         # create keys
         for k, v in qs.items():
             if k.startswith(prefix_match):
                 uid = k[len(prefix_match):]
                 issuetracker_entries[uid] = None
 
         def url_cleaner(input_str):
             input_str = input_str.replace('"', '').replace("'", '')
             input_str = sanitize_html(input_str, strip=True)
             return input_str
 
         # populate
         for uid in issuetracker_entries:
             url_data = qs.get(self._get_keyname('url', uid))
 
             pat = qs.get(self._get_keyname('pat', uid))
             try:
                 pat_compiled = re.compile(r'%s' % pat)
             except re.error:
                 pat_compiled = None
 
             issuetracker_entries[uid] = AttributeDict({
                 'pat': pat,
                 'pat_compiled': pat_compiled,
                 'url': url_cleaner(
                     qs.get(self._get_keyname('url', uid)) or ''),
                 'pref': sanitize_html(
                     qs.get(self._get_keyname('pref', uid)) or ''),
                 'desc': qs.get(
                     self._get_keyname('desc', uid)),
             })
 
         return issuetracker_entries
 
     def get_global_settings(self, cache=False):
         """
         Returns list of global issue tracker settings
         """
         defaults = self.global_settings.get_all_settings(cache=cache)
         settings = self._make_dict_for_settings(defaults)
         return settings
 
     def get_repo_settings(self, cache=False):
         """
         Returns list of issue tracker settings per repository
         """
         if not self.repo_settings:
             raise Exception('Repository is not specified')
         all_settings = self.repo_settings.get_all_settings(cache=cache)
         settings = self._make_dict_for_settings(all_settings)
         return settings
 
     def get_settings(self, cache=False):
         if self.inherit_global_settings:
             return self.get_global_settings(cache=cache)
         else:
             return self.get_repo_settings(cache=cache)
 
     def delete_entries(self, uid):
         if self.repo_settings:
             all_patterns = self.get_repo_settings()
             settings_model = self.repo_settings
         else:
             all_patterns = self.get_global_settings()
             settings_model = self.global_settings
         entries = all_patterns.get(uid, [])
 
         for del_key in entries:
             setting_name = self._get_keyname(del_key, uid, prefix='')
             entry = settings_model.get_setting_by_name(setting_name)
             if entry:
                 Session().delete(entry)
 
         Session().commit()
 
     def create_or_update_setting(
             self, name, val=Optional(''), type_=Optional('unicode')):
         if self.repo_settings:
             setting = self.repo_settings.create_or_update_setting(
                 name, val, type_)
         else:
             setting = self.global_settings.create_or_update_setting(
                 name, val, type_)
         return setting
 
 
 class VcsSettingsModel(object):
 
     INHERIT_SETTINGS = 'inherit_vcs_settings'
     GENERAL_SETTINGS = (
         'use_outdated_comments',
         'pr_merge_enabled',
         'hg_use_rebase_for_merging',
         'hg_close_branch_before_merging',
         'git_use_rebase_for_merging',
         'git_close_branch_before_merging',
         'diff_cache',
     )
 
     HOOKS_SETTINGS = (
         ('hooks', 'changegroup.repo_size'),
         ('hooks', 'changegroup.push_logger'),
         ('hooks', 'outgoing.pull_logger'),
     )
     HG_SETTINGS = (
         ('extensions', 'largefiles'),
         ('phases', 'publish'),
         ('extensions', 'evolve'),
         ('extensions', 'topic'),
         ('experimental', 'evolution'),
         ('experimental', 'evolution.exchange'),
     )
     GIT_SETTINGS = (
         ('vcs_git_lfs', 'enabled'),
     )
     GLOBAL_HG_SETTINGS = (
         ('extensions', 'largefiles'),
-        ('largefiles', 'usercache'),
         ('phases', 'publish'),
         ('extensions', 'evolve'),
         ('extensions', 'topic'),
         ('experimental', 'evolution'),
         ('experimental', 'evolution.exchange'),
     )
 
     GLOBAL_GIT_SETTINGS = (
         ('vcs_git_lfs', 'enabled'),
-        ('vcs_git_lfs', 'store_location')
     )
 
     SVN_BRANCH_SECTION = 'vcs_svn_branch'
     SVN_TAG_SECTION = 'vcs_svn_tag'
-    SSL_SETTING = ('web', 'push_ssl')
     PATH_SETTING = ('paths', '/')
 
     def __init__(self, sa=None, repo=None):
         self.global_settings = SettingsModel(sa=sa)
         self.repo_settings = SettingsModel(sa=sa, repo=repo) if repo else None
         self._ui_settings = (
             self.HG_SETTINGS + self.GIT_SETTINGS + self.HOOKS_SETTINGS)
         self._svn_sections = (self.SVN_BRANCH_SECTION, self.SVN_TAG_SECTION)
 
     @property
     @assert_repo_settings
     def inherit_global_settings(self):
         setting = self.repo_settings.get_setting_by_name(self.INHERIT_SETTINGS)
         return setting.app_settings_value if setting else True
 
     @inherit_global_settings.setter
     @assert_repo_settings
     def inherit_global_settings(self, value):
         self.repo_settings.create_or_update_setting(
             self.INHERIT_SETTINGS, value, type_='bool')
 
     def get_keyname(self, key_name, prefix='rhodecode_'):
         return f'{prefix}{key_name}'
 
     def get_global_svn_branch_patterns(self):
         return self.global_settings.get_ui_by_section(self.SVN_BRANCH_SECTION)
 
     @assert_repo_settings
     def get_repo_svn_branch_patterns(self):
         return self.repo_settings.get_ui_by_section(self.SVN_BRANCH_SECTION)
 
     def get_global_svn_tag_patterns(self):
         return self.global_settings.get_ui_by_section(self.SVN_TAG_SECTION)
 
     @assert_repo_settings
     def get_repo_svn_tag_patterns(self):
         return self.repo_settings.get_ui_by_section(self.SVN_TAG_SECTION)
 
     def get_global_settings(self):
         return self._collect_all_settings(global_=True)
 
     @assert_repo_settings
     def get_repo_settings(self):
         return self._collect_all_settings(global_=False)
 
     @assert_repo_settings
     def get_repo_settings_inherited(self):
         global_settings = self.get_global_settings()
         global_settings.update(self.get_repo_settings())
         return global_settings
 
     @assert_repo_settings
     def create_or_update_repo_settings(
             self, data, inherit_global_settings=False):
         from rhodecode.model.scm import ScmModel
 
         self.inherit_global_settings = inherit_global_settings
 
         repo = self.repo_settings.get_repo()
         if not inherit_global_settings:
             if repo.repo_type == 'svn':
                 self.create_repo_svn_settings(data)
             else:
                 self.create_or_update_repo_hook_settings(data)
                 self.create_or_update_repo_pr_settings(data)
 
             if repo.repo_type == 'hg':
                 self.create_or_update_repo_hg_settings(data)
 
             if repo.repo_type == 'git':
                 self.create_or_update_repo_git_settings(data)
 
         ScmModel().mark_for_invalidation(repo.repo_name, delete=True)
 
     @assert_repo_settings
     def create_or_update_repo_hook_settings(self, data):
         for section, key in self.HOOKS_SETTINGS:
             data_key = self._get_form_ui_key(section, key)
             if data_key not in data:
                 raise ValueError(
                     f'The given data does not contain {data_key} key')
 
             active = data.get(data_key)
             repo_setting = self.repo_settings.get_ui_by_section_and_key(
                 section, key)
             if not repo_setting:
                 global_setting = self.global_settings.\
                     get_ui_by_section_and_key(section, key)
                 self.repo_settings.create_ui_section_value(
                     section, global_setting.ui_value, key=key, active=active)
             else:
                 repo_setting.ui_active = active
                 Session().add(repo_setting)
 
     def update_global_hook_settings(self, data):
         for section, key in self.HOOKS_SETTINGS:
             data_key = self._get_form_ui_key(section, key)
             if data_key not in data:
                 raise ValueError(
                     f'The given data does not contain {data_key} key')
             active = data.get(data_key)
             repo_setting = self.global_settings.get_ui_by_section_and_key(
                 section, key)
             repo_setting.ui_active = active
             Session().add(repo_setting)
 
     @assert_repo_settings
     def create_or_update_repo_pr_settings(self, data):
         return self._create_or_update_general_settings(
             self.repo_settings, data)
 
     def create_or_update_global_pr_settings(self, data):
         return self._create_or_update_general_settings(
             self.global_settings, data)
 
     @assert_repo_settings
     def create_repo_svn_settings(self, data):
         return self._create_svn_settings(self.repo_settings, data)
 
     def _set_evolution(self, settings, is_enabled):
         if is_enabled:
             # if evolve is active set evolution=all
 
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution'), value='all',
                 active=True)
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution.exchange'), value='yes',
                 active=True)
             # if evolve is active set topics server support
             self._create_or_update_ui(
                 settings, *('extensions', 'topic'), value='',
                 active=True)
 
         else:
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution'), value='',
                 active=False)
             self._create_or_update_ui(
                 settings, *('experimental', 'evolution.exchange'), value='no',
                 active=False)
             self._create_or_update_ui(
                 settings, *('extensions', 'topic'), value='',
                 active=False)
 
     @assert_repo_settings
     def create_or_update_repo_hg_settings(self, data):
         largefiles, phases, evolve = \
             self.HG_SETTINGS[:3]
         largefiles_key, phases_key, evolve_key = \
             self._get_settings_keys(self.HG_SETTINGS[:3], data)
 
         self._create_or_update_ui(
             self.repo_settings, *largefiles, value='',
             active=data[largefiles_key])
         self._create_or_update_ui(
             self.repo_settings, *evolve, value='',
             active=data[evolve_key])
         self._set_evolution(self.repo_settings, is_enabled=data[evolve_key])
 
         self._create_or_update_ui(
             self.repo_settings, *phases, value=safe_str(data[phases_key]))
 
     def create_or_update_global_hg_settings(self, data):
-        opts_len = 4
-        largefiles, largefiles_store, phases, evolve \
+        opts_len = 3
+        largefiles, phases, evolve \
             = self.GLOBAL_HG_SETTINGS[:opts_len]
-        largefiles_key, largefiles_store_key, phases_key, evolve_key \
+        largefiles_key, phases_key, evolve_key \
             = self._get_settings_keys(self.GLOBAL_HG_SETTINGS[:opts_len], data)
 
         self._create_or_update_ui(
             self.global_settings, *largefiles, value='',
             active=data[largefiles_key])
         self._create_or_update_ui(
-            self.global_settings, *largefiles_store, value=data[largefiles_store_key])
-        self._create_or_update_ui(
             self.global_settings, *phases, value=safe_str(data[phases_key]))
         self._create_or_update_ui(
             self.global_settings, *evolve, value='',
             active=data[evolve_key])
         self._set_evolution(self.global_settings, is_enabled=data[evolve_key])
 
     def create_or_update_repo_git_settings(self, data):
         # NOTE(marcink): # comma makes unpack work properly
         lfs_enabled, \
             = self.GIT_SETTINGS
 
         lfs_enabled_key, \
             = self._get_settings_keys(self.GIT_SETTINGS, data)
 
         self._create_or_update_ui(
             self.repo_settings, *lfs_enabled, value=data[lfs_enabled_key],
             active=data[lfs_enabled_key])
 
     def create_or_update_global_git_settings(self, data):
-        lfs_enabled, lfs_store_location \
-            = self.GLOBAL_GIT_SETTINGS
-        lfs_enabled_key, lfs_store_location_key \
-            = self._get_settings_keys(self.GLOBAL_GIT_SETTINGS, data)
+        lfs_enabled = self.GLOBAL_GIT_SETTINGS[0]
+        lfs_enabled_key = self._get_settings_keys(self.GLOBAL_GIT_SETTINGS, data)[0]
 
         self._create_or_update_ui(
             self.global_settings, *lfs_enabled, value=data[lfs_enabled_key],
             active=data[lfs_enabled_key])
-        self._create_or_update_ui(
-            self.global_settings, *lfs_store_location,
-            value=data[lfs_store_location_key])
 
     def create_or_update_global_svn_settings(self, data):
         # branch/tags patterns
         self._create_svn_settings(self.global_settings, data)
 
-    def update_global_ssl_setting(self, value):
-        self._create_or_update_ui(
-            self.global_settings, *self.SSL_SETTING, value=value)
-
     @assert_repo_settings
     def delete_repo_svn_pattern(self, id_):
         ui = self.repo_settings.UiDbModel.get(id_)
         if ui and ui.repository.repo_name == self.repo_settings.repo:
             # only delete if it's the same repo as initialized settings
             self.repo_settings.delete_ui(id_)
         else:
             # raise error as if we wouldn't find this option
             self.repo_settings.delete_ui(-1)
 
     def delete_global_svn_pattern(self, id_):
         self.global_settings.delete_ui(id_)
 
     @assert_repo_settings
     def get_repo_ui_settings(self, section=None, key=None):
         global_uis = self.global_settings.get_ui(section, key)
         repo_uis = self.repo_settings.get_ui(section, key)
 
         filtered_repo_uis = self._filter_ui_settings(repo_uis)
         filtered_repo_uis_keys = [
             (s.section, s.key) for s in filtered_repo_uis]
 
         def _is_global_ui_filtered(ui):
             return (
                 (ui.section, ui.key) in filtered_repo_uis_keys
                 or ui.section in self._svn_sections)
 
         filtered_global_uis = [
             ui for ui in global_uis if not _is_global_ui_filtered(ui)]
 
         return filtered_global_uis + filtered_repo_uis
 
     def get_global_ui_settings(self, section=None, key=None):
         return self.global_settings.get_ui(section, key)
 
     def get_ui_settings_as_config_obj(self, section=None, key=None):
         config = base.Config()
 
         ui_settings = self.get_ui_settings(section=section, key=key)
 
         for entry in ui_settings:
             config.set(entry.section, entry.key, entry.value)
 
         return config
 
     def get_ui_settings(self, section=None, key=None):
         if not self.repo_settings or self.inherit_global_settings:
             return self.get_global_ui_settings(section, key)
         else:
             return self.get_repo_ui_settings(section, key)
 
     def get_svn_patterns(self, section=None):
         if not self.repo_settings:
             return self.get_global_ui_settings(section)
         else:
             return self.get_repo_ui_settings(section)
 
     @assert_repo_settings
     def get_repo_general_settings(self):
         global_settings = self.global_settings.get_all_settings()
         repo_settings = self.repo_settings.get_all_settings()
         filtered_repo_settings = self._filter_general_settings(repo_settings)
         global_settings.update(filtered_repo_settings)
         return global_settings
 
     def get_global_general_settings(self):
         return self.global_settings.get_all_settings()
 
     def get_general_settings(self):
         if not self.repo_settings or self.inherit_global_settings:
             return self.get_global_general_settings()
         else:
             return self.get_repo_general_settings()
 
     def _filter_ui_settings(self, settings):
         filtered_settings = [
             s for s in settings if self._should_keep_setting(s)]
         return filtered_settings
 
     def _should_keep_setting(self, setting):
         keep = (
             (setting.section, setting.key) in self._ui_settings or
             setting.section in self._svn_sections)
         return keep
 
     def _filter_general_settings(self, settings):
         keys = [self.get_keyname(key) for key in self.GENERAL_SETTINGS]
         return {
             k: settings[k]
             for k in settings if k in keys}
 
     def _collect_all_settings(self, global_=False):
         settings = self.global_settings if global_ else self.repo_settings
         result = {}
 
         for section, key in self._ui_settings:
             ui = settings.get_ui_by_section_and_key(section, key)
             result_key = self._get_form_ui_key(section, key)
 
             if ui:
                 if section in ('hooks', 'extensions'):
                     result[result_key] = ui.ui_active
                 elif result_key in ['vcs_git_lfs_enabled']:
                     result[result_key] = ui.ui_active
                 else:
                     result[result_key] = ui.ui_value
 
         for name in self.GENERAL_SETTINGS:
             setting = settings.get_setting_by_name(name)
             if setting:
                 result_key = self.get_keyname(name)
                 result[result_key] = setting.app_settings_value
 
         return result
 
     def _get_form_ui_key(self, section, key):
         return '{section}_{key}'.format(
             section=section, key=key.replace('.', '_'))
 
     def _create_or_update_ui(
             self, settings, section, key, value=None, active=None):
         ui = settings.get_ui_by_section_and_key(section, key)
         if not ui:
             active = True if active is None else active
             settings.create_ui_section_value(
                 section, value, key=key, active=active)
         else:
             if active is not None:
                 ui.ui_active = active
             if value is not None:
                 ui.ui_value = value
             Session().add(ui)
 
     def _create_svn_settings(self, settings, data):
         svn_settings = {
             'new_svn_branch': self.SVN_BRANCH_SECTION,
             'new_svn_tag': self.SVN_TAG_SECTION
         }
         for key in svn_settings:
             if data.get(key):
                 settings.create_ui_section_value(svn_settings[key], data[key])
 
     def _create_or_update_general_settings(self, settings, data):
         for name in self.GENERAL_SETTINGS:
             data_key = self.get_keyname(name)
             if data_key not in data:
                 raise ValueError(
                     f'The given data does not contain {data_key} key')
             setting = settings.create_or_update_setting(
                 name, data[data_key], 'bool')
             Session().add(setting)
 
     def _get_settings_keys(self, settings, data):
         data_keys = [self._get_form_ui_key(*s) for s in settings]
         for data_key in data_keys:
             if data_key not in data:
                 raise ValueError(
                     f'The given data does not contain {data_key} key')
         return data_keys
 
     def create_largeobjects_dirs_if_needed(self, repo_store_path):
         """
         This is subscribed to the `pyramid.events.ApplicationCreated` event. It
         does a repository scan if enabled in the settings.
         """
 
         from rhodecode.lib.vcs.backends.hg import largefiles_store
         from rhodecode.lib.vcs.backends.git import lfs_store
 
         paths = [
             largefiles_store(repo_store_path),
             lfs_store(repo_store_path)]
 
         for path in paths:
             if os.path.isdir(path):
                 continue
             if os.path.isfile(path):
                 continue
             # not a file nor dir, we try to create it
             try:
                 os.makedirs(path)
             except Exception:
                 log.warning('Failed to create largefiles dir:%s', path)

rhodecode/model/user.py

@@ -1,1055 +1,1055 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 users model for RhodeCode
 """
 
 import logging
 import traceback
 import datetime
 import ipaddress
 
 from pyramid.threadlocal import get_current_request
 from sqlalchemy.exc import DatabaseError
 
 from rhodecode import events
 from rhodecode.lib.user_log_filter import user_log_filter
 from rhodecode.lib.utils2 import (
     get_current_rhodecode_user, action_logger_generic,
     AttributeDict, str2bool)
 from rhodecode.lib.str_utils import safe_str
 from rhodecode.lib.exceptions import (
     DefaultUserException, UserOwnsReposException, UserOwnsRepoGroupsException,
     UserOwnsUserGroupsException, NotAllowedToCreateUserError,
     UserOwnsPullRequestsException, UserOwnsArtifactsException, DuplicateUpdateUserError)
 from rhodecode.lib.caching_query import FromCache
 from rhodecode.model import BaseModel
 from rhodecode.model.db import (
     _hash_key, func, true, false, or_, joinedload, User, UserToPerm,
     UserEmailMap, UserIpMap, UserLog)
 from rhodecode.model.meta import Session
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.repo_group import RepoGroupModel
 
 log = logging.getLogger(__name__)
 
 
 class UserModel(BaseModel):
     cls = User
 
     def get(self, user_id, cache=False):
         user = self.sa.query(User)
         if cache:
             user = user.options(
                 FromCache("sql_cache_short", f"get_user_{user_id}"))
         return user.get(user_id)
 
     def get_user(self, user):
         return self._get_user(user)
 
     def _serialize_user(self, user):
         import rhodecode.lib.helpers as h
 
         return {
             'id': user.user_id,
             'first_name': user.first_name,
             'last_name': user.last_name,
             'username': user.username,
             'email': user.email,
             'icon_link': h.gravatar_url(user.email, 30),
             'profile_link':  h.link_to_user(user),
             'value_display': h.escape(h.person(user)),
             'value': user.username,
             'value_type': 'user',
             'active': user.active,
         }
 
     def get_users(self, name_contains=None, limit=20, only_active=True):
 
         query = self.sa.query(User)
         if only_active:
             query = query.filter(User.active == true())
 
         if name_contains:
             ilike_expression = f'%{safe_str(name_contains)}%'
             query = query.filter(
                 or_(
                     User.name.ilike(ilike_expression),
                     User.lastname.ilike(ilike_expression),
                     User.username.ilike(ilike_expression)
                 )
             )
             # sort by len to have top most matches first
             query = query.order_by(func.length(User.username))\
                 .order_by(User.username)
             query = query.limit(limit)
 
         users = query.all()
 
         _users = [
             self._serialize_user(user) for user in users
         ]
         return _users
 
     def get_by_username(self, username, cache=False, case_insensitive=False):
 
         if case_insensitive:
             user = self.sa.query(User).filter(User.username.ilike(username))
         else:
             user = self.sa.query(User)\
                 .filter(User.username == username)
 
         if cache:
             name_key = _hash_key(username)
             user = user.options(
                 FromCache("sql_cache_short", f"get_user_{name_key}"))
         return user.scalar()
 
     def get_by_email(self, email, cache=False, case_insensitive=False):
         return User.get_by_email(email, case_insensitive, cache)
 
     def get_by_auth_token(self, auth_token, cache=False):
         return User.get_by_auth_token(auth_token, cache)
 
     def get_active_user_count(self, cache=False):
         qry = User.query().filter(
             User.active == true()).filter(
                 User.username != User.DEFAULT_USER)
         if cache:
             qry = qry.options(
                 FromCache("sql_cache_short", "get_active_users"))
         return qry.count()
 
     def create(self, form_data, cur_user=None):
         if not cur_user:
             cur_user = getattr(get_current_rhodecode_user(), 'username', None)
 
         user_data = {
             'username': form_data['username'],
             'password': form_data['password'],
             'email': form_data['email'],
             'firstname': form_data['firstname'],
             'lastname': form_data['lastname'],
             'active': form_data['active'],
             'extern_type': form_data['extern_type'],
             'extern_name': form_data['extern_name'],
             'admin': False,
             'cur_user': cur_user
         }
 
         if 'create_repo_group' in form_data:
             user_data['create_repo_group'] = str2bool(
                 form_data.get('create_repo_group'))
 
         try:
             if form_data.get('password_change'):
                 user_data['force_password_change'] = True
             return UserModel().create_or_update(**user_data)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def update_user(self, user, skip_attrs=None, **kwargs):
         from rhodecode.lib.auth import get_crypt_password
 
         user = self._get_user(user)
         if user.username == User.DEFAULT_USER:
             raise DefaultUserException(
                 "You can't edit this user (`%(username)s`) since it's "
                 "crucial for entire application" % {
                 'username': user.username})
 
         # first store only defaults
         user_attrs = {
             'updating_user_id': user.user_id,
             'username': user.username,
             'password': user.password,
             'email': user.email,
             'firstname': user.name,
             'lastname': user.lastname,
             'description': user.description,
             'active': user.active,
             'admin': user.admin,
             'extern_name': user.extern_name,
             'extern_type': user.extern_type,
             'language': user.user_data.get('language')
         }
 
         # in case there's new_password, that comes from form, use it to
         # store password
         if kwargs.get('new_password'):
             kwargs['password'] = kwargs['new_password']
 
         # cleanups, my_account password change form
         kwargs.pop('current_password', None)
         kwargs.pop('new_password', None)
 
         # cleanups, user edit password change form
         kwargs.pop('password_confirmation', None)
         kwargs.pop('password_change', None)
 
         # create repo group on user creation
         kwargs.pop('create_repo_group', None)
 
         # legacy forms send name, which is the firstname
         firstname = kwargs.pop('name', None)
         if firstname:
             kwargs['firstname'] = firstname
 
         for k, v in kwargs.items():
             # skip if we don't want to update this
             if skip_attrs and k in skip_attrs:
                 continue
 
             user_attrs[k] = v
 
         try:
             return self.create_or_update(**user_attrs)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def create_or_update(
             self, username, password, email, firstname='', lastname='',
             active=True, admin=False, extern_type=None, extern_name=None,
             cur_user=None, plugin=None, force_password_change=False,
             allow_to_create_user=True, create_repo_group=None,
             updating_user_id=None, language=None, description='',
             strict_creation_check=True):
         """
         Creates a new instance if not found, or updates current one
 
         :param username:
         :param password:
         :param email:
         :param firstname:
         :param lastname:
         :param active:
         :param admin:
         :param extern_type:
         :param extern_name:
         :param cur_user:
         :param plugin: optional plugin this method was called from
         :param force_password_change: toggles new or existing user flag
             for password change
         :param allow_to_create_user: Defines if the method can actually create
             new users
         :param create_repo_group: Defines if the method should also
             create an repo group with user name, and owner
         :param updating_user_id: if we set it up this is the user we want to
             update this allows to editing username.
         :param language: language of user from interface.
         :param description: user description
         :param strict_creation_check: checks for allowed creation license wise etc.
 
         :returns: new User object with injected `is_new_user` attribute.
         """
 
         if not cur_user:
             cur_user = getattr(get_current_rhodecode_user(), 'username', None)
 
         from rhodecode.lib.auth import (
             get_crypt_password, check_password)
         from rhodecode.lib import hooks_base
 
         def _password_change(new_user, password):
             old_password = new_user.password or ''
             # empty password
             if not old_password:
                 return False
 
             # password check is only needed for RhodeCode internal auth calls
             # in case it's a plugin we don't care
             if not plugin:
 
                 # first check if we gave crypted password back, and if it
                 # matches it's not password change
                 if new_user.password == password:
                     return False
 
                 password_match = check_password(password, old_password)
                 if not password_match:
                     return True
 
             return False
 
         # read settings on default personal repo group creation
         if create_repo_group is None:
             default_create_repo_group = RepoGroupModel()\
                 .get_default_create_personal_repo_group()
             create_repo_group = default_create_repo_group
 
         user_data = {
             'username': username,
             'password': password,
             'email': email,
             'firstname': firstname,
             'lastname': lastname,
             'active': active,
             'admin': admin
         }
 
         if updating_user_id:
             log.debug('Checking for existing account in RhodeCode '
                       'database with user_id `%s` ', updating_user_id)
             user = User.get(updating_user_id)
             # now also validate if USERNAME belongs to potentially other user
             maybe_other_user = User.get_by_username(username, case_insensitive=True)
             if maybe_other_user and maybe_other_user.user_id != updating_user_id:
                 raise DuplicateUpdateUserError(f'different user exists with the {username} username')
         else:
             log.debug('Checking for existing account in RhodeCode '
                       'database with username `%s` ', username)
             user = User.get_by_username(username, case_insensitive=True)
 
         if user is None:
             # we check internal flag if this method is actually allowed to
             # create new user
             if not allow_to_create_user:
                 msg = ('Method wants to create new user, but it is not '
                        'allowed to do so')
                 log.warning(msg)
                 raise NotAllowedToCreateUserError(msg)
 
             log.debug('Creating new user %s', username)
 
             # only if we create user that is active
             new_active_user = active
             if new_active_user and strict_creation_check:
                 # raises UserCreationError if it's not allowed for any reason to
                 # create new active user, this also executes pre-create hooks
                 hooks_base.check_allowed_create_user(user_data, cur_user, strict_check=True)
             events.trigger(events.UserPreCreate(user_data))
             new_user = User()
             edit = False
         else:
             log.debug('updating user `%s`', username)
             events.trigger(events.UserPreUpdate(user, user_data))
             new_user = user
             edit = True
 
             # we're not allowed to edit default user
             if user.username == User.DEFAULT_USER:
                 raise DefaultUserException(
                     "You can't edit this user (`%(username)s`) since it's "
                     "crucial for entire application"
                     % {'username': user.username})
 
         # inject special attribute that will tell us if User is new or old
         new_user.is_new_user = not edit
         # for users that didn's specify auth type, we use RhodeCode built in
         from rhodecode.authentication.plugins import auth_rhodecode
         extern_name = extern_name or auth_rhodecode.RhodeCodeAuthPlugin.uid
         extern_type = extern_type or auth_rhodecode.RhodeCodeAuthPlugin.uid
 
         try:
             new_user.username = username
             new_user.admin = admin
             new_user.email = email
             new_user.active = active
             new_user.extern_name = safe_str(extern_name)
             new_user.extern_type = safe_str(extern_type)
             new_user.name = firstname
             new_user.lastname = lastname
             new_user.description = description
 
             # set password only if creating an user or password is changed
             if not edit or _password_change(new_user, password):
                 reason = 'new password' if edit else 'new user'
                 log.debug('Updating password reason=>%s', reason)
                 new_user.password = get_crypt_password(password) if password else None
 
             if force_password_change:
                 new_user.update_userdata(force_password_change=True)
             if language:
                 new_user.update_userdata(language=language)
             new_user.update_userdata(notification_status=True)
 
             self.sa.add(new_user)
 
             if not edit and create_repo_group:
                 RepoGroupModel().create_personal_repo_group(
                     new_user, commit_early=False)
 
             if not edit:
                 # add the RSS token
                 self.add_auth_token(
                     user=username, lifetime_minutes=-1,
                     role=self.auth_token_role.ROLE_FEED,
                     description='Generated feed token')
 
                 kwargs = new_user.get_dict()
                 # backward compat, require api_keys present
                 kwargs['api_keys'] = kwargs['auth_tokens']
                 hooks_base.create_user(created_by=cur_user, **kwargs)
                 events.trigger(events.UserPostCreate(user_data))
             return new_user
         except (DatabaseError,):
             log.error(traceback.format_exc())
             raise
 
     def create_registration(self, form_data,
                             extern_name='rhodecode', extern_type='rhodecode'):
         from rhodecode.model.notification import NotificationModel
         from rhodecode.model.notification import EmailNotificationModel
 
         try:
             form_data['admin'] = False
             form_data['extern_name'] = extern_name
             form_data['extern_type'] = extern_type
             new_user = self.create(form_data)
 
             self.sa.add(new_user)
             self.sa.flush()
 
             user_data = new_user.get_dict()
             user_data.update({
                 'first_name': user_data.get('firstname'),
                 'last_name': user_data.get('lastname'),
             })
             kwargs = {
                 # use SQLALCHEMY safe dump of user data
                 'user': AttributeDict(user_data),
                 'date': datetime.datetime.now()
             }
             notification_type = EmailNotificationModel.TYPE_REGISTRATION
 
             # create notification objects, and emails
             NotificationModel().create(
                 created_by=new_user,
                 notification_subject='',  # Filled in based on the notification_type
                 notification_body='',  # Filled in based on the notification_type
                 notification_type=notification_type,
                 recipients=None,  # all admins
                 email_kwargs=kwargs,
             )
 
             return new_user
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def _handle_user_repos(self, username, repositories, handle_user,
                            handle_mode=None):
 
         left_overs = True
 
         from rhodecode.model.repo import RepoModel
 
         if handle_mode == 'detach':
             for obj in repositories:
                 obj.user = handle_user
                 # set description we know why we super admin now owns
                 # additional repositories that were orphaned !
                 obj.description += '  \n::detached repository from deleted user: %s' % (username,)
                 self.sa.add(obj)
             left_overs = False
         elif handle_mode == 'delete':
             for obj in repositories:
                 RepoModel().delete(obj, forks='detach')
             left_overs = False
 
         # if nothing is done we have left overs left
         return left_overs
 
     def _handle_user_repo_groups(self, username, repository_groups, handle_user,
                                  handle_mode=None):
 
         left_overs = True
 
         from rhodecode.model.repo_group import RepoGroupModel
 
         if handle_mode == 'detach':
             for r in repository_groups:
                 r.user = handle_user
                 # set description we know why we super admin now owns
                 # additional repositories that were orphaned !
                 r.group_description += '  \n::detached repository group from deleted user: %s' % (username,)
                 r.personal = False
                 self.sa.add(r)
             left_overs = False
         elif handle_mode == 'delete':
             for r in repository_groups:
                 RepoGroupModel().delete(r)
             left_overs = False
 
         # if nothing is done we have left overs left
         return left_overs
 
     def _handle_user_user_groups(self, username, user_groups, handle_user,
                                  handle_mode=None):
 
         left_overs = True
 
         from rhodecode.model.user_group import UserGroupModel
 
         if handle_mode == 'detach':
             for r in user_groups:
                 for user_user_group_to_perm in r.user_user_group_to_perm:
                     if user_user_group_to_perm.user.username == username:
                         user_user_group_to_perm.user = handle_user
                 r.user = handle_user
                 # set description we know why we super admin now owns
                 # additional repositories that were orphaned !
                 r.user_group_description += '  \n::detached user group from deleted user: %s' % (username,)
                 self.sa.add(r)
             left_overs = False
         elif handle_mode == 'delete':
             for r in user_groups:
                 UserGroupModel().delete(r)
             left_overs = False
 
         # if nothing is done we have left overs left
         return left_overs
 
     def _handle_user_pull_requests(self, username, pull_requests, handle_user,
                                    handle_mode=None):
         left_overs = True
 
         from rhodecode.model.pull_request import PullRequestModel
 
         if handle_mode == 'detach':
             for pr in pull_requests:
                 pr.user_id = handle_user.user_id
                 # set description we know why we super admin now owns
                 # additional repositories that were orphaned !
                 pr.description += '  \n::detached pull requests from deleted user: %s' % (username,)
                 self.sa.add(pr)
             left_overs = False
         elif handle_mode == 'delete':
             for pr in pull_requests:
                 PullRequestModel().delete(pr)
 
             left_overs = False
 
         # if nothing is done we have leftovers left
         return left_overs
 
     def _handle_user_artifacts(self, username, artifacts, handle_user,
                                handle_mode=None):
 
         left_overs = True
 
         if handle_mode == 'detach':
             for a in artifacts:
                 a.upload_user = handle_user
                 # set description we know why we super admin now owns
                 # additional artifacts that were orphaned !
                 a.file_description += '  \n::detached artifact from deleted user: %s' % (username,)
                 self.sa.add(a)
             left_overs = False
         elif handle_mode == 'delete':
             from rhodecode.apps.file_store import utils as store_utils
             request = get_current_request()
-            storage = store_utils.get_file_storage(request.registry.settings)
+            f_store = store_utils.get_filestore_backend(request.registry.settings)
             for a in artifacts:
                 file_uid = a.file_uid
-                storage.delete(file_uid)
+                f_store.delete(file_uid)
                 self.sa.delete(a)
 
             left_overs = False
 
         # if nothing is done we have left overs left
         return left_overs
 
     def delete(self, user, cur_user=None, handle_repos=None,
                handle_repo_groups=None, handle_user_groups=None,
                handle_pull_requests=None, handle_artifacts=None, handle_new_owner=None):
         from rhodecode.lib import hooks_base
 
         if not cur_user:
             cur_user = getattr(get_current_rhodecode_user(), 'username', None)
 
         user = self._get_user(user)
 
         try:
             if user.username == User.DEFAULT_USER:
                 raise DefaultUserException(
                     "You can't remove this user since it's"
                     " crucial for entire application")
             handle_user = handle_new_owner or self.cls.get_first_super_admin()
             log.debug('New detached objects owner %s', handle_user)
 
             left_overs = self._handle_user_repos(
                 user.username, user.repositories, handle_user, handle_repos)
             if left_overs and user.repositories:
                 repos = [x.repo_name for x in user.repositories]
                 raise UserOwnsReposException(
                     'user "%(username)s" still owns %(len_repos)s repositories and cannot be '
                     'removed. Switch owners or remove those repositories:%(list_repos)s'
                     % {'username': user.username, 'len_repos': len(repos),
                        'list_repos': ', '.join(repos)})
 
             left_overs = self._handle_user_repo_groups(
                 user.username, user.repository_groups, handle_user, handle_repo_groups)
             if left_overs and user.repository_groups:
                 repo_groups = [x.group_name for x in user.repository_groups]
                 raise UserOwnsRepoGroupsException(
                     'user "%(username)s" still owns %(len_repo_groups)s repository groups and cannot be '
                     'removed. Switch owners or remove those repository groups:%(list_repo_groups)s'
                     % {'username': user.username, 'len_repo_groups': len(repo_groups),
                        'list_repo_groups': ', '.join(repo_groups)})
 
             left_overs = self._handle_user_user_groups(
                 user.username, user.user_groups, handle_user, handle_user_groups)
             if left_overs and user.user_groups:
                 user_groups = [x.users_group_name for x in user.user_groups]
                 raise UserOwnsUserGroupsException(
                     'user "%s" still owns %s user groups and cannot be '
                     'removed. Switch owners or remove those user groups:%s'
                     % (user.username, len(user_groups), ', '.join(user_groups)))
 
             left_overs = self._handle_user_pull_requests(
                 user.username, user.user_pull_requests, handle_user, handle_pull_requests)
             if left_overs and user.user_pull_requests:
                 pull_requests = [f'!{x.pull_request_id}' for x in user.user_pull_requests]
                 raise UserOwnsPullRequestsException(
                     'user "%s" still owns %s pull requests and cannot be '
                     'removed. Switch owners or remove those pull requests:%s'
                     % (user.username, len(pull_requests), ', '.join(pull_requests)))
 
             left_overs = self._handle_user_artifacts(
                 user.username, user.artifacts, handle_user, handle_artifacts)
             if left_overs and user.artifacts:
                 artifacts = [x.file_uid for x in user.artifacts]
                 raise UserOwnsArtifactsException(
                     'user "%s" still owns %s artifacts and cannot be '
                     'removed. Switch owners or remove those artifacts:%s'
                     % (user.username, len(artifacts), ', '.join(artifacts)))
 
             user_data = user.get_dict()  # fetch user data before expire
 
             # we might change the user data with detach/delete, make sure
             # the object is marked as expired before actually deleting !
             self.sa.expire(user)
             self.sa.delete(user)
 
             hooks_base.delete_user(deleted_by=cur_user, **user_data)
         except Exception:
             log.error(traceback.format_exc())
             raise
 
     def reset_password_link(self, data, pwd_reset_url):
         from rhodecode.lib.celerylib import tasks, run_task
         from rhodecode.model.notification import EmailNotificationModel
         user_email = data['email']
         try:
             user = User.get_by_email(user_email)
             if user:
                 log.debug('password reset user found %s', user)
 
                 email_kwargs = {
                     'password_reset_url': pwd_reset_url,
                     'user': user,
                     'email': user_email,
                     'date': datetime.datetime.now(),
                     'first_admin_email': User.get_first_super_admin().email
                 }
 
                 (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
                     EmailNotificationModel.TYPE_PASSWORD_RESET, **email_kwargs)
 
                 recipients = [user_email]
 
                 action_logger_generic(
                     'sending password reset email to user: {}'.format(
                         user), namespace='security.password_reset')
 
                 run_task(tasks.send_email, recipients, subject,
                          email_body_plaintext, email_body)
 
             else:
                 log.debug("password reset email %s not found", user_email)
         except Exception:
             log.error(traceback.format_exc())
             return False
 
         return True
 
     def reset_password(self, data):
         from rhodecode.lib.celerylib import tasks, run_task
         from rhodecode.model.notification import EmailNotificationModel
         from rhodecode.lib import auth
         user_email = data['email']
         pre_db = True
         try:
             user = User.get_by_email(user_email)
             new_passwd = auth.PasswordGenerator().gen_password(
                 12, auth.PasswordGenerator.ALPHABETS_BIG_SMALL)
             if user:
                 user.password = auth.get_crypt_password(new_passwd)
                 # also force this user to reset his password !
                 user.update_userdata(force_password_change=True)
 
                 Session().add(user)
 
                 # now delete the token in question
                 UserApiKeys = AuthTokenModel.cls
                 UserApiKeys().query().filter(
                     UserApiKeys.api_key == data['token']).delete()
 
                 Session().commit()
                 log.info('successfully reset password for `%s`', user_email)
 
             if new_passwd is None:
                 raise Exception('unable to generate new password')
 
             pre_db = False
 
             email_kwargs = {
                 'new_password': new_passwd,
                 'user': user,
                 'email': user_email,
                 'date': datetime.datetime.now(),
                 'first_admin_email': User.get_first_super_admin().email
             }
 
             (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
                 EmailNotificationModel.TYPE_PASSWORD_RESET_CONFIRMATION,
                 **email_kwargs)
 
             recipients = [user_email]
 
             action_logger_generic(
                 'sent new password to user: {} with email: {}'.format(
                     user, user_email), namespace='security.password_reset')
 
             run_task(tasks.send_email, recipients, subject,
                      email_body_plaintext, email_body)
 
         except Exception:
             log.error('Failed to update user password')
             log.error(traceback.format_exc())
             if pre_db:
                 # we rollback only if local db stuff fails. If it goes into
                 # run_task, we're pass rollback state this wouldn't work then
                 Session().rollback()
 
         return True
 
     def fill_data(self, auth_user, user_id=None, api_key=None, username=None):
         """
         Fetches auth_user by user_id,or api_key if present.
         Fills auth_user attributes with those taken from database.
         Additionally set's is_authenitated if lookup fails
         present in database
 
         :param auth_user: instance of user to set attributes
         :param user_id: user id to fetch by
         :param api_key: api key to fetch by
         :param username: username to fetch by
         """
         def token_obfuscate(token):
             if token:
                 return token[:4] + "****"
 
         if user_id is None and api_key is None and username is None:
             raise Exception('You need to pass user_id, api_key or username')
 
         log.debug(
             'AuthUser: fill data execution based on: '
             'user_id:%s api_key:%s username:%s', user_id, api_key, username)
         try:
             found_with = ''
             dbuser = None
             if user_id:
                 dbuser = self.get(user_id)
                 found_with = 'user_id'
             elif api_key:
                 dbuser = self.get_by_auth_token(api_key)
                 found_with = 'auth_token'
             elif username:
                 dbuser = self.get_by_username(username)
                 found_with = 'username'
 
             if not dbuser:
                 log.warning(
                     'Unable to lookup user by id:%s api_key:%s username:%s, found with: %s',
                     user_id, token_obfuscate(api_key), username, found_with)
                 return False
             if not dbuser.active:
                 log.debug('User `%s:%s` is inactive, skipping fill data',
                           username, user_id)
                 return False
 
             log.debug('AuthUser: filling found user:%s data, found with: %s', dbuser, found_with)
 
             attrs = {
                 'user_id': dbuser.user_id,
                 'username': dbuser.username,
                 'name': dbuser.name,
                 'first_name': dbuser.first_name,
                 'firstname': dbuser.firstname,
                 'last_name': dbuser.last_name,
                 'lastname': dbuser.lastname,
                 'admin': dbuser.admin,
                 'active': dbuser.active,
 
                 'email': dbuser.email,
                 'emails': dbuser.emails_cached(),
                 'short_contact': dbuser.short_contact,
                 'full_contact': dbuser.full_contact,
                 'full_name': dbuser.full_name,
                 'full_name_or_username': dbuser.full_name_or_username,
 
                 '_api_key': dbuser._api_key,
                 '_user_data': dbuser._user_data,
 
                 'created_on': dbuser.created_on,
                 'extern_name': dbuser.extern_name,
                 'extern_type': dbuser.extern_type,
 
                 'inherit_default_permissions': dbuser.inherit_default_permissions,
 
                 'language': dbuser.language,
                 'last_activity': dbuser.last_activity,
                 'last_login': dbuser.last_login,
                 'password': dbuser.password,
             }
             auth_user.__dict__.update(attrs)
         except Exception:
             log.error(traceback.format_exc())
             auth_user.is_authenticated = False
             return False
 
         return True
 
     def has_perm(self, user, perm):
         perm = self._get_perm(perm)
         user = self._get_user(user)
 
         return UserToPerm.query().filter(UserToPerm.user == user)\
             .filter(UserToPerm.permission == perm).scalar() is not None
 
     def grant_perm(self, user, perm):
         """
         Grant user global permissions
 
         :param user:
         :param perm:
         """
         user = self._get_user(user)
         perm = self._get_perm(perm)
         # if this permission is already granted skip it
         _perm = UserToPerm.query()\
             .filter(UserToPerm.user == user)\
             .filter(UserToPerm.permission == perm)\
             .scalar()
         if _perm:
             return
         new = UserToPerm()
         new.user = user
         new.permission = perm
         self.sa.add(new)
         return new
 
     def revoke_perm(self, user, perm):
         """
         Revoke users global permissions
 
         :param user:
         :param perm:
         """
         user = self._get_user(user)
         perm = self._get_perm(perm)
 
         obj = UserToPerm.query()\
                 .filter(UserToPerm.user == user)\
                 .filter(UserToPerm.permission == perm)\
                 .scalar()
         if obj:
             self.sa.delete(obj)
 
     def add_extra_email(self, user, email):
         """
         Adds email address to UserEmailMap
 
         :param user:
         :param email:
         """
 
         user = self._get_user(user)
 
         obj = UserEmailMap()
         obj.user = user
         obj.email = email
         self.sa.add(obj)
         return obj
 
     def delete_extra_email(self, user, email_id):
         """
         Removes email address from UserEmailMap
 
         :param user:
         :param email_id:
         """
         user = self._get_user(user)
         obj = UserEmailMap.query().get(email_id)
         if obj and obj.user_id == user.user_id:
             self.sa.delete(obj)
 
     def parse_ip_range(self, ip_range):
         ip_list = []
 
         def make_unique(value):
             seen = []
             return [c for c in value if not (c in seen or seen.append(c))]
 
         # firsts split by commas
         for ip_range in ip_range.split(','):
             if not ip_range:
                 continue
             ip_range = ip_range.strip()
             if '-' in ip_range:
                 start_ip, end_ip = ip_range.split('-', 1)
                 start_ip = ipaddress.ip_address(safe_str(start_ip.strip()))
                 end_ip = ipaddress.ip_address(safe_str(end_ip.strip()))
                 parsed_ip_range = []
 
                 for index in range(int(start_ip), int(end_ip) + 1):
                     new_ip = ipaddress.ip_address(index)
                     parsed_ip_range.append(str(new_ip))
                 ip_list.extend(parsed_ip_range)
             else:
                 ip_list.append(ip_range)
 
         return make_unique(ip_list)
 
     def add_extra_ip(self, user, ip, description=None):
         """
         Adds ip address to UserIpMap
 
         :param user:
         :param ip:
         """
 
         user = self._get_user(user)
         obj = UserIpMap()
         obj.user = user
         obj.ip_addr = ip
         obj.description = description
         self.sa.add(obj)
         return obj
 
     auth_token_role = AuthTokenModel.cls
 
     def add_auth_token(self, user, lifetime_minutes, role, description='',
                        scope_callback=None):
         """
         Add AuthToken for user.
 
         :param user: username/user_id
         :param lifetime_minutes: in minutes the lifetime for token, -1 equals no limit
         :param role: one of AuthTokenModel.cls.ROLE_*
         :param description: optional string description
         """
 
         token = AuthTokenModel().create(
             user, description, lifetime_minutes, role)
         if scope_callback and callable(scope_callback):
             # call the callback if we provide, used to attach scope for EE edition
             scope_callback(token)
         return token
 
     def delete_extra_ip(self, user, ip_id):
         """
         Removes ip address from UserIpMap
 
         :param user:
         :param ip_id:
         """
         user = self._get_user(user)
         obj = UserIpMap.query().get(ip_id)
         if obj and obj.user_id == user.user_id:
             self.sa.delete(obj)
 
     def get_accounts_in_creation_order(self, current_user=None):
         """
         Get accounts in order of creation for deactivation for license limits
 
         pick currently logged in user, and append to the list in position 0
         pick all super-admins in order of creation date and add it to the list
         pick all other accounts in order of creation and add it to the list.
 
         Based on that list, the last accounts can be disabled as they are
         created at the end and don't include any of the super admins as well
         as the current user.
 
         :param current_user: optionally current user running this operation
         """
 
         if not current_user:
             current_user = get_current_rhodecode_user()
         active_super_admins = [
             x.user_id for x in User.query()
             .filter(User.user_id != current_user.user_id)
             .filter(User.active == true())
             .filter(User.admin == true())
             .order_by(User.created_on.asc())]
 
         active_regular_users = [
             x.user_id for x in User.query()
             .filter(User.user_id != current_user.user_id)
             .filter(User.active == true())
             .filter(User.admin == false())
             .order_by(User.created_on.asc())]
 
         list_of_accounts = [current_user.user_id]
         list_of_accounts += active_super_admins
         list_of_accounts += active_regular_users
 
         return list_of_accounts
 
     def deactivate_last_users(self, expected_users, current_user=None):
         """
         Deactivate accounts that are over the license limits.
         Algorithm of which accounts to disabled is based on the formula:
 
         Get current user, then super admins in creation order, then regular
         active users in creation order.
 
         Using that list we mark all accounts from the end of it as inactive.
         This way we block only latest created accounts.
 
         :param expected_users: list of users in special order, we deactivate
             the end N amount of users from that list
         """
 
         list_of_accounts = self.get_accounts_in_creation_order(
             current_user=current_user)
 
         for acc_id in list_of_accounts[expected_users + 1:]:
             user = User.get(acc_id)
             log.info('Deactivating account %s for license unlock', user)
             user.active = False
             Session().add(user)
             Session().commit()
 
         return
 
     def get_user_log(self, user, filter_term):
         user_log = UserLog.query()\
             .filter(or_(UserLog.user_id == user.user_id,
                         UserLog.username == user.username))\
             .options(joinedload(UserLog.user))\
             .options(joinedload(UserLog.repository))\
             .order_by(UserLog.action_date.desc())
 
         user_log = user_log_filter(user_log, filter_term)
         return user_log

rhodecode/public/js/rhodecode/routes.js

@@ -1,421 +1,422 b''
 
 /******************************************************************************
  *                                                                            *
  *                      DO NOT CHANGE THIS FILE MANUALLY                      *
  *                                                                            *
  *                                                                            *
  *        This file is automatically generated when the app starts up with    *
  *                         generate_js_files = true                           *
  *                                                                            *
  *  To add a route here pass jsroute=True to the route definition in the app  *
  *                                                                            *
  ******************************************************************************/
 function registerRCRoutes() {
     // routes registration
     pyroutes.register('admin_artifacts', '/_admin/artifacts', []);
     pyroutes.register('admin_artifacts_data', '/_admin/artifacts-data', []);
     pyroutes.register('admin_artifacts_delete', '/_admin/artifacts/%(uid)s/delete', ['uid']);
     pyroutes.register('admin_artifacts_show_all', '/_admin/artifacts', []);
     pyroutes.register('admin_artifacts_show_info', '/_admin/artifacts/%(uid)s', ['uid']);
     pyroutes.register('admin_artifacts_update', '/_admin/artifacts/%(uid)s/update', ['uid']);
     pyroutes.register('admin_audit_log_entry', '/_admin/audit_logs/%(audit_log_id)s', ['audit_log_id']);
     pyroutes.register('admin_audit_logs', '/_admin/audit_logs', []);
     pyroutes.register('admin_automation', '/_admin/automation', []);
     pyroutes.register('admin_automation_update', '/_admin/automation/%(entry_id)s/update', ['entry_id']);
     pyroutes.register('admin_defaults_repositories', '/_admin/defaults/repositories', []);
     pyroutes.register('admin_defaults_repositories_update', '/_admin/defaults/repositories/update', []);
     pyroutes.register('admin_home', '/_admin', []);
     pyroutes.register('admin_permissions_application', '/_admin/permissions/application', []);
     pyroutes.register('admin_permissions_application_update', '/_admin/permissions/application/update', []);
     pyroutes.register('admin_permissions_auth_token_access', '/_admin/permissions/auth_token_access', []);
     pyroutes.register('admin_permissions_branch', '/_admin/permissions/branch', []);
     pyroutes.register('admin_permissions_branch_update', '/_admin/permissions/branch/update', []);
     pyroutes.register('admin_permissions_global', '/_admin/permissions/global', []);
     pyroutes.register('admin_permissions_global_update', '/_admin/permissions/global/update', []);
     pyroutes.register('admin_permissions_ips', '/_admin/permissions/ips', []);
     pyroutes.register('admin_permissions_object', '/_admin/permissions/object', []);
     pyroutes.register('admin_permissions_object_update', '/_admin/permissions/object/update', []);
     pyroutes.register('admin_permissions_overview', '/_admin/permissions/overview', []);
     pyroutes.register('admin_permissions_ssh_keys', '/_admin/permissions/ssh_keys', []);
     pyroutes.register('admin_permissions_ssh_keys_data', '/_admin/permissions/ssh_keys/data', []);
     pyroutes.register('admin_permissions_ssh_keys_update', '/_admin/permissions/ssh_keys/update', []);
     pyroutes.register('admin_scheduler', '/_admin/scheduler', []);
     pyroutes.register('admin_scheduler_show_tasks', '/_admin/scheduler/_tasks', []);
     pyroutes.register('admin_settings', '/_admin/settings', []);
     pyroutes.register('admin_settings_email', '/_admin/settings/email', []);
     pyroutes.register('admin_settings_email_update', '/_admin/settings/email/update', []);
     pyroutes.register('admin_settings_exception_tracker', '/_admin/settings/exceptions', []);
     pyroutes.register('admin_settings_exception_tracker_delete', '/_admin/settings/exceptions/%(exception_id)s/delete', ['exception_id']);
     pyroutes.register('admin_settings_exception_tracker_delete_all', '/_admin/settings/exceptions_delete_all', []);
     pyroutes.register('admin_settings_exception_tracker_show', '/_admin/settings/exceptions/%(exception_id)s', ['exception_id']);
     pyroutes.register('admin_settings_global', '/_admin/settings/global', []);
     pyroutes.register('admin_settings_global_update', '/_admin/settings/global/update', []);
     pyroutes.register('admin_settings_hooks', '/_admin/settings/hooks', []);
     pyroutes.register('admin_settings_hooks_delete', '/_admin/settings/hooks/delete', []);
     pyroutes.register('admin_settings_hooks_update', '/_admin/settings/hooks/update', []);
     pyroutes.register('admin_settings_issuetracker', '/_admin/settings/issue-tracker', []);
     pyroutes.register('admin_settings_issuetracker_delete', '/_admin/settings/issue-tracker/delete', []);
     pyroutes.register('admin_settings_issuetracker_test', '/_admin/settings/issue-tracker/test', []);
     pyroutes.register('admin_settings_issuetracker_update', '/_admin/settings/issue-tracker/update', []);
     pyroutes.register('admin_settings_labs', '/_admin/settings/labs', []);
     pyroutes.register('admin_settings_labs_update', '/_admin/settings/labs/update', []);
     pyroutes.register('admin_settings_license', '/_admin/settings/license', []);
     pyroutes.register('admin_settings_license_unlock', '/_admin/settings/license_unlock', []);
     pyroutes.register('admin_settings_mapping', '/_admin/settings/mapping', []);
     pyroutes.register('admin_settings_mapping_update', '/_admin/settings/mapping/update', []);
     pyroutes.register('admin_settings_open_source', '/_admin/settings/open_source', []);
     pyroutes.register('admin_settings_process_management', '/_admin/settings/process_management', []);
     pyroutes.register('admin_settings_process_management_data', '/_admin/settings/process_management/data', []);
     pyroutes.register('admin_settings_process_management_master_signal', '/_admin/settings/process_management/master_signal', []);
     pyroutes.register('admin_settings_process_management_signal', '/_admin/settings/process_management/signal', []);
     pyroutes.register('admin_settings_scheduler_create', '/_admin/scheduler/create', []);
     pyroutes.register('admin_settings_scheduler_delete', '/_admin/scheduler/%(schedule_id)s/delete', ['schedule_id']);
     pyroutes.register('admin_settings_scheduler_edit', '/_admin/scheduler/%(schedule_id)s', ['schedule_id']);
     pyroutes.register('admin_settings_scheduler_execute', '/_admin/scheduler/%(schedule_id)s/execute', ['schedule_id']);
     pyroutes.register('admin_settings_scheduler_new', '/_admin/scheduler/new', []);
     pyroutes.register('admin_settings_scheduler_update', '/_admin/scheduler/%(schedule_id)s/update', ['schedule_id']);
     pyroutes.register('admin_settings_search', '/_admin/settings/search', []);
     pyroutes.register('admin_settings_sessions', '/_admin/settings/sessions', []);
     pyroutes.register('admin_settings_sessions_cleanup', '/_admin/settings/sessions/cleanup', []);
     pyroutes.register('admin_settings_system', '/_admin/settings/system', []);
     pyroutes.register('admin_settings_system_update', '/_admin/settings/system/updates', []);
     pyroutes.register('admin_settings_update', '/_admin/settings/update', []);
     pyroutes.register('admin_settings_vcs', '/_admin/settings/vcs', []);
     pyroutes.register('admin_settings_vcs_svn_generate_cfg', '/_admin/settings/vcs/svn_generate_cfg', []);
     pyroutes.register('admin_settings_vcs_svn_pattern_delete', '/_admin/settings/vcs/svn_pattern_delete', []);
     pyroutes.register('admin_settings_vcs_update', '/_admin/settings/vcs/update', []);
     pyroutes.register('admin_settings_visual', '/_admin/settings/visual', []);
     pyroutes.register('admin_settings_visual_update', '/_admin/settings/visual/update', []);
+    pyroutes.register('admin_security_modify_allowed_vcs_client_versions', '/_admin/security/modify/allowed_vcs_client_versions', []);
     pyroutes.register('apiv2', '/_admin/api', []);
     pyroutes.register('atom_feed_home', '/%(repo_name)s/feed-atom', ['repo_name']);
     pyroutes.register('atom_feed_home_old', '/%(repo_name)s/feed/atom', ['repo_name']);
     pyroutes.register('auth_home', '/_admin/auth*traverse', []);
     pyroutes.register('bookmarks_home', '/%(repo_name)s/bookmarks', ['repo_name']);
     pyroutes.register('branch_remove', '/%(repo_name)s/branches/%(branch_name)s/remove', ['repo_name', 'branch_name']);
     pyroutes.register('branches_home', '/%(repo_name)s/branches', ['repo_name']);
     pyroutes.register('channelstream_connect', '/_admin/channelstream/connect', []);
     pyroutes.register('channelstream_proxy', '/_channelstream', []);
     pyroutes.register('channelstream_subscribe', '/_admin/channelstream/subscribe', []);
     pyroutes.register('check_2fa', '/_admin/check_2fa', []);
     pyroutes.register('commit_draft_comments_submit', '/%(repo_name)s/changeset/%(commit_id)s/draft_comments_submit', ['repo_name', 'commit_id']);
     pyroutes.register('debug_style_email', '/_admin/debug_style/email/%(email_id)s', ['email_id']);
     pyroutes.register('debug_style_email_plain_rendered', '/_admin/debug_style/email-rendered/%(email_id)s', ['email_id']);
     pyroutes.register('debug_style_home', '/_admin/debug_style', []);
     pyroutes.register('debug_style_template', '/_admin/debug_style/t/%(t_path)s', ['t_path']);
     pyroutes.register('download_file', '/_file_store/download/%(fid)s', ['fid']);
     pyroutes.register('download_file_by_token', '/_file_store/token-download/%(_auth_token)s/%(fid)s', ['_auth_token', 'fid']);
     pyroutes.register('edit_repo', '/%(repo_name)s/settings', ['repo_name']);
     pyroutes.register('edit_repo_advanced', '/%(repo_name)s/settings/advanced', ['repo_name']);
     pyroutes.register('edit_repo_advanced_archive', '/%(repo_name)s/settings/advanced/archive', ['repo_name']);
     pyroutes.register('edit_repo_advanced_delete', '/%(repo_name)s/settings/advanced/delete', ['repo_name']);
     pyroutes.register('edit_repo_advanced_fork', '/%(repo_name)s/settings/advanced/fork', ['repo_name']);
     pyroutes.register('edit_repo_advanced_hooks', '/%(repo_name)s/settings/advanced/hooks', ['repo_name']);
     pyroutes.register('edit_repo_advanced_journal', '/%(repo_name)s/settings/advanced/journal', ['repo_name']);
     pyroutes.register('edit_repo_advanced_locking', '/%(repo_name)s/settings/advanced/locking', ['repo_name']);
     pyroutes.register('edit_repo_audit_logs', '/%(repo_name)s/settings/audit_logs', ['repo_name']);
     pyroutes.register('edit_repo_caches', '/%(repo_name)s/settings/caches', ['repo_name']);
     pyroutes.register('edit_repo_fields', '/%(repo_name)s/settings/fields', ['repo_name']);
     pyroutes.register('edit_repo_fields_create', '/%(repo_name)s/settings/fields/create', ['repo_name']);
     pyroutes.register('edit_repo_fields_delete', '/%(repo_name)s/settings/fields/%(field_id)s/delete', ['repo_name', 'field_id']);
     pyroutes.register('edit_repo_group', '/%(repo_group_name)s/_edit', ['repo_group_name']);
     pyroutes.register('edit_repo_group_advanced', '/%(repo_group_name)s/_settings/advanced', ['repo_group_name']);
     pyroutes.register('edit_repo_group_advanced_delete', '/%(repo_group_name)s/_settings/advanced/delete', ['repo_group_name']);
     pyroutes.register('edit_repo_group_perms', '/%(repo_group_name)s/_settings/permissions', ['repo_group_name']);
     pyroutes.register('edit_repo_group_perms_update', '/%(repo_group_name)s/_settings/permissions/update', ['repo_group_name']);
     pyroutes.register('edit_repo_issuetracker', '/%(repo_name)s/settings/issue_trackers', ['repo_name']);
     pyroutes.register('edit_repo_issuetracker_delete', '/%(repo_name)s/settings/issue_trackers/delete', ['repo_name']);
     pyroutes.register('edit_repo_issuetracker_test', '/%(repo_name)s/settings/issue_trackers/test', ['repo_name']);
     pyroutes.register('edit_repo_issuetracker_update', '/%(repo_name)s/settings/issue_trackers/update', ['repo_name']);
     pyroutes.register('edit_repo_maintenance', '/%(repo_name)s/settings/maintenance', ['repo_name']);
     pyroutes.register('edit_repo_maintenance_execute', '/%(repo_name)s/settings/maintenance/execute', ['repo_name']);
     pyroutes.register('edit_repo_perms', '/%(repo_name)s/settings/permissions', ['repo_name']);
     pyroutes.register('edit_repo_perms_branch', '/%(repo_name)s/settings/branch_permissions', ['repo_name']);
     pyroutes.register('edit_repo_perms_branch_delete', '/%(repo_name)s/settings/branch_permissions/%(rule_id)s/delete', ['repo_name', 'rule_id']);
     pyroutes.register('edit_repo_perms_set_private', '/%(repo_name)s/settings/permissions/set_private', ['repo_name']);
     pyroutes.register('edit_repo_remote', '/%(repo_name)s/settings/remote', ['repo_name']);
     pyroutes.register('edit_repo_remote_pull', '/%(repo_name)s/settings/remote/pull', ['repo_name']);
     pyroutes.register('edit_repo_remote_push', '/%(repo_name)s/settings/remote/push', ['repo_name']);
     pyroutes.register('edit_repo_statistics', '/%(repo_name)s/settings/statistics', ['repo_name']);
     pyroutes.register('edit_repo_statistics_reset', '/%(repo_name)s/settings/statistics/update', ['repo_name']);
     pyroutes.register('edit_repo_strip', '/%(repo_name)s/settings/strip', ['repo_name']);
     pyroutes.register('edit_repo_vcs', '/%(repo_name)s/settings/vcs', ['repo_name']);
     pyroutes.register('edit_repo_vcs_svn_pattern_delete', '/%(repo_name)s/settings/vcs/svn_pattern/delete', ['repo_name']);
     pyroutes.register('edit_repo_vcs_update', '/%(repo_name)s/settings/vcs/update', ['repo_name']);
     pyroutes.register('edit_user_audit_logs', '/_admin/users/%(user_id)s/edit/audit', ['user_id']);
     pyroutes.register('edit_user_audit_logs_download', '/_admin/users/%(user_id)s/edit/audit/download', ['user_id']);
     pyroutes.register('edit_user_auth_tokens', '/_admin/users/%(user_id)s/edit/auth_tokens', ['user_id']);
     pyroutes.register('edit_user_auth_tokens_add', '/_admin/users/%(user_id)s/edit/auth_tokens/new', ['user_id']);
     pyroutes.register('edit_user_auth_tokens_delete', '/_admin/users/%(user_id)s/edit/auth_tokens/delete', ['user_id']);
     pyroutes.register('edit_user_auth_tokens_view', '/_admin/users/%(user_id)s/edit/auth_tokens/view', ['user_id']);
     pyroutes.register('edit_user_caches', '/_admin/users/%(user_id)s/edit/caches', ['user_id']);
     pyroutes.register('edit_user_caches_update', '/_admin/users/%(user_id)s/edit/caches/update', ['user_id']);
     pyroutes.register('edit_user_emails', '/_admin/users/%(user_id)s/edit/emails', ['user_id']);
     pyroutes.register('edit_user_emails_add', '/_admin/users/%(user_id)s/edit/emails/new', ['user_id']);
     pyroutes.register('edit_user_emails_delete', '/_admin/users/%(user_id)s/edit/emails/delete', ['user_id']);
     pyroutes.register('edit_user_group', '/_admin/user_groups/%(user_group_id)s/edit', ['user_group_id']);
     pyroutes.register('edit_user_group_advanced', '/_admin/user_groups/%(user_group_id)s/edit/advanced', ['user_group_id']);
     pyroutes.register('edit_user_group_advanced_sync', '/_admin/user_groups/%(user_group_id)s/edit/advanced/sync', ['user_group_id']);
     pyroutes.register('edit_user_group_global_perms', '/_admin/user_groups/%(user_group_id)s/edit/global_permissions', ['user_group_id']);
     pyroutes.register('edit_user_group_global_perms_update', '/_admin/user_groups/%(user_group_id)s/edit/global_permissions/update', ['user_group_id']);
     pyroutes.register('edit_user_group_perms', '/_admin/user_groups/%(user_group_id)s/edit/permissions', ['user_group_id']);
     pyroutes.register('edit_user_group_perms_summary', '/_admin/user_groups/%(user_group_id)s/edit/permissions_summary', ['user_group_id']);
     pyroutes.register('edit_user_group_perms_summary_json', '/_admin/user_groups/%(user_group_id)s/edit/permissions_summary/json', ['user_group_id']);
     pyroutes.register('edit_user_group_perms_update', '/_admin/user_groups/%(user_group_id)s/edit/permissions/update', ['user_group_id']);
     pyroutes.register('edit_user_groups_management', '/_admin/users/%(user_id)s/edit/groups_management', ['user_id']);
     pyroutes.register('edit_user_groups_management_updates', '/_admin/users/%(user_id)s/edit/edit_user_groups_management/updates', ['user_id']);
     pyroutes.register('edit_user_ips', '/_admin/users/%(user_id)s/edit/ips', ['user_id']);
     pyroutes.register('edit_user_ips_add', '/_admin/users/%(user_id)s/edit/ips/new', ['user_id']);
     pyroutes.register('edit_user_ips_delete', '/_admin/users/%(user_id)s/edit/ips/delete', ['user_id']);
     pyroutes.register('edit_user_perms_summary', '/_admin/users/%(user_id)s/edit/permissions_summary', ['user_id']);
     pyroutes.register('edit_user_perms_summary_json', '/_admin/users/%(user_id)s/edit/permissions_summary/json', ['user_id']);
     pyroutes.register('edit_user_ssh_keys', '/_admin/users/%(user_id)s/edit/ssh_keys', ['user_id']);
     pyroutes.register('edit_user_ssh_keys_add', '/_admin/users/%(user_id)s/edit/ssh_keys/new', ['user_id']);
     pyroutes.register('edit_user_ssh_keys_delete', '/_admin/users/%(user_id)s/edit/ssh_keys/delete', ['user_id']);
     pyroutes.register('edit_user_ssh_keys_generate_keypair', '/_admin/users/%(user_id)s/edit/ssh_keys/generate', ['user_id']);
     pyroutes.register('favicon', '/favicon.ico', []);
     pyroutes.register('file_preview', '/_file_preview', []);
     pyroutes.register('gist_delete', '/_admin/gists/%(gist_id)s/delete', ['gist_id']);
     pyroutes.register('gist_edit', '/_admin/gists/%(gist_id)s/edit', ['gist_id']);
     pyroutes.register('gist_edit_check_revision', '/_admin/gists/%(gist_id)s/edit/check_revision', ['gist_id']);
     pyroutes.register('gist_show', '/_admin/gists/%(gist_id)s', ['gist_id']);
     pyroutes.register('gist_show_formatted', '/_admin/gists/%(gist_id)s/rev/%(revision)s/%(format)s', ['gist_id', 'revision', 'format']);
     pyroutes.register('gist_show_formatted_path', '/_admin/gists/%(gist_id)s/rev/%(revision)s/%(format)s/%(f_path)s', ['gist_id', 'revision', 'format', 'f_path']);
     pyroutes.register('gist_show_rev', '/_admin/gists/%(gist_id)s/rev/%(revision)s', ['gist_id', 'revision']);
     pyroutes.register('gist_update', '/_admin/gists/%(gist_id)s/update', ['gist_id']);
     pyroutes.register('gists_create', '/_admin/gists/create', []);
     pyroutes.register('gists_new', '/_admin/gists/new', []);
     pyroutes.register('gists_show', '/_admin/gists', []);
     pyroutes.register('global_integrations_create', '/_admin/integrations/%(integration)s/new', ['integration']);
     pyroutes.register('global_integrations_edit', '/_admin/integrations/%(integration)s/%(integration_id)s', ['integration', 'integration_id']);
     pyroutes.register('global_integrations_home', '/_admin/integrations', []);
     pyroutes.register('global_integrations_list', '/_admin/integrations/%(integration)s', ['integration']);
     pyroutes.register('global_integrations_new', '/_admin/integrations/new', []);
     pyroutes.register('goto_switcher_data', '/_goto_data', []);
     pyroutes.register('home', '/', []);
     pyroutes.register('hovercard_pull_request', '/_hovercard/pull_request/%(pull_request_id)s', ['pull_request_id']);
     pyroutes.register('hovercard_repo_commit', '/_hovercard/commit/%(repo_name)s/%(commit_id)s', ['repo_name', 'commit_id']);
     pyroutes.register('hovercard_user', '/_hovercard/user/%(user_id)s', ['user_id']);
     pyroutes.register('hovercard_user_group', '/_hovercard/user_group/%(user_group_id)s', ['user_group_id']);
     pyroutes.register('hovercard_username', '/_hovercard/username/%(username)s', ['username']);
     pyroutes.register('journal', '/_admin/journal', []);
     pyroutes.register('journal_atom', '/_admin/journal/atom', []);
     pyroutes.register('journal_public', '/_admin/public_journal', []);
     pyroutes.register('journal_public_atom', '/_admin/public_journal/atom', []);
     pyroutes.register('journal_public_atom_old', '/_admin/public_journal_atom', []);
     pyroutes.register('journal_public_rss', '/_admin/public_journal/rss', []);
     pyroutes.register('journal_public_rss_old', '/_admin/public_journal_rss', []);
     pyroutes.register('journal_rss', '/_admin/journal/rss', []);
     pyroutes.register('login', '/_admin/login', []);
     pyroutes.register('logout', '/_admin/logout', []);
     pyroutes.register('main_page_repo_groups_data', '/_home_repo_groups', []);
     pyroutes.register('main_page_repos_data', '/_home_repos', []);
     pyroutes.register('markup_preview', '/_markup_preview', []);
     pyroutes.register('my_account_auth_tokens', '/_admin/my_account/auth_tokens', []);
     pyroutes.register('my_account_auth_tokens_add', '/_admin/my_account/auth_tokens/new', []);
     pyroutes.register('my_account_auth_tokens_delete', '/_admin/my_account/auth_tokens/delete', []);
     pyroutes.register('my_account_auth_tokens_view', '/_admin/my_account/auth_tokens/view', []);
     pyroutes.register('my_account_bookmarks', '/_admin/my_account/bookmarks', []);
     pyroutes.register('my_account_bookmarks_update', '/_admin/my_account/bookmarks/update', []);
     pyroutes.register('my_account_configure_2fa', '/_admin/my_account/configure_2fa', []);
     pyroutes.register('my_account_configure_2fa_update', '/_admin/my_account/configure_2fa_update', []);
     pyroutes.register('my_account_edit', '/_admin/my_account/edit', []);
     pyroutes.register('my_account_emails', '/_admin/my_account/emails', []);
     pyroutes.register('my_account_emails_add', '/_admin/my_account/emails/new', []);
     pyroutes.register('my_account_emails_delete', '/_admin/my_account/emails/delete', []);
     pyroutes.register('my_account_external_identity', '/_admin/my_account/external-identity', []);
     pyroutes.register('my_account_external_identity_delete', '/_admin/my_account/external-identity/delete', []);
     pyroutes.register('my_account_goto_bookmark', '/_admin/my_account/bookmark/%(bookmark_id)s', ['bookmark_id']);
     pyroutes.register('my_account_notifications', '/_admin/my_account/notifications', []);
     pyroutes.register('my_account_notifications_test_channelstream', '/_admin/my_account/test_channelstream', []);
     pyroutes.register('my_account_notifications_toggle_visibility', '/_admin/my_account/toggle_visibility', []);
     pyroutes.register('my_account_password', '/_admin/my_account/password', []);
     pyroutes.register('my_account_password_update', '/_admin/my_account/password/update', []);
     pyroutes.register('my_account_perms', '/_admin/my_account/perms', []);
     pyroutes.register('my_account_profile', '/_admin/my_account/profile', []);
     pyroutes.register('my_account_pullrequests', '/_admin/my_account/pull_requests', []);
     pyroutes.register('my_account_pullrequests_data', '/_admin/my_account/pull_requests/data', []);
     pyroutes.register('my_account_regenerate_2fa_recovery_codes', '/_admin/my_account/regenerate_recovery_codes', []);
     pyroutes.register('my_account_repos', '/_admin/my_account/repos', []);
     pyroutes.register('my_account_show_2fa_recovery_codes', '/_admin/my_account/recovery_codes', []);
     pyroutes.register('my_account_ssh_keys', '/_admin/my_account/ssh_keys', []);
     pyroutes.register('my_account_ssh_keys_add', '/_admin/my_account/ssh_keys/new', []);
     pyroutes.register('my_account_ssh_keys_delete', '/_admin/my_account/ssh_keys/delete', []);
     pyroutes.register('my_account_ssh_keys_generate', '/_admin/my_account/ssh_keys/generate', []);
     pyroutes.register('my_account_update', '/_admin/my_account/update', []);
     pyroutes.register('my_account_user_group_membership', '/_admin/my_account/user_group_membership', []);
     pyroutes.register('my_account_watched', '/_admin/my_account/watched', []);
     pyroutes.register('notifications_delete', '/_admin/notifications/%(notification_id)s/delete', ['notification_id']);
     pyroutes.register('notifications_mark_all_read', '/_admin/notifications_mark_all_read', []);
     pyroutes.register('notifications_show', '/_admin/notifications/%(notification_id)s', ['notification_id']);
     pyroutes.register('notifications_show_all', '/_admin/notifications', []);
     pyroutes.register('notifications_update', '/_admin/notifications/%(notification_id)s/update', ['notification_id']);
     pyroutes.register('ops_celery_error_test', '/_admin/ops/error-celery', []);
     pyroutes.register('ops_error_test', '/_admin/ops/error', []);
     pyroutes.register('ops_healthcheck', '/_admin/ops/status', []);
     pyroutes.register('ops_ping', '/_admin/ops/ping', []);
     pyroutes.register('ops_redirect_test', '/_admin/ops/redirect', []);
     pyroutes.register('plugin_admin_chat', '/_admin/plugin_admin_chat/%(action)s', ['action']);
     pyroutes.register('pull_requests_global', '/_admin/pull-request/%(pull_request_id)s', ['pull_request_id']);
     pyroutes.register('pull_requests_global_0', '/_admin/pull_requests/%(pull_request_id)s', ['pull_request_id']);
     pyroutes.register('pull_requests_global_1', '/_admin/pull-requests/%(pull_request_id)s', ['pull_request_id']);
     pyroutes.register('pullrequest_comment_create', '/%(repo_name)s/pull-request/%(pull_request_id)s/comment', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_comment_delete', '/%(repo_name)s/pull-request/%(pull_request_id)s/comment/%(comment_id)s/delete', ['repo_name', 'pull_request_id', 'comment_id']);
     pyroutes.register('pullrequest_comment_edit', '/%(repo_name)s/pull-request/%(pull_request_id)s/comment/%(comment_id)s/edit', ['repo_name', 'pull_request_id', 'comment_id']);
     pyroutes.register('pullrequest_comments', '/%(repo_name)s/pull-request/%(pull_request_id)s/comments', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_create', '/%(repo_name)s/pull-request/create', ['repo_name']);
     pyroutes.register('pullrequest_delete', '/%(repo_name)s/pull-request/%(pull_request_id)s/delete', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_draft_comments_submit', '/%(repo_name)s/pull-request/%(pull_request_id)s/draft_comments_submit', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_drafts', '/%(repo_name)s/pull-request/%(pull_request_id)s/drafts', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_merge', '/%(repo_name)s/pull-request/%(pull_request_id)s/merge', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_new', '/%(repo_name)s/pull-request/new', ['repo_name']);
     pyroutes.register('pullrequest_repo_refs', '/%(repo_name)s/pull-request/refs/%(target_repo_name)s', ['repo_name', 'target_repo_name']);
     pyroutes.register('pullrequest_repo_targets', '/%(repo_name)s/pull-request/repo-targets', ['repo_name']);
     pyroutes.register('pullrequest_show', '/%(repo_name)s/pull-request/%(pull_request_id)s', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_show_all', '/%(repo_name)s/pull-request', ['repo_name']);
     pyroutes.register('pullrequest_show_all_data', '/%(repo_name)s/pull-request-data', ['repo_name']);
     pyroutes.register('pullrequest_todos', '/%(repo_name)s/pull-request/%(pull_request_id)s/todos', ['repo_name', 'pull_request_id']);
     pyroutes.register('pullrequest_update', '/%(repo_name)s/pull-request/%(pull_request_id)s/update', ['repo_name', 'pull_request_id']);
     pyroutes.register('register', '/_admin/register', []);
     pyroutes.register('repo_archivefile', '/%(repo_name)s/archive/%(fname)s', ['repo_name', 'fname']);
     pyroutes.register('repo_artifacts_data', '/%(repo_name)s/artifacts_data', ['repo_name']);
     pyroutes.register('repo_artifacts_delete', '/%(repo_name)s/artifacts/delete/%(uid)s', ['repo_name', 'uid']);
     pyroutes.register('repo_artifacts_get', '/%(repo_name)s/artifacts/download/%(uid)s', ['repo_name', 'uid']);
     pyroutes.register('repo_artifacts_info', '/%(repo_name)s/artifacts/info/%(uid)s', ['repo_name', 'uid']);
     pyroutes.register('repo_artifacts_list', '/%(repo_name)s/artifacts', ['repo_name']);
     pyroutes.register('repo_artifacts_new', '/%(repo_name)s/artifacts/new', ['repo_name']);
     pyroutes.register('repo_artifacts_store', '/%(repo_name)s/artifacts/store', ['repo_name']);
     pyroutes.register('repo_artifacts_stream_script', '/_file_store/stream-upload-script', []);
     pyroutes.register('repo_artifacts_stream_store', '/_file_store/stream-upload', []);
     pyroutes.register('repo_artifacts_update', '/%(repo_name)s/artifacts/update/%(uid)s', ['repo_name', 'uid']);
     pyroutes.register('repo_automation', '/%(repo_name)s/settings/automation', ['repo_name']);
     pyroutes.register('repo_automation_update', '/%(repo_name)s/settings/automation/%(entry_id)s/update', ['repo_name', 'entry_id']);
     pyroutes.register('repo_changelog', '/%(repo_name)s/changelog', ['repo_name']);
     pyroutes.register('repo_changelog_file', '/%(repo_name)s/changelog/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_commit', '/%(repo_name)s/changeset/%(commit_id)s', ['repo_name', 'commit_id']);
     pyroutes.register('repo_commit_children', '/%(repo_name)s/changeset_children/%(commit_id)s', ['repo_name', 'commit_id']);
     pyroutes.register('repo_commit_comment_attachment_upload', '/%(repo_name)s/changeset/%(commit_id)s/comment/attachment_upload', ['repo_name', 'commit_id']);
     pyroutes.register('repo_commit_comment_create', '/%(repo_name)s/changeset/%(commit_id)s/comment/create', ['repo_name', 'commit_id']);
     pyroutes.register('repo_commit_comment_delete', '/%(repo_name)s/changeset/%(commit_id)s/comment/%(comment_id)s/delete', ['repo_name', 'commit_id', 'comment_id']);
     pyroutes.register('repo_commit_comment_edit', '/%(repo_name)s/changeset/%(commit_id)s/comment/%(comment_id)s/edit', ['repo_name', 'commit_id', 'comment_id']);
     pyroutes.register('repo_commit_comment_history_view', '/%(repo_name)s/changeset/%(commit_id)s/comment/%(comment_id)s/history_view/%(comment_history_id)s', ['repo_name', 'commit_id', 'comment_id', 'comment_history_id']);
     pyroutes.register('repo_commit_comment_preview', '/%(repo_name)s/changeset/%(commit_id)s/comment/preview', ['repo_name', 'commit_id']);
     pyroutes.register('repo_commit_data', '/%(repo_name)s/changeset-data/%(commit_id)s', ['repo_name', 'commit_id']);
     pyroutes.register('repo_commit_download', '/%(repo_name)s/changeset-download/%(commit_id)s', ['repo_name', 'commit_id']);
     pyroutes.register('repo_commit_parents', '/%(repo_name)s/changeset_parents/%(commit_id)s', ['repo_name', 'commit_id']);
     pyroutes.register('repo_commit_patch', '/%(repo_name)s/changeset-patch/%(commit_id)s', ['repo_name', 'commit_id']);
     pyroutes.register('repo_commit_raw', '/%(repo_name)s/changeset-diff/%(commit_id)s', ['repo_name', 'commit_id']);
     pyroutes.register('repo_commit_raw_deprecated', '/%(repo_name)s/raw-changeset/%(commit_id)s', ['repo_name', 'commit_id']);
     pyroutes.register('repo_commits', '/%(repo_name)s/commits', ['repo_name']);
     pyroutes.register('repo_commits_elements', '/%(repo_name)s/commits_elements', ['repo_name']);
     pyroutes.register('repo_commits_elements_file', '/%(repo_name)s/commits_elements/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_commits_file', '/%(repo_name)s/commits/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_compare', '/%(repo_name)s/compare/%(source_ref_type)s@%(source_ref)s...%(target_ref_type)s@%(target_ref)s', ['repo_name', 'source_ref_type', 'source_ref', 'target_ref_type', 'target_ref']);
     pyroutes.register('repo_compare_select', '/%(repo_name)s/compare', ['repo_name']);
     pyroutes.register('repo_create', '/_admin/repos/create', []);
     pyroutes.register('repo_creating', '/%(repo_name)s/repo_creating', ['repo_name']);
     pyroutes.register('repo_creating_check', '/%(repo_name)s/repo_creating_check', ['repo_name']);
     pyroutes.register('repo_default_reviewers_data', '/%(repo_name)s/settings/review/default-reviewers', ['repo_name']);
     pyroutes.register('repo_file_authors', '/%(repo_name)s/authors/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_file_download', '/%(repo_name)s/download/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_file_download:legacy', '/%(repo_name)s/rawfile/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_file_history', '/%(repo_name)s/history/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_file_raw', '/%(repo_name)s/raw/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_files', '/%(repo_name)s/files/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_files:annotated', '/%(repo_name)s/annotate/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_files:annotated_previous', '/%(repo_name)s/annotate-previous/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_files:default_commit', '/%(repo_name)s/files', ['repo_name']);
     pyroutes.register('repo_files:default_path', '/%(repo_name)s/files/%(commit_id)s/', ['repo_name', 'commit_id']);
     pyroutes.register('repo_files:rendered', '/%(repo_name)s/render/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_files_add_file', '/%(repo_name)s/add_file/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_files_check_head', '/%(repo_name)s/check_head/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_files_create_file', '/%(repo_name)s/create_file/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_files_delete_file', '/%(repo_name)s/delete_file/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_files_diff', '/%(repo_name)s/diff/%(f_path)s', ['repo_name', 'f_path']);
     pyroutes.register('repo_files_diff_2way_redirect', '/%(repo_name)s/diff-2way/%(f_path)s', ['repo_name', 'f_path']);
     pyroutes.register('repo_files_edit_file', '/%(repo_name)s/edit_file/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_files_nodelist', '/%(repo_name)s/nodelist/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_files_remove_file', '/%(repo_name)s/remove_file/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_files_replace_binary', '/%(repo_name)s/replace_binary/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_files_update_file', '/%(repo_name)s/update_file/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_files_upload_file', '/%(repo_name)s/upload_file/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_fork_create', '/%(repo_name)s/fork/create', ['repo_name']);
     pyroutes.register('repo_fork_new', '/%(repo_name)s/fork', ['repo_name']);
     pyroutes.register('repo_forks_data', '/%(repo_name)s/forks/data', ['repo_name']);
     pyroutes.register('repo_forks_show_all', '/%(repo_name)s/forks', ['repo_name']);
     pyroutes.register('repo_group_create', '/_admin/repo_group/create', []);
     pyroutes.register('repo_group_home', '/%(repo_group_name)s', ['repo_group_name']);
     pyroutes.register('repo_group_home_slash', '/%(repo_group_name)s/', ['repo_group_name']);
     pyroutes.register('repo_group_integrations_create', '/%(repo_group_name)s/_settings/integrations/%(integration)s/new', ['repo_group_name', 'integration']);
     pyroutes.register('repo_group_integrations_edit', '/%(repo_group_name)s/_settings/integrations/%(integration)s/%(integration_id)s', ['repo_group_name', 'integration', 'integration_id']);
     pyroutes.register('repo_group_integrations_home', '/%(repo_group_name)s/_settings/integrations', ['repo_group_name']);
     pyroutes.register('repo_group_integrations_list', '/%(repo_group_name)s/_settings/integrations/%(integration)s', ['repo_group_name', 'integration']);
     pyroutes.register('repo_group_integrations_new', '/%(repo_group_name)s/_settings/integrations/new', ['repo_group_name']);
     pyroutes.register('repo_group_list_data', '/_repo_groups', []);
     pyroutes.register('repo_group_new', '/_admin/repo_group/new', []);
     pyroutes.register('repo_groups', '/_admin/repo_groups', []);
     pyroutes.register('repo_groups_data', '/_admin/repo_groups_data', []);
     pyroutes.register('repo_integrations_create', '/%(repo_name)s/settings/integrations/%(integration)s/new', ['repo_name', 'integration']);
     pyroutes.register('repo_integrations_edit', '/%(repo_name)s/settings/integrations/%(integration)s/%(integration_id)s', ['repo_name', 'integration', 'integration_id']);
     pyroutes.register('repo_integrations_home', '/%(repo_name)s/settings/integrations', ['repo_name']);
     pyroutes.register('repo_integrations_list', '/%(repo_name)s/settings/integrations/%(integration)s', ['repo_name', 'integration']);
     pyroutes.register('repo_integrations_new', '/%(repo_name)s/settings/integrations/new', ['repo_name']);
     pyroutes.register('repo_list_data', '/_repos', []);
     pyroutes.register('repo_new', '/_admin/repos/new', []);
     pyroutes.register('repo_nodetree_full', '/%(repo_name)s/nodetree_full/%(commit_id)s/%(f_path)s', ['repo_name', 'commit_id', 'f_path']);
     pyroutes.register('repo_nodetree_full:default_path', '/%(repo_name)s/nodetree_full/%(commit_id)s/', ['repo_name', 'commit_id']);
     pyroutes.register('repo_refs_changelog_data', '/%(repo_name)s/refs-data-changelog', ['repo_name']);
     pyroutes.register('repo_refs_data', '/%(repo_name)s/refs-data', ['repo_name']);
     pyroutes.register('repo_reviewers', '/%(repo_name)s/settings/review/rules', ['repo_name']);
     pyroutes.register('repo_reviewers_review_rule_delete', '/%(repo_name)s/settings/review/rules/%(rule_id)s/delete', ['repo_name', 'rule_id']);
     pyroutes.register('repo_reviewers_review_rule_edit', '/%(repo_name)s/settings/review/rules/%(rule_id)s', ['repo_name', 'rule_id']);
     pyroutes.register('repo_reviewers_review_rule_new', '/%(repo_name)s/settings/review/rules/new', ['repo_name']);
     pyroutes.register('repo_settings_quick_actions', '/%(repo_name)s/settings/quick-action', ['repo_name']);
     pyroutes.register('repo_stats', '/%(repo_name)s/repo_stats/%(commit_id)s', ['repo_name', 'commit_id']);
     pyroutes.register('repo_summary', '/%(repo_name)s', ['repo_name']);
     pyroutes.register('repo_summary_commits', '/%(repo_name)s/summary-commits', ['repo_name']);
     pyroutes.register('repo_summary_explicit', '/%(repo_name)s/summary', ['repo_name']);
     pyroutes.register('repo_summary_slash', '/%(repo_name)s/', ['repo_name']);
     pyroutes.register('repos', '/_admin/repos', []);
     pyroutes.register('repos_data', '/_admin/repos_data', []);
     pyroutes.register('reset_password', '/_admin/password_reset', []);
     pyroutes.register('reset_password_confirmation', '/_admin/password_reset_confirmation', []);
     pyroutes.register('robots', '/robots.txt', []);
     pyroutes.register('rss_feed_home', '/%(repo_name)s/feed-rss', ['repo_name']);
     pyroutes.register('rss_feed_home_old', '/%(repo_name)s/feed/rss', ['repo_name']);
     pyroutes.register('search', '/_admin/search', []);
     pyroutes.register('search_repo', '/%(repo_name)s/_search', ['repo_name']);
     pyroutes.register('search_repo_alt', '/%(repo_name)s/search', ['repo_name']);
     pyroutes.register('search_repo_group', '/%(repo_group_name)s/_search', ['repo_group_name']);
     pyroutes.register('setup_2fa', '/_admin/setup_2fa', []);
     pyroutes.register('store_user_session_value', '/_store_session_attr', []);
     pyroutes.register('strip_check', '/%(repo_name)s/settings/strip_check', ['repo_name']);
     pyroutes.register('strip_execute', '/%(repo_name)s/settings/strip_execute', ['repo_name']);
     pyroutes.register('tags_home', '/%(repo_name)s/tags', ['repo_name']);
     pyroutes.register('toggle_following', '/_admin/toggle_following', []);
     pyroutes.register('upload_file', '/_file_store/upload', []);
     pyroutes.register('user_autocomplete_data', '/_users', []);
     pyroutes.register('user_create_personal_repo_group', '/_admin/users/%(user_id)s/create_repo_group', ['user_id']);
     pyroutes.register('user_delete', '/_admin/users/%(user_id)s/delete', ['user_id']);
     pyroutes.register('user_disable_force_password_reset', '/_admin/users/%(user_id)s/password_reset_disable', ['user_id']);
     pyroutes.register('user_edit', '/_admin/users/%(user_id)s/edit', ['user_id']);
     pyroutes.register('user_edit_advanced', '/_admin/users/%(user_id)s/edit/advanced', ['user_id']);
     pyroutes.register('user_edit_global_perms', '/_admin/users/%(user_id)s/edit/global_permissions', ['user_id']);
     pyroutes.register('user_edit_global_perms_update', '/_admin/users/%(user_id)s/edit/global_permissions/update', ['user_id']);
     pyroutes.register('user_enable_force_password_reset', '/_admin/users/%(user_id)s/password_reset_enable', ['user_id']);
     pyroutes.register('user_group_autocomplete_data', '/_user_groups', []);
     pyroutes.register('user_group_members_data', '/_admin/user_groups/%(user_group_id)s/members', ['user_group_id']);
     pyroutes.register('user_group_profile', '/_profile_user_group/%(user_group_name)s', ['user_group_name']);
     pyroutes.register('user_groups', '/_admin/user_groups', []);
     pyroutes.register('user_groups_create', '/_admin/user_groups/create', []);
     pyroutes.register('user_groups_data', '/_admin/user_groups_data', []);
     pyroutes.register('user_groups_delete', '/_admin/user_groups/%(user_group_id)s/delete', ['user_group_id']);
     pyroutes.register('user_groups_new', '/_admin/user_groups/new', []);
     pyroutes.register('user_groups_update', '/_admin/user_groups/%(user_group_id)s/update', ['user_group_id']);
     pyroutes.register('user_notice_dismiss', '/_admin/users/%(user_id)s/notice_dismiss', ['user_id']);
     pyroutes.register('user_profile', '/_profiles/%(username)s', ['username']);
     pyroutes.register('user_update', '/_admin/users/%(user_id)s/update', ['user_id']);
     pyroutes.register('users', '/_admin/users', []);
     pyroutes.register('users_create', '/_admin/users/create', []);
     pyroutes.register('users_data', '/_admin/users_data', []);
     pyroutes.register('users_new', '/_admin/users/new', []);
 }

rhodecode/subscribers.py

@@ -1,399 +1,422 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import io
 import shlex
 
 import math
 import re
 import os
 import datetime
 import logging
 import queue
 import subprocess
 
 
 from dateutil.parser import parse
 from pyramid.interfaces import IRoutesMapper
 from pyramid.settings import asbool
 from pyramid.path import AssetResolver
 from threading import Thread
 
 from rhodecode.config.jsroutes import generate_jsroutes_content
 from rhodecode.lib.base import get_auth_user
 from rhodecode.lib.celerylib.loader import set_celery_conf
 
 import rhodecode
 
 
 log = logging.getLogger(__name__)
 
 
 def add_renderer_globals(event):
     from rhodecode.lib import helpers
 
     # TODO: When executed in pyramid view context the request is not available
     # in the event. Find a better solution to get the request.
     from pyramid.threadlocal import get_current_request
     request = event['request'] or get_current_request()
 
     # Add Pyramid translation as '_' to context
     event['_'] = request.translate
     event['_ungettext'] = request.plularize
     event['h'] = helpers
 
 
 def set_user_lang(event):
     request = event.request
     cur_user = getattr(request, 'user', None)
 
     if cur_user:
         user_lang = cur_user.get_instance().user_data.get('language')
         if user_lang:
             log.debug('lang: setting current user:%s language to: %s', cur_user, user_lang)
             event.request._LOCALE_ = user_lang
 
 
 def update_celery_conf(event):
     log.debug('Setting celery config from new request')
     set_celery_conf(request=event.request, registry=event.request.registry)
 
 
 def add_request_user_context(event):
     """
     Adds auth user into request context
     """
 
     request = event.request
     # access req_id as soon as possible
     req_id = request.req_id
 
     if hasattr(request, 'vcs_call'):
         # skip vcs calls
         return
 
     if hasattr(request, 'rpc_method'):
         # skip api calls
         return
 
     auth_user, auth_token = get_auth_user(request)
     request.user = auth_user
     request.user_auth_token = auth_token
     request.environ['rc_auth_user'] = auth_user
     request.environ['rc_auth_user_id'] = str(auth_user.user_id)
     request.environ['rc_req_id'] = req_id
 
 
 def reset_log_bucket(event):
     """
     reset the log bucket on new request
     """
     request = event.request
     request.req_id_records_init()
 
 
 def scan_repositories_if_enabled(event):
     """
     This is subscribed to the `pyramid.events.ApplicationCreated` event. It
     does a repository scan if enabled in the settings.
     """
+
     settings = event.app.registry.settings
     vcs_server_enabled = settings['vcs.server.enable']
     import_on_startup = settings['startup.import_repos']
+
     if vcs_server_enabled and import_on_startup:
         from rhodecode.model.scm import ScmModel
         from rhodecode.lib.utils import repo2db_mapper
         scm = ScmModel()
         repositories = scm.repo_scan(scm.repos_path)
         repo2db_mapper(repositories, remove_obsolete=False)
 
 
 def write_metadata_if_needed(event):
     """
     Writes upgrade metadata
     """
     import rhodecode
     from rhodecode.lib import system_info
     from rhodecode.lib import ext_json
 
     fname = '.rcmetadata.json'
     ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
     metadata_destination = os.path.join(ini_loc, fname)
 
     def get_update_age():
         now = datetime.datetime.utcnow()
 
         with open(metadata_destination, 'rb') as f:
             data = ext_json.json.loads(f.read())
             if 'created_on' in data:
                 update_date = parse(data['created_on'])
                 diff = now - update_date
                 return diff.total_seconds() / 60.0
 
         return 0
 
     def write():
         configuration = system_info.SysInfo(
             system_info.rhodecode_config)()['value']
         license_token = configuration['config']['license_token']
 
         setup = dict(
             workers=configuration['config']['server:main'].get(
                 'workers', '?'),
             worker_type=configuration['config']['server:main'].get(
                 'worker_class', 'sync'),
         )
         dbinfo = system_info.SysInfo(system_info.database_info)()['value']
         del dbinfo['url']
 
         metadata = dict(
             desc='upgrade metadata info',
             license_token=license_token,
             created_on=datetime.datetime.utcnow().isoformat(),
             usage=system_info.SysInfo(system_info.usage_info)()['value'],
             platform=system_info.SysInfo(system_info.platform_type)()['value'],
             database=dbinfo,
             cpu=system_info.SysInfo(system_info.cpu)()['value'],
             memory=system_info.SysInfo(system_info.memory)()['value'],
             setup=setup
         )
 
         with open(metadata_destination, 'wb') as f:
             f.write(ext_json.json.dumps(metadata))
 
     settings = event.app.registry.settings
     if settings.get('metadata.skip'):
         return
 
     # only write this every 24h, workers restart caused unwanted delays
     try:
         age_in_min = get_update_age()
     except Exception:
         age_in_min = 0
 
     if age_in_min > 60 * 60 * 24:
         return
 
     try:
         write()
     except Exception:
         pass
 
 
 def write_usage_data(event):
     import rhodecode
     from rhodecode.lib import system_info
     from rhodecode.lib import ext_json
 
     settings = event.app.registry.settings
     instance_tag = settings.get('metadata.write_usage_tag')
     if not settings.get('metadata.write_usage'):
         return
 
     def get_update_age(dest_file):
-        now = datetime.datetime.utcnow()
+        now = datetime.datetime.now(datetime.UTC)
 
         with open(dest_file, 'rb') as f:
             data = ext_json.json.loads(f.read())
             if 'created_on' in data:
                 update_date = parse(data['created_on'])
                 diff = now - update_date
                 return math.ceil(diff.total_seconds() / 60.0)
 
         return 0
 
-    utc_date = datetime.datetime.utcnow()
+    utc_date = datetime.datetime.now(datetime.UTC)
     hour_quarter = int(math.ceil((utc_date.hour + utc_date.minute/60.0) / 6.))
-    fname = '.rc_usage_{date.year}{date.month:02d}{date.day:02d}_{hour}.json'.format(
-        date=utc_date, hour=hour_quarter)
+    fname = f'.rc_usage_{utc_date.year}{utc_date.month:02d}{utc_date.day:02d}_{hour_quarter}.json'
     ini_loc = os.path.dirname(rhodecode.CONFIG.get('__file__'))
 
     usage_dir = os.path.join(ini_loc, '.rcusage')
     if not os.path.isdir(usage_dir):
         os.makedirs(usage_dir)
     usage_metadata_destination = os.path.join(usage_dir, fname)
 
     try:
         age_in_min = get_update_age(usage_metadata_destination)
     except Exception:
         age_in_min = 0
 
     # write every 6th hour
     if age_in_min and age_in_min < 60 * 6:
         log.debug('Usage file created %s minutes ago, skipping (threshold: %s minutes)...',
                   age_in_min, 60 * 6)
         return
 
     def write(dest_file):
         configuration = system_info.SysInfo(system_info.rhodecode_config)()['value']
         license_token = configuration['config']['license_token']
 
         metadata = dict(
             desc='Usage data',
             instance_tag=instance_tag,
             license_token=license_token,
             created_on=datetime.datetime.utcnow().isoformat(),
             usage=system_info.SysInfo(system_info.usage_info)()['value'],
         )
 
         with open(dest_file, 'wb') as f:
             f.write(ext_json.formatted_json(metadata))
 
     try:
         log.debug('Writing usage file at: %s', usage_metadata_destination)
         write(usage_metadata_destination)
     except Exception:
         pass
 
 
 def write_js_routes_if_enabled(event):
     registry = event.app.registry
 
     mapper = registry.queryUtility(IRoutesMapper)
     _argument_prog = re.compile(r'\{(.*?)\}|:\((.*)\)')
 
     def _extract_route_information(route):
         """
         Convert a route into tuple(name, path, args), eg:
             ('show_user', '/profile/%(username)s', ['username'])
         """
 
         route_path = route.pattern
         pattern = route.pattern
 
         def replace(matchobj):
             if matchobj.group(1):
                 return "%%(%s)s" % matchobj.group(1).split(':')[0]
             else:
                 return "%%(%s)s" % matchobj.group(2)
 
         route_path = _argument_prog.sub(replace, route_path)
 
         if not route_path.startswith('/'):
             route_path = f'/{route_path}'
 
         return (
             route.name,
             route_path,
             [(arg[0].split(':')[0] if arg[0] != '' else arg[1])
              for arg in _argument_prog.findall(pattern)]
         )
 
     def get_routes():
         # pyramid routes
         for route in mapper.get_routes():
             if not route.name.startswith('__'):
                 yield _extract_route_information(route)
 
     if asbool(registry.settings.get('generate_js_files', 'false')):
         static_path = AssetResolver().resolve('rhodecode:public').abspath()
         jsroutes = get_routes()
         jsroutes_file_content = generate_jsroutes_content(jsroutes)
         jsroutes_file_path = os.path.join(
             static_path, 'js', 'rhodecode', 'routes.js')
 
         try:
             with open(jsroutes_file_path, 'w', encoding='utf-8') as f:
                 f.write(jsroutes_file_content)
             log.debug('generated JS files in %s', jsroutes_file_path)
         except Exception:
             log.exception('Failed to write routes.js into %s', jsroutes_file_path)
 
 
+def import_license_if_present(event):
+    """
+    This is subscribed to the `pyramid.events.ApplicationCreated` event. It
+    does a import license key based on a presence of the file.
+    """
+    settings = event.app.registry.settings
+
+    rhodecode_edition_id = settings.get('rhodecode.edition_id')
+    license_file_path = settings.get('license.import_path')
+    force = settings.get('license.import_path_mode') == 'force'
+
+    if license_file_path and rhodecode_edition_id == 'EE':
+        log.debug('license.import_path= is set importing license from %s', license_file_path)
+        from rhodecode.model.meta import Session
+        from rhodecode.model.license import apply_license_from_file
+        try:
+            apply_license_from_file(license_file_path, force=force)
+            Session().commit()
+        except OSError:
+            log.exception('Failed to import license from %s, make sure this file exists', license_file_path)
+
+
 class Subscriber(object):
     """
     Base class for subscribers to the pyramid event system.
     """
     def __call__(self, event):
         self.run(event)
 
     def run(self, event):
         raise NotImplementedError('Subclass has to implement this.')
 
 
 class AsyncSubscriber(Subscriber):
     """
     Subscriber that handles the execution of events in a separate task to not
     block the execution of the code which triggers the event. It puts the
     received events into a queue from which the worker process takes them in
     order.
     """
     def __init__(self):
         self._stop = False
         self._eventq = queue.Queue()
         self._worker = self.create_worker()
         self._worker.start()
 
     def __call__(self, event):
         self._eventq.put(event)
 
     def create_worker(self):
         worker = Thread(target=self.do_work)
         worker.daemon = True
         return worker
 
     def stop_worker(self):
         self._stop = False
         self._eventq.put(None)
         self._worker.join()
 
     def do_work(self):
         while not self._stop:
             event = self._eventq.get()
             if event is not None:
                 self.run(event)
 
 
 class AsyncSubprocessSubscriber(AsyncSubscriber):
     """
     Subscriber that uses the subprocess module to execute a command if an
     event is received. Events are handled asynchronously::
 
         subscriber = AsyncSubprocessSubscriber('ls -la', timeout=10)
         subscriber(dummyEvent) # running __call__(event)
 
     """
 
     def __init__(self, cmd, timeout=None):
         if not isinstance(cmd, (list, tuple)):
             cmd = shlex.split(cmd)
         super().__init__()
         self._cmd = cmd
         self._timeout = timeout
 
     def run(self, event):
         cmd = self._cmd
         timeout = self._timeout
         log.debug('Executing command %s.', cmd)
 
         try:
             output = subprocess.check_output(
                 cmd, timeout=timeout, stderr=subprocess.STDOUT)
             log.debug('Command finished %s', cmd)
             if output:
                 log.debug('Command output: %s', output)
         except subprocess.TimeoutExpired as e:
             log.exception('Timeout while executing command.')
             if e.output:
                 log.error('Command output: %s', e.output)
         except subprocess.CalledProcessError as e:
             log.exception('Error while executing command.')
             if e.output:
                 log.error('Command output: %s', e.output)
         except Exception:
             log.exception(
                 'Exception while executing command %s.', cmd)

rhodecode/templates/admin/auth/auth_settings.mako

@@ -1,122 +1,127 b''
 <%inherit file="/base/base.mako"/>
 
 <%def name="title()">
     ${_('Authentication Settings')}
     %if c.rhodecode_name:
         &middot; ${h.branding(c.rhodecode_name)}}
     %endif
 </%def>
 
 <%def name="breadcrumbs_links()"></%def>
 
 <%def name="menu_bar_nav()">
     ${self.menu_items(active='admin')}
 </%def>
 
 <%def name="menu_bar_subnav()">
     ${self.admin_menu(active='authentication')}
 </%def>
 
 <%def name="main()">
 
 <div class="box">
 
   <div class='sidebar-col-wrapper'>
 
     <div class="sidebar">
         <ul class="nav nav-pills nav-stacked">
           % for item in resource.get_root().get_nav_list():
+
             <li ${('class=active' if item == resource else '')}>
-              <a href="${request.resource_path(item, route_name='auth_home')}">${item.display_name}</a>
+              % if getattr(item, 'is_root', False):
+                <a href="${request.resource_path(item, route_name='auth_home')}">${item.display_name}</a>
+              % else:
+                <a style="padding-left: 10px" href="${request.resource_path(item, route_name='auth_home')}">${item.display_name}</a>
+              % endif
             </li>
           % endfor
         </ul>
     </div>
 
     <div class="main-content-full-width">
       ${h.secure_form(request.resource_path(resource, route_name='auth_home'), request=request)}
         <div class="panel panel-default">
 
           <div class="panel-heading">
             <h3 class="panel-title">${_("Enabled and Available Plugins")}</h3>
           </div>
 
           <div class="panel-body">
 
 
               <div class="label">${_("Ordered Activated Plugins")}</div>
               <div class="textarea text-area editor">
                   ${h.textarea('auth_plugins',cols=120,rows=20,class_="medium")}
               </div>
               <div class="field">
                 <p class="help-block pre-formatting">${_('List of plugins, separated by commas.'
                   '\nThe order of the plugins is also the order in which '
                   'RhodeCode Enterprise will try to authenticate a user.')}
                 </p>
               </div>
 
               <table class="rctable">
                   <th>${_('Activate')}</th>
                   <th>${_('Plugin Name')}</th>
                   <th>${_('Documentation')}</th>
                   <th>${_('Plugin ID')}</th>
                   <th>${_('Enabled')}</th>
                   %for plugin in available_plugins:
                       <tr class="${('inactive' if (not plugin.is_active() and plugin.get_id() in enabled_plugins) else '')}">
                           <td>
                             <span plugin_id="${plugin.get_id()}" class="toggle-plugin btn ${('btn-success' if plugin.get_id() in enabled_plugins else '')}">
                               ${(_('activated') if plugin.get_id() in enabled_plugins else _('not active'))}
                             </span>
                           </td>
                           <td>${plugin.get_display_name()}</td>
                           <td>
                               % if plugin.docs():
                                 <a href="${plugin.docs()}">docs</a>
                               % endif
                           </td>
                           <td>${plugin.get_id()}</td>
                           <td>${h.bool2icon(plugin.is_active(),show_at_false=False)}</td>
                       </tr>
                   %endfor
               </table>
 
             <div class="buttons">
               ${h.submit('save',_('Save'),class_="btn")}
             </div>
           </div>
         </div>
       ${h.end_form()}
     </div>
   </div>
 </div>
 
 <script>
   $('.toggle-plugin').click(function(e){
     var auth_plugins_input = $('#auth_plugins');
     var elems = [];
 
     $.each(auth_plugins_input.val().split(',') , function (index, element) {
         if (element !== "") {
             elems.push(element.strip())
         }
     });
 
     var cur_button = e.currentTarget;
     var plugin_id = $(cur_button).attr('plugin_id');
     if($(cur_button).hasClass('btn-success')){
       elems.splice(elems.indexOf(plugin_id), 1);
       auth_plugins_input.val(elems.join(',\n'));
       $(cur_button).removeClass('btn-success');
       cur_button.innerHTML = _gettext('not active');
     }
     else{
       if (elems.indexOf(plugin_id) === -1) {
             elems.push(plugin_id);
       }
       auth_plugins_input.val(elems.join(',\n'));
       $(cur_button).addClass('btn-success');
       cur_button.innerHTML = _gettext('activated');
     }
   });
 </script>
 </%def>

rhodecode/templates/admin/auth/plugin_settings.mako

@@ -1,139 +1,146 b''
 <%inherit file="/base/base.mako"/>
 
 <%def name="title()">
     ${_('Authentication Settings')}
     %if c.rhodecode_name:
     &middot; ${h.branding(c.rhodecode_name)}}
     %endif
 </%def>
 
 <%def name="breadcrumbs_links()">
   ${h.link_to(_('Admin'),h.route_path('admin_home'))}
   &raquo;
   ${h.link_to(_('Authentication Plugins'),request.resource_path(resource.__parent__, route_name='auth_home'))}
   &raquo;
   ${resource.display_name}
 </%def>
 
 <%def name="menu_bar_nav()">
     ${self.menu_items(active='admin')}
 </%def>
 
 <%def name="menu_bar_subnav()">
     ${self.admin_menu(active='authentication')}
 </%def>
 
 <%def name="main()">
 
   <div class="box">
 
     <div class='sidebar-col-wrapper'>
 
       <div class="sidebar">
         <ul class="nav nav-pills nav-stacked">
           % for item in resource.get_root().get_nav_list():
             <li ${('class=active' if item == resource else '')}>
               <a href="${request.resource_path(item, route_name='auth_home')}">${item.display_name}</a>
             </li>
           % endfor
         </ul>
       </div>
 
       <div class="main-content-full-width">
         <div class="panel panel-default">
           <div class="panel-heading">
             <h3 class="panel-title">${_('Plugin')}: ${resource.display_name}</h3>
           </div>
           <div class="panel-body">
             <div class="plugin_form">
               <div class="fields">
                 ${h.secure_form(request.resource_path(resource, route_name='auth_home'), request=request)}
                 <div class="form">
 
+                  ## Allow derived templates to add something above the form
+                  ## input fields
+                  %if hasattr(next, 'above_form_fields'):
+                    ${next.above_form_fields()}
+                  %endif
+
+                  <h4>${_('Plugin Configuration')}</h4>
                   %for node in plugin.get_settings_schema():
                     <%
                       label_to_type = {'label-checkbox': 'bool', 'label-textarea': 'textarea'}
                     %>
 
                     <div class="field">
                       <div class="label ${label_to_type.get(node.widget)}"><label for="${node.name}">${node.title}</label></div>
                       <div class="input">
                         %if node.widget in ["string", "int", "unicode"]:
                           ${h.text(node.name, defaults.get(node.name), class_="large")}
                         %elif node.widget == "password":
                           ${h.password(node.name, defaults.get(node.name), class_="large")}
                         %elif node.widget == "bool":
                             %if node.name == "global_2fa" and c.rhodecode_edition_id != "EE":
                                 <input type="checkbox" disabled/>
                                 <%node.description = _('This feature is available in RhodeCode EE edition only. Contact {sales_email} to obtain a trial license.').format(sales_email='<a href="mailto:[e-mail]">[e-mail]</a>')%>
                             %else:
                                 <div class="checkbox" >${h.checkbox(node.name, True, checked=defaults.get(node.name))}</div>
                             %endif
                         %elif node.widget == "select":
                           ${h.select(node.name, defaults.get(node.name), node.validator.choices, class_="select2AuthSetting")}
                         %elif node.widget == "select_with_labels":
                           ${h.select(node.name, defaults.get(node.name), node.choices, class_="select2AuthSetting")}
                         %elif node.widget == "textarea":
                           <div class="textarea" style="margin-left: 0px">${h.textarea(node.name, defaults.get(node.name), rows=10)}</div>
                         %elif node.widget == "readonly":
                           ${node.default}
                         %else:
                           This field is of type ${node.typ}, which cannot be displayed. Must be one of [string|int|bool|select].
                         %endif
 
                         %if node.name in errors:
                           <span class="error-message">${errors.get(node.name)}</span>
                           <br />
                         %endif
                         <p class="help-block pre-formatting">${node.description | n}</p>
                       </div>
                     </div>
                   %endfor
 
                   ## Allow derived templates to add something below the form
                   ## input fields
                   %if hasattr(next, 'below_form_fields'):
                     ${next.below_form_fields()}
                   %endif
 
                   <div class="buttons">
                     ${h.submit('save',_('Save'),class_="btn")}
                   </div>
 
                 </div>
                 ${h.end_form()}
               </div>
             </div>
 
 % if request.GET.get('schema'):
 ## this is for development and creation of example configurations for documentation
 <pre>
     % for node in plugin.get_settings_schema():
     *option*: `${node.name}` => `${defaults.get(node.name)}`${'\n    # '.join(['']+node.description.splitlines())}
 
     % endfor
 </pre>
 
 % endif
 
           </div>
         </div>
       </div>
 
     </div>
   </div>
 
 
 <script>
 $(document).ready(function() {
   var select2Options = {
         containerCssClass: 'drop-menu',
         dropdownCssClass: 'drop-menu-dropdown',
         dropdownAutoWidth: true,
         minimumResultsForSearch: -1
   };
   $('.select2AuthSetting').select2(select2Options);
 
 });
 </script>
 </%def>

rhodecode/templates/admin/repos/repo_edit_advanced.mako

@@ -1,307 +1,324 b''
 <%namespace name="base" file="/base/base.mako"/>
 
 <%
  elems = [
     (_('Repository ID'), c.rhodecode_db_repo.repo_id, '', ''),
     (_('Owner'), lambda:base.gravatar_with_user(c.rhodecode_db_repo.user.email, tooltip=True), '', ''),
     (_('Created on'), h.format_date(c.rhodecode_db_repo.created_on), '', ''),
     (_('Updated on'), h.format_date(c.rhodecode_db_repo.updated_on), '', ''),
     (_('Cached Commit id'), lambda: h.link_to(c.rhodecode_db_repo.changeset_cache.get('short_id'), h.route_path('repo_commit',repo_name=c.repo_name,commit_id=c.rhodecode_db_repo.changeset_cache.get('raw_id'))), '', ''),
     (_('Cached Commit date'), c.rhodecode_db_repo.changeset_cache.get('date'), '', ''),
     (_('Cached Commit data'), lambda: h.link_to('refresh now', h.current_route_path(request, update_commit_cache=1)), '', ''),
     (_('Attached scoped tokens'), len(c.rhodecode_db_repo.scoped_tokens), '', [x.user for x in c.rhodecode_db_repo.scoped_tokens]),
     (_('Pull requests source'), len(c.rhodecode_db_repo.pull_requests_source), '', ['pr_id:{}, repo:{}'.format(x.pull_request_id,x.source_repo.repo_name) for x in c.rhodecode_db_repo.pull_requests_source]),
     (_('Pull requests target'), len(c.rhodecode_db_repo.pull_requests_target), '', ['pr_id:{}, repo:{}'.format(x.pull_request_id,x.target_repo.repo_name) for x in c.rhodecode_db_repo.pull_requests_target]),
     (_('Attached Artifacts'), len(c.rhodecode_db_repo.artifacts), '', ''),
  ]
 %>
 
 <div class="panel panel-default">
     <div class="panel-heading" id="advanced-info" >
         <h3 class="panel-title">${_('Repository: %s') % c.rhodecode_db_repo.repo_name} <a class="permalink" href="#advanced-info"> ¶</a></h3>
     </div>
     <div class="panel-body">
         ${base.dt_info_panel(elems)}
     </div>
 </div>
 
 
 <div class="panel panel-default">
     <div class="panel-heading" id="advanced-fork">
         <h3 class="panel-title">${_('Fork Reference')} <a class="permalink" href="#advanced-fork"> ¶</a></h3>
     </div>
     <div class="panel-body">
       ${h.secure_form(h.route_path('edit_repo_advanced_fork', repo_name=c.rhodecode_db_repo.repo_name), request=request)}
 
         % if c.rhodecode_db_repo.fork:
             <div class="panel-body-title-text">${h.literal(_('This repository is a fork of %(repo_link)s') % {'repo_link': h.link_to_if(c.has_origin_repo_read_perm,c.rhodecode_db_repo.fork.repo_name, h.route_path('repo_summary', repo_name=c.rhodecode_db_repo.fork.repo_name))})}
             | <button class="btn btn-link btn-danger" type="submit">Remove fork reference</button></div>
         % endif
 
          <div class="field">
              ${h.hidden('id_fork_of')}
              ${h.submit('set_as_fork_%s' % c.rhodecode_db_repo.repo_name,_('Set'),class_="btn btn-small",)}
          </div>
          <div class="field">
              <span class="help-block">${_('Manually set this repository as a fork of another from the list')}</span>
          </div>
       ${h.end_form()}
     </div>
 </div>
 
 
 <div class="panel panel-default">
     <div class="panel-heading" id="advanced-journal">
         <h3 class="panel-title">${_('Public Journal Visibility')} <a class="permalink" href="#advanced-journal"> ¶</a></h3>
     </div>
     <div class="panel-body">
       ${h.secure_form(h.route_path('edit_repo_advanced_journal', repo_name=c.rhodecode_db_repo.repo_name), request=request)}
         <div class="field">
         %if c.in_public_journal:
           <button class="btn btn-small" type="submit">
               ${_('Remove from Public Journal')}
           </button>
         %else:
           <button class="btn btn-small" type="submit">
               ${_('Add to Public Journal')}
           </button>
         %endif
         </div>
        <div class="field" >
        <span class="help-block">${_('All actions made on this repository will be visible to everyone following the public journal.')}</span>
        </div>
       ${h.end_form()}
     </div>
 </div>
 
 
 <div class="panel panel-default">
     <div class="panel-heading" id="advanced-locking">
         <h3 class="panel-title">${_('Locking state')} <a class="permalink" href="#advanced-locking"> ¶</a></h3>
     </div>
     <div class="panel-body">
         ${h.secure_form(h.route_path('edit_repo_advanced_locking', repo_name=c.rhodecode_db_repo.repo_name), request=request)}
 
         %if c.rhodecode_db_repo.locked[0]:
              <div class="panel-body-title-text">${'Locked by %s on %s. Lock reason: %s' % (h.person_by_id(c.rhodecode_db_repo.locked[0]),
              h.format_date(h. time_to_datetime(c.rhodecode_db_repo.locked[1])), c.rhodecode_db_repo.locked[2])}</div>
         %else:
             <div class="panel-body-title-text">${_('This Repository is not currently locked.')}</div>
         %endif
 
         <div class="field" >
             %if c.rhodecode_db_repo.locked[0]:
               ${h.hidden('set_unlock', '1')}
               <button class="btn btn-small" type="submit"
                       onclick="submitConfirm(event, this, _gettext('Confirm to unlock this repository'), _gettext('Unlock'), '${c.rhodecode_db_repo.repo_name}')"
               >
                   <i class="icon-unlock"></i>
                   ${_('Unlock repository')}
               </button>
             %else:
               ${h.hidden('set_lock', '1')}
               <button class="btn btn-small" type="submit"
                       onclick="submitConfirm(event, this, _gettext('Confirm to lock this repository'), _gettext('lock'), '${c.rhodecode_db_repo.repo_name}')"
               >
                   <i class="icon-lock"></i>
                   ${_('Lock repository')}
               </button>
             %endif
          </div>
          <div class="field" >
             <span class="help-block">
                 ${_('Force repository locking. This only works when anonymous access is disabled. Pulling from the repository locks the repository to that user until the same user pushes to that repository again.')}
             </span>
          </div>
         ${h.end_form()}
     </div>
 </div>
 
 
 <div class="panel panel-default">
     <div class="panel-heading" id="advanced-hooks">
         <h3 class="panel-title">${_('Hooks')} <a class="permalink" href="#advanced-hooks"> ¶</a></h3>
     </div>
     <div class="panel-body">
         <table class="rctable">
             <th>${_('Hook type')}</th>
             <th>${_('Hook version')}</th>
             <th>${_('Current version')}</th>
             % if c.ver_info_dict:
             <tr>
                 <td>${_('PRE HOOK')}</td>
                 <td>${c.ver_info_dict['pre_version']}</td>
                 <td>${c.rhodecode_version}</td>
             </tr>
             <tr>
                 <td>${_('POST HOOK')}</td>
                 <td>${c.ver_info_dict['post_version']}</td>
                 <td>${c.rhodecode_version}</td>
             </tr>
             % else:
                 <tr>
                     <td>${_('Unable to read hook information from VCS Server')}</td>
                 </tr>
             % endif
         </table>
 
         <a class="btn btn-primary" href="${h.route_path('edit_repo_advanced_hooks', repo_name=c.repo_name)}"
            onclick="return confirm('${_('Confirm to reinstall hooks for this repository.')}');">
             ${_('Update Hooks')}
         </a>
         % if c.hooks_outdated:
             <span class="alert-error" style="padding: 10px">
                 ${_('Outdated hooks detected, please update hooks using `Update Hooks` action.')}
             </span>
         % endif
     </div>
 </div>
 
 <div class="panel panel-warning">
     <div class="panel-heading" id="advanced-archive">
         <h3 class="panel-title">${_('Archive repository')} <a class="permalink" href="#advanced-archive"> ¶</a></h3>
     </div>
     <div class="panel-body">
       ${h.secure_form(h.route_path('edit_repo_advanced_archive', repo_name=c.repo_name), request=request)}
 
         <div style="margin: 0 0 20px 0" class="fake-space"></div>
 
         <div class="field">
         % if c.rhodecode_db_repo.archived:
             This repository is already archived. Only super-admin users can un-archive this repository.
         % else:
             <button class="btn btn-small btn-warning" type="submit"
                     onclick="submitConfirm(event, this, _gettext('Confirm to archive this repository. <br/>This action is irreversible !'), _gettext('Archive'), '${c.rhodecode_db_repo.repo_name}')"
             >
                 ${_('Archive this repository')}
             </button>
         % endif
 
         </div>
         <div class="field">
             <span class="help-block">
                 <strong>
                     ${_('This action is irreversible')} !
                 </strong><br/>
                 ${_('Archiving the repository will make it entirely read-only. The repository cannot be committed to.'
                     'It is hidden from the search results and dashboard. ')}
             </span>
         </div>
 
       ${h.end_form()}
     </div>
 </div>
 
 
 <div class="panel panel-danger">
     <div class="panel-heading" id="advanced-delete">
         <h3 class="panel-title">${_('Delete repository')} <a class="permalink" href="#advanced-delete"> ¶</a></h3>
     </div>
     <div class="panel-body">
       ${h.secure_form(h.route_path('edit_repo_advanced_delete', repo_name=c.repo_name), request=request)}
         <table class="display">
             <tr>
                 <td>
                     ${_ungettext('This repository has %s fork.', 'This repository has %s forks.', c.rhodecode_db_repo.forks.count()) % c.rhodecode_db_repo.forks.count()}
                 </td>
                 <td>
                     %if c.rhodecode_db_repo.forks.count():
                         <input type="radio" name="forks" value="detach_forks" checked="checked"/> <label for="forks">${_('Detach forks')}</label>
                     %endif
                 </td>
                 <td>
                     %if c.rhodecode_db_repo.forks.count():
                         <input type="radio" name="forks" value="delete_forks"/> <label for="forks">${_('Delete forks')}</label>
                     %endif
                 </td>
             </tr>
+
             <% attached_prs = len(c.rhodecode_db_repo.pull_requests_source + c.rhodecode_db_repo.pull_requests_target) %>
             % if c.rhodecode_db_repo.pull_requests_source or c.rhodecode_db_repo.pull_requests_target:
             <tr>
                 <td>
                     ${_ungettext('This repository has %s attached pull request.', 'This repository has %s attached pull requests.', attached_prs) % attached_prs}
                     <br/>
-                    ${_('Consider to archive this repository instead.')}
+                    <br/>
+                    <strong>${_('Consider to archive this repository instead.')}</strong>
                 </td>
                 <td></td>
                 <td></td>
             </tr>
             % endif
+
+            <% attached_artifacts = len(c.rhodecode_db_repo.artifacts) %>
+            % if attached_artifacts:
+            <tr>
+                <td>
+                    ${_ungettext('This repository has %s attached artifact.', 'This repository has %s attached artifacts.', attached_artifacts) % attached_artifacts}
+                    <br/>
+                    <br/>
+                    <strong>${_('Consider to archive this repository instead.')}</strong>
+                </td>
+                <td></td>
+                <td></td>
+            </tr>
+            % endif
+
         </table>
         <div style="margin: 0 0 20px 0" class="fake-space"></div>
 
         <div class="field">
             <button class="btn btn-small btn-danger" type="submit"
                     onclick="submitConfirm(event, this, _gettext('Confirm to delete this repository'), _gettext('Delete'), '${c.rhodecode_db_repo.repo_name}')"
             >
                 ${_('Delete this repository')}
             </button>
         </div>
         <div class="field">
             <span class="help-block">
                 ${_('This repository will be renamed in a special way in order to make it inaccessible to RhodeCode Enterprise and its VCS systems. If you need to fully delete it from the file system, please do it manually, or with rhodecode-cleanup-repos command available in rhodecode-tools.')}
             </span>
         </div>
 
       ${h.end_form()}
     </div>
 </div>
 
 
 <script>
 
 var currentRepoId = ${c.rhodecode_db_repo.repo_id};
 
 var repoTypeFilter = function(data) {
     var results = [];
 
     if (!data.results[0]) {
         return data
     }
 
     $.each(data.results[0].children, function() {
         // filter out the SAME repo, it cannot be used as fork of itself
         if (this.repo_id != currentRepoId) {
             this.id = this.repo_id;
             results.push(this)
         }
     });
     data.results[0].children = results;
     return data;
 };
 
 $("#id_fork_of").select2({
     cachedDataSource: {},
     minimumInputLength: 2,
     placeholder: "${_('Change repository') if c.rhodecode_db_repo.fork else _('Pick repository')}",
     dropdownAutoWidth: true,
     containerCssClass: "drop-menu",
     dropdownCssClass: "drop-menu-dropdown",
     formatResult: formatRepoResult,
     query: $.debounce(250, function(query){
         self = this;
         var cacheKey = query.term;
         var cachedData = self.cachedDataSource[cacheKey];
 
         if (cachedData) {
             query.callback({results: cachedData.results});
         } else {
             $.ajax({
                 url: pyroutes.url('repo_list_data'),
                 data: {'query': query.term, repo_type: '${c.rhodecode_db_repo.repo_type}'},
                 dataType: 'json',
                 type: 'GET',
                 success: function(data) {
                     data = repoTypeFilter(data);
                     self.cachedDataSource[cacheKey] = data;
                     query.callback({results: data.results});
                 },
                 error: function(data, textStatus, errorThrown) {
                     alert("Error while fetching entries.\nError code {0} ({1}).".format(data.status, data.statusText));
                 }
             })
         }
     })
 });
 </script>
 

rhodecode/templates/base/base.mako

@@ -1,1263 +1,1264 b''
 
 <%!
     from rhodecode.lib import html_filters
 %>
 
 <%inherit file="root.mako"/>
 
 <%include file="/ejs_templates/templates.html"/>
 
 <div class="outerwrapper">
   <!-- HEADER -->
   <div class="header">
       <div id="header-inner" class="wrapper">
           <div id="logo">
               <div class="logo-wrapper">
                   <a href="${h.route_path('home')}"><img src="${h.asset('images/rhodecode-logo-white-60x60.png')}" alt="RhodeCode"/></a>
               </div>
               % if c.rhodecode_name:
                <div class="branding">
                    <a href="${h.route_path('home')}">${h.branding(c.rhodecode_name)}</a>
                </div>
               % endif
           </div>
           <!-- MENU BAR NAV -->
           ${self.menu_bar_nav()}
           <!-- END MENU BAR NAV -->
       </div>
   </div>
   ${self.menu_bar_subnav()}
   <!-- END HEADER -->
 
   <!-- CONTENT -->
   <div id="content" class="wrapper">
 
       <rhodecode-toast id="notifications"></rhodecode-toast>
 
       <div class="main">
           ${next.main()}
       </div>
 
   </div>
   <!-- END CONTENT -->
 
 </div>
 
 <!-- FOOTER -->
 <div id="footer">
    <div id="footer-inner" class="title wrapper">
        <div>
            <% sid = 'block' if request.GET.get('showrcid') else 'none' %>
 
            <p class="footer-link-right">
                <a class="grey-link-action" href="${h.route_path('home', _query={'showrcid': 1})}">
                RhodeCode
                % if c.visual.show_version:
                    ${c.rhodecode_version}
                % endif
                ${c.rhodecode_edition}
                </a> |
 
                % if c.visual.rhodecode_support_url:
                   <a class="grey-link-action" href="${c.visual.rhodecode_support_url}" target="_blank">${_('Support')}</a> |
                   <a class="grey-link-action" href="https://docs.rhodecode.com" target="_blank">${_('Documentation')}</a>
                % endif
 
            </p>
 
            <p class="server-instance" style="display:${sid}">
                ## display hidden instance ID if specially defined
                &copy; 2010-${h.datetime.today().year}, <a href="${h.route_url('rhodecode_official')}" target="_blank">RhodeCode GmbH</a>. All rights reserved.
                % if c.rhodecode_instanceid:
                    ${_('RhodeCode instance id: {}').format(c.rhodecode_instanceid)}
                % endif
            </p>
        </div>
    </div>
 </div>
 
 <!-- END FOOTER -->
 
 ### MAKO DEFS ###
 
 <%def name="menu_bar_subnav()">
 </%def>
 
 <%def name="breadcrumbs(class_='breadcrumbs')">
     <div class="${class_}">
     ${self.breadcrumbs_links()}
     </div>
 </%def>
 
 <%def name="admin_menu(active=None)">
 
   <div id="context-bar">
     <div class="wrapper">
       <div class="title">
         <div class="title-content">
           <div class="title-main">
             % if c.is_super_admin:
                 ${_('Super-admin Panel')}
             % else:
                 ${_('Delegated Admin Panel')}
             % endif
           </div>
         </div>
       </div>
 
     <ul id="context-pages" class="navigation horizontal-list">
 
         ## super-admin case (Top Menu)
         % if c.is_super_admin:
           <li class="${h.is_active('audit_logs', active)}"><a href="${h.route_path('admin_audit_logs')}">${_('Admin audit logs')}</a></li>
           <li class="${h.is_active('repositories', active)}"><a href="${h.route_path('repos')}">${_('Repositories')}</a></li>
           <li class="${h.is_active('repository_groups', active)}"><a href="${h.route_path('repo_groups')}">${_('Repository groups')}</a></li>
           <li class="${h.is_active('users', active)}"><a href="${h.route_path('users')}">${_('Users')}</a></li>
           <li class="${h.is_active('user_groups', active)}"><a href="${h.route_path('user_groups')}">${_('User groups')}</a></li>
+          <li class="${h.is_active('security', active)}"><a href="${h.route_path('admin_security')}">${_('Security')}</a></li>
           <li class="${h.is_active('artifacts', active)}"><a href="${h.route_path('admin_artifacts')}">${_('Artifacts')}</a></li>
           <li class="${h.is_active('automation', active)}"><a href="${h.route_path('admin_automation')}">${_('Automation')}</a></li>
           <li class="${h.is_active('scheduler', active)}"><a href="${h.route_path('admin_scheduler')}">${_('Scheduler')}</a></li>
           <li class="${h.is_active('permissions', active)}"><a href="${h.route_path('admin_permissions_application')}">${_('Permissions')}</a></li>
           <li class="${h.is_active('authentication', active)}"><a href="${h.route_path('auth_home', traverse='')}">${_('Authentication')}</a></li>
           <li class="${h.is_active('integrations', active)}"><a href="${h.route_path('global_integrations_home')}">${_('Integrations')}</a></li>
           <li class="${h.is_active('defaults', active)}"><a href="${h.route_path('admin_defaults_repositories')}">${_('Defaults')}</a></li>
           <li class="${h.is_active('settings', active)}"><a href="${h.route_path('admin_settings')}">${_('Settings')}</a></li>
 
         ## delegated admin
         % elif c.is_delegated_admin:
            <%
            repositories=c.auth_user.repositories_admin or c.can_create_repo
            repository_groups=c.auth_user.repository_groups_admin or c.can_create_repo_group
            user_groups=c.auth_user.user_groups_admin or c.can_create_user_group
            %>
 
            %if repositories:
               <li class="${h.is_active('repositories', active)} local-admin-repos"><a href="${h.route_path('repos')}">${_('Repositories')}</a></li>
            %endif
            %if repository_groups:
               <li class="${h.is_active('repository_groups', active)} local-admin-repo-groups"><a href="${h.route_path('repo_groups')}">${_('Repository groups')}</a></li>
            %endif
            %if user_groups:
               <li class="${h.is_active('user_groups', active)} local-admin-user-groups"><a href="${h.route_path('user_groups')}">${_('User groups')}</a></li>
            %endif
         % endif
     </ul>
 
     </div>
     <div class="clear"></div>
   </div>
 </%def>
 
 <%def name="dt_info_panel(elements)">
     <dl class="dl-horizontal">
     %for dt, dd, title, show_items in elements:
       <dt>${dt}:</dt>
       <dd title="${h.tooltip(title)}">
       %if callable(dd):
           ## allow lazy evaluation of elements
           ${dd()}
       %else:
           ${dd}
       %endif
       %if show_items:
           <span class="btn-collapse" data-toggle="item-${h.md5_safe(dt)[:6]}-details">${_('Show More')} </span>
       %endif
       </dd>
 
       %if show_items:
           <div class="collapsable-content" data-toggle="item-${h.md5_safe(dt)[:6]}-details" style="display: none">
           %for item in show_items:
               <dt></dt>
               <dd>${item}</dd>
           %endfor
           </div>
       %endif
 
     %endfor
     </dl>
 </%def>
 
 <%def name="tr_info_entry(element)">
     <% key, val, title, show_items = element %>
 
     <tr>
         <td style="vertical-align: top">${key}</td>
         <td title="${h.tooltip(title)}">
           %if callable(val):
               ## allow lazy evaluation of elements
               ${val()}
           %else:
               ${val}
           %endif
           %if show_items:
               <div class="collapsable-content" data-toggle="item-${h.md5_safe(h.safe_str(val))[:6]}-details" style="display: none">
               % for item in show_items:
                   <dt></dt>
                   <dd>${item}</dd>
               % endfor
               </div>
           %endif
         </td>
         <td style="vertical-align: top">
           %if show_items:
               <span class="btn-collapse" data-toggle="item-${h.md5_safe(h.safe_str(val))[:6]}-details">${_('Show More')} </span>
           %endif
         </td>
     </tr>
 
 </%def>
 
 <%def name="gravatar(email, size=16, tooltip=False, tooltip_alt=None, user=None, extra_class=None)">
   <%
     if size > 16:
         gravatar_class = ['gravatar','gravatar-large']
     else:
         gravatar_class = ['gravatar']
 
     data_hovercard_url = ''
     data_hovercard_alt = tooltip_alt.replace('<', '&lt;').replace('>', '&gt;') if tooltip_alt else ''
 
     if tooltip:
         gravatar_class += ['tooltip-hovercard']
     if extra_class:
         gravatar_class += extra_class
     if tooltip and user:
         if user.username == h.DEFAULT_USER:
             gravatar_class.pop(-1)
         else:
             data_hovercard_url = request.route_path('hovercard_user', user_id=getattr(user, 'user_id', ''))
     gravatar_class = ' '.join(gravatar_class)
 
   %>
   <%doc>
     TODO: johbo: For now we serve double size images to make it smooth
     for retina. This is how it worked until now. Should be replaced
     with a better solution at some point.
   </%doc>
 
   <img class="${gravatar_class}" height="${size}" width="${size}" data-hovercard-url="${data_hovercard_url}" data-hovercard-alt="${data_hovercard_alt}" src="${h.gravatar_url(email, size * 2, request=request)}" />
 </%def>
 
 
 <%def name="gravatar_with_user(contact, size=16, show_disabled=False, tooltip=False, _class='rc-user')">
   <%
       email = h.email_or_none(contact)
       rc_user = h.discover_user(contact)
   %>
 
   <div class="${_class}">
     ${self.gravatar(email, size, tooltip=tooltip, tooltip_alt=contact, user=rc_user)}
     <span class="${('user user-disabled' if show_disabled else 'user')}">
         ${h.link_to_user(rc_user or contact)}
     </span>
   </div>
 </%def>
 
 
 <%def name="user_group_icon(user_group=None, size=16, tooltip=False)">
   <%
     if (size > 16):
         gravatar_class = 'icon-user-group-alt'
     else:
         gravatar_class = 'icon-user-group-alt'
 
     if tooltip:
         gravatar_class += ' tooltip-hovercard'
 
     data_hovercard_url = request.route_path('hovercard_user_group', user_group_id=user_group.users_group_id)
   %>
   <%doc>
     TODO: johbo: For now we serve double size images to make it smooth
     for retina. This is how it worked until now. Should be replaced
     with a better solution at some point.
   </%doc>
 
   <i style="font-size: ${size}px" class="${gravatar_class} x-icon-size-${size}" data-hovercard-url="${data_hovercard_url}"></i>
 </%def>
 
 <%def name="repo_page_title(repo_instance)">
 <div class="title-content repo-title">
 
     <div class="title-main">
         ## SVN/HG/GIT icons
         %if h.is_hg(repo_instance):
             <i class="icon-hg"></i>
         %endif
         %if h.is_git(repo_instance):
             <i class="icon-git"></i>
         %endif
         %if h.is_svn(repo_instance):
             <i class="icon-svn"></i>
         %endif
 
         ## public/private
         %if repo_instance.private:
             <i class="icon-repo-private"></i>
         %else:
             <i class="icon-repo-public"></i>
         %endif
 
         ## repo name with group name
         ${h.breadcrumb_repo_link(repo_instance)}
 
         ## Context Actions
         <div class="pull-right">
             %if c.rhodecode_user.username != h.DEFAULT_USER:
                 <a href="${h.route_path('atom_feed_home', repo_name=c.rhodecode_db_repo.repo_uid, _query=dict(auth_token=c.rhodecode_user.feed_token))}" title="${_('RSS Feed')}" class="btn btn-sm"><i class="icon-rss-sign"></i>RSS</a>
 
                 <a href="#WatchRepo" onclick="toggleFollowingRepo(this, templateContext.repo_id); return false" title="${_('Watch this Repository and actions on it in your personalized journal')}" class="btn btn-sm ${('watching' if c.repository_is_user_following else '')}">
                     % if c.repository_is_user_following:
                         <i class="icon-eye-off"></i>${_('Unwatch')}
                     % else:
                         <i class="icon-eye"></i>${_('Watch')}
                     % endif
 
                 </a>
             %else:
                 <a href="${h.route_path('atom_feed_home', repo_name=c.rhodecode_db_repo.repo_uid)}" title="${_('RSS Feed')}" class="btn btn-sm"><i class="icon-rss-sign"></i>RSS</a>
             %endif
         </div>
 
     </div>
 
     ## FORKED
     %if repo_instance.fork:
     <p class="discreet">
         <i class="icon-code-fork"></i> ${_('Fork of')}
         ${h.link_to_if(c.has_origin_repo_read_perm,repo_instance.fork.repo_name, h.route_path('repo_summary', repo_name=repo_instance.fork.repo_name))}
     </p>
     %endif
 
     ## IMPORTED FROM REMOTE
     %if repo_instance.clone_uri:
     <p class="discreet">
        <i class="icon-code-fork"></i> ${_('Clone from')}
        <a href="${h.safe_str(h.hide_credentials(repo_instance.clone_uri))}">${h.hide_credentials(repo_instance.clone_uri)}</a>
     </p>
     %endif
 
     ## LOCKING STATUS
      %if repo_instance.locked[0]:
        <p class="locking_locked discreet">
            <i class="icon-repo-lock"></i>
            ${_('Repository locked by %(user)s') % {'user': h.person_by_id(repo_instance.locked[0])}}
        </p>
      %elif repo_instance.enable_locking:
          <p class="locking_unlocked discreet">
              ${_('Repository not locked. Pull repository to lock it.')}
          </p>
      %endif
 
 </div>
 </%def>
 
 <%def name="repo_menu(active=None)">
     <%
     ## determine if we have "any" option available
     can_lock = h.HasRepoPermissionAny('repository.write','repository.admin')(c.repo_name) and c.rhodecode_db_repo.enable_locking
     has_actions = can_lock
 
     %>
     % if c.rhodecode_db_repo.archived:
     <div class="alert alert-warning text-center">
         <strong>${_('This repository has been archived. It is now read-only.')}</strong>
     </div>
     % endif
 
   <!--- REPO CONTEXT BAR -->
   <div id="context-bar">
     <div class="wrapper">
 
       <div class="title">
           ${self.repo_page_title(c.rhodecode_db_repo)}
       </div>
 
       <ul id="context-pages" class="navigation horizontal-list">
         <li class="${h.is_active('summary', active)}"><a class="menulink" href="${h.route_path('repo_summary_explicit', repo_name=c.repo_name)}"><div class="menulabel">${_('Summary')}</div></a></li>
         <li class="${h.is_active('commits', active)}"><a class="menulink" href="${h.route_path('repo_commits', repo_name=c.repo_name)}"><div class="menulabel">${_('Commits')}</div></a></li>
         <li class="${h.is_active('files', active)}"><a class="menulink" href="${h.repo_files_by_ref_url(c.repo_name, c.rhodecode_db_repo.repo_type, f_path='', ref_name=c.rhodecode_db_repo.landing_ref_name, commit_id='tip', query={'at':c.rhodecode_db_repo.landing_ref_name})}"><div class="menulabel">${_('Files')}</div></a></li>
         <li class="${h.is_active('compare', active)}"><a class="menulink" href="${h.route_path('repo_compare_select',repo_name=c.repo_name)}"><div class="menulabel">${_('Compare')}</div></a></li>
 
         ## TODO: anderson: ideally it would have a function on the scm_instance "enable_pullrequest() and enable_fork()"
         %if c.rhodecode_db_repo.repo_type in ['git','hg']:
           <li class="${h.is_active('showpullrequest', active)}">
             <a class="menulink" href="${h.route_path('pullrequest_show_all', repo_name=c.repo_name)}" title="${h.tooltip(_('Show Pull Requests for %s') % c.repo_name)}">
               <div class="menulabel">
                   ${_('Pull Requests')} <span class="menulink-counter">${c.repository_pull_requests}</span>
               </div>
             </a>
           </li>
         %endif
 
         <li class="${h.is_active('artifacts', active)}">
             <a class="menulink" href="${h.route_path('repo_artifacts_list',repo_name=c.repo_name)}">
                 <div class="menulabel">
                     ${_('Artifacts')}  <span class="menulink-counter">${c.repository_artifacts}</span>
                 </div>
             </a>
         </li>
 
         %if not c.rhodecode_db_repo.archived and h.HasRepoPermissionAll('repository.admin')(c.repo_name):
             <li class="${h.is_active('settings', active)}"><a class="menulink" href="${h.route_path('edit_repo',repo_name=c.repo_name)}"><div class="menulabel">${_('Repository Settings')}</div></a></li>
         %endif
 
         <li class="${h.is_active('options', active)}">
           % if has_actions:
             <a class="menulink dropdown">
               <div class="menulabel">${_('Quick Actions')}<div class="show_more"></div></div>
             </a>
             <ul class="submenu">
                 %if can_lock:
                     %if c.rhodecode_db_repo.locked[0]:
                       <li><a class="locking_del" href="${h.route_path('repo_settings_quick_actions',repo_name=c.repo_name, _query={'action': 'toggle-lock', 'set_unlock': 1})}">${_('Unlock Repository')}</a></li>
                     %else:
                       <li><a class="locking_add" href="${h.route_path('repo_settings_quick_actions',repo_name=c.repo_name, _query={'action': 'toggle-lock', 'set_lock': 1})}">${_('Lock Repository')}</a></li>
                     %endif
                 %endif
             </ul>
           % endif
         </li>
 
       </ul>
     </div>
     <div class="clear"></div>
   </div>
 
   <!--- REPO END CONTEXT BAR -->
 
 </%def>
 
 <%def name="repo_group_page_title(repo_group_instance)">
 <div class="title-content">
     <div class="title-main">
         ## Repository Group icon
         <i class="icon-repo-group"></i>
 
         ## repo name with group name
         ${h.breadcrumb_repo_group_link(repo_group_instance)}
     </div>
 
     <%namespace name="dt" file="/data_table/_dt_elements.mako"/>
     <div class="repo-group-desc discreet">
     ${dt.repo_group_desc(repo_group_instance.description_safe, repo_group_instance.personal, c.visual.stylify_metatags)}
     </div>
 
 </div>
 </%def>
 
 
 <%def name="repo_group_menu(active=None)">
     <%
     gr_name = c.repo_group.group_name if c.repo_group else None
     # create repositories with write permission on group is set to true
     group_admin = h.HasRepoGroupPermissionAny('group.admin')(gr_name, 'group admin index page')
 
     %>
 
 
   <!--- REPO GROUP CONTEXT BAR -->
   <div id="context-bar">
     <div class="wrapper">
       <div class="title">
           ${self.repo_group_page_title(c.repo_group)}
       </div>
 
       <ul id="context-pages" class="navigation horizontal-list">
         <li class="${h.is_active('home', active)}">
             <a class="menulink" href="${h.route_path('repo_group_home', repo_group_name=c.repo_group.group_name)}"><div class="menulabel">${_('Group Home')}</div></a>
         </li>
         % if c.is_super_admin or group_admin:
             <li class="${h.is_active('settings', active)}">
                 <a class="menulink" href="${h.route_path('edit_repo_group',repo_group_name=c.repo_group.group_name)}" title="${_('You have admin right to this group, and can edit it')}"><div class="menulabel">${_('Group Settings')}</div></a>
             </li>
         % endif
 
       </ul>
     </div>
     <div class="clear"></div>
   </div>
 
   <!--- REPO GROUP CONTEXT BAR -->
 
 </%def>
 
 
 <%def name="usermenu(active=False)">
     <%
     not_anonymous = c.rhodecode_user.username != h.DEFAULT_USER
 
     gr_name = c.repo_group.group_name if (hasattr(c, 'repo_group') and c.repo_group) else None
     # create repositories with write permission on group is set to true
 
     can_fork = c.is_super_admin or h.HasPermissionAny('hg.fork.repository')()
     create_on_write = h.HasPermissionAny('hg.create.write_on_repogroup.true')()
     group_write = h.HasRepoGroupPermissionAny('group.write')(gr_name, 'can write into group index page')
     group_admin = h.HasRepoGroupPermissionAny('group.admin')(gr_name, 'group admin index page')
 
     can_create_repos = c.is_super_admin or c.can_create_repo
     can_create_repo_groups = c.is_super_admin or c.can_create_repo_group
 
     can_create_repos_in_group = c.is_super_admin or group_admin or (group_write and create_on_write)
     can_create_repo_groups_in_group = c.is_super_admin or group_admin
     %>
 
     % if not_anonymous:
     <%
     default_target_group = dict()
     if c.rhodecode_user.personal_repo_group:
         default_target_group = dict(parent_group=c.rhodecode_user.personal_repo_group.group_id)
     %>
 
     ## create action
     <li>
        <a href="#create-actions" onclick="return false;" class="menulink childs">
         <i class="icon-plus-circled"></i>
        </a>
 
        <div class="action-menu submenu">
 
         <ol>
             ## scope of within a repository
             % if hasattr(c, 'rhodecode_db_repo') and c.rhodecode_db_repo:
                 <li class="submenu-title">${_('This Repository')}</li>
                 <li>
                     <a href="${h.route_path('pullrequest_new',repo_name=c.repo_name)}">${_('Create Pull Request')}</a>
                 </li>
                 % if can_fork:
                 <li>
                     <a href="${h.route_path('repo_fork_new',repo_name=c.repo_name,_query=default_target_group)}">${_('Fork this repository')}</a>
                 </li>
                 % endif
             % endif
 
             ## scope of within repository groups
             % if hasattr(c, 'repo_group') and c.repo_group and (can_create_repos_in_group or can_create_repo_groups_in_group):
                 <li class="submenu-title">${_('This Repository Group')}</li>
 
                 % if can_create_repos_in_group:
                     <li>
                     <a href="${h.route_path('repo_new',_query=dict(parent_group=c.repo_group.group_id))}">${_('New Repository')}</a>
                     </li>
                 % endif
 
                 % if can_create_repo_groups_in_group:
                     <li>
                         <a href="${h.route_path('repo_group_new',_query=dict(parent_group=c.repo_group.group_id))}">${_('New Repository Group')}</a>
                     </li>
                 % endif
             % endif
 
             ## personal group
             % if c.rhodecode_user.personal_repo_group:
                 <li class="submenu-title">Personal Group</li>
 
                 <li>
                 <a href="${h.route_path('repo_new',_query=dict(parent_group=c.rhodecode_user.personal_repo_group.group_id))}" >${_('New Repository')} </a>
                 </li>
 
                 <li>
                 <a href="${h.route_path('repo_group_new',_query=dict(parent_group=c.rhodecode_user.personal_repo_group.group_id))}">${_('New Repository Group')} </a>
                 </li>
             % endif
 
             ## Global actions
             <li class="submenu-title">RhodeCode</li>
             % if can_create_repos:
                 <li>
                 <a href="${h.route_path('repo_new')}" >${_('New Repository')}</a>
                 </li>
             % endif
 
             % if can_create_repo_groups:
                 <li>
                 <a href="${h.route_path('repo_group_new')}" >${_('New Repository Group')}</a>
                 </li>
             % endif
 
             <li>
                 <a href="${h.route_path('gists_new')}">${_('New Gist')}</a>
             </li>
 
         </ol>
 
        </div>
     </li>
 
     ## notifications
     <li>
        <a class="${('empty' if c.unread_notifications == 0 else '')}" href="${h.route_path('notifications_show_all')}">
            ${c.unread_notifications}
        </a>
     </li>
    % endif
 
     ## USER MENU
     <li id="quick_login_li" class="${'active' if active else ''}">
         % if c.rhodecode_user.username == h.DEFAULT_USER:
           <a id="quick_login_link" class="menulink childs" href="${h.route_path('login', _query={'came_from': h.current_route_path(request)})}">
             ${gravatar(c.rhodecode_user.email, 20)}
             <span class="user">
                 <span>${_('Sign in')}</span>
             </span>
           </a>
         % else:
           ## logged in user
           <a id="quick_login_link" class="menulink childs">
             ${gravatar(c.rhodecode_user.email, 20)}
             <span class="user">
                 <span class="menu_link_user">${c.rhodecode_user.username}</span>
                 <div class="show_more"></div>
             </span>
           </a>
           ## subnav with menu for logged in user
           <div class="user-menu submenu">
               <div id="quick_login">
                 %if c.rhodecode_user.username != h.DEFAULT_USER:
                     <div class="">
                         <div class="big_gravatar">${gravatar(c.rhodecode_user.email, 48)}</div>
                         <div class="full_name">${c.rhodecode_user.full_name_or_username}</div>
                         <div class="email">${c.rhodecode_user.email}</div>
                     </div>
                     <div class="">
                     <ol class="links">
                       <li>${h.link_to(_(u'My account'),h.route_path('my_account_profile'))}</li>
                       % if c.rhodecode_user.personal_repo_group:
                       <li>${h.link_to(_(u'My personal group'), h.route_path('repo_group_home', repo_group_name=c.rhodecode_user.personal_repo_group.group_name))}</li>
                       % endif
                       <li>${h.link_to(_(u'Pull Requests'), h.route_path('my_account_pullrequests'))}</li>
 
                       % if c.debug_style:
                       <li>
                           <a class="menulink" title="${_('Style')}" href="${h.route_path('debug_style_home')}">
                             <div class="menulabel">${_('[Style]')}</div>
                           </a>
                       </li>
                       % endif
 
                       ## bookmark-items
                       <li class="bookmark-items">
                           ${_('Bookmarks')}
                           <div class="pull-right">
                               <a href="${h.route_path('my_account_bookmarks')}">
 
                                   <i class="icon-cog"></i>
                               </a>
                           </div>
                       </li>
                       % if not c.bookmark_items:
                           <li>
                               <a href="${h.route_path('my_account_bookmarks')}">${_('No Bookmarks yet.')}</a>
                           </li>
                       % endif
                       % for item in c.bookmark_items:
                       <li>
                           % if item.repo_id:
                               <div>
                                 <a class="bookmark-item" href="${h.route_path('my_account_goto_bookmark', bookmark_id=item.position)}">
                                 <code>${item.position}</code>
                                 % if item.repo_type == 'hg':
                                     <i class="icon-hg" title="${_('Repository')}" style="font-size: 16px"></i>
                                 % elif item.repo_type == 'git':
                                     <i class="icon-git" title="${_('Repository')}" style="font-size: 16px"></i>
                                 % elif item.repo_type == 'svn':
                                     <i class="icon-svn" title="${_('Repository')}" style="font-size: 16px"></i>
                                 % endif
                                 ${(item.title or h.shorter(item.repo_name, 30))}
                               </a>
                               </div>
                           % elif item.group_id:
                               <div>
                                 <a class="bookmark-item" href="${h.route_path('my_account_goto_bookmark', bookmark_id=item.position)}">
                                 <code>${item.position}</code>
                                 <i class="icon-repo-group" title="${_('Repository group')}" style="font-size: 14px"></i>
                                 ${(item.title or h.shorter(item.group_name, 30))}
                               </a>
                               </div>
                           % else:
                               <a class="bookmark-item" href="${h.route_path('my_account_goto_bookmark', bookmark_id=item.position)}">
                                 <code>${item.position}</code>
                                 ${item.title}
                               </a>
                           % endif
                       </li>
                       % endfor
 
                       <li class="logout">
                       ${h.secure_form(h.route_path('logout'), request=request)}
                           ${h.submit('log_out', _(u'Sign Out'),class_="btn btn-primary")}
                       ${h.end_form()}
                       </li>
                     </ol>
                     </div>
                 %endif
               </div>
           </div>
 
         % endif
     </li>
 </%def>
 
 <%def name="menu_items(active=None)">
     <%
         notice_messages, notice_level = c.rhodecode_user.get_notice_messages()
         notice_display = 'none' if len(notice_messages) == 0 else ''
     %>
 
     <ul id="quick" class="main_nav navigation horizontal-list">
        ## notice box for important system messages
        <li style="display: ${notice_display}">
           <a class="notice-box" href="#openNotice" onclick="$('.notice-messages-container').toggle(); return false">
             <div class="menulabel-notice ${notice_level}" >
                 ${len(notice_messages)}
             </div>
           </a>
        </li>
         <div class="notice-messages-container" style="display: none">
         <div class="notice-messages">
             <table class="rctable">
             % for notice in notice_messages:
                 <tr id="notice-message-${notice['msg_id']}" class="notice-message-${notice['level']}">
                     <td style="vertical-align: text-top; width: 20px">
                         <i class="tooltip icon-info notice-color-${notice['level']}" title="${notice['level']}"></i>
                     </td>
                     <td>
                         <span><i class="icon-plus-squared cursor-pointer" onclick="$('#notice-${notice['msg_id']}').toggle()"></i> </span>
                         ${notice['subject']}
 
                         <div id="notice-${notice['msg_id']}" style="display: none">
                             ${h.render(notice['body'], renderer='markdown')}
                         </div>
                     </td>
                     <td style="vertical-align: text-top; width: 35px;">
                         <a class="tooltip" title="${_('dismiss')}" href="#dismiss" onclick="dismissNotice(${notice['msg_id']});return false">
                             <i class="icon-remove icon-filled-red"></i>
                         </a>
                     </td>
                 </tr>
 
             % endfor
             </table>
         </div>
         </div>
         ## Main filter
        <li>
         <div class="menulabel main_filter_box">
             <div class="main_filter_input_box">
                 <ul class="searchItems">
 
                     <li class="searchTag searchTagIcon">
                         <i class="icon-search"></i>
                     </li>
 
                     % if c.template_context['search_context']['repo_id']:
                         <li class="searchTag searchTagFilter searchTagHidable" >
                         ##<a href="${h.route_path('search_repo',repo_name=c.template_context['search_context']['repo_name'])}">
                             <span class="tag">
                                 This repo
                                 <a href="#removeGoToFilter" onclick="removeGoToFilter(); return false"><i class="icon-cancel-circled"></i></a>
                             </span>
                         ##</a>
                         </li>
                     % elif c.template_context['search_context']['repo_group_id']:
                         <li class="searchTag searchTagFilter searchTagHidable">
                         ##<a href="${h.route_path('search_repo_group',repo_group_name=c.template_context['search_context']['repo_group_name'])}">
                             <span class="tag">
                                 This group
                                 <a href="#removeGoToFilter" onclick="removeGoToFilter(); return false"><i class="icon-cancel-circled"></i></a>
                             </span>
                         ##</a>
                         </li>
                     % endif
 
                     <li class="searchTagInput">
                         <input class="main_filter_input" id="main_filter" size="25" type="text" name="main_filter" placeholder="${_('search / go to...')}" value="" />
                     </li>
                     <li class="searchTag searchTagHelp">
                         <a href="#showFilterHelp" onclick="showMainFilterBox(); return false">?</a>
                     </li>
                 </ul>
             </div>
         </div>
 
         <div id="main_filter_help" style="display: none">
 - Use '/' key to quickly access this field.
 
 - Enter a name of repository, or repository group for quick search.
 
 - Prefix query to allow special search:
 
    <strong>user:</strong>admin, to search for usernames, always global
 
    <strong>user_group:</strong>devops, to search for user groups, always global
 
    <strong>pr:</strong>303, to search for pull request number, title, or description, always global
 
    <strong>commit:</strong>efced4, to search for commits, scoped to repositories or groups
 
    <strong>file:</strong>models.py, to search for file paths, scoped to repositories or groups
 
 % if c.template_context['search_context']['repo_id']:
    For advanced full text search visit: <a href="${h.route_path('search_repo',repo_name=c.template_context['search_context']['repo_name'])}">repository search</a>
 % elif c.template_context['search_context']['repo_group_id']:
    For advanced full text search visit: <a href="${h.route_path('search_repo_group',repo_group_name=c.template_context['search_context']['repo_group_name'])}">repository group search</a>
 % else:
    For advanced full text search visit: <a href="${h.route_path('search')}">global search</a>
 % endif
         </div>
        </li>
 
       ## ROOT MENU
         <li class="${h.is_active('home', active)}">
           <a class="menulink" title="${_('Home')}" href="${h.route_path('home')}">
             <div class="menulabel">${_('Home')}</div>
           </a>
         </li>
 
       %if c.rhodecode_user.username != h.DEFAULT_USER:
         <li class="${h.is_active('journal', active)}">
           <a class="menulink" title="${_('Show activity journal')}" href="${h.route_path('journal')}">
             <div class="menulabel">${_('Journal')}</div>
           </a>
         </li>
       %else:
         <li class="${h.is_active('journal', active)}">
           <a class="menulink" title="${_('Show Public activity journal')}" href="${h.route_path('journal_public')}">
             <div class="menulabel">${_('Public journal')}</div>
           </a>
         </li>
       %endif
 
         <li class="${h.is_active('gists', active)}">
           <a class="menulink childs" title="${_('Show Gists')}" href="${h.route_path('gists_show')}">
             <div class="menulabel">${_('Gists')}</div>
           </a>
         </li>
 
         % if c.is_super_admin or c.is_delegated_admin:
         <li class="${h.is_active('admin', active)}">
           <a class="menulink childs" title="${_('Admin settings')}" href="${h.route_path('admin_home')}">
             <div class="menulabel">${_('Admin')} </div>
           </a>
         </li>
         % endif
 
       ## render extra user menu
       ${usermenu(active=(active=='my_account'))}
 
     </ul>
 
     <script type="text/javascript">
         var visualShowPublicIcon = "${c.visual.show_public_icon}" == "True";
 
         var formatRepoResult = function(result, container, query, escapeMarkup) {
             return function(data, escapeMarkup) {
                 if (!data.repo_id){
                   return data.text; // optgroup text Repositories
                 }
 
                 var tmpl = '';
                 var repoType = data['repo_type'];
                 var repoName = data['text'];
 
                 if(data && data.type == 'repo'){
                     if(repoType === 'hg'){
                         tmpl += '<i class="icon-hg"></i> ';
                     }
                     else if(repoType === 'git'){
                         tmpl += '<i class="icon-git"></i> ';
                     }
                     else if(repoType === 'svn'){
                         tmpl += '<i class="icon-svn"></i> ';
                     }
                     if(data['private']){
                         tmpl += '<i class="icon-lock" ></i> ';
                     }
                     else if(visualShowPublicIcon){
                         tmpl += '<i class="icon-unlock-alt"></i> ';
                     }
                 }
                 tmpl += escapeMarkup(repoName);
                 return tmpl;
 
             }(result, escapeMarkup);
         };
 
         var formatRepoGroupResult = function(result, container, query, escapeMarkup) {
             return function(data, escapeMarkup) {
                 if (!data.repo_group_id){
                   return data.text; // optgroup text Repositories
                 }
 
                 var tmpl = '';
                 var repoGroupName = data['text'];
 
                 if(data){
 
                     tmpl += '<i class="icon-repo-group"></i> ';
 
                 }
                 tmpl += escapeMarkup(repoGroupName);
                 return tmpl;
 
             }(result, escapeMarkup);
         };
 
         var escapeRegExChars = function (value) {
             return value.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&");
         };
 
         var getRepoIcon = function(repo_type) {
             if (repo_type === 'hg') {
                 return '<i class="icon-hg"></i> ';
             }
             else if (repo_type === 'git') {
                 return '<i class="icon-git"></i> ';
             }
             else if (repo_type === 'svn') {
                 return '<i class="icon-svn"></i> ';
             }
             return ''
         };
 
         var autocompleteMainFilterFormatResult = function (data, value, org_formatter) {
 
             if (value.split(':').length === 2) {
                 value = value.split(':')[1]
             }
 
             var searchType = data['type'];
             var searchSubType = data['subtype'];
             var valueDisplay = data['value_display'];
             var valueIcon = data['value_icon'];
 
             var pattern = '(' + escapeRegExChars(value) + ')';
 
             valueDisplay = Select2.util.escapeMarkup(valueDisplay);
 
             // highlight match
             if (searchType != 'text') {
                 valueDisplay = valueDisplay.replace(new RegExp(pattern, 'gi'), '<strong>$1<\/strong>');
             }
 
             var icon = '';
 
             if (searchType === 'hint') {
                 icon += '<i class="icon-repo-group"></i> ';
             }
             // full text search/hints
             else if (searchType === 'search') {
                 if (valueIcon === undefined) {
                     icon += '<i class="icon-more"></i> ';
                 } else {
                     icon += valueIcon + ' ';
                 }
 
                 if (searchSubType !== undefined && searchSubType == 'repo') {
                     valueDisplay += '<div class="pull-right tag">repository</div>';
                 }
                 else if (searchSubType !== undefined && searchSubType == 'repo_group') {
                     valueDisplay += '<div class="pull-right tag">repo group</div>';
                 }
             }
             // repository
             else if (searchType === 'repo') {
 
                 var repoIcon = getRepoIcon(data['repo_type']);
                 icon += repoIcon;
 
                 if (data['private']) {
                     icon += '<i class="icon-lock" ></i> ';
                 }
                 else if (visualShowPublicIcon) {
                     icon += '<i class="icon-unlock-alt"></i> ';
                 }
             }
             // repository groups
             else if (searchType === 'repo_group') {
                 icon += '<i class="icon-repo-group"></i> ';
             }
             // user group
             else if (searchType === 'user_group') {
                 icon += '<i class="icon-group"></i> ';
             }
             // user
             else if (searchType === 'user') {
                 icon += '<img class="gravatar" src="{0}"/>'.format(data['icon_link']);
             }
             // pull request
             else if (searchType === 'pull_request') {
                 icon += '<i class="icon-merge"></i> ';
             }
             // commit
             else if (searchType === 'commit') {
                 var repo_data = data['repo_data'];
                 var repoIcon = getRepoIcon(repo_data['repository_type']);
                 if (repoIcon) {
                     icon += repoIcon;
                 } else {
                     icon += '<i class="icon-tag"></i>';
                 }
             }
             // file
             else if (searchType === 'file') {
                 var repo_data = data['repo_data'];
                 var repoIcon = getRepoIcon(repo_data['repository_type']);
                 if (repoIcon) {
                     icon += repoIcon;
                 } else {
                     icon += '<i class="icon-tag"></i>';
                 }
             }
             // generic text
             else if (searchType === 'text') {
                 icon = '';
             }
 
             var tmpl = '<div class="ac-container-wrap">{0}{1}</div>';
             return tmpl.format(icon, valueDisplay);
         };
 
         var handleSelect = function(element, suggestion) {
             if (suggestion.type === "hint") {
                 // we skip action
                 $('#main_filter').focus();
             }
             else if (suggestion.type === "text") {
                 // we skip action
                 $('#main_filter').focus();
 
             } else {
               window.location = suggestion['url'];
             }
         };
 
         var autocompleteMainFilterResult = function (suggestion, originalQuery, queryLowerCase) {
             if (queryLowerCase.split(':').length === 2) {
                 queryLowerCase = queryLowerCase.split(':')[1]
             }
             if (suggestion.type === "text") {
                 // special case we don't want to "skip" display for
                 return true
             }
             return suggestion.value_display.toLowerCase().indexOf(queryLowerCase) !== -1;
         };
 
         var cleanContext = {
             repo_view_type: null,
 
             repo_id: null,
             repo_name: "",
 
             repo_group_id: null,
             repo_group_name: null
         };
         var removeGoToFilter = function () {
             $('.searchTagHidable').hide();
             $('#main_filter').autocomplete(
                 'setOptions', {params:{search_context: cleanContext}});
         };
 
         $('#main_filter').autocomplete({
             serviceUrl: pyroutes.url('goto_switcher_data'),
             params: {
                 "search_context": templateContext.search_context
             },
             minChars:2,
             maxHeight:400,
             deferRequestBy: 300, //miliseconds
             tabDisabled: true,
             autoSelectFirst: false,
             containerClass: 'autocomplete-qfilter-suggestions',
             formatResult: autocompleteMainFilterFormatResult,
             lookupFilter: autocompleteMainFilterResult,
             onSelect: function (element, suggestion) {
                 handleSelect(element, suggestion);
                 return false;
             },
             onSearchError: function (element, query, jqXHR, textStatus, errorThrown) {
                 if (jqXHR !== 'abort') {
                     var message = formatErrorMessage(jqXHR, textStatus, errorThrown);
                     SwalNoAnimation.fire({
                         icon: 'error',
                         title: _gettext('Error during search operation'),
                         html: '<span style="white-space: pre-line">{0}</span>'.format(message),
                     }).then(function(result) {
                         window.location.reload();
                     })
                 }
             },
             onSearchStart: function (params) {
                 $('.searchTag.searchTagIcon').html('<i class="icon-spin animate-spin"></i>')
             },
             onSearchComplete: function (query, suggestions) {
                 $('.searchTag.searchTagIcon').html('<i class="icon-search"></i>')
             },
         });
 
         showMainFilterBox = function () {
             $('#main_filter_help').toggle();
         };
 
         $('#main_filter').on('keydown.autocomplete', function (e) {
 
             var BACKSPACE = 8;
             var el = $(e.currentTarget);
             if(e.which === BACKSPACE){
                 var inputVal = el.val();
                 if (inputVal === ""){
                     removeGoToFilter()
                 }
             }
         });
 
         var dismissNotice = function(noticeId) {
 
             var url = pyroutes.url('user_notice_dismiss',
                     {"user_id": templateContext.rhodecode_user.user_id});
 
             var postData = {
                 'csrf_token': CSRF_TOKEN,
                 'notice_id': noticeId,
             };
 
             var success = function(response) {
                 $('#notice-message-' + noticeId).remove();
                 return false;
             };
             var failure = function(data, textStatus, xhr) {
                 alert("error processing request: " + textStatus);
                 return false;
             };
             ajaxPOST(url, postData, success, failure);
         }
 
         var hideLicenseWarning = function () {
             var fingerprint = templateContext.session_attrs.license_fingerprint;
             storeUserSessionAttr('rc_user_session_attr.hide_license_warning', fingerprint);
             $('#notifications').hide();
         }
 
         var hideLicenseError = function () {
             var fingerprint = templateContext.session_attrs.license_fingerprint;
             storeUserSessionAttr('rc_user_session_attr.hide_license_error', fingerprint);
             $('#notifications').hide();
         }
 
     </script>
     <script src="${h.asset('js/rhodecode/base/keyboard-bindings.js', ver=c.rhodecode_version_hash)}"></script>
 </%def>
 
 <div class="modal" id="help_kb" tabindex="-1" role="dialog" aria-labelledby="myModalLabel" aria-hidden="true">
     <div class="modal-dialog">
       <div class="modal-content">
         <div class="modal-header">
           <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
           <h4 class="modal-title" id="myModalLabel">${_('Keyboard shortcuts')}</h4>
         </div>
         <div class="modal-body">
               <div class="block-left">
                 <table class="keyboard-mappings">
                     <tbody>
                   <tr>
                     <th></th>
                     <th>${_('Site-wide shortcuts')}</th>
                   </tr>
                   <%
                      elems = [
                          ('/', 'Use quick search box'),
                          ('g h', 'Goto home page'),
                          ('g g', 'Goto my private gists page'),
                          ('g G', 'Goto my public gists page'),
                          ('g 0-9', 'Goto bookmarked items from 0-9'),
                          ('n r', 'New repository page'),
                          ('n g', 'New gist page'),
                      ]
                   %>
                   %for key, desc in elems:
                   <tr>
                     <td class="keys">
                       <span class="key tag">${key}</span>
                     </td>
                     <td>${desc}</td>
                   </tr>
                 %endfor
                 </tbody>
                   </table>
               </div>
               <div class="block-left">
                 <table class="keyboard-mappings">
                 <tbody>
                   <tr>
                     <th></th>
                     <th>${_('Repositories')}</th>
                   </tr>
                   <%
                      elems = [
                          ('g s', 'Goto summary page'),
                          ('g c', 'Goto changelog page'),
                          ('g f', 'Goto files page'),
                          ('g F', 'Goto files page with file search activated'),
                          ('g p', 'Goto pull requests page'),
                          ('g o', 'Goto repository settings'),
                          ('g O', 'Goto repository access permissions settings'),
                          ('t s', 'Toggle sidebar on some pages'),
                      ]
                   %>
                   %for key, desc in elems:
                   <tr>
                     <td class="keys">
                       <span class="key tag">${key}</span>
                     </td>
                     <td>${desc}</td>
                   </tr>
                 %endfor
                 </tbody>
               </table>
             </div>
         </div>
         <div class="modal-footer">
         </div>
       </div><!-- /.modal-content -->
     </div><!-- /.modal-dialog -->
 </div><!-- /.modal -->
 
 
 <script type="text/javascript">
     (function () {
         "use sctrict";
 
         // details block auto-hide menu
         $(document).mouseup(function(e) {
             var container = $('.details-inline-block');
             if (!container.is(e.target) && container.has(e.target).length === 0) {
                 $('.details-inline-block[open]').removeAttr('open')
             }
         });
 
         var $sideBar = $('.right-sidebar');
         var expanded = $sideBar.hasClass('right-sidebar-expanded');
         var sidebarState = templateContext.session_attrs.sidebarState;
         var sidebarEnabled = $('aside.right-sidebar').get(0);
 
         if (sidebarState === 'expanded') {
             expanded = true
         } else if (sidebarState === 'collapsed') {
             expanded = false
         }
         if (sidebarEnabled) {
             // show sidebar since it's hidden on load
             $('.right-sidebar').show();
 
             // init based on set initial class, or if defined user session attrs
             if (expanded) {
                 window.expandSidebar();
                 window.updateStickyHeader();
 
             } else {
                 window.collapseSidebar();
                 window.updateStickyHeader();
             }
         }
     })()
 
 </script>

rhodecode/templates/base/vcs_settings.mako

@@ -1,345 +1,308 b''
 ## snippet for displaying vcs settings
 ## usage:
 ##    <%namespace name="vcss" file="/base/vcssettings.mako"/>
 ##    ${vcss.vcs_settings_fields()}
 
 <%def name="vcs_settings_fields(suffix='', svn_branch_patterns=None, svn_tag_patterns=None, repo_type=None, display_globals=False, **kwargs)">
     % if display_globals:
-        <div class="panel panel-default">
-            <div class="panel-heading" id="general">
-                <h3 class="panel-title">${_('General')}<a class="permalink" href="#general"> ¶</a></h3>
-            </div>
-            <div class="panel-body">
-                <div class="field">
-                    <div class="checkbox">
-                        ${h.checkbox('web_push_ssl' + suffix, 'True')}
-                        <label for="web_push_ssl${suffix}">${_('Require SSL for vcs operations')}</label>
-                    </div>
-                    <div class="label">
-                        <span class="help-block">${_('Activate to set RhodeCode to require SSL for pushing or pulling. If SSL certificate is missing it will return a HTTP Error 406: Not Acceptable.')}</span>
-                    </div>
-                 </div>
-            </div>
-        </div>
+
     % endif
 
     % if display_globals or repo_type in ['git', 'hg']:
         <div class="panel panel-default">
             <div class="panel-heading" id="vcs-hooks-options">
                 <h3 class="panel-title">${_('Internal Hooks')}<a class="permalink" href="#vcs-hooks-options"> ¶</a></h3>
             </div>
             <div class="panel-body">
                 <div class="field">
                     <div class="checkbox">
                         ${h.checkbox('hooks_changegroup_repo_size' + suffix, 'True', **kwargs)}
                         <label for="hooks_changegroup_repo_size${suffix}">${_('Show repository size after push')}</label>
                     </div>
 
                     <div class="label">
                         <span class="help-block">${_('Trigger a hook that calculates repository size after each push.')}</span>
                     </div>
                     <div class="checkbox">
                         ${h.checkbox('hooks_changegroup_push_logger' + suffix, 'True', **kwargs)}
                         <label for="hooks_changegroup_push_logger${suffix}">${_('Execute pre/post push hooks')}</label>
                     </div>
                     <div class="label">
                         <span class="help-block">${_('Execute Built in pre/post push hooks. This also executes rcextensions hooks.')}</span>
                     </div>
                     <div class="checkbox">
                         ${h.checkbox('hooks_outgoing_pull_logger' + suffix, 'True', **kwargs)}
                         <label for="hooks_outgoing_pull_logger${suffix}">${_('Execute pre/post pull hooks')}</label>
                     </div>
                     <div class="label">
                         <span class="help-block">${_('Execute Built in pre/post pull hooks. This also executes rcextensions hooks.')}</span>
                     </div>
                 </div>
             </div>
         </div>
     % endif
 
     % if display_globals or repo_type in ['hg']:
         <div class="panel panel-default">
             <div class="panel-heading" id="vcs-hg-options">
                 <h3 class="panel-title">${_('Mercurial Settings')}<a class="permalink" href="#vcs-hg-options"> ¶</a></h3>
             </div>
             <div class="panel-body">
                 <div class="checkbox">
                     ${h.checkbox('extensions_largefiles' + suffix, 'True', **kwargs)}
                     <label for="extensions_largefiles${suffix}">${_('Enable largefiles extension')}</label>
                 </div>
                 <div class="label">
                     % if display_globals:
                         <span class="help-block">${_('Enable Largefiles extensions for all repositories.')}</span>
                     % else:
                         <span class="help-block">${_('Enable Largefiles extensions for this repository.')}</span>
                     % endif
                 </div>
 
-                % if display_globals:
-                <div class="field">
-                    <div class="input">
-                        ${h.text('largefiles_usercache' + suffix, size=59)}
-                    </div>
-                </div>
-                <div class="label">
-                    <span class="help-block">${_('Filesystem location where Mercurial largefile objects should be stored.')}</span>
-                </div>
-                % endif
-
                 <div class="checkbox">
                     ${h.checkbox('phases_publish' + suffix, 'True', **kwargs)}
                     <label for="phases_publish${suffix}">${_('Set repositories as publishing') if display_globals else _('Set repository as publishing')}</label>
                 </div>
                 <div class="label">
                     <span class="help-block">${_('When this is enabled all commits in the repository are seen as public commits by clients.')}</span>
                 </div>
 
                 <div class="checkbox">
                     ${h.checkbox('extensions_evolve' + suffix, 'True', **kwargs)}
                     <label for="extensions_evolve${suffix}">${_('Enable Evolve and Topic extension')}</label>
                 </div>
                 <div class="label">
                     % if display_globals:
                         <span class="help-block">${_('Enable Evolve and Topic extensions for all repositories.')}</span>
                     % else:
                         <span class="help-block">${_('Enable Evolve and Topic extensions for this repository.')}</span>
                     % endif
                 </div>
 
             </div>
         </div>
     % endif
 
     % if display_globals or repo_type in ['git']:
         <div class="panel panel-default">
             <div class="panel-heading" id="vcs-git-options">
                 <h3 class="panel-title">${_('Git Settings')}<a class="permalink" href="#vcs-git-options"> ¶</a></h3>
             </div>
             <div class="panel-body">
                 <div class="checkbox">
                     ${h.checkbox('vcs_git_lfs_enabled' + suffix, 'True', **kwargs)}
                     <label for="vcs_git_lfs_enabled${suffix}">${_('Enable lfs extension')}</label>
                 </div>
                 <div class="label">
                     % if display_globals:
                         <span class="help-block">${_('Enable lfs extensions for all repositories.')}</span>
                     % else:
                         <span class="help-block">${_('Enable lfs extensions for this repository.')}</span>
                     % endif
                 </div>
-
-                % if display_globals:
-                <div class="field">
-                    <div class="input">
-                        ${h.text('vcs_git_lfs_store_location' + suffix, size=59)}
-                    </div>
-                </div>
-                <div class="label">
-                    <span class="help-block">${_('Filesystem location where Git lfs objects should be stored.')}</span>
-                </div>
-                % endif
             </div>
         </div>
     % endif
 
     % if display_globals or repo_type in ['svn']:
         <div class="panel panel-default">
             <div class="panel-heading" id="vcs-svn-options">
                 <h3 class="panel-title">${_('Subversion Settings')}<a class="permalink" href="#vcs-svn-options"> ¶</a></h3>
             </div>
             <div class="panel-body">
                 % if display_globals:
                 <div class="field">
                     <div class="content" >
                         <label>${_('mod_dav config')}</label><br/>
                         <code>path: ${c.svn_config_path}</code>
                     </div>
                     <br/>
 
                     <div>
 
                     % if c.svn_generate_config:
                         <span class="buttons">
                             <button class="btn btn-primary" id="vcs_svn_generate_cfg">${_('Re-generate Apache Config')}</button>
                         </span>
                     % endif
                     </div>
                 </div>
                 % endif
 
                 <div class="field">
                     <div class="content" >
                         <label>${_('Repository patterns')}</label><br/>
                     </div>
                 </div>
                 <div class="label">
                     <span class="help-block">${_('Patterns for identifying SVN branches and tags. For recursive search, use "*". Eg.: "/branches/*"')}</span>
                 </div>
 
                 <div class="field branch_patterns">
                     <div class="input" >
                         <label>${_('Branches')}:</label><br/>
                     </div>
                     % if svn_branch_patterns:
                         % for branch in svn_branch_patterns:
                         <div class="input adjacent"   id="${'id%s' % branch.ui_id}">
                             ${h.hidden('branch_ui_key' + suffix, branch.ui_key)}
                             ${h.text('branch_value_%d' % branch.ui_id + suffix, branch.ui_value, size=59, readonly="readonly", class_='disabled')}
                             % if kwargs.get('disabled') != 'disabled':
                                 <span class="btn btn-x" onclick="ajaxDeletePattern(${branch.ui_id},'${'id%s' % branch.ui_id}')">
                                     ${_('Delete')}
                                 </span>
                             % endif
                         </div>
                         % endfor
                     %endif
                 </div>
                 % if kwargs.get('disabled') != 'disabled':
                     <div class="field branch_patterns">
                         <div class="input" >
                             ${h.text('new_svn_branch',size=59,placeholder='New branch pattern')}
                         </div>
                     </div>
                 % endif
                 <div class="field tag_patterns">
                     <div class="input" >
                         <label>${_('Tags')}:</label><br/>
                     </div>
                     % if svn_tag_patterns:
                         % for tag in svn_tag_patterns:
                         <div class="input"  id="${'id%s' % tag.ui_id + suffix}">
                             ${h.hidden('tag_ui_key' + suffix, tag.ui_key)}
                             ${h.text('tag_ui_value_new_%d' % tag.ui_id + suffix, tag.ui_value, size=59, readonly="readonly", class_='disabled tag_input')}
                             % if kwargs.get('disabled') != 'disabled':
                                 <span class="btn btn-x" onclick="ajaxDeletePattern(${tag.ui_id},'${'id%s' % tag.ui_id}')">
                                     ${_('Delete')}
                                 </span>
                             %endif
                         </div>
                         % endfor
                     % endif
                 </div>
                 % if kwargs.get('disabled') != 'disabled':
                     <div class="field tag_patterns">
                         <div class="input" >
                             ${h.text('new_svn_tag' + suffix, size=59, placeholder='New tag pattern')}
                         </div>
                     </div>
                 %endif
             </div>
         </div>
     % else:
         ${h.hidden('new_svn_branch' + suffix, '')}
         ${h.hidden('new_svn_tag' + suffix, '')}
     % endif
 
 
     % if display_globals or repo_type in ['hg', 'git']:
         <div class="panel panel-default">
             <div class="panel-heading" id="vcs-pull-requests-options">
                 <h3 class="panel-title">${_('Pull Request Settings')}<a class="permalink" href="#vcs-pull-requests-options"> ¶</a></h3>
             </div>
             <div class="panel-body">
                 <div class="checkbox">
                     ${h.checkbox('rhodecode_pr_merge_enabled' + suffix, 'True', **kwargs)}
                     <label for="rhodecode_pr_merge_enabled${suffix}">${_('Enable server-side merge for pull requests')}</label>
                 </div>
                 <div class="label">
                     <span class="help-block">${_('Note: when this feature is enabled, it only runs hooks defined in the rcextension package. Custom hooks added on the Admin -> Settings -> Hooks page will not be run when pull requests are automatically merged from the web interface.')}</span>
                 </div>
                 <div class="checkbox">
                     ${h.checkbox('rhodecode_use_outdated_comments' + suffix, 'True', **kwargs)}
                     <label for="rhodecode_use_outdated_comments${suffix}">${_('Invalidate and relocate inline comments during update')}</label>
                 </div>
                 <div class="label">
                     <span class="help-block">${_('During the update of a pull request, the position of inline comments will be updated and outdated inline comments will be hidden.')}</span>
                 </div>
             </div>
         </div>
     % endif
 
     % if display_globals or repo_type in ['hg', 'git', 'svn']:
         <div class="panel panel-default">
             <div class="panel-heading" id="vcs-pull-requests-options">
                 <h3 class="panel-title">${_('Diff cache')}<a class="permalink" href="#vcs-pull-requests-options"> ¶</a></h3>
             </div>
             <div class="panel-body">
                 <div class="checkbox">
                     ${h.checkbox('rhodecode_diff_cache' + suffix, 'True', **kwargs)}
                     <label for="rhodecode_diff_cache${suffix}">${_('Enable caching diffs for pull requests cache and commits')}</label>
                 </div>
             </div>
         </div>
     % endif
 
     % if display_globals or repo_type in ['hg',]:
         <div class="panel panel-default">
             <div class="panel-heading" id="vcs-pull-requests-options">
                 <h3 class="panel-title">${_('Mercurial Pull Request Settings')}<a class="permalink" href="#vcs-hg-pull-requests-options"> ¶</a></h3>
             </div>
             <div class="panel-body">
                 ## Specific HG settings
                 <div class="checkbox">
                     ${h.checkbox('rhodecode_hg_use_rebase_for_merging' + suffix, 'True', **kwargs)}
                     <label for="rhodecode_hg_use_rebase_for_merging${suffix}">${_('Use rebase as merge strategy')}</label>
                 </div>
                 <div class="label">
                     <span class="help-block">${_('Use rebase instead of creating a merge commit when merging via web interface.')}</span>
                 </div>
 
                 <div class="checkbox">
                     ${h.checkbox('rhodecode_hg_close_branch_before_merging' + suffix, 'True', **kwargs)}
                     <label for="rhodecode_hg_close_branch_before_merging{suffix}">${_('Close branch before merging it')}</label>
                 </div>
                 <div class="label">
                     <span class="help-block">${_('Close branch before merging it into destination branch. No effect when rebase strategy is use.')}</span>
                 </div>
 
 
             </div>
         </div>
     % endif
 
 % if display_globals or repo_type in ['git']:
     <div class="panel panel-default">
         <div class="panel-heading" id="vcs-pull-requests-options">
             <h3 class="panel-title">${_('Git Pull Request Settings')}<a class="permalink" href="#vcs-git-pull-requests-options"> ¶</a></h3>
         </div>
         <div class="panel-body">
 ##                <div class="checkbox">
 ##                    ${h.checkbox('rhodecode_git_use_rebase_for_merging' + suffix, 'True', **kwargs)}
 ##                    <label for="rhodecode_git_use_rebase_for_merging${suffix}">${_('Use rebase as merge strategy')}</label>
 ##                </div>
 ##                <div class="label">
 ##                  <span class="help-block">${_('Use rebase instead of creating a merge commit when merging via web interface.')}</span>
 ##               </div>
 
             <div class="checkbox">
                 ${h.checkbox('rhodecode_git_close_branch_before_merging' + suffix, 'True', **kwargs)}
                 <label for="rhodecode_git_close_branch_before_merging{suffix}">${_('Delete branch after merging it')}</label>
             </div>
             <div class="label">
                 <span class="help-block">${_('Delete branch after merging it into destination branch.')}</span>
             </div>
         </div>
     </div>
 % endif
 
 <script type="text/javascript">
 
 $(document).ready(function() {
     /* On click handler for the `Generate Apache Config` button. It sends a
     POST request to trigger the (re)generation of the mod_dav_svn config. */
     $('#vcs_svn_generate_cfg').on('click', function(event) {
         event.preventDefault();
         var url = "${h.route_path('admin_settings_vcs_svn_generate_cfg')}";
         var jqxhr = $.post(url, {'csrf_token': CSRF_TOKEN});
         jqxhr.done(function(data) {
             $.Topic('/notifications').publish(data);
         });
     });
 });
 
 </script>
 </%def>
 

rhodecode/tests/config/test_sanitize_settings.py

@@ -1,205 +1,205 b''
 
 
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import pytest
 
 from rhodecode.tests import no_newline_id_generator
 from rhodecode.config.middleware import sanitize_settings_and_apply_defaults
 from rhodecode.config.settings_maker import SettingsMaker
 
 
 class TestHelperFunctions(object):
     @pytest.mark.parametrize('raw, expected', [
         ('true', True), (u'true', True),
         ('yes', True), (u'yes', True),
         ('on', True), (u'on', True),
         ('false', False), (u'false', False),
         ('no', False), (u'no', False),
         ('off', False), (u'off', False),
         ('invalid-bool-value', False),
         ('invalid-∫øø@-√å@¨€', False),
         (u'invalid-∫øø@-√å@¨€', False),
     ])
     def test_bool_func_helper(self, raw, expected):
         val = SettingsMaker._bool_func(raw)
         assert val == expected
 
     @pytest.mark.parametrize('raw, expected', [
         ('', ''),
         ('test-string', 'test-string'),
         ('CaSe-TeSt', 'case-test'),
         ('test-string-烩€', 'test-string-烩€'),
         (u'test-string-烩€', u'test-string-烩€'),
     ])
     def test_string_func_helper(self, raw, expected):
         val = SettingsMaker._string_func(raw)
         assert val == expected
 
     @pytest.mark.parametrize('raw, expected', [
         ('', []),
         ('test', ['test']),
         ('CaSe-TeSt', ['CaSe-TeSt']),
         ('test-string-烩€', ['test-string-烩€']),
         (u'test-string-烩€', [u'test-string-烩€']),
         ('hg,git,svn', ['hg', 'git', 'svn']),
         ('hg, git, svn', ['hg', 'git', 'svn']),
 
         (', hg , git , svn , ', ['', 'hg', 'git', 'svn', '']),
         ('cheese,free node,other', ['cheese', 'free node', 'other']),
     ], ids=no_newline_id_generator)
     def test_list_setting_helper(self, raw, expected):
         val = SettingsMaker._list_func(raw)
         assert val == expected
 
     @pytest.mark.parametrize('raw, expected', [
         ('hg git svn', ['hg', 'git', 'svn']),
     ], ids=no_newline_id_generator)
     def test_list_setting_spaces_helper(self, raw, expected):
         val = SettingsMaker._list_func(raw, sep=' ')
         assert val == expected
 
     @pytest.mark.parametrize('raw, expected', [
         ('hg\ngit\nsvn', ['hg', 'git', 'svn']),
         (' hg\n git\n svn ', ['hg', 'git', 'svn']),
     ], ids=no_newline_id_generator)
     def test_list_setting_newlines_helper(self, raw, expected):
         val = SettingsMaker._list_func(raw, sep='\n')
         assert val == expected
 
     @pytest.mark.parametrize('raw, expected', [
         ('0', 0),
         ('-0', 0),
         ('12345', 12345),
         ('-12345', -12345),
         (u'-12345', -12345),
     ])
     def test_int_setting_helper(self, raw, expected):
         val = SettingsMaker._int_func(raw)
         assert val == expected
 
     @pytest.mark.parametrize('raw', [
         ('0xff'),
         (''),
         ('invalid-int'),
         ('invalid-⁄~†'),
         (u'invalid-⁄~†'),
     ])
     def test_int_setting_helper_invalid_input(self, raw):
         with pytest.raises(Exception):
             SettingsMaker._int_func(raw)
 
 
 class TestSanitizeVcsSettings(object):
     _bool_funcs = [
         ('vcs.hooks.direct_calls', False),
         ('vcs.server.enable', True),
         ('vcs.start_server', False),
         ('startup.import_repos', False),
     ]
 
     _string_funcs = [
         ('vcs.svn.compatible_version', ''),
-        ('vcs.hooks.protocol', 'http'),
+        ('vcs.hooks.protocol.v2', 'celery'),
         ('vcs.hooks.host', '*'),
         ('vcs.scm_app_implementation', 'http'),
         ('vcs.server', ''),
         ('vcs.server.protocol', 'http'),
     ]
 
     _list_settings = [
         ('vcs.backends', 'hg git'),
     ]
 
     # @pytest.mark.parametrize('key, default', _list_settings)
     # def test_list_setting_spacesep_list(self, key, default):
     #     test_list = ['test', 'list', 'values', 'for', key]
     #     input_value = ' '.join(test_list)
     #     settings = {key: input_value}
     #     sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
     #     assert settings[key] == test_list
     #
     # @pytest.mark.parametrize('key, default', _list_settings)
     # def test_list_setting_newlinesep_list(self, key, default):
     #     test_list = ['test', 'list', 'values', 'for', key]
     #     input_value = '\n'.join(test_list)
     #     settings = {key: input_value}
     #     sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
     #     assert settings[key] == test_list
 
     @pytest.mark.parametrize('key, default', _list_settings)
     def test_list_setting_commasep_list(self, key, default):
         test_list = ['test', 'list', 'values', 'for', key]
         input_value = ','.join(test_list)
         settings = {key: input_value}
         sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] == test_list
 
     @pytest.mark.parametrize('key, default', _list_settings)
     def test_list_setting_comma_and_space_sep_list(self, key, default):
         test_list = ['test', 'list', 'values', 'for', key]
         input_value = ', '.join(test_list)
         settings = {key: input_value}
         sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] == test_list
 
     @pytest.mark.parametrize('key, default', _string_funcs)
     def test_string_func_string(self, key, default):
         test_value = 'test-string-for-{}'.format(key)
         settings = {key: test_value}
         sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] == test_value
 
     @pytest.mark.parametrize('key, default', _string_funcs)
     def test_string_func_default(self, key, default):
         settings = {}
         sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] == default
 
     # @pytest.mark.parametrize('key, default', _string_funcs)
     # def test_string_func_lowercase(self, key, default):
     #     test_value = 'Test-String-For-{}'.format(key)
     #     settings = {key: test_value}
     #     sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
     #     assert settings[key] == test_value.lower()
 
     @pytest.mark.parametrize('key, default', _bool_funcs)
     def test_bool_func_true(self, key, default):
         settings = {key: 'true'}
         sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] is True
 
     @pytest.mark.parametrize('key, default', _bool_funcs)
     def test_bool_func_false(self, key, default):
         settings = {key: 'false'}
         sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] is False
 
     @pytest.mark.parametrize('key, default', _bool_funcs)
     def test_bool_func_invalid_string(self, key, default):
         settings = {key: 'no-bool-val-string'}
         sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] is False
 
     @pytest.mark.parametrize('key, default', _bool_funcs)
     def test_bool_func_default(self, key, default):
         settings = {}
         sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] is default

rhodecode/tests/fixture.py

@@ -1,428 +1,428 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Helpers for fixture generation
 """
 
 import os
 import time
 import tempfile
 import shutil
 import configparser
 
 from rhodecode.model.settings import SettingsModel
 from rhodecode.model.db import Repository, User, RepoGroup, UserGroup, Gist, UserEmailMap
 from rhodecode.model.meta import Session
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.user import UserModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.user_group import UserGroupModel
 from rhodecode.model.gist import GistModel
 from rhodecode.model.auth_token import AuthTokenModel
 from rhodecode.model.scm import ScmModel
 from rhodecode.authentication.plugins.auth_rhodecode import \
     RhodeCodeAuthPlugin
 
 from rhodecode.tests import TEST_USER_ADMIN_LOGIN
 
 dn = os.path.dirname
 FIXTURES = os.path.join(dn(dn(os.path.abspath(__file__))), 'tests', 'fixtures')
 
 
 def error_function(*args, **kwargs):
     raise Exception('Total Crash !')
 
 
 class TestINI(object):
     """
     Allows to create a new test.ini file as a copy of existing one with edited
     data. Example usage::
 
         with TestINI('test.ini', [{'section':{'key':val'}]) as new_test_ini_path:
             print('paster server %s' % new_test_ini)
     """
 
     def __init__(self, ini_file_path, ini_params, new_file_prefix='DEFAULT',
                  destroy=True, dir=None):
         self.ini_file_path = ini_file_path
         self.ini_params = ini_params
         self.new_path = None
         self.new_path_prefix = new_file_prefix
         self._destroy = destroy
         self._dir = dir
 
     def __enter__(self):
         return self.create()
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.destroy()
 
     def create(self):
         parser = configparser.ConfigParser()
         parser.read(self.ini_file_path)
 
         for data in self.ini_params:
             section, ini_params = list(data.items())[0]
 
             for key, val in ini_params.items():
                 parser[section][key] = str(val)
 
         with tempfile.NamedTemporaryFile(
                 mode='w',
                 prefix=self.new_path_prefix, suffix='.ini', dir=self._dir,
                 delete=False) as new_ini_file:
             parser.write(new_ini_file)
             self.new_path = new_ini_file.name
 
         return self.new_path
 
     def destroy(self):
         if self._destroy:
             os.remove(self.new_path)
 
 
 class Fixture(object):
 
     def anon_access(self, status):
         """
         Context process for disabling anonymous access. use like:
         fixture = Fixture()
         with fixture.anon_access(False):
             #tests
 
         after this block anon access will be set to `not status`
         """
 
         class context(object):
             def __enter__(self):
                 anon = User.get_default_user()
                 anon.active = status
                 Session().add(anon)
                 Session().commit()
                 time.sleep(1.5)  # must sleep for cache (1s to expire)
 
             def __exit__(self, exc_type, exc_val, exc_tb):
                 anon = User.get_default_user()
                 anon.active = not status
                 Session().add(anon)
                 Session().commit()
 
         return context()
 
     def auth_restriction(self, registry, auth_restriction):
         """
         Context process for changing the builtin rhodecode plugin auth restrictions.
         Use like:
         fixture = Fixture()
         with fixture.auth_restriction('super_admin'):
             #tests
 
         after this block auth restriction will be taken off
         """
 
         class context(object):
             def _get_plugin(self):
                 plugin_id = 'egg:rhodecode-enterprise-ce#{}'.format(RhodeCodeAuthPlugin.uid)
                 plugin = RhodeCodeAuthPlugin(plugin_id)
                 return plugin
 
             def __enter__(self):
 
                 plugin = self._get_plugin()
                 plugin.create_or_update_setting('auth_restriction', auth_restriction)
                 Session().commit()
                 SettingsModel().invalidate_settings_cache(hard=True)
 
             def __exit__(self, exc_type, exc_val, exc_tb):
 
                 plugin = self._get_plugin()
                 plugin.create_or_update_setting(
                     'auth_restriction', RhodeCodeAuthPlugin.AUTH_RESTRICTION_NONE)
                 Session().commit()
                 SettingsModel().invalidate_settings_cache(hard=True)
 
         return context()
 
     def scope_restriction(self, registry, scope_restriction):
         """
         Context process for changing the builtin rhodecode plugin scope restrictions.
         Use like:
         fixture = Fixture()
         with fixture.scope_restriction('scope_http'):
             #tests
 
         after this block scope restriction will be taken off
         """
 
         class context(object):
             def _get_plugin(self):
                 plugin_id = 'egg:rhodecode-enterprise-ce#{}'.format(RhodeCodeAuthPlugin.uid)
                 plugin = RhodeCodeAuthPlugin(plugin_id)
                 return plugin
 
             def __enter__(self):
                 plugin = self._get_plugin()
                 plugin.create_or_update_setting('scope_restriction', scope_restriction)
                 Session().commit()
                 SettingsModel().invalidate_settings_cache(hard=True)
 
             def __exit__(self, exc_type, exc_val, exc_tb):
                 plugin = self._get_plugin()
                 plugin.create_or_update_setting(
                     'scope_restriction', RhodeCodeAuthPlugin.AUTH_RESTRICTION_SCOPE_ALL)
                 Session().commit()
                 SettingsModel().invalidate_settings_cache(hard=True)
 
         return context()
 
     def _get_repo_create_params(self, **custom):
         repo_type = custom.get('repo_type') or 'hg'
 
         default_landing_ref, landing_ref_lbl = ScmModel.backend_landing_ref(repo_type)
 
         defs = {
             'repo_name': None,
             'repo_type': repo_type,
             'clone_uri': '',
             'push_uri': '',
             'repo_group': '-1',
             'repo_description': 'DESC',
             'repo_private': False,
             'repo_landing_commit_ref': default_landing_ref,
             'repo_copy_permissions': False,
             'repo_state': Repository.STATE_CREATED,
         }
         defs.update(custom)
         if 'repo_name_full' not in custom:
             defs.update({'repo_name_full': defs['repo_name']})
 
         # fix the repo name if passed as repo_name_full
         if defs['repo_name']:
             defs['repo_name'] = defs['repo_name'].split('/')[-1]
 
         return defs
 
     def _get_group_create_params(self, **custom):
         defs = {
             'group_name': None,
             'group_description': 'DESC',
             'perm_updates': [],
             'perm_additions': [],
             'perm_deletions': [],
             'group_parent_id': -1,
             'enable_locking': False,
             'recursive': False,
         }
         defs.update(custom)
 
         return defs
 
     def _get_user_create_params(self, name, **custom):
         defs = {
             'username': name,
             'password': 'qweqwe',
             'email': '[e-mail]' % name,
             'firstname': 'TestUser',
             'lastname': 'Test',
             'description': 'test description',
             'active': True,
             'admin': False,
             'extern_type': 'rhodecode',
             'extern_name': None,
         }
         defs.update(custom)
 
         return defs
 
     def _get_user_group_create_params(self, name, **custom):
         defs = {
             'users_group_name': name,
             'user_group_description': 'DESC',
             'users_group_active': True,
             'user_group_data': {},
         }
         defs.update(custom)
 
         return defs
 
     def create_repo(self, name, **kwargs):
         repo_group = kwargs.get('repo_group')
         if isinstance(repo_group, RepoGroup):
             kwargs['repo_group'] = repo_group.group_id
             name = name.split(Repository.NAME_SEP)[-1]
             name = Repository.NAME_SEP.join((repo_group.group_name, name))
 
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             r = Repository.get_by_repo_name(name)
             if r:
                 return r
 
         form_data = self._get_repo_create_params(repo_name=name, **kwargs)
         cur_user = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         RepoModel().create(form_data, cur_user)
         Session().commit()
         repo = Repository.get_by_repo_name(name)
         assert repo
         return repo
 
     def create_fork(self, repo_to_fork, fork_name, **kwargs):
         repo_to_fork = Repository.get_by_repo_name(repo_to_fork)
 
         form_data = self._get_repo_create_params(
             repo_name=fork_name,
             fork_parent_id=repo_to_fork.repo_id,
             repo_type=repo_to_fork.repo_type,
             **kwargs)
 
         # TODO: fix it !!
         form_data['description'] = form_data['repo_description']
         form_data['private'] = form_data['repo_private']
         form_data['landing_rev'] = form_data['repo_landing_commit_ref']
 
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         RepoModel().create_fork(form_data, cur_user=owner)
         Session().commit()
         r = Repository.get_by_repo_name(fork_name)
         assert r
         return r
 
     def destroy_repo(self, repo_name, **kwargs):
-        RepoModel().delete(repo_name, pull_requests='delete', **kwargs)
+        RepoModel().delete(repo_name, pull_requests='delete', artifacts='delete', **kwargs)
         Session().commit()
 
     def destroy_repo_on_filesystem(self, repo_name):
         rm_path = os.path.join(RepoModel().repos_path, repo_name)
         if os.path.isdir(rm_path):
             shutil.rmtree(rm_path)
 
     def create_repo_group(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             gr = RepoGroup.get_by_group_name(group_name=name)
             if gr:
                 return gr
         form_data = self._get_group_create_params(group_name=name, **kwargs)
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         gr = RepoGroupModel().create(
             group_name=form_data['group_name'],
             group_description=form_data['group_name'],
             owner=owner)
         Session().commit()
         gr = RepoGroup.get_by_group_name(gr.group_name)
         return gr
 
     def destroy_repo_group(self, repogroupid):
         RepoGroupModel().delete(repogroupid)
         Session().commit()
 
     def create_user(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             user = User.get_by_username(name)
             if user:
                 return user
         form_data = self._get_user_create_params(name, **kwargs)
         user = UserModel().create(form_data)
 
         # create token for user
         AuthTokenModel().create(
                 user=user, description=u'TEST_USER_TOKEN')
 
         Session().commit()
         user = User.get_by_username(user.username)
         return user
 
     def destroy_user(self, userid):
         UserModel().delete(userid)
         Session().commit()
 
     def create_additional_user_email(self, user, email):
         uem = UserEmailMap()
         uem.user = user
         uem.email = email
         Session().add(uem)
         return uem
 
     def destroy_users(self, userid_iter):
         for user_id in userid_iter:
             if User.get_by_username(user_id):
                 UserModel().delete(user_id)
                 Session().commit()
 
     def create_user_group(self, name, **kwargs):
         if 'skip_if_exists' in kwargs:
             del kwargs['skip_if_exists']
             gr = UserGroup.get_by_group_name(group_name=name)
             if gr:
                 return gr
         # map active flag to the real attribute. For API consistency of fixtures
         if 'active' in kwargs:
             kwargs['users_group_active'] = kwargs['active']
             del kwargs['active']
         form_data = self._get_user_group_create_params(name, **kwargs)
         owner = kwargs.get('cur_user', TEST_USER_ADMIN_LOGIN)
         user_group = UserGroupModel().create(
             name=form_data['users_group_name'],
             description=form_data['user_group_description'],
             owner=owner, active=form_data['users_group_active'],
             group_data=form_data['user_group_data'])
         Session().commit()
         user_group = UserGroup.get_by_group_name(user_group.users_group_name)
         return user_group
 
     def destroy_user_group(self, usergroupid):
         UserGroupModel().delete(user_group=usergroupid, force=True)
         Session().commit()
 
     def create_gist(self, **kwargs):
         form_data = {
             'description': 'new-gist',
             'owner': TEST_USER_ADMIN_LOGIN,
             'gist_type': GistModel.cls.GIST_PUBLIC,
             'lifetime': -1,
             'acl_level': Gist.ACL_LEVEL_PUBLIC,
             'gist_mapping': {b'filename1.txt': {'content': b'hello world'},}
         }
         form_data.update(kwargs)
         gist = GistModel().create(
             description=form_data['description'], owner=form_data['owner'],
             gist_mapping=form_data['gist_mapping'], gist_type=form_data['gist_type'],
             lifetime=form_data['lifetime'], gist_acl_level=form_data['acl_level']
         )
         Session().commit()
         return gist
 
     def destroy_gists(self, gistid=None):
         for g in GistModel.cls.get_all():
             if gistid:
                 if gistid == g.gist_access_id:
                     GistModel().delete(g)
             else:
                 GistModel().delete(g)
         Session().commit()
 
     def load_resource(self, resource_name, strip=False):
         with open(os.path.join(FIXTURES, resource_name), 'rb') as f:
             source = f.read()
             if strip:
                 source = source.strip()
 
         return source

rhodecode/tests/fixture_mods/fixture_pyramid.py

@@ -1,226 +1,226 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
 
 from rhodecode.lib.config_utils import get_app_config
 from rhodecode.tests.fixture import TestINI
 from rhodecode.tests import TESTS_TMP_PATH
 from rhodecode.tests.server_utils import RcVCSServer
 
 
 @pytest.fixture(scope='session')
 def vcsserver(request, vcsserver_port, vcsserver_factory):
     """
     Session scope VCSServer.
 
     Tests which need the VCSServer have to rely on this fixture in order
     to ensure it will be running.
 
     For specific needs, the fixture vcsserver_factory can be used. It allows to
     adjust the configuration file for the test run.
 
     Command line args:
 
     --without-vcsserver: Allows to switch this fixture off. You have to
     manually start the server.
 
     --vcsserver-port: Will expect the VCSServer to listen on this port.
     """
 
     if not request.config.getoption('with_vcsserver'):
         return None
 
     return vcsserver_factory(
         request, vcsserver_port=vcsserver_port)
 
 
 @pytest.fixture(scope='session')
 def vcsserver_factory(tmpdir_factory):
     """
     Use this if you need a running vcsserver with a special configuration.
     """
 
     def factory(request, overrides=(), vcsserver_port=None,
                 log_file=None, workers='3'):
 
         if vcsserver_port is None:
             vcsserver_port = get_available_port()
 
         overrides = list(overrides)
         overrides.append({'server:main': {'port': vcsserver_port}})
 
         option_name = 'vcsserver_config_http'
         override_option_name = 'vcsserver_config_override'
         config_file = get_config(
             request.config, option_name=option_name,
             override_option_name=override_option_name, overrides=overrides,
             basetemp=tmpdir_factory.getbasetemp().strpath,
             prefix='test_vcs_')
 
         server = RcVCSServer(config_file, log_file, workers)
         server.start()
 
         @request.addfinalizer
         def cleanup():
             server.shutdown()
 
         server.wait_until_ready()
         return server
 
     return factory
 
 
 def _use_log_level(config):
     level = config.getoption('test_loglevel') or 'critical'
     return level.upper()
 
 
 @pytest.fixture(scope='session')
 def ini_config(request, tmpdir_factory, rcserver_port, vcsserver_port):
     option_name = 'pyramid_config'
     log_level = _use_log_level(request.config)
 
     overrides = [
         {'server:main': {'port': rcserver_port}},
         {'app:main': {
             'cache_dir': '%(here)s/rc-tests/rc_data',
             'vcs.server': f'localhost:{vcsserver_port}',
             # johbo: We will always start the VCSServer on our own based on the
             # fixtures of the test cases. For the test run it must always be
             # off in the INI file.
             'vcs.start_server': 'false',
 
             'vcs.server.protocol': 'http',
             'vcs.scm_app_implementation': 'http',
             'vcs.svn.proxy.enabled': 'true',
-            'vcs.hooks.protocol': 'http',
+            'vcs.hooks.protocol.v2': 'celery',
             'vcs.hooks.host': '*',
             'repo_store.path': TESTS_TMP_PATH,
             'app.service_api.token': 'service_secret_token',
         }},
 
         {'handler_console': {
             'class': 'StreamHandler',
             'args': '(sys.stderr,)',
             'level': log_level,
         }},
 
     ]
 
     filename = get_config(
         request.config, option_name=option_name,
         override_option_name='{}_override'.format(option_name),
         overrides=overrides,
         basetemp=tmpdir_factory.getbasetemp().strpath,
         prefix='test_rce_')
     return filename
 
 
 @pytest.fixture(scope='session')
 def ini_settings(ini_config):
     ini_path = ini_config
     return get_app_config(ini_path)
 
 
 def get_available_port(min_port=40000, max_port=55555):
     from rhodecode.lib.utils2 import get_available_port as _get_port
     return _get_port(min_port, max_port)
 
 
 @pytest.fixture(scope='session')
 def rcserver_port(request):
     port = get_available_port()
     print(f'Using rhodecode port {port}')
     return port
 
 
 @pytest.fixture(scope='session')
 def vcsserver_port(request):
     port = request.config.getoption('--vcsserver-port')
     if port is None:
         port = get_available_port()
         print(f'Using vcsserver port {port}')
     return port
 
 
 @pytest.fixture(scope='session')
 def available_port_factory() -> get_available_port:
     """
     Returns a callable which returns free port numbers.
     """
     return get_available_port
 
 
 @pytest.fixture()
 def available_port(available_port_factory):
     """
     Gives you one free port for the current test.
 
     Uses "available_port_factory" to retrieve the port.
     """
     return available_port_factory()
 
 
 @pytest.fixture(scope='session')
 def testini_factory(tmpdir_factory, ini_config):
     """
     Factory to create an INI file based on TestINI.
 
     It will make sure to place the INI file in the correct directory.
     """
     basetemp = tmpdir_factory.getbasetemp().strpath
     return TestIniFactory(basetemp, ini_config)
 
 
 class TestIniFactory(object):
 
     def __init__(self, basetemp, template_ini):
         self._basetemp = basetemp
         self._template_ini = template_ini
 
     def __call__(self, ini_params, new_file_prefix='test'):
         ini_file = TestINI(
             self._template_ini, ini_params=ini_params,
             new_file_prefix=new_file_prefix, dir=self._basetemp)
         result = ini_file.create()
         return result
 
 
 def get_config(
         config, option_name, override_option_name, overrides=None,
         basetemp=None, prefix='test'):
     """
     Find a configuration file and apply overrides for the given `prefix`.
     """
     config_file = (
         config.getoption(option_name) or config.getini(option_name))
     if not config_file:
         pytest.exit(
             "Configuration error, could not extract {}.".format(option_name))
 
     overrides = overrides or []
     config_override = config.getoption(override_option_name)
     if config_override:
         overrides.append(config_override)
     temp_ini_file = TestINI(
         config_file, ini_params=overrides, new_file_prefix=prefix,
         dir=basetemp)
 
     return temp_ini_file.create()

rhodecode/tests/lib/middleware/test_simplehg.py

@@ -1,155 +1,154 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import urllib.parse
 
 import mock
 import pytest
 import simplejson as json
 
 from rhodecode.lib.vcs.backends.base import Config
 from rhodecode.tests.lib.middleware import mock_scm_app
 import rhodecode.lib.middleware.simplehg as simplehg
 
 
 def get_environ(url):
     """Construct a minimum WSGI environ based on the URL."""
     parsed_url = urllib.parse.urlparse(url)
     environ = {
         'PATH_INFO': parsed_url.path,
         'QUERY_STRING': parsed_url.query,
     }
 
     return environ
 
 
 @pytest.mark.parametrize(
     'url, expected_action',
     [
         ('/foo/bar?cmd=unbundle&key=tip', 'push'),
         ('/foo/bar?cmd=pushkey&key=tip', 'push'),
         ('/foo/bar?cmd=listkeys&key=tip', 'pull'),
         ('/foo/bar?cmd=changegroup&key=tip', 'pull'),
         ('/foo/bar?cmd=hello', 'pull'),
         ('/foo/bar?cmd=batch', 'push'),
         ('/foo/bar?cmd=putlfile', 'push'),
         # Edge case: unknown argument: assume push
         ('/foo/bar?cmd=unknown&key=tip', 'push'),
         ('/foo/bar?cmd=&key=tip', 'push'),
         # Edge case: not cmd argument
         ('/foo/bar?key=tip', 'push'),
     ])
 def test_get_action(url, expected_action, request_stub):
     app = simplehg.SimpleHg(config={'auth_ret_code': '', 'base_path': ''},
                             registry=request_stub.registry)
     assert expected_action == app._get_action(get_environ(url))
 
 
 @pytest.mark.parametrize(
     'environ, expected_xargs, expected_batch',
     [
         ({},
          [''], ['push']),
 
         ({'HTTP_X_HGARG_1': ''},
          [''], ['push']),
 
         ({'HTTP_X_HGARG_1': 'cmds=listkeys+namespace%3Dphases'},
          ['listkeys namespace=phases'], ['pull']),
 
         ({'HTTP_X_HGARG_1': 'cmds=pushkey+namespace%3Dbookmarks%2Ckey%3Dbm%2Cold%3D%2Cnew%3Dcb9a9f314b8b07ba71012fcdbc544b5a4d82ff5b'},
          ['pushkey namespace=bookmarks,key=bm,old=,new=cb9a9f314b8b07ba71012fcdbc544b5a4d82ff5b'], ['push']),
 
         ({'HTTP_X_HGARG_1': 'namespace=phases'},
          ['namespace=phases'], ['push']),
 
     ])
 def test_xarg_and_batch_commands(environ, expected_xargs, expected_batch):
     app = simplehg.SimpleHg
 
     result = app._get_xarg_headers(environ)
     result_batch = app._get_batch_cmd(environ)
     assert expected_xargs == result
     assert expected_batch == result_batch
 
 
 @pytest.mark.parametrize(
     'url, expected_repo_name',
     [
         ('/foo?cmd=unbundle&key=tip', 'foo'),
         ('/foo/bar?cmd=pushkey&key=tip', 'foo/bar'),
         ('/foo/bar/baz?cmd=listkeys&key=tip', 'foo/bar/baz'),
         # Repos with trailing slashes.
         ('/foo/?cmd=unbundle&key=tip', 'foo'),
         ('/foo/bar/?cmd=pushkey&key=tip', 'foo/bar'),
         ('/foo/bar/baz/?cmd=listkeys&key=tip', 'foo/bar/baz'),
     ])
 def test_get_repository_name(url, expected_repo_name, request_stub):
     app = simplehg.SimpleHg(config={'auth_ret_code': '', 'base_path': ''},
                             registry=request_stub.registry)
     assert expected_repo_name == app._get_repository_name(get_environ(url))
 
 
 def test_get_config(user_util, baseapp, request_stub):
     repo = user_util.create_repo(repo_type='git')
     app = simplehg.SimpleHg(config={'auth_ret_code': '', 'base_path': ''},
                             registry=request_stub.registry)
     extras = [('foo', 'FOO', 'bar', 'BAR')]
 
     hg_config = app._create_config(extras, repo_name=repo.repo_name)
 
     config = simplehg.utils.make_db_config(repo=repo.repo_name)
     config.set('rhodecode', 'RC_SCM_DATA', json.dumps(extras))
     hg_config_org = config
 
     expected_config = [
         ('vcs_svn_tag', 'ff89f8c714d135d865f44b90e5413b88de19a55f', '/tags/*'),
-        ('web', 'push_ssl', 'False'),
         ('web', 'allow_push', '*'),
         ('web', 'allow_archive', 'gz zip bz2'),
         ('web', 'baseurl', '/'),
         ('vcs_git_lfs', 'store_location', hg_config_org.get('vcs_git_lfs', 'store_location')),
         ('vcs_svn_branch', '9aac1a38c3b8a0cdc4ae0f960a5f83332bc4fa5e', '/branches/*'),
         ('vcs_svn_branch', 'c7e6a611c87da06529fd0dd733308481d67c71a8', '/trunk'),
         ('largefiles', 'usercache', hg_config_org.get('largefiles', 'usercache')),
         ('hooks', 'preoutgoing.pre_pull', 'python:vcsserver.hooks.pre_pull'),
         ('hooks', 'prechangegroup.pre_push', 'python:vcsserver.hooks.pre_push'),
         ('hooks', 'outgoing.pull_logger', 'python:vcsserver.hooks.log_pull_action'),
         ('hooks', 'pretxnchangegroup.pre_push', 'python:vcsserver.hooks.pre_push'),
         ('hooks', 'changegroup.push_logger', 'python:vcsserver.hooks.log_push_action'),
         ('hooks', 'changegroup.repo_size', 'python:vcsserver.hooks.repo_size'),
         ('phases', 'publish', 'True'),
         ('extensions', 'largefiles', ''),
         ('paths', '/', hg_config_org.get('paths', '/')),
         ('rhodecode', 'RC_SCM_DATA', '[["foo","FOO","bar","BAR"]]')
     ]
     for entry in expected_config:
         assert entry in hg_config
 
 
 def test_create_wsgi_app_uses_scm_app_from_simplevcs(request_stub):
     config = {
         'auth_ret_code': '',
         'base_path': '',
         'vcs.scm_app_implementation':
             'rhodecode.tests.lib.middleware.mock_scm_app',
     }
     app = simplehg.SimpleHg(config=config, registry=request_stub.registry)
     wsgi_app = app._create_wsgi_app('/tmp/test', 'test_repo', {})
     assert wsgi_app is mock_scm_app.mock_hg_wsgi

rhodecode/tests/lib/middleware/test_simplevcs.py

@@ -1,451 +1,448 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import mock
 import pytest
 
 from rhodecode.lib.str_utils import base64_to_str
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.tests.utils import CustomTestApp
 
 from rhodecode.lib.caching_query import FromCache
 from rhodecode.lib.middleware import simplevcs
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.middleware.utils import scm_app_http
 from rhodecode.model.db import User, _hash_key
 from rhodecode.model.meta import Session, cache as db_cache
 from rhodecode.tests import (
     HG_REPO, TEST_USER_ADMIN_LOGIN, TEST_USER_ADMIN_PASS)
 from rhodecode.tests.lib.middleware import mock_scm_app
 
 
 class StubVCSController(simplevcs.SimpleVCS):
 
     SCM = 'hg'
     stub_response_body = tuple()
 
     def __init__(self, *args, **kwargs):
         super(StubVCSController, self).__init__(*args, **kwargs)
         self._action = 'pull'
         self._is_shadow_repo_dir = True
         self._name = HG_REPO
         self.set_repo_names(None)
 
     @property
     def is_shadow_repo_dir(self):
         return self._is_shadow_repo_dir
 
     def _get_repository_name(self, environ):
         return self._name
 
     def _get_action(self, environ):
         return self._action
 
     def _create_wsgi_app(self, repo_path, repo_name, config):
         def fake_app(environ, start_response):
             headers = [
                 ('Http-Accept', 'application/mercurial')
             ]
             start_response('200 OK', headers)
             return self.stub_response_body
         return fake_app
 
     def _create_config(self, extras, repo_name, scheme='http'):
         return None
 
 
 @pytest.fixture()
 def vcscontroller(baseapp, config_stub, request_stub):
     from rhodecode.config.middleware import ce_auth_resources
 
     config_stub.testing_securitypolicy()
     config_stub.include('rhodecode.authentication')
 
     for resource in ce_auth_resources:
         config_stub.include(resource)
 
     controller = StubVCSController(
         baseapp.config.get_settings(), request_stub.registry)
     app = HttpsFixup(controller, baseapp.config.get_settings())
     app = CustomTestApp(app)
 
     _remove_default_user_from_query_cache()
 
     # Sanity checks that things are set up correctly
     app.get('/' + HG_REPO, status=200)
 
     app.controller = controller
     return app
 
 
 def _remove_default_user_from_query_cache():
     user = User.get_default_user(cache=True)
     query = Session().query(User).filter(User.username == user.username)
     query = query.options(
         FromCache("sql_cache_short", f"get_user_{_hash_key(user.username)}"))
 
     db_cache.invalidate(
         query, {},
         FromCache("sql_cache_short", f"get_user_{_hash_key(user.username)}"))
 
     Session().expire(user)
 
 
 def test_handles_exceptions_during_permissions_checks(
         vcscontroller, disable_anonymous_user, enable_auth_plugins, test_user_factory):
 
     test_password = 'qweqwe'
     test_user = test_user_factory(password=test_password, extern_type='headers', extern_name='headers')
     test_username = test_user.username
 
     enable_auth_plugins.enable([
         'egg:rhodecode-enterprise-ce#headers',
         'egg:rhodecode-enterprise-ce#token',
         'egg:rhodecode-enterprise-ce#rhodecode'],
         override={
             'egg:rhodecode-enterprise-ce#headers': {'auth_headers_header': 'REMOTE_USER'}
         })
 
     user_and_pass = f'{test_username}:{test_password}'
     auth_password = base64_to_str(user_and_pass)
 
     extra_environ = {
         'AUTH_TYPE': 'Basic',
         'HTTP_AUTHORIZATION': f'Basic {auth_password}',
         'REMOTE_USER': test_username,
     }
 
     # Verify that things are hooked up correctly, we pass user with headers bound auth, and headers filled in
     vcscontroller.get('/', status=200, extra_environ=extra_environ)
 
     # Simulate trouble during permission checks
     with mock.patch('rhodecode.model.db.User.get_by_username',
                     side_effect=Exception('permission_error_test')) as get_user:
         # Verify that a correct 500 is returned and check that the expected
         # code path was hit.
         vcscontroller.get('/', status=500, extra_environ=extra_environ)
         assert get_user.called
 
 
 class StubFailVCSController(simplevcs.SimpleVCS):
     def _handle_request(self, environ, start_response):
         raise Exception("BOOM")
 
 
 @pytest.fixture(scope='module')
 def fail_controller(baseapp):
     controller = StubFailVCSController(
         baseapp.config.get_settings(), baseapp.config)
     controller = HttpsFixup(controller, baseapp.config.get_settings())
     controller = CustomTestApp(controller)
     return controller
 
 
 def test_handles_exceptions_as_internal_server_error(fail_controller):
     fail_controller.get('/', status=500)
 
 
 def test_provides_traceback_for_appenlight(fail_controller):
     response = fail_controller.get(
         '/', status=500, extra_environ={'appenlight.client': 'fake'})
     assert 'appenlight.__traceback' in response.request.environ
 
 
 def test_provides_utils_scm_app_as_scm_app_by_default(baseapp, request_stub):
     controller = StubVCSController(baseapp.config.get_settings(), request_stub.registry)
     assert controller.scm_app is scm_app_http
 
 
 def test_allows_to_override_scm_app_via_config(baseapp, request_stub):
     config = baseapp.config.get_settings().copy()
     config['vcs.scm_app_implementation'] = (
         'rhodecode.tests.lib.middleware.mock_scm_app')
     controller = StubVCSController(config, request_stub.registry)
     assert controller.scm_app is mock_scm_app
 
 
 @pytest.mark.parametrize('query_string, expected', [
     ('cmd=stub_command', True),
     ('cmd=listkeys', False),
 ])
 def test_should_check_locking(query_string, expected):
     result = simplevcs._should_check_locking(query_string)
     assert result == expected
 
 
 class TestShadowRepoRegularExpression(object):
     pr_segment = 'pull-request'
     shadow_segment = 'repository'
 
     @pytest.mark.parametrize('url, expected', [
         # repo with/without groups
         ('My-Repo/{pr_segment}/1/{shadow_segment}', True),
         ('Group/My-Repo/{pr_segment}/2/{shadow_segment}', True),
         ('Group/Sub-Group/My-Repo/{pr_segment}/3/{shadow_segment}', True),
         ('Group/Sub-Group1/Sub-Group2/My-Repo/{pr_segment}/3/{shadow_segment}', True),
 
         # pull request ID
         ('MyRepo/{pr_segment}/1/{shadow_segment}', True),
         ('MyRepo/{pr_segment}/1234567890/{shadow_segment}', True),
         ('MyRepo/{pr_segment}/-1/{shadow_segment}', False),
         ('MyRepo/{pr_segment}/invalid/{shadow_segment}', False),
 
         # unicode
         (u'Sp€çîál-Repö/{pr_segment}/1/{shadow_segment}', True),
         (u'Sp€çîál-Gröüp/Sp€çîál-Repö/{pr_segment}/1/{shadow_segment}', True),
 
         # trailing/leading slash
         ('/My-Repo/{pr_segment}/1/{shadow_segment}', False),
         ('My-Repo/{pr_segment}/1/{shadow_segment}/', False),
         ('/My-Repo/{pr_segment}/1/{shadow_segment}/', False),
 
         # misc
         ('My-Repo/{pr_segment}/1/{shadow_segment}/extra', False),
         ('My-Repo/{pr_segment}/1/{shadow_segment}extra', False),
     ])
     def test_shadow_repo_regular_expression(self, url, expected):
         from rhodecode.lib.middleware.simplevcs import SimpleVCS
         url = url.format(
             pr_segment=self.pr_segment,
             shadow_segment=self.shadow_segment)
         match_obj = SimpleVCS.shadow_repo_re.match(url)
         assert (match_obj is not None) == expected
 
 
 @pytest.mark.backends('git', 'hg')
 class TestShadowRepoExposure(object):
 
     def test_pull_on_shadow_repo_propagates_to_wsgi_app(
             self, baseapp, request_stub):
         """
         Check that a pull action to a shadow repo is propagated to the
         underlying wsgi app.
         """
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
-        controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'pull'
         controller._is_shadow_repo_dir = True
         controller.stub_response_body = (b'dummy body value',)
         controller._get_default_cache_ttl = mock.Mock(
             return_value=(False, 0))
 
         environ_stub = {
             'HTTP_HOST': 'test.example.com',
             'HTTP_ACCEPT': 'application/mercurial',
             'REQUEST_METHOD': 'GET',
             'wsgi.url_scheme': 'http',
         }
 
         response = controller(environ_stub, mock.Mock())
         response_body = b''.join(response)
 
         # Assert that we got the response from the wsgi app.
         assert response_body == b''.join(controller.stub_response_body)
 
     def test_pull_on_shadow_repo_that_is_missing(self, baseapp, request_stub):
         """
         Check that a pull action to a shadow repo is propagated to the
         underlying wsgi app.
         """
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
-        controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'pull'
         controller._is_shadow_repo_dir = False
         controller.stub_response_body = (b'dummy body value',)
         environ_stub = {
             'HTTP_HOST': 'test.example.com',
             'HTTP_ACCEPT': 'application/mercurial',
             'REQUEST_METHOD': 'GET',
             'wsgi.url_scheme': 'http',
         }
 
         response = controller(environ_stub, mock.Mock())
         response_body = b''.join(response)
 
         # Assert that we got the response from the wsgi app.
         assert b'404 Not Found' in response_body
 
     def test_push_on_shadow_repo_raises(self, baseapp, request_stub):
         """
         Check that a push action to a shadow repo is aborted.
         """
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
-        controller._check_ssl = mock.Mock()
         controller.is_shadow_repo = True
         controller._action = 'push'
         controller.stub_response_body = (b'dummy body value',)
         environ_stub = {
             'HTTP_HOST': 'test.example.com',
             'HTTP_ACCEPT': 'application/mercurial',
             'REQUEST_METHOD': 'GET',
             'wsgi.url_scheme': 'http',
         }
 
         response = controller(environ_stub, mock.Mock())
         response_body = b''.join(response)
 
         assert response_body != controller.stub_response_body
         # Assert that a 406 error is returned.
         assert b'406 Not Acceptable' in response_body
 
     def test_set_repo_names_no_shadow(self, baseapp, request_stub):
         """
         Check that the set_repo_names method sets all names to the one returned
         by the _get_repository_name method on a request to a non shadow repo.
         """
         environ_stub = {}
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._name = 'RepoGroup/MyRepo'
         controller.set_repo_names(environ_stub)
         assert not controller.is_shadow_repo
         assert (controller.url_repo_name ==
                 controller.acl_repo_name ==
                 controller.vcs_repo_name ==
                 controller._get_repository_name(environ_stub))
 
     def test_set_repo_names_with_shadow(
             self, baseapp, pr_util, config_stub, request_stub):
         """
         Check that the set_repo_names method sets correct names on a request
         to a shadow repo.
         """
         from rhodecode.model.pull_request import PullRequestModel
 
         pull_request = pr_util.create_pull_request()
         shadow_url = '{target}/{pr_segment}/{pr_id}/{shadow_segment}'.format(
             target=pull_request.target_repo.repo_name,
             pr_id=pull_request.pull_request_id,
             pr_segment=TestShadowRepoRegularExpression.pr_segment,
             shadow_segment=TestShadowRepoRegularExpression.shadow_segment)
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._name = shadow_url
         controller.set_repo_names({})
 
         # Get file system path to shadow repo for assertions.
         workspace_id = PullRequestModel()._workspace_id(pull_request)
         vcs_repo_name = pull_request.target_repo.get_shadow_repository_path(workspace_id)
 
         assert controller.vcs_repo_name == vcs_repo_name
         assert controller.url_repo_name == shadow_url
         assert controller.acl_repo_name == pull_request.target_repo.repo_name
         assert controller.is_shadow_repo
 
     def test_set_repo_names_with_shadow_but_missing_pr(
             self, baseapp, pr_util, config_stub, request_stub):
         """
         Checks that the set_repo_names method enforces matching target repos
         and pull request IDs.
         """
         pull_request = pr_util.create_pull_request()
         shadow_url = '{target}/{pr_segment}/{pr_id}/{shadow_segment}'.format(
             target=pull_request.target_repo.repo_name,
             pr_id=999999999,
             pr_segment=TestShadowRepoRegularExpression.pr_segment,
             shadow_segment=TestShadowRepoRegularExpression.shadow_segment)
         controller = StubVCSController(
             baseapp.config.get_settings(), request_stub.registry)
         controller._name = shadow_url
         controller.set_repo_names({})
 
         assert not controller.is_shadow_repo
         assert (controller.url_repo_name ==
                 controller.acl_repo_name ==
                 controller.vcs_repo_name)
 
 
 @pytest.mark.usefixtures('baseapp')
 class TestGenerateVcsResponse(object):
 
     def test_ensures_that_start_response_is_called_early_enough(self):
         self.call_controller_with_response_body(iter(['a', 'b']))
         assert self.start_response.called
 
     def test_invalidates_cache_after_body_is_consumed(self):
         result = self.call_controller_with_response_body(iter(['a', 'b']))
         assert not self.was_cache_invalidated()
         # Consume the result
         list(result)
         assert self.was_cache_invalidated()
 
     def test_raises_unknown_exceptions(self):
         result = self.call_controller_with_response_body(
             self.raise_result_iter(vcs_kind='unknown'))
         with pytest.raises(Exception):
             list(result)
 
     def call_controller_with_response_body(self, response_body):
         settings = {
             'base_path': 'fake_base_path',
-            'vcs.hooks.protocol': 'http',
+            'vcs.hooks.protocol.v2': 'celery',
             'vcs.hooks.direct_calls': False,
         }
         registry = AttributeDict()
         controller = StubVCSController(settings, registry)
         controller._invalidate_cache = mock.Mock()
         controller.stub_response_body = response_body
         self.start_response = mock.Mock()
         result = controller._generate_vcs_response(
             environ={}, start_response=self.start_response,
             repo_path='fake_repo_path',
             extras={}, action='push')
         self.controller = controller
         return result
 
     def raise_result_iter(self, vcs_kind='repo_locked'):
         """
         Simulates an exception due to a vcs raised exception if kind vcs_kind
         """
         raise self.vcs_exception(vcs_kind=vcs_kind)
         yield "never_reached"
 
     def vcs_exception(self, vcs_kind='repo_locked'):
         locked_exception = Exception('TEST_MESSAGE')
         locked_exception._vcs_kind = vcs_kind
         return locked_exception
 
     def was_cache_invalidated(self):
         return self.controller._invalidate_cache.called
 
 
 class TestInitializeGenerator(object):
 
     def test_drains_first_element(self):
         gen = self.factory(['__init__', 1, 2])
         result = list(gen)
         assert result == [1, 2]
 
     @pytest.mark.parametrize('values', [
         [],
         [1, 2],
     ])
     def test_raises_value_error(self, values):
         with pytest.raises(ValueError):
             self.factory(values)
 
     @simplevcs.initialize_generator
     def factory(self, iterable):
         for elem in iterable:
             yield elem

rhodecode/tests/lib/test_utils.py

@@ -1,489 +1,490 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import multiprocessing
 import os
 
 import mock
 import py
 import pytest
 
 from rhodecode.lib import caching_query
 from rhodecode.lib import utils
 from rhodecode.lib.str_utils import safe_bytes
 from rhodecode.model import settings
 from rhodecode.model import db
 from rhodecode.model import meta
 from rhodecode.model.repo import RepoModel
 from rhodecode.model.repo_group import RepoGroupModel
 from rhodecode.model.settings import UiSetting, SettingsModel
 from rhodecode.tests.fixture import Fixture
 from rhodecode_tools.lib.hash_utils import md5_safe
 from rhodecode.lib.ext_json import json
 
 fixture = Fixture()
 
 
 def extract_hooks(config):
     """Return a dictionary with the hook entries of the given config."""
     hooks = {}
     config_items = config.serialize()
     for section, name, value in config_items:
         if section != 'hooks':
             continue
         hooks[name] = value
 
     return hooks
 
 
 def disable_hooks(request, hooks):
     """Disables the given hooks from the UI settings."""
     session = meta.Session()
 
     model = SettingsModel()
     for hook_key in hooks:
         sett = model.get_ui_by_key(hook_key)
         sett.ui_active = False
         session.add(sett)
 
     # Invalidate cache
     ui_settings = session.query(db.RhodeCodeUi).options(
         caching_query.FromCache('sql_cache_short', 'get_hg_ui_settings'))
 
     meta.cache.invalidate(
         ui_settings, {},
         caching_query.FromCache('sql_cache_short', 'get_hg_ui_settings'))
 
     ui_settings = session.query(db.RhodeCodeUi).options(
         caching_query.FromCache('sql_cache_short', 'get_hook_settings'))
 
     meta.cache.invalidate(
         ui_settings, {},
         caching_query.FromCache('sql_cache_short', 'get_hook_settings'))
 
     @request.addfinalizer
     def rollback():
         session.rollback()
 
 
 HOOK_PRE_PUSH = db.RhodeCodeUi.HOOK_PRE_PUSH
 HOOK_PRETX_PUSH = db.RhodeCodeUi.HOOK_PRETX_PUSH
 HOOK_PUSH = db.RhodeCodeUi.HOOK_PUSH
 HOOK_PRE_PULL = db.RhodeCodeUi.HOOK_PRE_PULL
 HOOK_PULL = db.RhodeCodeUi.HOOK_PULL
 HOOK_REPO_SIZE = db.RhodeCodeUi.HOOK_REPO_SIZE
 HOOK_PUSH_KEY = db.RhodeCodeUi.HOOK_PUSH_KEY
 
 HG_HOOKS = frozenset(
     (HOOK_PRE_PULL, HOOK_PULL, HOOK_PRE_PUSH, HOOK_PRETX_PUSH, HOOK_PUSH,
      HOOK_REPO_SIZE, HOOK_PUSH_KEY))
 
 
 @pytest.mark.parametrize('disabled_hooks,expected_hooks', [
     ([], HG_HOOKS),
     (HG_HOOKS, []),
 
     ([HOOK_PRE_PUSH, HOOK_PRETX_PUSH, HOOK_REPO_SIZE, HOOK_PUSH_KEY], [HOOK_PRE_PULL, HOOK_PULL, HOOK_PUSH]),
 
     # When a pull/push hook is disabled, its pre-pull/push counterpart should
     # be disabled too.
     ([HOOK_PUSH], [HOOK_PRE_PULL, HOOK_PULL, HOOK_REPO_SIZE]),
     ([HOOK_PULL], [HOOK_PRE_PUSH, HOOK_PRETX_PUSH, HOOK_PUSH, HOOK_REPO_SIZE,
                    HOOK_PUSH_KEY]),
 ])
 def test_make_db_config_hg_hooks(baseapp, request, disabled_hooks,
                                  expected_hooks):
     disable_hooks(request, disabled_hooks)
 
     config = utils.make_db_config()
     hooks = extract_hooks(config)
 
     assert set(hooks.keys()).intersection(HG_HOOKS) == set(expected_hooks)
 
 
 @pytest.mark.parametrize('disabled_hooks,expected_hooks', [
     ([], ['pull', 'push']),
     ([HOOK_PUSH], ['pull']),
     ([HOOK_PULL], ['push']),
     ([HOOK_PULL, HOOK_PUSH], []),
 ])
 def test_get_enabled_hook_classes(disabled_hooks, expected_hooks):
     hook_keys = (HOOK_PUSH, HOOK_PULL)
     ui_settings = [
         ('hooks', key, 'some value', key not in disabled_hooks)
         for key in hook_keys]
 
     result = utils.get_enabled_hook_classes(ui_settings)
     assert sorted(result) == expected_hooks
 
 
 def test_get_filesystem_repos_finds_repos(tmpdir, baseapp):
     _stub_git_repo(tmpdir.ensure('repo', dir=True))
     repos = list(utils.get_filesystem_repos(str(tmpdir)))
     assert repos == [('repo', ('git', tmpdir.join('repo')))]
 
 
 def test_get_filesystem_repos_skips_directories(tmpdir, baseapp):
     tmpdir.ensure('not-a-repo', dir=True)
     repos = list(utils.get_filesystem_repos(str(tmpdir)))
     assert repos == []
 
 
 def test_get_filesystem_repos_skips_directories_with_repos(tmpdir, baseapp):
     _stub_git_repo(tmpdir.ensure('subdir/repo', dir=True))
     repos = list(utils.get_filesystem_repos(str(tmpdir)))
     assert repos == []
 
 
 def test_get_filesystem_repos_finds_repos_in_subdirectories(tmpdir, baseapp):
     _stub_git_repo(tmpdir.ensure('subdir/repo', dir=True))
     repos = list(utils.get_filesystem_repos(str(tmpdir), recursive=True))
     assert repos == [('subdir/repo', ('git', tmpdir.join('subdir', 'repo')))]
 
 
 def test_get_filesystem_repos_skips_names_starting_with_dot(tmpdir):
     _stub_git_repo(tmpdir.ensure('.repo', dir=True))
     repos = list(utils.get_filesystem_repos(str(tmpdir)))
     assert repos == []
 
 
 def test_get_filesystem_repos_skips_files(tmpdir):
     tmpdir.ensure('test-file')
     repos = list(utils.get_filesystem_repos(str(tmpdir)))
     assert repos == []
 
 
 def test_get_filesystem_repos_skips_removed_repositories(tmpdir):
     removed_repo_name = 'rm__00000000_000000_000000__.stub'
     assert utils.REMOVED_REPO_PAT.match(removed_repo_name)
     _stub_git_repo(tmpdir.ensure(removed_repo_name, dir=True))
     repos = list(utils.get_filesystem_repos(str(tmpdir)))
     assert repos == []
 
 
 def _stub_git_repo(repo_path):
     """
     Make `repo_path` look like a Git repository.
     """
     repo_path.ensure('.git', dir=True)
 
 
 def test_get_dirpaths_returns_all_paths_on_str(tmpdir):
     tmpdir.ensure('test-file')
     tmpdir.ensure('test-file-1')
     tmp_path = str(tmpdir)
     dirpaths = utils.get_dirpaths(tmp_path)
     assert list(sorted(dirpaths)) == ['test-file', 'test-file-1']
 
 
 def test_get_dirpaths_returns_all_paths_on_bytes(tmpdir):
     tmpdir.ensure('test-file-bytes')
     tmp_path = str(tmpdir)
     dirpaths = utils.get_dirpaths(safe_bytes(tmp_path))
     assert list(sorted(dirpaths)) == [b'test-file-bytes']
 
 
 def test_get_dirpaths_returns_all_paths_bytes(
         tmpdir, platform_encodes_filenames):
     if platform_encodes_filenames:
         pytest.skip("This platform seems to encode filenames.")
     tmpdir.ensure('repo-a-umlaut-\xe4')
     dirpaths = utils.get_dirpaths(str(tmpdir))
     assert dirpaths == ['repo-a-umlaut-\xe4']
 
 
 def test_get_dirpaths_skips_paths_it_cannot_decode(
         tmpdir, platform_encodes_filenames):
     if platform_encodes_filenames:
         pytest.skip("This platform seems to encode filenames.")
     path_with_latin1 = 'repo-a-umlaut-\xe4'
     tmp_path = str(tmpdir.ensure(path_with_latin1))
     dirpaths = utils.get_dirpaths(tmp_path)
     assert dirpaths == []
 
 
 @pytest.fixture(scope='session')
 def platform_encodes_filenames():
     """
     Boolean indicator if the current platform changes filename encodings.
     """
     path_with_latin1 = 'repo-a-umlaut-\xe4'
     tmpdir = py.path.local.mkdtemp()
     tmpdir.ensure(path_with_latin1)
     read_path = tmpdir.listdir()[0].basename
     tmpdir.remove()
     return path_with_latin1 != read_path
 
 
 def test_repo2db_mapper_groups(repo_groups):
     session = meta.Session()
     zombie_group, parent_group, child_group = repo_groups
     zombie_path = os.path.join(
         RepoGroupModel().repos_path, zombie_group.full_path)
     os.rmdir(zombie_path)
 
     # Avoid removing test repos when calling repo2db_mapper
     repo_list = {
         repo.repo_name: 'test' for repo in session.query(db.Repository).all()
     }
     utils.repo2db_mapper(repo_list, remove_obsolete=True)
 
     groups_in_db = session.query(db.RepoGroup).all()
     assert child_group in groups_in_db
     assert parent_group in groups_in_db
     assert zombie_path not in groups_in_db
 
 
 def test_repo2db_mapper_enables_largefiles(backend):
     repo = backend.create_repo()
     repo_list = {repo.repo_name: 'test'}
     with mock.patch('rhodecode.model.db.Repository.scm_instance') as scm_mock:
         utils.repo2db_mapper(repo_list, remove_obsolete=False)
         _, kwargs = scm_mock.call_args
         assert kwargs['config'].get('extensions', 'largefiles') == ''
 
 
 @pytest.mark.backends("git", "svn")
 def test_repo2db_mapper_installs_hooks_for_repos_in_db(backend):
     repo = backend.create_repo()
     repo_list = {repo.repo_name: 'test'}
     utils.repo2db_mapper(repo_list, remove_obsolete=False)
 
 
 @pytest.mark.backends("git", "svn")
 def test_repo2db_mapper_installs_hooks_for_newly_added_repos(backend):
     repo = backend.create_repo()
     RepoModel().delete(repo, fs_remove=False)
     meta.Session().commit()
     repo_list = {repo.repo_name: repo.scm_instance()}
     utils.repo2db_mapper(repo_list, remove_obsolete=False)
 
 
 class TestPasswordChanged(object):
 
     def setup_method(self):
         self.session = {
             'rhodecode_user': {
                 'password': '0cc175b9c0f1b6a831c399e269772661'
             }
         }
         self.auth_user = mock.Mock()
         self.auth_user.userame = 'test'
         self.auth_user.password = 'abc123'
 
     def test_returns_false_for_default_user(self):
         self.auth_user.username = db.User.DEFAULT_USER
         result = utils.password_changed(self.auth_user, self.session)
         assert result is False
 
     def test_returns_false_if_password_was_not_changed(self):
         self.session['rhodecode_user']['password'] = md5_safe(
             self.auth_user.password)
         result = utils.password_changed(self.auth_user, self.session)
         assert result is False
 
     def test_returns_true_if_password_was_changed(self):
         result = utils.password_changed(self.auth_user, self.session)
         assert result is True
 
     def test_returns_true_if_auth_user_password_is_empty(self):
         self.auth_user.password = None
         result = utils.password_changed(self.auth_user, self.session)
         assert result is True
 
     def test_returns_true_if_session_password_is_empty(self):
         self.session['rhodecode_user'].pop('password')
         result = utils.password_changed(self.auth_user, self.session)
         assert result is True
 
 
 class TestReadOpenSourceLicenses(object):
     def test_success(self):
         utils._license_cache = None
         json_data = '''
         {
             "python2.7-pytest-2.7.1": {"UNKNOWN": null},
             "python2.7-Markdown-2.6.2": {
                 "BSD-3-Clause": "http://spdx.org/licenses/BSD-3-Clause"
             }
         }
         '''
         resource_string_patch = mock.patch.object(
             utils.pkg_resources, 'resource_string', return_value=json_data)
         with resource_string_patch:
             result = utils.read_opensource_licenses()
         assert result == json.loads(json_data)
 
     def test_caching(self):
         utils._license_cache = {
             "python2.7-pytest-2.7.1": {
                 "UNKNOWN": None
             },
             "python2.7-Markdown-2.6.2": {
                 "BSD-3-Clause": "http://spdx.org/licenses/BSD-3-Clause"
             }
         }
         resource_patch = mock.patch.object(
             utils.pkg_resources, 'resource_string', side_effect=Exception)
         json_patch = mock.patch.object(
             utils.json, 'loads', side_effect=Exception)
 
         with resource_patch as resource_mock, json_patch as json_mock:
             result = utils.read_opensource_licenses()
 
         assert resource_mock.call_count == 0
         assert json_mock.call_count == 0
         assert result == utils._license_cache
 
     def test_licenses_file_contains_no_unknown_licenses(self):
         utils._license_cache = None
         result = utils.read_opensource_licenses()
 
         for license_data in result:
             if isinstance(license_data["license"], list):
                 for lic_data in license_data["license"]:
                     assert 'UNKNOWN' not in lic_data["fullName"]
             else:
                 full_name = license_data.get("fullName") or license_data
                 assert 'UNKNOWN' not in full_name
 
 
 class TestMakeDbConfig(object):
     def test_data_from_config_data_from_db_returned(self):
         test_data = [
             ('section1', 'option1', 'value1'),
             ('section2', 'option2', 'value2'),
             ('section3', 'option3', 'value3'),
         ]
-        with mock.patch.object(utils, 'config_data_from_db') as config_mock:
+        with mock.patch.object(utils, 'prepare_config_data') as config_mock:
             config_mock.return_value = test_data
             kwargs = {'clear_session': False, 'repo': 'test_repo'}
             result = utils.make_db_config(**kwargs)
         config_mock.assert_called_once_with(**kwargs)
         for section, option, expected_value in test_data:
             value = result.get(section, option)
             assert value == expected_value
 
 
-class TestConfigDataFromDb(object):
-    def test_config_data_from_db_returns_active_settings(self):
+class TestPrepareConfigData(object):
+    def test_prepare_config_data_returns_active_settings(self):
         test_data = [
             UiSetting('section1', 'option1', 'value1', True),
             UiSetting('section2', 'option2', 'value2', True),
             UiSetting('section3', 'option3', 'value3', False),
         ]
         repo_name = 'test_repo'
 
         model_patch = mock.patch.object(settings, 'VcsSettingsModel')
         hooks_patch = mock.patch.object(
             utils, 'get_enabled_hook_classes',
             return_value=['pull', 'push', 'repo_size'])
         with model_patch as model_mock, hooks_patch:
             instance_mock = mock.Mock()
             model_mock.return_value = instance_mock
             instance_mock.get_ui_settings.return_value = test_data
-            result = utils.config_data_from_db(
+            result = utils.prepare_config_data(
                 clear_session=False, repo=repo_name)
 
         self._assert_repo_name_passed(model_mock, repo_name)
 
         expected_result = [
             ('section1', 'option1', 'value1'),
             ('section2', 'option2', 'value2'),
         ]
-        assert result == expected_result
+        # We have extra config items returned, so we're ignoring two last items
+        assert result[:2] == expected_result
 
     def _assert_repo_name_passed(self, model_mock, repo_name):
         assert model_mock.call_count == 1
         call_args, call_kwargs = model_mock.call_args
         assert call_kwargs['repo'] == repo_name
 
 
 class TestIsDirWritable(object):
     def test_returns_false_when_not_writable(self):
         with mock.patch('builtins.open', side_effect=OSError):
             assert not utils._is_dir_writable('/stub-path')
 
     def test_returns_true_when_writable(self, tmpdir):
         assert utils._is_dir_writable(str(tmpdir))
 
     def test_is_safe_against_race_conditions(self, tmpdir):
         workers = multiprocessing.Pool()
         directories = [str(tmpdir)] * 10
         workers.map(utils._is_dir_writable, directories)
 
 
 class TestGetEnabledHooks(object):
     def test_only_active_hooks_are_enabled(self):
         ui_settings = [
             UiSetting('hooks', db.RhodeCodeUi.HOOK_PUSH, 'value', True),
             UiSetting('hooks', db.RhodeCodeUi.HOOK_REPO_SIZE, 'value', True),
             UiSetting('hooks', db.RhodeCodeUi.HOOK_PULL, 'value', False)
         ]
         result = utils.get_enabled_hook_classes(ui_settings)
         assert result == ['push', 'repo_size']
 
     def test_all_hooks_are_enabled(self):
         ui_settings = [
             UiSetting('hooks', db.RhodeCodeUi.HOOK_PUSH, 'value', True),
             UiSetting('hooks', db.RhodeCodeUi.HOOK_REPO_SIZE, 'value', True),
             UiSetting('hooks', db.RhodeCodeUi.HOOK_PULL, 'value', True)
         ]
         result = utils.get_enabled_hook_classes(ui_settings)
         assert result == ['push', 'repo_size', 'pull']
 
     def test_no_enabled_hooks_when_no_hook_settings_are_found(self):
         ui_settings = []
         result = utils.get_enabled_hook_classes(ui_settings)
         assert result == []
 
 
 def test_obfuscate_url_pw():
     from rhodecode.lib.utils2 import obfuscate_url_pw
     engine = u'/home/repos/malmö'
     assert obfuscate_url_pw(engine)
 
 
 @pytest.mark.parametrize("test_ua, expected", [
     ("", ""),
     ('"quoted"', 'quoted'),
     ('internal-merge', 'internal-merge'),
     ('hg/internal-merge', 'hg/internal-merge'),
     ('git/internal-merge', 'git/internal-merge'),
 
     # git
     ('git/2.10.1 (Apple Git-78)', 'git/2.10.1'),
     ('GiT/2.37.2.windows.2', 'git/2.37.2'),
     ('git/2.35.1 (Microsoft Windows NT 10.0.19044.0; Win32NT x64) CLR/4.0.30319 VS16/16.0.0', 'git/2.35.1'),
     ('ssh-user-agent', 'ssh-user-agent'),
     ('git/ssh-user-agent', 'git/ssh-user-agent'),
 
 
     # hg
     ('mercurial/proto-1.0 (Mercurial 4.2)', 'mercurial/4.2'),
     ('mercurial/proto-1.0', ''),
     ('mercurial/proto-1.0 (Mercurial 3.9.2)', 'mercurial/3.9.2'),
     ('mercurial/ssh-user-agent', 'mercurial/ssh-user-agent'),
     ('mercurial/proto-1.0 (Mercurial 5.8rc0)', 'mercurial/5.8rc0'),
 
 
 ])
 def test_user_agent_normalizer(test_ua, expected):
     from rhodecode.lib.utils2 import user_agent_normalizer
     assert user_agent_normalizer(test_ua, safe=False) == expected

rhodecode/tests/models/settings/test_vcs_settings.py

@@ -1,1114 +1,1097 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import mock
 import pytest
 
 from rhodecode.lib.utils2 import str2bool
 from rhodecode.model.meta import Session
 from rhodecode.model.settings import VcsSettingsModel, UiSetting
 
 
 HOOKS_FORM_DATA = {
     'hooks_changegroup_repo_size': True,
     'hooks_changegroup_push_logger': True,
     'hooks_outgoing_pull_logger': True
 }
 
 SVN_FORM_DATA = {
     'new_svn_branch': 'test-branch',
     'new_svn_tag': 'test-tag'
 }
 
 GENERAL_FORM_DATA = {
     'rhodecode_pr_merge_enabled': True,
     'rhodecode_use_outdated_comments': True,
     'rhodecode_hg_use_rebase_for_merging': True,
     'rhodecode_hg_close_branch_before_merging': True,
     'rhodecode_git_use_rebase_for_merging': True,
     'rhodecode_git_close_branch_before_merging': True,
     'rhodecode_diff_cache': True,
 }
 
 
 class TestInheritGlobalSettingsProperty(object):
     def test_get_raises_exception_when_repository_not_specified(self):
         model = VcsSettingsModel()
         with pytest.raises(Exception) as exc_info:
             model.inherit_global_settings
         assert str(exc_info.value) == 'Repository is not specified'
 
     def test_true_is_returned_when_value_is_not_found(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         assert model.inherit_global_settings is True
 
     def test_value_is_returned(self, repo_stub, settings_util):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         settings_util.create_repo_rhodecode_setting(
             repo_stub, VcsSettingsModel.INHERIT_SETTINGS, False, 'bool')
         assert model.inherit_global_settings is False
 
     def test_value_is_set(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         model.inherit_global_settings = False
         setting = model.repo_settings.get_setting_by_name(
             VcsSettingsModel.INHERIT_SETTINGS)
         try:
             assert setting.app_settings_type == 'bool'
             assert setting.app_settings_value is False
         finally:
             Session().delete(setting)
             Session().commit()
 
     def test_set_raises_exception_when_repository_not_specified(self):
         model = VcsSettingsModel()
         with pytest.raises(Exception) as exc_info:
             model.inherit_global_settings = False
         assert str(exc_info.value) == 'Repository is not specified'
 
 
 class TestVcsSettingsModel(object):
     def test_global_svn_branch_patterns(self):
         model = VcsSettingsModel()
         expected_result = {'test': 'test'}
         with mock.patch.object(model, 'global_settings') as settings_mock:
             get_settings = settings_mock.get_ui_by_section
             get_settings.return_value = expected_result
             settings_mock.return_value = expected_result
             result = model.get_global_svn_branch_patterns()
 
         get_settings.assert_called_once_with(model.SVN_BRANCH_SECTION)
         assert expected_result == result
 
     def test_repo_svn_branch_patterns(self):
         model = VcsSettingsModel()
         expected_result = {'test': 'test'}
         with mock.patch.object(model, 'repo_settings') as settings_mock:
             get_settings = settings_mock.get_ui_by_section
             get_settings.return_value = expected_result
             settings_mock.return_value = expected_result
             result = model.get_repo_svn_branch_patterns()
 
         get_settings.assert_called_once_with(model.SVN_BRANCH_SECTION)
         assert expected_result == result
 
     def test_repo_svn_branch_patterns_raises_exception_when_repo_is_not_set(
             self):
         model = VcsSettingsModel()
         with pytest.raises(Exception) as exc_info:
             model.get_repo_svn_branch_patterns()
         assert str(exc_info.value) == 'Repository is not specified'
 
     def test_global_svn_tag_patterns(self):
         model = VcsSettingsModel()
         expected_result = {'test': 'test'}
         with mock.patch.object(model, 'global_settings') as settings_mock:
             get_settings = settings_mock.get_ui_by_section
             get_settings.return_value = expected_result
             settings_mock.return_value = expected_result
             result = model.get_global_svn_tag_patterns()
 
         get_settings.assert_called_once_with(model.SVN_TAG_SECTION)
         assert expected_result == result
 
     def test_repo_svn_tag_patterns(self):
         model = VcsSettingsModel()
         expected_result = {'test': 'test'}
         with mock.patch.object(model, 'repo_settings') as settings_mock:
             get_settings = settings_mock.get_ui_by_section
             get_settings.return_value = expected_result
             settings_mock.return_value = expected_result
             result = model.get_repo_svn_tag_patterns()
 
         get_settings.assert_called_once_with(model.SVN_TAG_SECTION)
         assert expected_result == result
 
     def test_repo_svn_tag_patterns_raises_exception_when_repo_is_not_set(self):
         model = VcsSettingsModel()
         with pytest.raises(Exception) as exc_info:
             model.get_repo_svn_tag_patterns()
         assert str(exc_info.value) == 'Repository is not specified'
 
     def test_get_global_settings(self):
         expected_result = {'test': 'test'}
         model = VcsSettingsModel()
         with mock.patch.object(model, '_collect_all_settings') as collect_mock:
             collect_mock.return_value = expected_result
             result = model.get_global_settings()
 
         collect_mock.assert_called_once_with(global_=True)
         assert result == expected_result
 
     def test_get_repo_settings(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         expected_result = {'test': 'test'}
         with mock.patch.object(model, '_collect_all_settings') as collect_mock:
             collect_mock.return_value = expected_result
             result = model.get_repo_settings()
 
         collect_mock.assert_called_once_with(global_=False)
         assert result == expected_result
 
     @pytest.mark.parametrize('settings, global_', [
         ('global_settings', True),
         ('repo_settings', False)
     ])
     def test_collect_all_settings(self, settings, global_):
         model = VcsSettingsModel()
         result_mock = self._mock_result()
 
         settings_patch = mock.patch.object(model, settings)
         with settings_patch as settings_mock:
             settings_mock.get_ui_by_section_and_key.return_value = result_mock
             settings_mock.get_setting_by_name.return_value = result_mock
             result = model._collect_all_settings(global_=global_)
 
         ui_settings = model.HG_SETTINGS + model.GIT_SETTINGS + model.HOOKS_SETTINGS
         self._assert_get_settings_calls(
             settings_mock, ui_settings, model.GENERAL_SETTINGS)
         self._assert_collect_all_settings_result(
             ui_settings, model.GENERAL_SETTINGS, result)
 
     @pytest.mark.parametrize('settings, global_', [
         ('global_settings', True),
         ('repo_settings', False)
     ])
     def test_collect_all_settings_without_empty_value(self, settings, global_):
         model = VcsSettingsModel()
 
         settings_patch = mock.patch.object(model, settings)
         with settings_patch as settings_mock:
             settings_mock.get_ui_by_section_and_key.return_value = None
             settings_mock.get_setting_by_name.return_value = None
             result = model._collect_all_settings(global_=global_)
 
         assert result == {}
 
     def _mock_result(self):
         result_mock = mock.Mock()
         result_mock.ui_value = 'ui_value'
         result_mock.ui_active = True
         result_mock.app_settings_value = 'setting_value'
         return result_mock
 
     def _assert_get_settings_calls(
             self, settings_mock, ui_settings, general_settings):
         assert (
             settings_mock.get_ui_by_section_and_key.call_count ==
             len(ui_settings))
         assert (
             settings_mock.get_setting_by_name.call_count ==
             len(general_settings))
 
         for section, key in ui_settings:
             expected_call = mock.call(section, key)
             assert (
                 expected_call in
                 settings_mock.get_ui_by_section_and_key.call_args_list)
 
         for name in general_settings:
             expected_call = mock.call(name)
             assert (
                 expected_call in
                 settings_mock.get_setting_by_name.call_args_list)
 
     def _assert_collect_all_settings_result(
             self, ui_settings, general_settings, result):
         expected_result = {}
         for section, key in ui_settings:
             key = '{}_{}'.format(section, key.replace('.', '_'))
 
             if section in ('extensions', 'hooks'):
                 value = True
             elif key in ['vcs_git_lfs_enabled']:
                 value = True
             else:
                 value = 'ui_value'
             expected_result[key] = value
 
         for name in general_settings:
             key = 'rhodecode_' + name
             expected_result[key] = 'setting_value'
 
         assert expected_result == result
 
 
 class TestCreateOrUpdateRepoHookSettings(object):
     def test_create_when_no_repo_object_found(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
 
         self._create_settings(model, HOOKS_FORM_DATA)
 
         cleanup = []
         try:
             for section, key in model.HOOKS_SETTINGS:
                 ui = model.repo_settings.get_ui_by_section_and_key(
                     section, key)
                 assert ui.ui_active is True
                 cleanup.append(ui)
         finally:
             for ui in cleanup:
                 Session().delete(ui)
             Session().commit()
 
     def test_create_raises_exception_when_data_incomplete(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
 
         deleted_key = 'hooks_changegroup_repo_size'
         data = HOOKS_FORM_DATA.copy()
         data.pop(deleted_key)
 
         with pytest.raises(ValueError) as exc_info:
             model.create_or_update_repo_hook_settings(data)
             Session().commit()
 
         msg = 'The given data does not contain {} key'.format(deleted_key)
         assert str(exc_info.value) == msg
 
     def test_update_when_repo_object_found(self, repo_stub, settings_util):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         for section, key in model.HOOKS_SETTINGS:
             settings_util.create_repo_rhodecode_ui(
                 repo_stub, section, None, key=key, active=False)
         model.create_or_update_repo_hook_settings(HOOKS_FORM_DATA)
         Session().commit()
 
         for section, key in model.HOOKS_SETTINGS:
             ui = model.repo_settings.get_ui_by_section_and_key(section, key)
             assert ui.ui_active is True
 
     def _create_settings(self, model, data):
         global_patch = mock.patch.object(model, 'global_settings')
         global_setting = mock.Mock()
         global_setting.ui_value = 'Test value'
         with global_patch as global_mock:
             global_mock.get_ui_by_section_and_key.return_value = global_setting
             model.create_or_update_repo_hook_settings(HOOKS_FORM_DATA)
             Session().commit()
 
 
 class TestUpdateGlobalHookSettings(object):
     def test_update_raises_exception_when_data_incomplete(self):
         model = VcsSettingsModel()
 
         deleted_key = 'hooks_changegroup_repo_size'
         data = HOOKS_FORM_DATA.copy()
         data.pop(deleted_key)
 
         with pytest.raises(ValueError) as exc_info:
             model.update_global_hook_settings(data)
             Session().commit()
 
         msg = 'The given data does not contain {} key'.format(deleted_key)
         assert str(exc_info.value) == msg
 
     def test_update_global_hook_settings(self, settings_util):
         model = VcsSettingsModel()
         setting_mock = mock.MagicMock()
         setting_mock.ui_active = False
         get_settings_patcher = mock.patch.object(
             model.global_settings, 'get_ui_by_section_and_key',
             return_value=setting_mock)
         session_patcher = mock.patch('rhodecode.model.settings.Session')
         with get_settings_patcher as get_settings_mock, session_patcher:
             model.update_global_hook_settings(HOOKS_FORM_DATA)
             Session().commit()
 
         assert setting_mock.ui_active is True
         assert get_settings_mock.call_count == 3
 
 
 class TestCreateOrUpdateRepoGeneralSettings(object):
     def test_calls_create_or_update_general_settings(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         create_patch = mock.patch.object(
             model, '_create_or_update_general_settings')
         with create_patch as create_mock:
             model.create_or_update_repo_pr_settings(GENERAL_FORM_DATA)
             Session().commit()
 
         create_mock.assert_called_once_with(
             model.repo_settings, GENERAL_FORM_DATA)
 
     def test_raises_exception_when_repository_is_not_specified(self):
         model = VcsSettingsModel()
         with pytest.raises(Exception) as exc_info:
             model.create_or_update_repo_pr_settings(GENERAL_FORM_DATA)
         assert str(exc_info.value) == 'Repository is not specified'
 
 
 class TestCreateOrUpdatGlobalGeneralSettings(object):
     def test_calls_create_or_update_general_settings(self):
         model = VcsSettingsModel()
         create_patch = mock.patch.object(
             model, '_create_or_update_general_settings')
         with create_patch as create_mock:
             model.create_or_update_global_pr_settings(GENERAL_FORM_DATA)
         create_mock.assert_called_once_with(
             model.global_settings, GENERAL_FORM_DATA)
 
 
 class TestCreateOrUpdateGeneralSettings(object):
     def test_create_when_no_repo_settings_found(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         model._create_or_update_general_settings(
             model.repo_settings, GENERAL_FORM_DATA)
 
         cleanup = []
         try:
             for name in model.GENERAL_SETTINGS:
                 setting = model.repo_settings.get_setting_by_name(name)
                 assert setting.app_settings_value is True
                 cleanup.append(setting)
         finally:
             for setting in cleanup:
                 Session().delete(setting)
             Session().commit()
 
     def test_create_raises_exception_when_data_incomplete(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
 
         deleted_key = 'rhodecode_pr_merge_enabled'
         data = GENERAL_FORM_DATA.copy()
         data.pop(deleted_key)
 
         with pytest.raises(ValueError) as exc_info:
             model._create_or_update_general_settings(model.repo_settings, data)
         Session().commit()
 
         msg = 'The given data does not contain {} key'.format(deleted_key)
         assert str(exc_info.value) == msg
 
     def test_update_when_repo_setting_found(self, repo_stub, settings_util):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         for name in model.GENERAL_SETTINGS:
             settings_util.create_repo_rhodecode_setting(
                 repo_stub, name, False, 'bool')
 
         model._create_or_update_general_settings(
             model.repo_settings, GENERAL_FORM_DATA)
         Session().commit()
 
         for name in model.GENERAL_SETTINGS:
             setting = model.repo_settings.get_setting_by_name(name)
             assert setting.app_settings_value is True
 
 
 class TestCreateRepoSvnSettings(object):
     def test_calls_create_svn_settings(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         with mock.patch.object(model, '_create_svn_settings') as create_mock:
             model.create_repo_svn_settings(SVN_FORM_DATA)
             Session().commit()
 
         create_mock.assert_called_once_with(model.repo_settings, SVN_FORM_DATA)
 
     def test_raises_exception_when_repository_is_not_specified(self):
         model = VcsSettingsModel()
         with pytest.raises(Exception) as exc_info:
             model.create_repo_svn_settings(SVN_FORM_DATA)
             Session().commit()
 
         assert str(exc_info.value) == 'Repository is not specified'
 
 
 class TestCreateSvnSettings(object):
     def test_create(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         model._create_svn_settings(model.repo_settings, SVN_FORM_DATA)
         Session().commit()
 
         branch_ui = model.repo_settings.get_ui_by_section(
             model.SVN_BRANCH_SECTION)
         tag_ui = model.repo_settings.get_ui_by_section(
             model.SVN_TAG_SECTION)
 
         try:
             assert len(branch_ui) == 1
             assert len(tag_ui) == 1
         finally:
             Session().delete(branch_ui[0])
             Session().delete(tag_ui[0])
             Session().commit()
 
     def test_create_tag(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         data = SVN_FORM_DATA.copy()
         data.pop('new_svn_branch')
         model._create_svn_settings(model.repo_settings, data)
         Session().commit()
 
         branch_ui = model.repo_settings.get_ui_by_section(
             model.SVN_BRANCH_SECTION)
         tag_ui = model.repo_settings.get_ui_by_section(
             model.SVN_TAG_SECTION)
 
         try:
             assert len(branch_ui) == 0
             assert len(tag_ui) == 1
         finally:
             Session().delete(tag_ui[0])
             Session().commit()
 
     def test_create_nothing_when_no_svn_settings_specified(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         model._create_svn_settings(model.repo_settings, {})
         Session().commit()
 
         branch_ui = model.repo_settings.get_ui_by_section(
             model.SVN_BRANCH_SECTION)
         tag_ui = model.repo_settings.get_ui_by_section(
             model.SVN_TAG_SECTION)
 
         assert len(branch_ui) == 0
         assert len(tag_ui) == 0
 
     def test_create_nothing_when_empty_settings_specified(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         data = {
             'new_svn_branch': '',
             'new_svn_tag': ''
         }
         model._create_svn_settings(model.repo_settings, data)
         Session().commit()
 
         branch_ui = model.repo_settings.get_ui_by_section(
             model.SVN_BRANCH_SECTION)
         tag_ui = model.repo_settings.get_ui_by_section(
             model.SVN_TAG_SECTION)
 
         assert len(branch_ui) == 0
         assert len(tag_ui) == 0
 
 
 class TestCreateOrUpdateUi(object):
     def test_create(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         model._create_or_update_ui(
             model.repo_settings, 'test-section', 'test-key', active=False,
             value='False')
         Session().commit()
 
         created_ui = model.repo_settings.get_ui_by_section_and_key(
             'test-section', 'test-key')
 
         try:
             assert created_ui.ui_active is False
             assert str2bool(created_ui.ui_value) is False
         finally:
             Session().delete(created_ui)
             Session().commit()
 
     def test_update(self, repo_stub, settings_util):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         # care about only 3 first settings
         largefiles, phases, evolve = model.HG_SETTINGS[:3]
 
         section = 'test-section'
         key = 'test-key'
         settings_util.create_repo_rhodecode_ui(
             repo_stub, section, 'True', key=key, active=True)
 
         model._create_or_update_ui(
             model.repo_settings, section, key, active=False, value='False')
         Session().commit()
 
         created_ui = model.repo_settings.get_ui_by_section_and_key(
             section, key)
         assert created_ui.ui_active is False
         assert str2bool(created_ui.ui_value) is False
 
 
 class TestCreateOrUpdateRepoHgSettings(object):
     FORM_DATA = {
         'extensions_largefiles': False,
         'extensions_evolve': False,
         'phases_publish': False
     }
 
     def test_creates_repo_hg_settings_when_data_is_correct(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         with mock.patch.object(model, '_create_or_update_ui') as create_mock:
             model.create_or_update_repo_hg_settings(self.FORM_DATA)
         expected_calls = [
             mock.call(model.repo_settings, 'extensions', 'largefiles', active=False, value=''),
             mock.call(model.repo_settings, 'extensions', 'evolve', active=False, value=''),
             mock.call(model.repo_settings, 'experimental', 'evolution', active=False, value=''),
             mock.call(model.repo_settings, 'experimental', 'evolution.exchange', active=False, value='no'),
             mock.call(model.repo_settings, 'extensions', 'topic', active=False, value=''),
             mock.call(model.repo_settings, 'phases', 'publish', value='False'),
         ]
         assert expected_calls == create_mock.call_args_list
 
     @pytest.mark.parametrize('field_to_remove', FORM_DATA.keys())
     def test_key_is_not_found(self, repo_stub, field_to_remove):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         data = self.FORM_DATA.copy()
         data.pop(field_to_remove)
         with pytest.raises(ValueError) as exc_info:
             model.create_or_update_repo_hg_settings(data)
             Session().commit()
 
         expected_message = 'The given data does not contain {} key'.format(
             field_to_remove)
         assert str(exc_info.value) == expected_message
 
     def test_create_raises_exception_when_repository_not_specified(self):
         model = VcsSettingsModel()
         with pytest.raises(Exception) as exc_info:
             model.create_or_update_repo_hg_settings(self.FORM_DATA)
             Session().commit()
 
         assert str(exc_info.value) == 'Repository is not specified'
 
 
-class TestUpdateGlobalSslSetting(object):
-    def test_updates_global_hg_settings(self):
-        model = VcsSettingsModel()
-        with mock.patch.object(model, '_create_or_update_ui') as create_mock:
-            model.update_global_ssl_setting('False')
-            Session().commit()
-
-        create_mock.assert_called_once_with(
-            model.global_settings, 'web', 'push_ssl', value='False')
-
-
 class TestCreateOrUpdateGlobalHgSettings(object):
     FORM_DATA = {
         'extensions_largefiles': False,
-        'largefiles_usercache': '/example/largefiles-store',
         'phases_publish': False,
         'extensions_evolve': False
     }
 
     def test_creates_repo_hg_settings_when_data_is_correct(self):
         model = VcsSettingsModel()
         with mock.patch.object(model, '_create_or_update_ui') as create_mock:
             model.create_or_update_global_hg_settings(self.FORM_DATA)
             Session().commit()
 
         expected_calls = [
             mock.call(model.global_settings, 'extensions', 'largefiles', active=False, value=''),
-            mock.call(model.global_settings, 'largefiles', 'usercache', value='/example/largefiles-store'),
             mock.call(model.global_settings, 'phases', 'publish', value='False'),
             mock.call(model.global_settings, 'extensions', 'evolve', active=False, value=''),
             mock.call(model.global_settings, 'experimental', 'evolution', active=False, value=''),
             mock.call(model.global_settings, 'experimental', 'evolution.exchange', active=False, value='no'),
             mock.call(model.global_settings, 'extensions', 'topic', active=False, value=''),
         ]
 
         assert expected_calls == create_mock.call_args_list
 
     @pytest.mark.parametrize('field_to_remove', FORM_DATA.keys())
     def test_key_is_not_found(self, repo_stub, field_to_remove):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         data = self.FORM_DATA.copy()
         data.pop(field_to_remove)
         with pytest.raises(Exception) as exc_info:
             model.create_or_update_global_hg_settings(data)
             Session().commit()
 
         expected_message = 'The given data does not contain {} key'.format(
             field_to_remove)
         assert str(exc_info.value) == expected_message
 
 
 class TestCreateOrUpdateGlobalGitSettings(object):
     FORM_DATA = {
         'vcs_git_lfs_enabled': False,
-        'vcs_git_lfs_store_location': '/example/lfs-store',
     }
 
     def test_creates_repo_hg_settings_when_data_is_correct(self):
         model = VcsSettingsModel()
         with mock.patch.object(model, '_create_or_update_ui') as create_mock:
             model.create_or_update_global_git_settings(self.FORM_DATA)
             Session().commit()
 
         expected_calls = [
             mock.call(model.global_settings, 'vcs_git_lfs', 'enabled', active=False, value=False),
-            mock.call(model.global_settings, 'vcs_git_lfs', 'store_location', value='/example/lfs-store'),
         ]
         assert expected_calls == create_mock.call_args_list
 
 
 class TestDeleteRepoSvnPattern(object):
     def test_success_when_repo_is_set(self, backend_svn, settings_util):
         repo = backend_svn.create_repo()
         repo_name = repo.repo_name
 
         model = VcsSettingsModel(repo=repo_name)
         entry = settings_util.create_repo_rhodecode_ui(
             repo, VcsSettingsModel.SVN_BRANCH_SECTION, 'svn-branch')
         Session().commit()
 
         model.delete_repo_svn_pattern(entry.ui_id)
 
     def test_fail_when_delete_id_from_other_repo(self, backend_svn):
         repo_name = backend_svn.repo_name
         model = VcsSettingsModel(repo=repo_name)
         delete_ui_patch = mock.patch.object(model.repo_settings, 'delete_ui')
         with delete_ui_patch as delete_ui_mock:
             model.delete_repo_svn_pattern(123)
             Session().commit()
 
         delete_ui_mock.assert_called_once_with(-1)
 
     def test_raises_exception_when_repository_is_not_specified(self):
         model = VcsSettingsModel()
         with pytest.raises(Exception) as exc_info:
             model.delete_repo_svn_pattern(123)
         assert str(exc_info.value) == 'Repository is not specified'
 
 
 class TestDeleteGlobalSvnPattern(object):
     def test_delete_global_svn_pattern_calls_delete_ui(self):
         model = VcsSettingsModel()
         delete_ui_patch = mock.patch.object(model.global_settings, 'delete_ui')
         with delete_ui_patch as delete_ui_mock:
             model.delete_global_svn_pattern(123)
         delete_ui_mock.assert_called_once_with(123)
 
 
 class TestFilterUiSettings(object):
     def test_settings_are_filtered(self):
         model = VcsSettingsModel()
         repo_settings = [
             UiSetting('extensions', 'largefiles', '', True),
             UiSetting('phases', 'publish', 'True', True),
             UiSetting('hooks', 'changegroup.repo_size', 'hook', True),
             UiSetting('hooks', 'changegroup.push_logger', 'hook', True),
             UiSetting('hooks', 'outgoing.pull_logger', 'hook', True),
             UiSetting(
                 'vcs_svn_branch', '84223c972204fa545ca1b22dac7bef5b68d7442d',
                 'test_branch', True),
             UiSetting(
                 'vcs_svn_tag', '84229c972204fa545ca1b22dac7bef5b68d7442d',
                 'test_tag', True),
         ]
         non_repo_settings = [
             UiSetting('largefiles', 'usercache', '/example/largefiles-store', True),
             UiSetting('test', 'outgoing.pull_logger', 'hook', True),
             UiSetting('hooks', 'test2', 'hook', True),
             UiSetting(
                 'vcs_svn_repo', '84229c972204fa545ca1b22dac7bef5b68d7442d',
                 'test_tag', True),
         ]
         settings = repo_settings + non_repo_settings
         filtered_settings = model._filter_ui_settings(settings)
         assert sorted(filtered_settings) == sorted(repo_settings)
 
 
 class TestFilterGeneralSettings(object):
     def test_settings_are_filtered(self):
         model = VcsSettingsModel()
         settings = {
             'rhodecode_abcde': 'value1',
             'rhodecode_vwxyz': 'value2',
         }
         general_settings = {
             'rhodecode_{}'.format(key): 'value'
             for key in VcsSettingsModel.GENERAL_SETTINGS
         }
         settings.update(general_settings)
 
         filtered_settings = model._filter_general_settings(general_settings)
         assert sorted(filtered_settings) == sorted(general_settings)
 
 
 class TestGetRepoUiSettings(object):
     def test_global_uis_are_returned_when_no_repo_uis_found(
             self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         result = model.get_repo_ui_settings()
         svn_sections = (
             VcsSettingsModel.SVN_TAG_SECTION,
             VcsSettingsModel.SVN_BRANCH_SECTION)
         expected_result = [
             s for s in model.global_settings.get_ui()
             if s.section not in svn_sections]
         assert sorted(result) == sorted(expected_result)
 
     def test_repo_uis_are_overriding_global_uis(
             self, repo_stub, settings_util):
         for section, key in VcsSettingsModel.HOOKS_SETTINGS:
             settings_util.create_repo_rhodecode_ui(
                 repo_stub, section, 'repo', key=key, active=False)
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         result = model.get_repo_ui_settings()
         for setting in result:
             locator = (setting.section, setting.key)
             if locator in VcsSettingsModel.HOOKS_SETTINGS:
                 assert setting.value == 'repo'
 
                 assert setting.active is False
 
     def test_global_svn_patterns_are_not_in_list(
             self, repo_stub, settings_util):
         svn_sections = (
             VcsSettingsModel.SVN_TAG_SECTION,
             VcsSettingsModel.SVN_BRANCH_SECTION)
         for section in svn_sections:
             settings_util.create_rhodecode_ui(
                 section, 'repo', key='deadbeef' + section, active=False)
             Session().commit()
 
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         result = model.get_repo_ui_settings()
         for setting in result:
             assert setting.section not in svn_sections
 
     def test_repo_uis_filtered_by_section_are_returned(
             self, repo_stub, settings_util):
         for section, key in VcsSettingsModel.HOOKS_SETTINGS:
             settings_util.create_repo_rhodecode_ui(
                 repo_stub, section, 'repo', key=key, active=False)
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         section, key = VcsSettingsModel.HOOKS_SETTINGS[0]
         result = model.get_repo_ui_settings(section=section)
         for setting in result:
             assert setting.section == section
 
     def test_repo_uis_filtered_by_key_are_returned(
             self, repo_stub, settings_util):
         for section, key in VcsSettingsModel.HOOKS_SETTINGS:
             settings_util.create_repo_rhodecode_ui(
                 repo_stub, section, 'repo', key=key, active=False)
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         section, key = VcsSettingsModel.HOOKS_SETTINGS[0]
         result = model.get_repo_ui_settings(key=key)
         for setting in result:
             assert setting.key == key
 
     def test_raises_exception_when_repository_is_not_specified(self):
         model = VcsSettingsModel()
         with pytest.raises(Exception) as exc_info:
             model.get_repo_ui_settings()
         assert str(exc_info.value) == 'Repository is not specified'
 
 
 class TestGetRepoGeneralSettings(object):
     def test_global_settings_are_returned_when_no_repo_settings_found(
             self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         result = model.get_repo_general_settings()
         expected_result = model.global_settings.get_all_settings()
         assert sorted(result) == sorted(expected_result)
 
     def test_repo_uis_are_overriding_global_uis(
             self, repo_stub, settings_util):
         for key in VcsSettingsModel.GENERAL_SETTINGS:
             settings_util.create_repo_rhodecode_setting(
                 repo_stub, key, 'abcde', type_='unicode')
             Session().commit()
 
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         result = model.get_repo_ui_settings()
         for key in result:
             if key in VcsSettingsModel.GENERAL_SETTINGS:
                 assert result[key] == 'abcde'
 
     def test_raises_exception_when_repository_is_not_specified(self):
         model = VcsSettingsModel()
         with pytest.raises(Exception) as exc_info:
             model.get_repo_general_settings()
         assert str(exc_info.value) == 'Repository is not specified'
 
 
 class TestGetGlobalGeneralSettings(object):
     def test_global_settings_are_returned(self, repo_stub):
         model = VcsSettingsModel()
         result = model.get_global_general_settings()
         expected_result = model.global_settings.get_all_settings()
         assert sorted(result) == sorted(expected_result)
 
     def test_repo_uis_are_not_overriding_global_uis(
             self, repo_stub, settings_util):
         for key in VcsSettingsModel.GENERAL_SETTINGS:
             settings_util.create_repo_rhodecode_setting(
                 repo_stub, key, 'abcde', type_='unicode')
             Session().commit()
 
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         result = model.get_global_general_settings()
         expected_result = model.global_settings.get_all_settings()
         assert sorted(result) == sorted(expected_result)
 
 
 class TestGetGlobalUiSettings(object):
     def test_global_uis_are_returned(self, repo_stub):
         model = VcsSettingsModel()
         result = model.get_global_ui_settings()
         expected_result = model.global_settings.get_ui()
         assert sorted(result) == sorted(expected_result)
 
     def test_repo_uis_are_not_overriding_global_uis(
             self, repo_stub, settings_util):
         for section, key in VcsSettingsModel.HOOKS_SETTINGS:
             settings_util.create_repo_rhodecode_ui(
                 repo_stub, section, 'repo', key=key, active=False)
             Session().commit()
 
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         result = model.get_global_ui_settings()
         expected_result = model.global_settings.get_ui()
         assert sorted(result) == sorted(expected_result)
 
     def test_ui_settings_filtered_by_section(
             self, repo_stub, settings_util):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         section, key = VcsSettingsModel.HOOKS_SETTINGS[0]
         result = model.get_global_ui_settings(section=section)
         expected_result = model.global_settings.get_ui(section=section)
         assert sorted(result) == sorted(expected_result)
 
     def test_ui_settings_filtered_by_key(
             self, repo_stub, settings_util):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         section, key = VcsSettingsModel.HOOKS_SETTINGS[0]
         result = model.get_global_ui_settings(key=key)
         expected_result = model.global_settings.get_ui(key=key)
         assert sorted(result) == sorted(expected_result)
 
 
 class TestGetGeneralSettings(object):
     def test_global_settings_are_returned_when_inherited_is_true(
             self, repo_stub, settings_util):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         model.inherit_global_settings = True
         for key in VcsSettingsModel.GENERAL_SETTINGS:
             settings_util.create_repo_rhodecode_setting(
                 repo_stub, key, 'abcde', type_='unicode')
             Session().commit()
 
         result = model.get_general_settings()
         expected_result = model.get_global_general_settings()
         assert sorted(result) == sorted(expected_result)
 
     def test_repo_settings_are_returned_when_inherited_is_false(
             self, repo_stub, settings_util):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         model.inherit_global_settings = False
         for key in VcsSettingsModel.GENERAL_SETTINGS:
             settings_util.create_repo_rhodecode_setting(
                 repo_stub, key, 'abcde', type_='unicode')
             Session().commit()
 
         result = model.get_general_settings()
         expected_result = model.get_repo_general_settings()
         assert sorted(result) == sorted(expected_result)
 
     def test_global_settings_are_returned_when_no_repository_specified(self):
         model = VcsSettingsModel()
         result = model.get_general_settings()
         expected_result = model.get_global_general_settings()
         assert sorted(result) == sorted(expected_result)
 
 
 class TestGetUiSettings(object):
     def test_global_settings_are_returned_when_inherited_is_true(
             self, repo_stub, settings_util):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         model.inherit_global_settings = True
         for section, key in VcsSettingsModel.HOOKS_SETTINGS:
             settings_util.create_repo_rhodecode_ui(
                 repo_stub, section, 'repo', key=key, active=True)
             Session().commit()
 
         result = model.get_ui_settings()
         expected_result = model.get_global_ui_settings()
         assert sorted(result) == sorted(expected_result)
 
     def test_repo_settings_are_returned_when_inherited_is_false(
             self, repo_stub, settings_util):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         model.inherit_global_settings = False
         for section, key in VcsSettingsModel.HOOKS_SETTINGS:
             settings_util.create_repo_rhodecode_ui(
                 repo_stub, section, 'repo', key=key, active=True)
             Session().commit()
 
         result = model.get_ui_settings()
         expected_result = model.get_repo_ui_settings()
         assert sorted(result) == sorted(expected_result)
 
     def test_repo_settings_filtered_by_section_and_key(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         model.inherit_global_settings = False
 
         args = ('section', 'key')
         with mock.patch.object(model, 'get_repo_ui_settings') as settings_mock:
             model.get_ui_settings(*args)
         Session().commit()
 
         settings_mock.assert_called_once_with(*args)
 
     def test_global_settings_filtered_by_section_and_key(self):
         model = VcsSettingsModel()
         args = ('section', 'key')
         with mock.patch.object(model, 'get_global_ui_settings') as (
                 settings_mock):
             model.get_ui_settings(*args)
         settings_mock.assert_called_once_with(*args)
 
     def test_global_settings_are_returned_when_no_repository_specified(self):
         model = VcsSettingsModel()
         result = model.get_ui_settings()
         expected_result = model.get_global_ui_settings()
         assert sorted(result) == sorted(expected_result)
 
 
 class TestGetSvnPatterns(object):
     def test_repo_settings_filtered_by_section_and_key(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub.repo_name)
         args = ('section', )
         with mock.patch.object(model, 'get_repo_ui_settings') as settings_mock:
             model.get_svn_patterns(*args)
 
         Session().commit()
         settings_mock.assert_called_once_with(*args)
 
     def test_global_settings_filtered_by_section_and_key(self):
         model = VcsSettingsModel()
         args = ('section', )
         with mock.patch.object(model, 'get_global_ui_settings') as (
                 settings_mock):
             model.get_svn_patterns(*args)
         settings_mock.assert_called_once_with(*args)
 
 
 class TestCreateOrUpdateRepoSettings(object):
     FORM_DATA = {
         'inherit_global_settings': False,
         'hooks_changegroup_repo_size': False,
         'hooks_changegroup_push_logger': False,
         'hooks_outgoing_pull_logger': False,
         'extensions_largefiles': False,
         'extensions_evolve': False,
-        'largefiles_usercache': '/example/largefiles-store',
         'vcs_git_lfs_enabled': False,
-        'vcs_git_lfs_store_location': '/',
         'phases_publish': 'False',
         'rhodecode_pr_merge_enabled': False,
         'rhodecode_use_outdated_comments': False,
         'new_svn_branch': '',
         'new_svn_tag': ''
     }
 
     def test_get_raises_exception_when_repository_not_specified(self):
         model = VcsSettingsModel()
         with pytest.raises(Exception) as exc_info:
             model.create_or_update_repo_settings(data=self.FORM_DATA)
             Session().commit()
 
         assert str(exc_info.value) == 'Repository is not specified'
 
     def test_only_svn_settings_are_updated_when_type_is_svn(self, backend_svn):
         repo = backend_svn.create_repo()
         model = VcsSettingsModel(repo=repo)
         with self._patch_model(model) as mocks:
             model.create_or_update_repo_settings(
                 data=self.FORM_DATA, inherit_global_settings=False)
             Session().commit()
 
         mocks['create_repo_svn_settings'].assert_called_once_with(
             self.FORM_DATA)
         non_called_methods = (
             'create_or_update_repo_hook_settings',
             'create_or_update_repo_pr_settings',
             'create_or_update_repo_hg_settings')
         for method in non_called_methods:
             assert mocks[method].call_count == 0
 
     def test_non_svn_settings_are_updated_when_type_is_hg(self, backend_hg):
         repo = backend_hg.create_repo()
         model = VcsSettingsModel(repo=repo)
         with self._patch_model(model) as mocks:
             model.create_or_update_repo_settings(
                 data=self.FORM_DATA, inherit_global_settings=False)
             Session().commit()
 
         assert mocks['create_repo_svn_settings'].call_count == 0
         called_methods = (
             'create_or_update_repo_hook_settings',
             'create_or_update_repo_pr_settings',
             'create_or_update_repo_hg_settings')
         for method in called_methods:
             mocks[method].assert_called_once_with(self.FORM_DATA)
 
     def test_non_svn_and_hg_settings_are_updated_when_type_is_git(
             self, backend_git):
         repo = backend_git.create_repo()
         model = VcsSettingsModel(repo=repo)
         with self._patch_model(model) as mocks:
             model.create_or_update_repo_settings(
                 data=self.FORM_DATA, inherit_global_settings=False)
 
         assert mocks['create_repo_svn_settings'].call_count == 0
         called_methods = (
             'create_or_update_repo_hook_settings',
             'create_or_update_repo_pr_settings')
         non_called_methods = (
             'create_repo_svn_settings',
             'create_or_update_repo_hg_settings'
         )
         for method in called_methods:
             mocks[method].assert_called_once_with(self.FORM_DATA)
         for method in non_called_methods:
             assert mocks[method].call_count == 0
 
     def test_no_methods_are_called_when_settings_are_inherited(
             self, backend):
         repo = backend.create_repo()
         model = VcsSettingsModel(repo=repo)
         with self._patch_model(model) as mocks:
             model.create_or_update_repo_settings(
                 data=self.FORM_DATA, inherit_global_settings=True)
         for method_name in mocks:
             assert mocks[method_name].call_count == 0
 
     def test_cache_is_marked_for_invalidation(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub)
         invalidation_patcher = mock.patch(
             'rhodecode.model.scm.ScmModel.mark_for_invalidation')
         with invalidation_patcher as invalidation_mock:
             model.create_or_update_repo_settings(
                 data=self.FORM_DATA, inherit_global_settings=True)
         Session().commit()
 
         invalidation_mock.assert_called_once_with(
             repo_stub.repo_name, delete=True)
 
     def test_inherit_flag_is_saved(self, repo_stub):
         model = VcsSettingsModel(repo=repo_stub)
         model.inherit_global_settings = True
         with self._patch_model(model):
             model.create_or_update_repo_settings(
                 data=self.FORM_DATA, inherit_global_settings=False)
             Session().commit()
 
         assert model.inherit_global_settings is False
 
     def _patch_model(self, model):
         return mock.patch.multiple(
             model,
             create_repo_svn_settings=mock.DEFAULT,
             create_or_update_repo_hook_settings=mock.DEFAULT,
             create_or_update_repo_pr_settings=mock.DEFAULT,
             create_or_update_repo_hg_settings=mock.DEFAULT)

rhodecode/tests/models/test_pullrequest.py

@@ -1,978 +1,978 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import mock
 import pytest
 import textwrap
 
 import rhodecode
 from rhodecode.lib.vcs.backends import get_backend
 from rhodecode.lib.vcs.backends.base import (
     MergeResponse, MergeFailureReason, Reference)
 from rhodecode.lib.vcs.exceptions import RepositoryError
 from rhodecode.lib.vcs.nodes import FileNode
 from rhodecode.model.comment import CommentsModel
 from rhodecode.model.db import PullRequest, Session
 from rhodecode.model.pull_request import PullRequestModel
 from rhodecode.model.user import UserModel
 from rhodecode.tests import TEST_USER_ADMIN_LOGIN
 from rhodecode.lib.str_utils import safe_str
 
 pytestmark = [
     pytest.mark.backends("git", "hg"),
 ]
 
 
 @pytest.mark.usefixtures('config_stub')
 class TestPullRequestModel(object):
 
     @pytest.fixture()
     def pull_request(self, request, backend, pr_util):
         """
         A pull request combined with multiples patches.
         """
         BackendClass = get_backend(backend.alias)
         merge_resp = MergeResponse(
             False, False, None, MergeFailureReason.UNKNOWN,
             metadata={'exception': 'MockError'})
         self.merge_patcher = mock.patch.object(
             BackendClass, 'merge', return_value=merge_resp)
         self.workspace_remove_patcher = mock.patch.object(
             BackendClass, 'cleanup_merge_workspace')
 
         self.workspace_remove_mock = self.workspace_remove_patcher.start()
         self.merge_mock = self.merge_patcher.start()
         self.comment_patcher = mock.patch(
             'rhodecode.model.changeset_status.ChangesetStatusModel.set_status')
         self.comment_patcher.start()
         self.notification_patcher = mock.patch(
             'rhodecode.model.notification.NotificationModel.create')
         self.notification_patcher.start()
         self.helper_patcher = mock.patch(
             'rhodecode.lib.helpers.route_path')
         self.helper_patcher.start()
 
         self.hook_patcher = mock.patch.object(PullRequestModel,
                                               'trigger_pull_request_hook')
         self.hook_mock = self.hook_patcher.start()
 
         self.invalidation_patcher = mock.patch(
             'rhodecode.model.pull_request.ScmModel.mark_for_invalidation')
         self.invalidation_mock = self.invalidation_patcher.start()
 
         self.pull_request = pr_util.create_pull_request(
             mergeable=True, name_suffix=u'ąć')
         self.source_commit = self.pull_request.source_ref_parts.commit_id
         self.target_commit = self.pull_request.target_ref_parts.commit_id
         self.workspace_id = 'pr-%s' % self.pull_request.pull_request_id
         self.repo_id = self.pull_request.target_repo.repo_id
 
         @request.addfinalizer
         def cleanup_pull_request():
             calls = [mock.call(
                 self.pull_request, self.pull_request.author, 'create')]
             self.hook_mock.assert_has_calls(calls)
 
             self.workspace_remove_patcher.stop()
             self.merge_patcher.stop()
             self.comment_patcher.stop()
             self.notification_patcher.stop()
             self.helper_patcher.stop()
             self.hook_patcher.stop()
             self.invalidation_patcher.stop()
 
         return self.pull_request
 
     def test_get_all(self, pull_request):
         prs = PullRequestModel().get_all(pull_request.target_repo)
         assert isinstance(prs, list)
         assert len(prs) == 1
 
     def test_count_all(self, pull_request):
         pr_count = PullRequestModel().count_all(pull_request.target_repo)
         assert pr_count == 1
 
     def test_get_awaiting_review(self, pull_request):
         prs = PullRequestModel().get_awaiting_review(pull_request.target_repo)
         assert isinstance(prs, list)
         assert len(prs) == 1
 
     def test_count_awaiting_review(self, pull_request):
         pr_count = PullRequestModel().count_awaiting_review(
             pull_request.target_repo)
         assert pr_count == 1
 
     def test_get_awaiting_my_review(self, pull_request):
         PullRequestModel().update_reviewers(
             pull_request, [(pull_request.author, ['author'], False, 'reviewer', [])],
             pull_request.author)
         Session().commit()
 
         prs = PullRequestModel().get_awaiting_my_review(
             pull_request.target_repo.repo_name, user_id=pull_request.author.user_id)
         assert isinstance(prs, list)
         assert len(prs) == 1
 
     def test_count_awaiting_my_review(self, pull_request):
         PullRequestModel().update_reviewers(
             pull_request, [(pull_request.author, ['author'], False, 'reviewer', [])],
             pull_request.author)
         Session().commit()
 
         pr_count = PullRequestModel().count_awaiting_my_review(
             pull_request.target_repo.repo_name, user_id=pull_request.author.user_id)
         assert pr_count == 1
 
     def test_delete_calls_cleanup_merge(self, pull_request):
         repo_id = pull_request.target_repo.repo_id
         PullRequestModel().delete(pull_request, pull_request.author)
         Session().commit()
 
         self.workspace_remove_mock.assert_called_once_with(
             repo_id, self.workspace_id)
 
     def test_close_calls_cleanup_and_hook(self, pull_request):
         PullRequestModel().close_pull_request(
             pull_request, pull_request.author)
         Session().commit()
 
         repo_id = pull_request.target_repo.repo_id
 
         self.workspace_remove_mock.assert_called_once_with(
             repo_id, self.workspace_id)
         self.hook_mock.assert_called_with(
             self.pull_request, self.pull_request.author, 'close')
 
     def test_merge_status(self, pull_request):
         self.merge_mock.return_value = MergeResponse(
             True, False, None, MergeFailureReason.NONE)
 
         assert pull_request._last_merge_source_rev is None
         assert pull_request._last_merge_target_rev is None
         assert pull_request.last_merge_status is None
 
         merge_response, status, msg = PullRequestModel().merge_status(pull_request)
         assert status is True
         assert msg == 'This pull request can be automatically merged.'
         self.merge_mock.assert_called_with(
             self.repo_id, self.workspace_id,
             pull_request.target_ref_parts,
             pull_request.source_repo.scm_instance(),
             pull_request.source_ref_parts, dry_run=True,
             use_rebase=False, close_branch=False)
 
         assert pull_request._last_merge_source_rev == self.source_commit
         assert pull_request._last_merge_target_rev == self.target_commit
         assert pull_request.last_merge_status is MergeFailureReason.NONE
 
         self.merge_mock.reset_mock()
         merge_response, status, msg = PullRequestModel().merge_status(pull_request)
         assert status is True
         assert msg == 'This pull request can be automatically merged.'
         assert self.merge_mock.called is False
 
     def test_merge_status_known_failure(self, pull_request):
         self.merge_mock.return_value = MergeResponse(
             False, False, None, MergeFailureReason.MERGE_FAILED,
             metadata={'unresolved_files': 'file1'})
 
         assert pull_request._last_merge_source_rev is None
         assert pull_request._last_merge_target_rev is None
         assert pull_request.last_merge_status is None
 
         merge_response, status, msg = PullRequestModel().merge_status(pull_request)
         assert status is False
         assert msg == 'This pull request cannot be merged because of merge conflicts. file1'
         self.merge_mock.assert_called_with(
             self.repo_id, self.workspace_id,
             pull_request.target_ref_parts,
             pull_request.source_repo.scm_instance(),
             pull_request.source_ref_parts, dry_run=True,
             use_rebase=False, close_branch=False)
 
         assert pull_request._last_merge_source_rev == self.source_commit
         assert pull_request._last_merge_target_rev == self.target_commit
         assert pull_request.last_merge_status is MergeFailureReason.MERGE_FAILED
 
         self.merge_mock.reset_mock()
         merge_response, status, msg = PullRequestModel().merge_status(pull_request)
         assert status is False
         assert msg == 'This pull request cannot be merged because of merge conflicts. file1'
         assert self.merge_mock.called is False
 
     def test_merge_status_unknown_failure(self, pull_request):
         self.merge_mock.return_value = MergeResponse(
             False, False, None, MergeFailureReason.UNKNOWN,
             metadata={'exception': 'MockError'})
 
         assert pull_request._last_merge_source_rev is None
         assert pull_request._last_merge_target_rev is None
         assert pull_request.last_merge_status is None
 
         merge_response, status, msg = PullRequestModel().merge_status(pull_request)
         assert status is False
         assert msg == (
             'This pull request cannot be merged because of an unhandled exception. '
             'MockError')
         self.merge_mock.assert_called_with(
             self.repo_id, self.workspace_id,
             pull_request.target_ref_parts,
             pull_request.source_repo.scm_instance(),
             pull_request.source_ref_parts, dry_run=True,
             use_rebase=False, close_branch=False)
 
         assert pull_request._last_merge_source_rev is None
         assert pull_request._last_merge_target_rev is None
         assert pull_request.last_merge_status is None
 
         self.merge_mock.reset_mock()
         merge_response, status, msg = PullRequestModel().merge_status(pull_request)
         assert status is False
         assert msg == (
             'This pull request cannot be merged because of an unhandled exception. '
             'MockError')
         assert self.merge_mock.called is True
 
     def test_merge_status_when_target_is_locked(self, pull_request):
         pull_request.target_repo.locked = [1, u'12345.50', 'lock_web']
         merge_response, status, msg = PullRequestModel().merge_status(pull_request)
         assert status is False
         assert msg == (
             'This pull request cannot be merged because the target repository '
             'is locked by user:1.')
 
     def test_merge_status_requirements_check_target(self, pull_request):
 
         def has_largefiles(self, repo):
             return repo == pull_request.source_repo
 
         patcher = mock.patch.object(PullRequestModel, '_has_largefiles', has_largefiles)
         with patcher:
             merge_response, status, msg = PullRequestModel().merge_status(pull_request)
 
         assert status is False
         assert msg == 'Target repository large files support is disabled.'
 
     def test_merge_status_requirements_check_source(self, pull_request):
 
         def has_largefiles(self, repo):
             return repo == pull_request.target_repo
 
         patcher = mock.patch.object(PullRequestModel, '_has_largefiles', has_largefiles)
         with patcher:
             merge_response, status, msg = PullRequestModel().merge_status(pull_request)
 
         assert status is False
         assert msg == 'Source repository large files support is disabled.'
 
     def test_merge(self, pull_request, merge_extras):
         user = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)
         merge_ref = Reference(
             'type', 'name', '6126b7bfcc82ad2d3deaee22af926b082ce54cc6')
         self.merge_mock.return_value = MergeResponse(
             True, True, merge_ref, MergeFailureReason.NONE)
 
         merge_extras['repository'] = pull_request.target_repo.repo_name
         PullRequestModel().merge_repo(
             pull_request, pull_request.author, extras=merge_extras)
         Session().commit()
 
         message = (
             u'Merge pull request !{pr_id} from {source_repo} {source_ref_name}'
             u'\n\n {pr_title}'.format(
                 pr_id=pull_request.pull_request_id,
                 source_repo=safe_str(
                     pull_request.source_repo.scm_instance().name),
                 source_ref_name=pull_request.source_ref_parts.name,
                 pr_title=safe_str(pull_request.title)
             )
         )
         self.merge_mock.assert_called_with(
             self.repo_id, self.workspace_id,
             pull_request.target_ref_parts,
             pull_request.source_repo.scm_instance(),
             pull_request.source_ref_parts,
             user_name=user.short_contact, user_email=user.email, message=message,
             use_rebase=False, close_branch=False
         )
         self.invalidation_mock.assert_called_once_with(
             pull_request.target_repo.repo_name)
 
         self.hook_mock.assert_called_with(
             self.pull_request, self.pull_request.author, 'merge')
 
         pull_request = PullRequest.get(pull_request.pull_request_id)
         assert pull_request.merge_rev == '6126b7bfcc82ad2d3deaee22af926b082ce54cc6'
 
     def test_merge_with_status_lock(self, pull_request, merge_extras):
         user = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)
         merge_ref = Reference(
             'type', 'name', '6126b7bfcc82ad2d3deaee22af926b082ce54cc6')
         self.merge_mock.return_value = MergeResponse(
             True, True, merge_ref, MergeFailureReason.NONE)
 
         merge_extras['repository'] = pull_request.target_repo.repo_name
 
         with pull_request.set_state(PullRequest.STATE_UPDATING):
             assert pull_request.pull_request_state == PullRequest.STATE_UPDATING
             PullRequestModel().merge_repo(
                 pull_request, pull_request.author, extras=merge_extras)
             Session().commit()
 
         assert pull_request.pull_request_state == PullRequest.STATE_CREATED
 
         message = (
             u'Merge pull request !{pr_id} from {source_repo} {source_ref_name}'
             u'\n\n {pr_title}'.format(
                 pr_id=pull_request.pull_request_id,
                 source_repo=safe_str(
                     pull_request.source_repo.scm_instance().name),
                 source_ref_name=pull_request.source_ref_parts.name,
                 pr_title=safe_str(pull_request.title)
             )
         )
         self.merge_mock.assert_called_with(
             self.repo_id, self.workspace_id,
             pull_request.target_ref_parts,
             pull_request.source_repo.scm_instance(),
             pull_request.source_ref_parts,
             user_name=user.short_contact, user_email=user.email, message=message,
             use_rebase=False, close_branch=False
         )
         self.invalidation_mock.assert_called_once_with(
             pull_request.target_repo.repo_name)
 
         self.hook_mock.assert_called_with(
             self.pull_request, self.pull_request.author, 'merge')
 
         pull_request = PullRequest.get(pull_request.pull_request_id)
         assert pull_request.merge_rev == '6126b7bfcc82ad2d3deaee22af926b082ce54cc6'
 
     def test_merge_failed(self, pull_request, merge_extras):
         user = UserModel().get_by_username(TEST_USER_ADMIN_LOGIN)
         merge_ref = Reference(
             'type', 'name', '6126b7bfcc82ad2d3deaee22af926b082ce54cc6')
         self.merge_mock.return_value = MergeResponse(
             False, False, merge_ref, MergeFailureReason.MERGE_FAILED)
 
         merge_extras['repository'] = pull_request.target_repo.repo_name
         PullRequestModel().merge_repo(
             pull_request, pull_request.author, extras=merge_extras)
         Session().commit()
 
         message = (
             u'Merge pull request !{pr_id} from {source_repo} {source_ref_name}'
             u'\n\n {pr_title}'.format(
                 pr_id=pull_request.pull_request_id,
                 source_repo=safe_str(
                     pull_request.source_repo.scm_instance().name),
                 source_ref_name=pull_request.source_ref_parts.name,
                 pr_title=safe_str(pull_request.title)
             )
         )
         self.merge_mock.assert_called_with(
             self.repo_id, self.workspace_id,
             pull_request.target_ref_parts,
             pull_request.source_repo.scm_instance(),
             pull_request.source_ref_parts,
             user_name=user.short_contact, user_email=user.email, message=message,
             use_rebase=False, close_branch=False
         )
 
         pull_request = PullRequest.get(pull_request.pull_request_id)
         assert self.invalidation_mock.called is False
         assert pull_request.merge_rev is None
 
     def test_get_commit_ids(self, pull_request):
         # The PR has been not merged yet, so expect an exception
         with pytest.raises(ValueError):
             PullRequestModel()._get_commit_ids(pull_request)
 
         # Merge revision is in the revisions list
         pull_request.merge_rev = pull_request.revisions[0]
         commit_ids = PullRequestModel()._get_commit_ids(pull_request)
         assert commit_ids == pull_request.revisions
 
         # Merge revision is not in the revisions list
         pull_request.merge_rev = 'f000' * 10
         commit_ids = PullRequestModel()._get_commit_ids(pull_request)
         assert commit_ids == pull_request.revisions + [pull_request.merge_rev]
 
     def test_get_diff_from_pr_version(self, pull_request):
         source_repo = pull_request.source_repo
         source_ref_id = pull_request.source_ref_parts.commit_id
         target_ref_id = pull_request.target_ref_parts.commit_id
         diff = PullRequestModel()._get_diff_from_pr_or_version(
             source_repo, source_ref_id, target_ref_id,
             hide_whitespace_changes=False, diff_context=6)
         assert b'file_1' in diff.raw.tobytes()
 
     def test_generate_title_returns_unicode(self):
         title = PullRequestModel().generate_pullrequest_title(
             source='source-dummy',
             source_ref='source-ref-dummy',
             target='target-dummy',
         )
         assert type(title) == str
 
     @pytest.mark.parametrize('title, has_wip', [
         ('hello', False),
         ('hello wip', False),
         ('hello wip: xxx', False),
         ('[wip] hello', True),
         ('[wip] hello', True),
         ('wip: hello', True),
         ('wip hello', True),
 
     ])
     def test_wip_title_marker(self, pull_request, title, has_wip):
         pull_request.title = title
         assert pull_request.work_in_progress == has_wip
 
 
 @pytest.mark.usefixtures('config_stub')
 class TestIntegrationMerge(object):
     @pytest.mark.parametrize('extra_config', (
-        {'vcs.hooks.protocol': 'http', 'vcs.hooks.direct_calls': False},
+        {'vcs.hooks.protocol.v2': 'celery', 'vcs.hooks.direct_calls': False},
     ))
     def test_merge_triggers_push_hooks(
             self, pr_util, user_admin, capture_rcextensions, merge_extras,
             extra_config):
 
         pull_request = pr_util.create_pull_request(
             approved=True, mergeable=True)
         # TODO: johbo: Needed for sqlite, try to find an automatic way for it
         merge_extras['repository'] = pull_request.target_repo.repo_name
         Session().commit()
 
         with mock.patch.dict(rhodecode.CONFIG, extra_config, clear=False):
             merge_state = PullRequestModel().merge_repo(
                 pull_request, user_admin, extras=merge_extras)
             Session().commit()
 
         assert merge_state.executed
         assert '_pre_push_hook' in capture_rcextensions
         assert '_push_hook' in capture_rcextensions
 
     def test_merge_can_be_rejected_by_pre_push_hook(
             self, pr_util, user_admin, capture_rcextensions, merge_extras):
         pull_request = pr_util.create_pull_request(
             approved=True, mergeable=True)
         # TODO: johbo: Needed for sqlite, try to find an automatic way for it
         merge_extras['repository'] = pull_request.target_repo.repo_name
         Session().commit()
 
         with mock.patch('rhodecode.EXTENSIONS.PRE_PUSH_HOOK') as pre_pull:
             pre_pull.side_effect = RepositoryError("Disallow push!")
             merge_status = PullRequestModel().merge_repo(
                 pull_request, user_admin, extras=merge_extras)
             Session().commit()
 
         assert not merge_status.executed
         assert 'pre_push' not in capture_rcextensions
         assert 'post_push' not in capture_rcextensions
 
     def test_merge_fails_if_target_is_locked(
             self, pr_util, user_regular, merge_extras):
         pull_request = pr_util.create_pull_request(
             approved=True, mergeable=True)
         locked_by = [user_regular.user_id + 1, 12345.50, 'lock_web']
         pull_request.target_repo.locked = locked_by
         # TODO: johbo: Check if this can work based on the database, currently
         # all data is pre-computed, that's why just updating the DB is not
         # enough.
         merge_extras['locked_by'] = locked_by
         merge_extras['repository'] = pull_request.target_repo.repo_name
         # TODO: johbo: Needed for sqlite, try to find an automatic way for it
         Session().commit()
         merge_status = PullRequestModel().merge_repo(
             pull_request, user_regular, extras=merge_extras)
         Session().commit()
 
         assert not merge_status.executed
 
 
 @pytest.mark.parametrize('use_outdated, inlines_count, outdated_count', [
     (False, 1, 0),
     (True, 0, 1),
 ])
 def test_outdated_comments(
         pr_util, use_outdated, inlines_count, outdated_count, config_stub):
     pull_request = pr_util.create_pull_request()
     pr_util.create_inline_comment(file_path='not_in_updated_diff')
 
     with outdated_comments_patcher(use_outdated) as outdated_comment_mock:
         pr_util.add_one_commit()
         assert_inline_comments(
             pull_request, visible=inlines_count, outdated=outdated_count)
     outdated_comment_mock.assert_called_with(pull_request)
 
 
 @pytest.mark.parametrize('mr_type, expected_msg', [
     (MergeFailureReason.NONE,
      'This pull request can be automatically merged.'),
     (MergeFailureReason.UNKNOWN,
      'This pull request cannot be merged because of an unhandled exception. CRASH'),
     (MergeFailureReason.MERGE_FAILED,
      'This pull request cannot be merged because of merge conflicts. CONFLICT_FILE'),
     (MergeFailureReason.PUSH_FAILED,
      'This pull request could not be merged because push to target:`some-repo@merge_commit` failed.'),
     (MergeFailureReason.TARGET_IS_NOT_HEAD,
      'This pull request cannot be merged because the target `ref_name` is not a head.'),
     (MergeFailureReason.HG_SOURCE_HAS_MORE_BRANCHES,
      'This pull request cannot be merged because the source contains more branches than the target.'),
     (MergeFailureReason.HG_TARGET_HAS_MULTIPLE_HEADS,
      'This pull request cannot be merged because the target `ref_name` has multiple heads: `a,b,c`.'),
     (MergeFailureReason.TARGET_IS_LOCKED,
      'This pull request cannot be merged because the target repository is locked by user:123.'),
     (MergeFailureReason.MISSING_TARGET_REF,
      'This pull request cannot be merged because the target reference `ref_name` is missing.'),
     (MergeFailureReason.MISSING_SOURCE_REF,
      'This pull request cannot be merged because the source reference `ref_name` is missing.'),
     (MergeFailureReason.SUBREPO_MERGE_FAILED,
      'This pull request cannot be merged because of conflicts related to sub repositories.'),
 
 ])
 def test_merge_response_message(mr_type, expected_msg):
     merge_ref = Reference('type', 'ref_name', '6126b7bfcc82ad2d3deaee22af926b082ce54cc6')
     metadata = {
         'unresolved_files': 'CONFLICT_FILE',
         'exception': "CRASH",
         'target': 'some-repo',
         'merge_commit': 'merge_commit',
         'target_ref': merge_ref,
         'source_ref': merge_ref,
         'heads': ','.join(['a', 'b', 'c']),
         'locked_by': 'user:123'
     }
 
     merge_response = MergeResponse(True, True, merge_ref, mr_type, metadata=metadata)
     assert merge_response.merge_status_message == expected_msg
 
 
 @pytest.fixture()
 def merge_extras(user_regular):
     """
     Context for the vcs operation when running a merge.
     """
     extras = {
         'ip': '127.0.0.1',
         'username': user_regular.username,
         'user_id': user_regular.user_id,
         'action': 'push',
         'repository': 'fake_target_repo_name',
         'scm': 'git',
         'config': 'fake_config_ini_path',
         'repo_store': '',
         'make_lock': None,
         'locked_by': [None, None, None],
         'server_url': 'http://test.example.com:5000',
         'hooks': ['push', 'pull'],
         'is_shadow_repo': False,
     }
     return extras
 
 
 @pytest.mark.usefixtures('config_stub')
 class TestUpdateCommentHandling(object):
 
     @pytest.fixture(autouse=True, scope='class')
     def enable_outdated_comments(self, request, baseapp):
         config_patch = mock.patch.dict(
             'rhodecode.CONFIG', {'rhodecode_use_outdated_comments': True})
         config_patch.start()
 
         @request.addfinalizer
         def cleanup():
             config_patch.stop()
 
     def test_comment_stays_unflagged_on_unchanged_diff(self, pr_util):
         commits = [
             {'message': 'a'},
             {'message': 'b', 'added': [FileNode(b'file_b', b'test_content\n')]},
             {'message': 'c', 'added': [FileNode(b'file_c', b'test_content\n')]},
         ]
         pull_request = pr_util.create_pull_request(
             commits=commits, target_head='a', source_head='b', revisions=['b'])
         pr_util.create_inline_comment(file_path='file_b')
         pr_util.add_one_commit(head='c')
 
         assert_inline_comments(pull_request, visible=1, outdated=0)
 
     def test_comment_stays_unflagged_on_change_above(self, pr_util):
         original_content = b''.join((b'line %d\n' % x for x in range(1, 11)))
         updated_content = b'new_line_at_top\n' + original_content
         commits = [
             {'message': 'a'},
             {'message': 'b', 'added': [FileNode(b'file_b', original_content)]},
             {'message': 'c', 'changed': [FileNode(b'file_b', updated_content)]},
         ]
         pull_request = pr_util.create_pull_request(
             commits=commits, target_head='a', source_head='b', revisions=['b'])
 
         with outdated_comments_patcher():
             comment = pr_util.create_inline_comment(
                 line_no=u'n8', file_path='file_b')
             pr_util.add_one_commit(head='c')
 
         assert_inline_comments(pull_request, visible=1, outdated=0)
         assert comment.line_no == u'n9'
 
     def test_comment_stays_unflagged_on_change_below(self, pr_util):
         original_content = b''.join([b'line %d\n' % x for x in range(10)])
         updated_content = original_content + b'new_line_at_end\n'
         commits = [
             {'message': 'a'},
             {'message': 'b', 'added': [FileNode(b'file_b', original_content)]},
             {'message': 'c', 'changed': [FileNode(b'file_b', updated_content)]},
         ]
         pull_request = pr_util.create_pull_request(
             commits=commits, target_head='a', source_head='b', revisions=['b'])
         pr_util.create_inline_comment(file_path='file_b')
         pr_util.add_one_commit(head='c')
 
         assert_inline_comments(pull_request, visible=1, outdated=0)
 
     @pytest.mark.parametrize('line_no', ['n4', 'o4', 'n10', 'o9'])
     def test_comment_flagged_on_change_around_context(self, pr_util, line_no):
         base_lines = [b'line %d\n' % x for x in range(1, 13)]
         change_lines = list(base_lines)
         change_lines.insert(6, b'line 6a added\n')
 
         # Changes on the last line of sight
         update_lines = list(change_lines)
         update_lines[0] = b'line 1 changed\n'
         update_lines[-1] = b'line 12 changed\n'
 
         def file_b(lines):
             return FileNode(b'file_b', b''.join(lines))
 
         commits = [
             {'message': 'a', 'added': [file_b(base_lines)]},
             {'message': 'b', 'changed': [file_b(change_lines)]},
             {'message': 'c', 'changed': [file_b(update_lines)]},
         ]
 
         pull_request = pr_util.create_pull_request(
             commits=commits, target_head='a', source_head='b', revisions=['b'])
         pr_util.create_inline_comment(line_no=line_no, file_path='file_b')
 
         with outdated_comments_patcher():
             pr_util.add_one_commit(head='c')
             assert_inline_comments(pull_request, visible=0, outdated=1)
 
     @pytest.mark.parametrize("change, content", [
         ('changed', b'changed\n'),
         ('removed', b''),
     ], ids=['changed', b'removed'])
     def test_comment_flagged_on_change(self, pr_util, change, content):
         commits = [
             {'message': 'a'},
             {'message': 'b', 'added': [FileNode(b'file_b', b'test_content\n')]},
             {'message': 'c', change: [FileNode(b'file_b', content)]},
         ]
         pull_request = pr_util.create_pull_request(
             commits=commits, target_head='a', source_head='b', revisions=['b'])
         pr_util.create_inline_comment(file_path='file_b')
 
         with outdated_comments_patcher():
             pr_util.add_one_commit(head='c')
             assert_inline_comments(pull_request, visible=0, outdated=1)
 
 
 @pytest.mark.usefixtures('config_stub')
 class TestUpdateChangedFiles(object):
 
     def test_no_changes_on_unchanged_diff(self, pr_util):
         commits = [
             {'message': 'a'},
             {'message': 'b',
              'added': [FileNode(b'file_b', b'test_content b\n')]},
             {'message': 'c',
              'added': [FileNode(b'file_c', b'test_content c\n')]},
         ]
         # open a PR from a to b, adding file_b
         pull_request = pr_util.create_pull_request(
             commits=commits, target_head='a', source_head='b', revisions=['b'],
             name_suffix='per-file-review')
 
         # modify PR adding new file file_c
         pr_util.add_one_commit(head='c')
 
         assert_pr_file_changes(
             pull_request,
             added=['file_c'],
             modified=[],
             removed=[])
 
     def test_modify_and_undo_modification_diff(self, pr_util):
         commits = [
             {'message': 'a'},
             {'message': 'b',
              'added': [FileNode(b'file_b', b'test_content b\n')]},
             {'message': 'c',
              'changed': [FileNode(b'file_b', b'test_content b modified\n')]},
             {'message': 'd',
              'changed': [FileNode(b'file_b', b'test_content b\n')]},
         ]
         # open a PR from a to b, adding file_b
         pull_request = pr_util.create_pull_request(
             commits=commits, target_head='a', source_head='b', revisions=['b'],
             name_suffix='per-file-review')
 
         # modify PR modifying file file_b
         pr_util.add_one_commit(head='c')
 
         assert_pr_file_changes(
             pull_request,
             added=[],
             modified=['file_b'],
             removed=[])
 
         # move the head again to d, which rollbacks change,
         # meaning we should indicate no changes
         pr_util.add_one_commit(head='d')
 
         assert_pr_file_changes(
             pull_request,
             added=[],
             modified=[],
             removed=[])
 
     def test_updated_all_files_in_pr(self, pr_util):
         commits = [
             {'message': 'a'},
             {'message': 'b', 'added': [
                 FileNode(b'file_a', b'test_content a\n'),
                 FileNode(b'file_b', b'test_content b\n'),
                 FileNode(b'file_c', b'test_content c\n')]},
             {'message': 'c', 'changed': [
                 FileNode(b'file_a', b'test_content a changed\n'),
                 FileNode(b'file_b', b'test_content b changed\n'),
                 FileNode(b'file_c', b'test_content c changed\n')]},
         ]
         # open a PR from a to b, changing 3 files
         pull_request = pr_util.create_pull_request(
             commits=commits, target_head='a', source_head='b', revisions=['b'],
             name_suffix='per-file-review')
 
         pr_util.add_one_commit(head='c')
 
         assert_pr_file_changes(
             pull_request,
             added=[],
             modified=['file_a', 'file_b', 'file_c'],
             removed=[])
 
     def test_updated_and_removed_all_files_in_pr(self, pr_util):
         commits = [
             {'message': 'a'},
             {'message': 'b', 'added': [
                 FileNode(b'file_a', b'test_content a\n'),
                 FileNode(b'file_b', b'test_content b\n'),
                 FileNode(b'file_c', b'test_content c\n')]},
             {'message': 'c', 'removed': [
                 FileNode(b'file_a', b'test_content a changed\n'),
                 FileNode(b'file_b', b'test_content b changed\n'),
                 FileNode(b'file_c', b'test_content c changed\n')]},
         ]
         # open a PR from a to b, removing 3 files
         pull_request = pr_util.create_pull_request(
             commits=commits, target_head='a', source_head='b', revisions=['b'],
             name_suffix='per-file-review')
 
         pr_util.add_one_commit(head='c')
 
         assert_pr_file_changes(
             pull_request,
             added=[],
             modified=[],
             removed=['file_a', 'file_b', 'file_c'])
 
 
 def test_update_writes_snapshot_into_pull_request_version(pr_util, config_stub):
     model = PullRequestModel()
     pull_request = pr_util.create_pull_request()
     pr_util.update_source_repository()
 
     model.update_commits(pull_request, pull_request.author)
 
     # Expect that it has a version entry now
     assert len(model.get_versions(pull_request)) == 1
 
 
 def test_update_skips_new_version_if_unchanged(pr_util, config_stub):
     pull_request = pr_util.create_pull_request()
     model = PullRequestModel()
     model.update_commits(pull_request, pull_request.author)
 
     # Expect that it still has no versions
     assert len(model.get_versions(pull_request)) == 0
 
 
 def test_update_assigns_comments_to_the_new_version(pr_util, config_stub):
     model = PullRequestModel()
     pull_request = pr_util.create_pull_request()
     comment = pr_util.create_comment()
     pr_util.update_source_repository()
 
     model.update_commits(pull_request, pull_request.author)
 
     # Expect that the comment is linked to the pr version now
     assert comment.pull_request_version == model.get_versions(pull_request)[0]
 
 
 def test_update_adds_a_comment_to_the_pull_request_about_the_change(pr_util, config_stub):
     model = PullRequestModel()
     pull_request = pr_util.create_pull_request()
     pr_util.update_source_repository()
     pr_util.update_source_repository()
 
     update_response = model.update_commits(pull_request, pull_request.author)
 
     commit_id = update_response.common_ancestor_id
     # Expect to find a new comment about the change
     expected_message = textwrap.dedent(
         """\
         Pull request updated. Auto status change to |under_review|
 
         .. role:: added
         .. role:: removed
         .. parsed-literal::
 
           Changed commits:
             * :added:`1 added`
             * :removed:`0 removed`
 
           Changed files:
             * `A file_2 <#a_c-{}-92ed3b5f07b4>`_
 
         .. |under_review| replace:: *"Under Review"*"""
     ).format(commit_id[:12])
     pull_request_comments = sorted(
         pull_request.comments, key=lambda c: c.modified_at)
     update_comment = pull_request_comments[-1]
     assert update_comment.text == expected_message
 
 
 def test_create_version_from_snapshot_updates_attributes(pr_util, config_stub):
     pull_request = pr_util.create_pull_request()
 
     # Avoiding default values
     pull_request.status = PullRequest.STATUS_CLOSED
     pull_request._last_merge_source_rev = "0" * 40
     pull_request._last_merge_target_rev = "1" * 40
     pull_request.last_merge_status = 1
     pull_request.merge_rev = "2" * 40
 
     # Remember automatic values
     created_on = pull_request.created_on
     updated_on = pull_request.updated_on
 
     # Create a new version of the pull request
     version = PullRequestModel()._create_version_from_snapshot(pull_request)
 
     # Check attributes
     assert version.title == pr_util.create_parameters['title']
     assert version.description == pr_util.create_parameters['description']
     assert version.status == PullRequest.STATUS_CLOSED
 
     # versions get updated created_on
     assert version.created_on != created_on
 
     assert version.updated_on == updated_on
     assert version.user_id == pull_request.user_id
     assert version.revisions == pr_util.create_parameters['revisions']
     assert version.source_repo == pr_util.source_repository
     assert version.source_ref == pr_util.create_parameters['source_ref']
     assert version.target_repo == pr_util.target_repository
     assert version.target_ref == pr_util.create_parameters['target_ref']
     assert version._last_merge_source_rev == pull_request._last_merge_source_rev
     assert version._last_merge_target_rev == pull_request._last_merge_target_rev
     assert version.last_merge_status == pull_request.last_merge_status
     assert version.merge_rev == pull_request.merge_rev
     assert version.pull_request == pull_request
 
 
 def test_link_comments_to_version_only_updates_unlinked_comments(pr_util, config_stub):
     version1 = pr_util.create_version_of_pull_request()
     comment_linked = pr_util.create_comment(linked_to=version1)
     comment_unlinked = pr_util.create_comment()
     version2 = pr_util.create_version_of_pull_request()
 
     PullRequestModel()._link_comments_to_version(version2)
     Session().commit()
 
     # Expect that only the new comment is linked to version2
     assert (
         comment_unlinked.pull_request_version_id ==
         version2.pull_request_version_id)
     assert (
         comment_linked.pull_request_version_id ==
         version1.pull_request_version_id)
     assert (
         comment_unlinked.pull_request_version_id !=
         comment_linked.pull_request_version_id)
 
 
 def test_calculate_commits():
     old_ids = [1, 2, 3]
     new_ids = [1, 3, 4, 5]
     change = PullRequestModel()._calculate_commit_id_changes(old_ids, new_ids)
     assert change.added == [4, 5]
     assert change.common == [1, 3]
     assert change.removed == [2]
     assert change.total == [1, 3, 4, 5]
 
 
 def assert_inline_comments(pull_request, visible=None, outdated=None):
     if visible is not None:
         inline_comments = CommentsModel().get_inline_comments(
             pull_request.target_repo.repo_id, pull_request=pull_request)
         inline_cnt = len(CommentsModel().get_inline_comments_as_list(
             inline_comments))
         assert inline_cnt == visible
     if outdated is not None:
         outdated_comments = CommentsModel().get_outdated_comments(
             pull_request.target_repo.repo_id, pull_request)
         assert len(outdated_comments) == outdated
 
 
 def assert_pr_file_changes(
         pull_request, added=None, modified=None, removed=None):
     pr_versions = PullRequestModel().get_versions(pull_request)
     # always use first version, ie original PR to calculate changes
     pull_request_version = pr_versions[0]
     old_diff_data, new_diff_data = PullRequestModel()._generate_update_diffs(
         pull_request, pull_request_version)
     file_changes = PullRequestModel()._calculate_file_changes(
         old_diff_data, new_diff_data)
 
     assert added == file_changes.added, \
         'expected added:%s vs value:%s' % (added, file_changes.added)
     assert modified == file_changes.modified, \
         'expected modified:%s vs value:%s' % (modified, file_changes.modified)
     assert removed == file_changes.removed, \
         'expected removed:%s vs value:%s' % (removed, file_changes.removed)
 
 
 def outdated_comments_patcher(use_outdated=True):
     return mock.patch.object(
         CommentsModel, 'use_outdated_comments',
         return_value=use_outdated)

rhodecode/tests/rhodecode.ini

@@ -1,834 +1,881 b''
 
 ; #########################################
 ; RHODECODE COMMUNITY EDITION CONFIGURATION
 ; #########################################
 
 [DEFAULT]
 ; Debug flag sets all loggers to debug, and enables request tracking
 debug = true
 
 ; ########################################################################
 ; EMAIL CONFIGURATION
 ; These settings will be used by the RhodeCode mailing system
 ; ########################################################################
 
 ; prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
 ; email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 
 [server:main]
 ; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
 ; Host port for gunicorn are controlled by gunicorn_conf.py
 host = 127.0.0.1
 port = 10020
 
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
-; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
+; run with gunicorn  --config gunicorn_conf.py --paste rhodecode.ini
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
 
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.
 ; allows to set RhodeCode under a prefix in server.
 ; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ; And set your prefix like: `prefix = /custom_prefix`
 ; be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ; to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
 ; Each option in the app:main can be override by an environmental variable
 ;
 ;To override an option:
 ;
 ;RC_<KeyName>
 ;Everything should be uppercase, . and - should be replaced by _.
 ;For example, if you have these configuration settings:
 ;rc_cache.repo_object.backend = foo
 ;can be overridden by
 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
 
 use = egg:rhodecode-enterprise-ce
 
 ; enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 ; encryption key used to encrypt social plugin tokens,
 ; remote_urls with credentials etc, if not set it defaults to
 ; `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
 ; decryption strict mode (enabled by default). It controls if decryption raises
 ; `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
 ; Pick algorithm for encryption. Either fernet (more secure) or aes (default)
 ; fernet is safer, and we strongly recommend switching to it.
 ; Due to backward compatibility aes is used as default.
 #rhodecode.encrypted_values.algorithm = fernet
 
 ; Return gzipped responses from RhodeCode (static files/application)
 gzip_responses = false
 
 ; Auto-generate javascript routes file on startup
 generate_js_files = false
 
 ; System global default language.
 ; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ; Perform a full repository scan and import on each server start.
 ; Settings this to true could lead to very long startup time.
 startup.import_repos = true
 
 ; URL at which the application is running. This is used for Bootstrapping
 ; requests in context when no web request is available. Used in ishell, or
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
 ; Host at which the Service API is running.
 app.service_api.host = http://rhodecode.local:10020
 
 ; Secret for Service API authentication.
 app.service_api.token =
 
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
 ; Cut off limit for large diffs (size in bytes). If overall diff size on
 ; commit, or pull request exceeds this limit this diff will be displayed
 ; partially. E.g 512000 == 512Kb
 cut_off_limit_diff = 1024000
 
 ; Cut off limit for large files inside diffs (size in bytes). Each individual
 ; file inside diff which exceeds this limit will be displayed partially.
 ;  E.g 128000 == 128Kb
 cut_off_limit_file = 256000
 
 ; Use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = false
 
 ; Force https in RhodeCode, fixes https redirects, assumes it's always https.
 ; Normally this is controlled by proper flags sent from http server such as Nginx or Apache
 force_https = false
 
 ; use Strict-Transport-Security headers
 use_htsts = false
 
 ; Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ; RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ; gist URL alias, used to create nicer urls for gist. This should be an
 ; url that does rewrites to _admin/gists/{gistid}.
 ; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ; List of views (using glob pattern syntax) that AUTH TOKENS could be
 ; used for access.
 ; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ; came from the the logged in user who own this authentication token.
 ; Additionally @TOKEN syntax can be used to bound the view to specific
 ; authentication token. Such view would be only accessible when used together
 ; with this authentication token
 ; list of all views can be found under `/_admin/permissions/auth_token_access`
 ; The list should be "," separated and on a single line.
 ; Most common views to enable:
 
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
 ; Default encoding used to convert from and to unicode
 ; can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ; instance-id prefix
 ; a prefix key for this instance used for cache invalidation when running
 ; multiple instances of RhodeCode, make sure it's globally unique for
 ; all running RhodeCode instances. Leave empty if you don't use it
 instance_id =
 
 ; Fallback authentication plugin. Set this to a plugin ID to force the usage
 ; of an authentication plugin also if it is disabled by it's settings.
 ; This could be useful if you are unable to log in to the system due to broken
 ; authentication settings. Then you can enable e.g. the internal RhodeCode auth
 ; module to log in again and fix the settings.
 ; Available builtin plugin IDs (hash is part of the ID):
 ; egg:rhodecode-enterprise-ce#rhodecode
 ; egg:rhodecode-enterprise-ce#pam
 ; egg:rhodecode-enterprise-ce#ldap
 ; egg:rhodecode-enterprise-ce#jasig_cas
 ; egg:rhodecode-enterprise-ce#headers
 ; egg:rhodecode-enterprise-ce#crowd
 
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
 ; Flag to control loading of legacy plugins in py:/path format
 auth_plugin.import_legacy_plugins = true
 
 ; alternative return HTTP header for failed authentication. Default HTTP
 ; response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ; handling that causing a series of failed authentication calls.
 ; Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ; This will be served instead of default 401 on bad authentication
 auth_ret_code =
 
 ; use special detection method when serving auth_ret_code, instead of serving
 ; ret_code directly, use 401 initially (Which triggers credentials prompt)
 ; and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ; locking return code. When repository is locked return this HTTP code. 2XX
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ; Filesystem location were repositories should be stored
 repo_store.path = /var/opt/rhodecode_repo_store
 
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ; Generated license token required for EE edition license.
 ; New generated token value can be found in Admin > settings > license page.
 license_token = abra-cada-bra1-rce3
 
 ; This flag hides sensitive information on the license page such as token, and license data
 license.hide_license_info = false
 
 ; supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 
 ; supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = dev
 
 ; Display extended labs settings
 labs_settings_active = true
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ; Send email with exception details when it happens
 #exception_tracker.send_email = false
 
 ; Comma separated list of recipients for exception emails,
 ; e.g [e-mail],[e-mail]
 ; Can be left empty, then emails will be sent to ALL super-admins
 #exception_tracker.send_email_recipients =
 
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
-; File store configuration. This is used to store and serve uploaded files
-file_store.enabled = true
+; NOTE: this setting IS DEPRECATED:
+; file_store backend is always enabled
+#file_store.enabled = true
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.backend = X -> use `file_store.backend.type = filesystem_v2` instead
 ; Storage backend, available options are: local
-file_store.backend = local
+#file_store.backend = local
 
+; NOTE: this setting IS DEPRECATED:
+; file_store.storage_path = X -> use `file_store.filesystem_v2.storage_path = X` instead
 ; path to store the uploaded binaries and artifacts
-file_store.storage_path = /var/opt/rhodecode_data/file_store
+#file_store.storage_path = /var/opt/rhodecode_data/file_store
+
+; Artifacts file-store, is used to store comment attachments and artifacts uploads.
+; file_store backend type: filesystem_v1, filesystem_v2 or objectstore (s3-based) are available as options
+; filesystem_v1 is backwards compat with pre 5.1 storage changes
+; new installations should choose filesystem_v2 or objectstore (s3-based), pick filesystem when migrating from
+; previous installations to keep the artifacts without a need of migration
+file_store.backend.type = filesystem_v1
+
+; filesystem options...
+file_store.filesystem_v1.storage_path = /var/opt/rhodecode_data/test_artifacts_file_store
+
+; filesystem_v2 options...
+file_store.filesystem_v2.storage_path = /var/opt/rhodecode_data/test_artifacts_file_store_2
+file_store.filesystem_v2.shards = 8
 
+; objectstore options...
+; url for s3 compatible storage that allows to upload artifacts
+; e.g http://minio:9000
+#file_store.backend.type = objectstore
+file_store.objectstore.url = http://s3-minio:9000
+
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-file-store/shard-N based on the bucket_shards number
+file_store.objectstore.bucket = rhodecode-file-store-tests
+
+; number of sharded buckets to create to distribute archives across
+; default is 8 shards
+file_store.objectstore.bucket_shards = 8
+
+; key for s3 auth
+file_store.objectstore.key = s3admin
+
+; secret for s3 auth
+file_store.objectstore.secret = s3secret4
+
+;region for s3 storage
+file_store.objectstore.region = eu-central-1
 
 ; Redis url to acquire/check generation of archives locks
 archive_cache.locking.url = redis://redis:6379/1
 
 ; Storage backend, only 'filesystem' and 'objectstore' are available now
 archive_cache.backend.type = filesystem
 
 ; url for s3 compatible storage that allows to upload artifacts
 ; e.g http://minio:9000
 archive_cache.objectstore.url = http://s3-minio:9000
 
 ; key for s3 auth
 archive_cache.objectstore.key = key
 
 ; secret for s3 auth
 archive_cache.objectstore.secret = secret
 
 ;region for s3 storage
 archive_cache.objectstore.region = eu-central-1
 
 ; number of sharded buckets to create to distribute archives across
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8
 
 ; a top-level bucket to put all other shards in
 ; objects will be stored in rhodecode-archive-cache/shard-N based on the bucket_shards number
 archive_cache.objectstore.bucket = rhodecode-archive-cache
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.objectstore.retry = false
 
 ; number of seconds to wait for next try using retry
 archive_cache.objectstore.retry_backoff = 1
 
 ; how many tries do do a retry fetch from this backend
 archive_cache.objectstore.retry_attempts = 10
 
 ; Default is $cache_dir/archive_cache if not set
 ; Generated repo archives will be cached at this location
 ; and served from the cache during subsequent requests for the same archive of
 ; the repository. This path is important to be shared across filesystems and with
 ; RhodeCode and vcsserver
 archive_cache.filesystem.store_dir = %(here)s/rc-tests/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
 archive_cache.filesystem.cache_size_gb = 2
 
 ; Eviction policy used to clear out after cache_size_gb limit is reached
 archive_cache.filesystem.eviction_policy = least-recently-stored
 
 ; By default cache uses sharding technique, this specifies how many shards are there
 ; default is 8 shards
 archive_cache.filesystem.cache_shards = 8
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.filesystem.retry = false
 
 ; number of seconds to wait for next try using retry
 archive_cache.filesystem.retry_backoff = 1
 
 ; how many tries do do a retry fetch from this backend
 archive_cache.filesystem.retry_attempts = 10
 
 
 ; #############
 ; CELERY CONFIG
 ; #############
 
 ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
 
 use_celery = false
 
 ; path to store schedule database
 #celerybeat-schedule.path =
 
 ; connection url to the message broker (default redis)
 celery.broker_url = redis://redis:6379/8
 
 ; results backend to get results for (default redis)
 celery.result_backend = redis://redis:6379/8
 
 ; rabbitmq example
 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
 ; maximum tasks to execute before worker restart
 celery.max_tasks_per_child = 20
 
 ; tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = true
 celery.task_store_eager_result = true
 
 ; #############
 ; DOGPILE CACHE
 ; #############
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
 cache_dir = %(here)s/rc-test-data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
 ; Only supported backend is `memory_lru`
 ; *********************************************
 rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
 rc_cache.sql_cache_short.expiration_time = 0
 
 
 ; *****************************************************
 ; `cache_repo_longterm` cache for repo object instances
 ; Only supported backend is `memory_lru`
 ; *****************************************************
 rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
 ; by default we use 30 Days, cache is still invalidated on push
 rc_cache.cache_repo_longterm.expiration_time = 2592000
 ; max items in LRU cache, set to smaller number to save memory, and expire last used caches
 rc_cache.cache_repo_longterm.max_size = 10000
 
 
 ; *********************************************
 ; `cache_general` cache for general purpose use
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *********************************************
 rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_general.expiration_time = 43200
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 rc_cache.cache_general.arguments.filename = %(here)s/rc-tests/cache-backend/cache_general_db
 
 ; alternative `cache_general` redis backend with distributed lock
 #rc_cache.cache_general.backend = dogpile.cache.rc.redis
 #rc_cache.cache_general.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_general.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_general.arguments.host = localhost
 #rc_cache.cache_general.arguments.port = 6379
 #rc_cache.cache_general.arguments.db = 0
 #rc_cache.cache_general.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_general.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_general.arguments.lock_auto_renewal = true
 
 ; *************************************************
 ; `cache_perms` cache for permission tree, auth TTL
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *************************************************
 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_perms.expiration_time = 0
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 rc_cache.cache_perms.arguments.filename = %(here)s/rc-tests/cache-backend/cache_perms_db
 
 ; alternative `cache_perms` redis backend with distributed lock
 #rc_cache.cache_perms.backend = dogpile.cache.rc.redis
 #rc_cache.cache_perms.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_perms.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_perms.arguments.host = localhost
 #rc_cache.cache_perms.arguments.port = 6379
 #rc_cache.cache_perms.arguments.db = 0
 #rc_cache.cache_perms.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_perms.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_perms.arguments.lock_auto_renewal = true
 
 ; ***************************************************
 ; `cache_repo` cache for file tree, Readme, RSS FEEDS
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; ***************************************************
 rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_repo.expiration_time = 2592000
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 rc_cache.cache_repo.arguments.filename = %(here)s/rc-tests/cache-backend/cache_repo_db
 
 ; alternative `cache_repo` redis backend with distributed lock
 #rc_cache.cache_repo.backend = dogpile.cache.rc.redis
 #rc_cache.cache_repo.expiration_time = 2592000
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
 
 #rc_cache.cache_repo.arguments.host = localhost
 #rc_cache.cache_repo.arguments.port = 6379
 #rc_cache.cache_repo.arguments.db = 1
 #rc_cache.cache_repo.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_repo.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_repo.arguments.lock_auto_renewal = true
 
 ; ##############
 ; BEAKER SESSION
 ; ##############
 
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
 ; types are file, ext:redis, ext:database, ext:memcached
 ; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session
 beaker.session.type = file
 beaker.session.data_dir = %(here)s/rc-tests/data/sessions
 
 ; Redis based sessions
 #beaker.session.type = ext:redis
 #beaker.session.url = redis://redis:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = test-rc-uytcxaz
 beaker.session.lock_dir = %(here)s/rc-tests/data/sessions/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
 ; Sets session as invalid (also logging out user) if it haven not been
 ; accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 
 ; Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
 ; Set https secure cookie
 beaker.session.secure = false
 
 ; default cookie expiration time in seconds, set to `true` to set expire
 ; at browser close
 #beaker.session.cookie_expires = 3600
 
 ; #############################
 ; SEARCH INDEXING CONFIGURATION
 ; #############################
 
 ; Full text search indexer is available in rhodecode-tools under
 ; `rhodecode-tools index` command
 
 ; WHOOSH Backend, doesn't require additional services to run
 ; it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/rc-tests/data/index
 
 ; ####################
 ; CHANNELSTREAM CONFIG
 ; ####################
 
 ; channelstream enables persistent connections and live notification
 ; in the system. It's also used by the chat system
 
 channelstream.enabled = false
 
 ; server address for channelstream server on the backend
 channelstream.server = channelstream:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = ENV_GENERATED
 channelstream.history.location = %(here)s/rc-tests/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
 ; ##############################
 ; MAIN RHODECODE DATABASE CONFIG
 ; ##############################
 
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8
 ; pymysql is an alternative driver for MySQL, use in case of problems with default one
 #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
 sqlalchemy.db1.url = sqlite:///%(here)s/rc-tests/rhodecode_test.db?timeout=30
 
 ; see sqlalchemy docs for other advanced settings
 ; print the sql statements to output
 sqlalchemy.db1.echo = false
 
 ; recycle the connections after this amount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 
 ; the number of connections to keep open inside the connection pool.
 ; 0 indicates no limit
 ; the general calculus with gevent is:
 ; if your system allows 500 concurrent greenlets (max_connections) that all do database access,
 ; then increase pool size + max overflow so that they add up to 500.
 #sqlalchemy.db1.pool_size = 5
 
 ; The number of connections to allow in connection pool "overflow", that is
 ; connections that can be opened above and beyond the pool_size setting,
 ; which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 ; Connection check ping, used to detect broken database connections
 ; could be enabled to better handle cases if MySQL has gone away errors
 #sqlalchemy.db1.ping_connection = true
 
 ; ##########
 ; VCS CONFIG
 ; ##########
 vcs.server.enable = true
 vcs.server = vcsserver:10010
 
 ; Web server connectivity protocol, responsible for web based VCS operations
 ; Available protocols are:
 ; `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
 ; Push/Pull operations protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
 ; `celery` - use celery based hooks
+#DEPRECATED:vcs.hooks.protocol = http
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
 ; accessible via network.
 ; Use vcs.hooks.host = "*" to bind to current hostname (for Docker)
 vcs.hooks.host = *
 
 ; Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
 ; List of enabled VCS backends, available options are:
 ; `hg`  - mercurial
 ; `git` - git
 ; `svn` - subversion
 vcs.backends = hg, git, svn
 
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
 ; Cache flag to cache vcsserver remote calls locally
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = false
 
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
 ; Maps RhodeCode repo groups into SVN paths for Apache
 ; ####################################################
 
 ; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ; Set a numeric version for your current SVN e.g 1.8, or 1.12
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
+; Redis connection settings for svn integrations logic
+; This connection string needs to be the same on ce and vcsserver
+vcs.svn.redis_conn = redis://redis:6379/0
+
 ; Enable SVN proxy of requests over HTTP
 vcs.svn.proxy.enabled = true
 
 ; host to connect to running SVN subsystem
 vcs.svn.proxy.host = http://svn:8090
 
 ; Enable or disable the config file generation.
 svn.proxy.generate_config = false
 
 ; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
 svn.proxy.config_file_path = %(here)s/rc-tests/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
 ; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako
 #svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako
 
 ; Used as a prefix to the `Location` block in the generated config file.
 ; In most cases it should be set to `/`.
 svn.proxy.location_root = /
 
 ; Command to reload the mod dav svn configuration on change.
 ; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh
 ; Make sure user who runs RhodeCode process is allowed to reload Apache
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
 
 ; If the timeout expires before the reload command finishes, the command will
 ; be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
 ; ####################
 ; SSH Support Settings
 ; ####################
 
 ; Defines if a custom authorized_keys file should be created and written on
 ; any change user ssh keys. Setting this to false also disables possibility
 ; of adding SSH keys by users from web interface. Super admins can still
 ; manage SSH Keys.
 ssh.generate_authorized_keyfile = true
 
 ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 ssh.authorized_keys_file_path = %(here)s/rc-tests/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
 ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
-ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+#DEPRECATED: ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
+ssh.wrapper_cmd.v2 = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
 ; Enables logging, and detailed output send back to the client during SSH
 ; operations. Useful for debugging, shouldn't be used in production.
 ssh.enable_debug_logging = true
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
 ssh.executable.hg = /usr/local/bin/rhodecode_bin/vcs_bin/hg
 ssh.executable.git = /usr/local/bin/rhodecode_bin/vcs_bin/git
 ssh.executable.svn = /usr/local/bin/rhodecode_bin/vcs_bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.
 ssh.enable_ui_key_generator = true
 
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Prometheus
 #statsd.enabled = false
 #statsd.statsd_host = 0.0.0.0
 #statsd.statsd_port = 8125
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
 ; configure logging automatically at server startup set to false
 ; to use the below custom logging config.
 ; RC_LOGGING_FORMATTER
 ; RC_LOGGING_LEVEL
 ; env variables can control the settings for logging in case of autoconfigure
 
 logging.autoconfigure = false
 
 ; specify your own custom logging config file to configure logging
 #logging.logging_conf_file = /path/to/custom_logging.ini
 
 ; Dummy marker to add new entries after.
 ; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1
 
 
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
 
 [loggers]
 keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper, dogpile
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, json, color_formatter, color_formatter_sql
 
 ; #######
 ; LOGGERS
 ; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_routes]
 level = DEBUG
 handlers =
 qualname = routes.middleware
 ## "level = DEBUG" logs the route matched and routing variables.
 propagate = 1
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_dogpile]
 level = INFO
 handlers = console
 qualname = dogpile
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
 [logger_celery]
 level = DEBUG
 handlers =
 qualname = celery
 
 
 ; ########
 ; HANDLERS
 ; ########
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr, )
 level = DEBUG
 ; To enable JSON formatted logs replace 'generic/color_formatter' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 [handler_console_sql]
 ; "level = DEBUG" logs SQL queries and results.
 ; "level = INFO" logs SQL queries.
 ; "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
 args = (sys.stderr, )
 level = WARN
 ; To enable JSON formatted logs replace 'generic/color_formatter_sql' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_json]
 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
 class = rhodecode.lib._vendor.jsonlogger.JsonFormatter

setup.py

@@ -1,200 +1,201 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 # Import early to make sure things are patched up properly
 from setuptools import setup, find_packages, Extension
 
 import os
 import re
 import sys
 import pkgutil
 import platform
 import codecs
 
 import pip
 
 pip_major_version = int(pip.__version__.split(".")[0])
 if pip_major_version >= 20:
     from pip._internal.req import parse_requirements
     from pip._internal.network.session import PipSession
 elif pip_major_version >= 10:
     from pip._internal.req import parse_requirements
     from pip._internal.download import PipSession
 else:
     from pip.req import parse_requirements
     from pip.download import PipSession
 
 
 def get_package_name(req_object):
     package_name = None
     try:
         from pip._internal.req.constructors import install_req_from_parsed_requirement
     except ImportError:
         install_req_from_parsed_requirement = None
 
     # In 20.1 of pip, the requirements object changed
     if hasattr(req_object, 'req'):
         package_name = req_object.req.name
 
     if package_name is None:
         if install_req_from_parsed_requirement:
             package = install_req_from_parsed_requirement(req_object)
             package_name = package.req.name
 
     if package_name is None:
         # fallback for older pip
         package_name = re.split('===|<=|!=|==|>=|~=|<|>', req_object.requirement)[0]
 
     return package_name
 
 
 if sys.version_info < (3, 10):
     raise Exception('RhodeCode requires Python 3.10 or later')
 
 here = os.path.abspath(os.path.dirname(__file__))
 
 # defines current platform
 __platform__ = platform.system()
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'
 __url__ = 'https://code.rhodecode.com'
 is_windows = __platform__ in ('Windows',)
 
 
 def _get_requirements(req_filename, exclude=None, extras=None):
     extras = extras or []
     exclude = exclude or []
 
     try:
         parsed = parse_requirements(
             os.path.join(here, req_filename), session=PipSession())
     except TypeError:
         # try pip < 6.0.0, that doesn't support session
         parsed = parse_requirements(os.path.join(here, req_filename))
 
     requirements = []
     for int_req in parsed:
         req_name = get_package_name(int_req)
         if req_name not in exclude:
             requirements.append(req_name)
     return requirements + extras
 
 
 # requirements extract
 setup_requirements = ['PasteScript']
 install_requirements = _get_requirements(
     'requirements.txt', exclude=['setuptools', 'entrypoints'])
 test_requirements = _get_requirements(
     'requirements_test.txt')
 
 
 def get_version():
     here = os.path.abspath(os.path.dirname(__file__))
     ver_file = os.path.join(here, "rhodecode", "VERSION")
     with open(ver_file, "rt") as f:
         version = f.read().strip()
 
     return version
 
 
 # additional files that goes into package itself
 package_data = {
     '': ['*.txt', '*.rst'],
     'configs': ['*.ini'],
     'rhodecode': ['VERSION', 'i18n/*/LC_MESSAGES/*.mo', ],
 }
 
 description = 'Source Code Management Platform'
 keywords = ' '.join([
     'rhodecode', 'mercurial', 'git', 'svn',
     'code review',
     'repo groups', 'ldap', 'repository management', 'hgweb',
     'hgwebdir', 'gitweb', 'serving hgweb',
 ])
 
 
 # README/DESCRIPTION generation
 readme_file = 'README.rst'
 changelog_file = 'CHANGES.rst'
 try:
     long_description = codecs.open(readme_file).read() + '\n\n' + \
                        codecs.open(changelog_file).read()
 except IOError as err:
     sys.stderr.write(
         "[WARNING] Cannot find file specified as long_description (%s)\n "
         "or changelog (%s) skipping that file" % (readme_file, changelog_file))
     long_description = description
 
 
 setup(
     name='rhodecode-enterprise-ce',
     version=get_version(),
     description=description,
     long_description=long_description,
     keywords=keywords,
     license=__license__,
     author=__author__,
     author_email='[e-mail]',
     url=__url__,
     setup_requires=setup_requirements,
     install_requires=install_requirements,
     tests_require=test_requirements,
     zip_safe=False,
     packages=find_packages(exclude=["docs", "tests*"]),
     package_data=package_data,
     include_package_data=True,
     classifiers=[
         'Development Status :: 6 - Mature',
         'Environment :: Web Environment',
         'Intended Audience :: Developers',
         'Operating System :: OS Independent',
         'Topic :: Software Development :: Version Control',
         'License :: OSI Approved :: Affero GNU General Public License v3 or later (AGPLv3+)',
         'Programming Language :: Python :: 3.10',
     ],
     message_extractors={
         'rhodecode': [
             ('**.py', 'python', None),
             ('**.js', 'javascript', None),
             ('templates/**.mako', 'mako', {'input_encoding': 'utf-8'}),
             ('templates/**.html', 'mako', {'input_encoding': 'utf-8'}),
             ('public/**', 'ignore', None),
         ]
     },
 
     entry_points={
         'paste.app_factory': [
             'main=rhodecode.config.middleware:make_pyramid_app',
         ],
         'pyramid.pshell_runner': [
             'ipython = rhodecode.lib.pyramid_shell:ipython_shell_runner',
         ],
         'console_scripts': [
             'rc-setup-app=rhodecode.lib.rc_commands.setup_rc:main',
             'rc-upgrade-db=rhodecode.lib.rc_commands.upgrade_db:main',
             'rc-ishell=rhodecode.lib.rc_commands.ishell:main',
             'rc-add-artifact=rhodecode.lib.rc_commands.add_artifact:main',
+            'rc-migrate-artifact=rhodecode.lib.rc_commands.migrate_artifact:main',
             'rc-ssh-wrapper=rhodecode.apps.ssh_support.lib.ssh_wrapper_v1:main',
             'rc-ssh-wrapper-v2=rhodecode.apps.ssh_support.lib.ssh_wrapper_v2:main',
         ],
         'beaker.backends': [
             'memorylru_base=rhodecode.lib.memory_lru_dict:MemoryLRUNamespaceManagerBase',
             'memorylru_debug=rhodecode.lib.memory_lru_dict:MemoryLRUNamespaceManagerDebug'
         ]
     },
 )

configs/init.d/.readme.txt

@@ -1,1 +0,0 b''
-Example init scripts.
  No newline at end of file

configs/init.d/supervisord.conf

@@ -1,61 +0,0 b''
-; Sample supervisor RhodeCode config file.
-;
-; For more information on the config file, please see:
-; http://supervisord.org/configuration.html
-;
-; Note: shell expansion ("~" or "$HOME") is not supported.  Environment
-; variables can be expanded using this syntax: "%(ENV_HOME)s".
-
-[unix_http_server]
-file=/tmp/supervisor.sock   ; (the path to the socket file)
-;chmod=0700                 ; socket file mode (default 0700)
-;chown=nobody:nogroup       ; socket file uid:gid owner
-;username=user              ; (default is no username (open server))
-;password=123               ; (default is no password (open server))
-
-[supervisord]
-logfile=/home/ubuntu/rhodecode/supervisord.log ; (main log file;default $CWD/supervisord.log)
-logfile_maxbytes=50MB        ; (max main logfile bytes b4 rotation;default 50MB)
-logfile_backups=10           ; (num of main logfile rotation backups;default 10)
-loglevel=info                ; (log level;default info; others: debug,warn,trace)
-pidfile=/home/ubuntu/rhodecode/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
-nodaemon=true               ; (start in foreground if true;default false)
-minfds=1024                  ; (min. avail startup file descriptors;default 1024)
-minprocs=200                 ; (min. avail process descriptors;default 200)
-;umask=022                   ; (process file creation umask;default 022)
-user=ubuntu                 ; (default is current user, required if root)
-;identifier=supervisor       ; (supervisord identifier, default is 'supervisor')
-;directory=/tmp              ; (default is not to cd during start)
-;nocleanup=true              ; (don't clean up tempfiles at start;default false)
-;childlogdir=/tmp            ; ('AUTO' child log dir, default $TEMP)
-environment=HOME=/home/ubuntu,LANG=en_US.UTF-8     ; (key value pairs to add to environment)
-;strip_ansi=false            ; (strip ansi escape codes in logs; def. false)
-
-; the below section must remain in the config file for RPC
-; (supervisorctl/web interface) to work, additional interfaces may be
-; added by defining them in separate rpcinterface: sections
-[rpcinterface:supervisor]
-supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
-
-[supervisorctl]
-serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL  for a unix socket
-;username=chris              ; should be same as http_username if set
-;password=123                ; should be same as http_password if set
-
-
-; restart with supervisorctl restart rhodecode:*
-[program:rhodecode]
-numprocs = 1
-numprocs_start = 5000
-directory=/home/ubuntu/rhodecode/source
-command = /home/ubuntu/rhodecode/venv/bin/paster serve /home/ubuntu/rhodecode/source/prod.ini
-process_name = %(program_name)s_%(process_num)04d
-redirect_stderr=true
-stdout_logfile=/home/ubuntu/rhodecode/rhodecode.log
-
-[program:rhodecode_workers]
-numproces = 1
-directory = /home/ubuntu/rhodecode/source
-command = /home/ubuntu/rhodecode/venv/bin/paster celeryd /home/ubuntu/rhodecode/source/prod.ini --autoscale=10,2
-redirect_stderr=true
-stdout_logfile=/%(here)s/rhodecode_workers.log

rhodecode/apps/file_store/backends/local_store.py

@@ -1,268 +0,0 b''
-# Copyright (C) 2016-2023 RhodeCode GmbH
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License, version 3
-# (only), as published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# This program is dual-licensed. If you wish to learn more about the
-# RhodeCode Enterprise Edition, including its added features, Support services,
-# and proprietary license terms, please see https://rhodecode.com/licenses/
-
-import os
-import time
-import errno
-import hashlib
-
-from rhodecode.lib.ext_json import json
-from rhodecode.apps.file_store import utils
-from rhodecode.apps.file_store.extensions import resolve_extensions
-from rhodecode.apps.file_store.exceptions import (
-    FileNotAllowedException, FileOverSizeException)
-
-METADATA_VER = 'v1'
-
-
-def safe_make_dirs(dir_path):
-    if not os.path.exists(dir_path):
-        try:
-            os.makedirs(dir_path)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
-            return
-
-
-class LocalFileStorage(object):
-
-    @classmethod
-    def apply_counter(cls, counter, filename):
-        name_counted = '%d-%s' % (counter, filename)
-        return name_counted
-
-    @classmethod
-    def resolve_name(cls, name, directory):
-        """
-        Resolves a unique name and the correct path. If a filename
-        for that path already exists then a numeric prefix with values > 0 will be
-        added, for example test.jpg -> 1-test.jpg etc. initially file would have 0 prefix.
-
-        :param name: base name of file
-        :param directory: absolute directory path
-        """
-
-        counter = 0
-        while True:
-            name_counted = cls.apply_counter(counter, name)
-
-            # sub_store prefix to optimize disk usage, e.g some_path/ab/final_file
-            sub_store = cls._sub_store_from_filename(name_counted)
-            sub_store_path = os.path.join(directory, sub_store)
-            safe_make_dirs(sub_store_path)
-
-            path = os.path.join(sub_store_path, name_counted)
-            if not os.path.exists(path):
-                return name_counted, path
-            counter += 1
-
-    @classmethod
-    def _sub_store_from_filename(cls, filename):
-        return filename[:2]
-
-    @classmethod
-    def calculate_path_hash(cls, file_path):
-        """
-        Efficient calculation of file_path sha256 sum
-
-        :param file_path:
-        :return: sha256sum
-        """
-        digest = hashlib.sha256()
-        with open(file_path, 'rb') as f:
-            for chunk in iter(lambda: f.read(1024 * 100), b""):
-                digest.update(chunk)
-
-        return digest.hexdigest()
-
-    def __init__(self, base_path, extension_groups=None):
-
-        """
-        Local file storage
-
-        :param base_path: the absolute base path where uploads are stored
-        :param extension_groups: extensions string
-        """
-
-        extension_groups = extension_groups or ['any']
-        self.base_path = base_path
-        self.extensions = resolve_extensions([], groups=extension_groups)
-
-    def __repr__(self):
-        return f'{self.__class__}@{self.base_path}'
-
-    def store_path(self, filename):
-        """
-        Returns absolute file path of the filename, joined to the
-        base_path.
-
-        :param filename: base name of file
-        """
-        prefix_dir = ''
-        if '/' in filename:
-            prefix_dir, filename = filename.split('/')
-            sub_store = self._sub_store_from_filename(filename)
-        else:
-            sub_store = self._sub_store_from_filename(filename)
-        return os.path.join(self.base_path, prefix_dir, sub_store, filename)
-
-    def delete(self, filename):
-        """
-        Deletes the filename. Filename is resolved with the
-        absolute path based on base_path. If file does not exist,
-        returns **False**, otherwise **True**
-
-        :param filename: base name of file
-        """
-        if self.exists(filename):
-            os.remove(self.store_path(filename))
-            return True
-        return False
-
-    def exists(self, filename):
-        """
-        Checks if file exists. Resolves filename's absolute
-        path based on base_path.
-
-        :param filename: file_uid name of file, e.g 0-f62b2b2d-9708-4079-a071-ec3f958448d4.svg
-        """
-        return os.path.exists(self.store_path(filename))
-
-    def filename_allowed(self, filename, extensions=None):
-        """Checks if a filename has an allowed extension
-
-        :param filename: base name of file
-        :param extensions: iterable of extensions (or self.extensions)
-        """
-        _, ext = os.path.splitext(filename)
-        return self.extension_allowed(ext, extensions)
-
-    def extension_allowed(self, ext, extensions=None):
-        """
-        Checks if an extension is permitted. Both e.g. ".jpg" and
-        "jpg" can be passed in. Extension lookup is case-insensitive.
-
-        :param ext: extension to check
-        :param extensions: iterable of extensions to validate against (or self.extensions)
-        """
-        def normalize_ext(_ext):
-            if _ext.startswith('.'):
-                _ext = _ext[1:]
-            return _ext.lower()
-
-        extensions = extensions or self.extensions
-        if not extensions:
-            return True
-
-        ext = normalize_ext(ext)
-
-        return ext in [normalize_ext(x) for x in extensions]
-
-    def save_file(self, file_obj, filename, directory=None, extensions=None,
-                  extra_metadata=None, max_filesize=None, randomized_name=True, **kwargs):
-        """
-        Saves a file object to the uploads location.
-        Returns the resolved filename, i.e. the directory +
-        the (randomized/incremented) base name.
-
-        :param file_obj: **cgi.FieldStorage** object (or similar)
-        :param filename: original filename
-        :param directory: relative path of sub-directory
-        :param extensions: iterable of allowed extensions, if not default
-        :param max_filesize: maximum size of file that should be allowed
-        :param randomized_name: generate random generated UID or fixed based on the filename
-        :param extra_metadata: extra JSON metadata to store next to the file with .meta suffix
-
-        """
-
-        extensions = extensions or self.extensions
-
-        if not self.filename_allowed(filename, extensions):
-            raise FileNotAllowedException()
-
-        if directory:
-            dest_directory = os.path.join(self.base_path, directory)
-        else:
-            dest_directory = self.base_path
-
-        safe_make_dirs(dest_directory)
-
-        uid_filename = utils.uid_filename(filename, randomized=randomized_name)
-
-        # resolve also produces special sub-dir for file optimized store
-        filename, path = self.resolve_name(uid_filename, dest_directory)
-        stored_file_dir = os.path.dirname(path)
-
-        no_body_seek = kwargs.pop('no_body_seek', False)
-        if no_body_seek:
-            pass
-        else:
-            file_obj.seek(0)
-
-        with open(path, "wb") as dest:
-            length = 256 * 1024
-            while 1:
-                buf = file_obj.read(length)
-                if not buf:
-                    break
-                dest.write(buf)
-
-        metadata = {}
-        if extra_metadata:
-            metadata = extra_metadata
-
-        size = os.stat(path).st_size
-
-        if max_filesize and size > max_filesize:
-            # free up the copied file, and raise exc
-            os.remove(path)
-            raise FileOverSizeException()
-
-        file_hash = self.calculate_path_hash(path)
-
-        metadata.update({
-            "filename": filename,
-             "size": size,
-             "time": time.time(),
-             "sha256": file_hash,
-             "meta_ver": METADATA_VER
-        })
-
-        filename_meta = filename + '.meta'
-        with open(os.path.join(stored_file_dir, filename_meta), "wb") as dest_meta:
-            dest_meta.write(json.dumps(metadata))
-
-        if directory:
-            filename = os.path.join(directory, filename)
-
-        return filename, metadata
-
-    def get_metadata(self, filename, ignore_missing=False):
-        """
-        Reads JSON stored metadata for a file
-
-        :param filename:
-        :return:
-        """
-        filename = self.store_path(filename)
-        filename_meta = filename + '.meta'
-        if ignore_missing and not os.path.isfile(filename_meta):
-            return {}
-        with open(filename_meta, "rb") as source_meta:
-            return json.loads(source_meta.read())