fix(ssh): Added alternative SshWrapper and changes needed to support it + service api. Fixes: RCCE-6

rhodecode/api/tests/test_service_api.py

@@ -0,0 +1,55 b''
+
+# Copyright (C) 2010-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import pytest
+
+from rhodecode.api.tests.utils import (
+    build_data, api_call)
+
+
[e-mail]("app")
+class TestServiceApi:
+
+    def test_service_api_with_wrong_secret(self):
+        id, payload = build_data("wrong_api_key", 'service_get_repo_name_by_id')
+        response = api_call(self.app, payload)
+
+        assert 'Invalid API KEY' == response.json['error']
+
+    def test_service_api_with_legit_secret(self):
+        id, payload = build_data(self.app.app.config.get_settings()['app.service_api.token'],
+                                 'service_get_repo_name_by_id', repo_id='1')
+        response = api_call(self.app, payload)
+        assert not response.json['error']
+
+    def test_service_api_not_a_part_of_public_api_suggestions(self):
+        id, payload = build_data("secret", 'some_random_guess_method')
+        response = api_call(self.app, payload)
+        assert 'service_' not in response.json['error']
+
+    def test_service_get_data_for_ssh_wrapper_output(self):
+        id, payload = build_data(
+            self.app.app.config.get_settings()['app.service_api.token'],
+            'service_get_data_for_ssh_wrapper',
+            user_id=1,
+            repo_name='vcs_test_git')
+        response = api_call(self.app, payload)
+
+        assert ['branch_permissions', 'repo_permissions', 'repos_path', 'user_id', 'username']\
+               == list(response.json['result'].keys())

rhodecode/api/views/service_api.py

@@ -0,0 +1,125 b''
+# Copyright (C) 2011-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import logging
+import datetime
+from collections import defaultdict
+
+from sqlalchemy import Table
+from rhodecode.api import jsonrpc_method, SERVICE_API_IDENTIFIER
+
+
+log = logging.getLogger(__name__)
+
+
+@jsonrpc_method()
+def service_get_data_for_ssh_wrapper(request, apiuser, user_id, repo_name, key_id=None):
+    from rhodecode.model.db import User
+    from rhodecode.model.scm import ScmModel
+    from rhodecode.model.meta import raw_query_executor, Base
+
+    if key_id:
+        table = Table('user_ssh_keys', Base.metadata, autoload=False)
+        atime = datetime.datetime.utcnow()
+        stmt = (
+            table.update()
+                .where(table.c.ssh_key_id == key_id)
+                .values(accessed_on=atime)
+        )
+
+        res_count = None
+        with raw_query_executor() as session:
+            result = session.execute(stmt)
+            if result.rowcount:
+                res_count = result.rowcount
+
+        if res_count:
+            log.debug(f'Update key id:{key_id} access time')
+    db_user = User.get(user_id)
+    if not db_user:
+        return None
+    auth_user = db_user.AuthUser()
+
+    return {
+        'user_id': db_user.user_id,
+        'username': db_user.username,
+        'repo_permissions': auth_user.permissions['repositories'],
+        "branch_permissions": auth_user.get_branch_permissions(repo_name),
+        "repos_path": ScmModel().repos_path
+    }
+
+
+@jsonrpc_method()
+def service_get_repo_name_by_id(request, apiuser, repo_id):
+    from rhodecode.model.repo import RepoModel
+    by_id_match = RepoModel().get_repo_by_id(repo_id)
+    if by_id_match:
+        repo_name = by_id_match.repo_name
+        return {
+            'repo_name': repo_name
+        }
+    return None
+
+
+@jsonrpc_method()
+def service_mark_for_invalidation(request, apiuser, repo_name):
+    from rhodecode.model.scm import ScmModel
+    ScmModel().mark_for_invalidation(repo_name)
+    return {'msg': "Applied"}
+
+
+@jsonrpc_method()
+def service_config_to_hgrc(request, apiuser, cli_flags, repo_name):
+    from rhodecode.model.db import RhodeCodeUi
+    from rhodecode.model.settings import VcsSettingsModel
+
+    ui_sections = defaultdict(list)
+    ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
+
+    default_hooks = [
+        ('pretxnchangegroup.ssh_auth', 'python:vcsserver.hooks.pre_push_ssh_auth'),
+        ('pretxnchangegroup.ssh', 'python:vcsserver.hooks.pre_push_ssh'),
+        ('changegroup.ssh', 'python:vcsserver.hooks.post_push_ssh'),
+
+        ('preoutgoing.ssh', 'python:vcsserver.hooks.pre_pull_ssh'),
+        ('outgoing.ssh', 'python:vcsserver.hooks.post_pull_ssh'),
+    ]
+
+    for k, v in default_hooks:
+        ui_sections['hooks'].append((k, v))
+
+    for entry in ui:
+        if not entry.active:
+            continue
+        sec = entry.section
+        key = entry.key
+
+        if sec in cli_flags:
+            # we want only custom hooks, so we skip builtins
+            if sec == 'hooks' and key in RhodeCodeUi.HOOKS_BUILTIN:
+                continue
+
+            ui_sections[sec].append([key, entry.value])
+
+    flags = []
+    for _sec, key_val in ui_sections.items():
+        flags.append(' ')
+        flags.append(f'[{_sec}]')
+        for key, val in key_val:
+            flags.append(f'{key}= {val}')
+    return {'flags': flags}

rhodecode/apps/ssh_support/lib/ssh_wrapper_v2.py

@@ -0,0 +1,72 b''
+# Copyright (C) 2016-2023 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import sys
+import time
+import logging
+
+import click
+
+from pyramid.paster import setup_logging
+
+from rhodecode.lib.statsd_client import StatsdClient
+from .backends import SshWrapperStandalone
+from .ssh_wrapper_v1 import setup_custom_logging
+
+log = logging.getLogger(__name__)
+
+
[e-mail]()
[e-mail]('ini_path', type=click.Path(exists=True))
[e-mail](
+    '--mode', '-m', required=False, default='auto',
+    type=click.Choice(['auto', 'vcs', 'git', 'hg', 'svn', 'test']),
+    help='mode of operation')
[e-mail]('--user', help='Username for which the command will be executed')
[e-mail]('--user-id', help='User ID for which the command will be executed')
[e-mail]('--key-id', help='ID of the key from the database')
[e-mail]('--shell', '-s', is_flag=True, help='Allow Shell')
[e-mail]('--debug', is_flag=True, help='Enabled detailed output logging')
+def main(ini_path, mode, user, user_id, key_id, shell, debug):
+    setup_custom_logging(ini_path, debug)
+
+    command = os.environ.get('SSH_ORIGINAL_COMMAND', '')
+    if not command and mode not in ['test']:
+        raise ValueError(
+            'Unable to fetch SSH_ORIGINAL_COMMAND from environment.'
+            'Please make sure this is set and available during execution '
+            'of this script.')
+    connection_info = os.environ.get('SSH_CONNECTION', '')
+    time_start = time.time()
+    env = {'RC_CMD_SSH_WRAPPER': '1'}
+    statsd = StatsdClient.statsd
+    try:
+        ssh_wrapper = SshWrapperStandalone(
+            command, connection_info, mode,
+            user, user_id, key_id, shell, ini_path, env)
+    except Exception:
+        log.exception('Failed to execute SshWrapper')
+        sys.exit(-5)
+    return_code = ssh_wrapper.wrap()
+    operation_took = time.time() - time_start
+    if statsd:
+        operation_took_ms = round(1000.0 * operation_took)
+        statsd.timing("rhodecode_ssh_wrapper_timing.histogram", operation_took_ms,
+                      use_decimals=False)
+    sys.exit(return_code)

configs/development.ini

@@ -1,865 +1,871 b''
 
 ; #########################################
 ; RHODECODE COMMUNITY EDITION CONFIGURATION
 ; #########################################
 
 [DEFAULT]
 ; Debug flag sets all loggers to debug, and enables request tracking
 debug = true
 
 ; ########################################################################
 ; EMAIL CONFIGURATION
 ; These settings will be used by the RhodeCode mailing system
 ; ########################################################################
 
 ; prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
 ; email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 
 [server:main]
 ; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
 ; Host port for gunicorn are controlled by gunicorn_conf.py
 host = 127.0.0.1
 port = 10020
 
 ; ##################################################
 ; WAITRESS WSGI SERVER - Recommended for Development
 ; ##################################################
 
 ; use server type
 use = egg:waitress#main
 
 ; number of worker threads
 threads = 5
 
 ; MAX BODY SIZE 100GB
 max_request_body_size = 107374182400
 
 ; Use poll instead of select, fixes file descriptors limits problems.
 ; May not work on old windows systems.
 asyncore_use_poll = true
 
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
 ; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
 
 ; Module to use, this setting shouldn't be changed
 #use = egg:gunicorn#main
 
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.
 ; allows to set RhodeCode under a prefix in server.
 ; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ; And set your prefix like: `prefix = /custom_prefix`
 ; be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ; to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
 ; Each option in the app:main can be override by an environmental variable
 ;
 ;To override an option:
 ;
 ;RC_<KeyName>
 ;Everything should be uppercase, . and - should be replaced by _.
 ;For example, if you have these configuration settings:
 ;rc_cache.repo_object.backend = foo
 ;can be overridden by
 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
 
 use = egg:rhodecode-enterprise-ce
 
 ; enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 ; #############
 ; DEBUG OPTIONS
 ; #############
 
 pyramid.reload_templates = true
 
 # During development the we want to have the debug toolbar enabled
 pyramid.includes =
     pyramid_debugtoolbar
 
 debugtoolbar.hosts = 0.0.0.0/0
 debugtoolbar.exclude_prefixes =
     /css
     /fonts
     /images
     /js
 
 ## RHODECODE PLUGINS ##
 rhodecode.includes =
     rhodecode.api
 
 
 # api prefix url
 rhodecode.api.url = /_admin/api
 
 ; enable debug style page
 debug_style = true
 
 ; #################
 ; END DEBUG OPTIONS
 ; #################
 
 ; encryption key used to encrypt social plugin tokens,
 ; remote_urls with credentials etc, if not set it defaults to
 ; `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
 ; decryption strict mode (enabled by default). It controls if decryption raises
 ; `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
 ; Pick algorithm for encryption. Either fernet (more secure) or aes (default)
 ; fernet is safer, and we strongly recommend switching to it.
 ; Due to backward compatibility aes is used as default.
 #rhodecode.encrypted_values.algorithm = fernet
 
 ; Return gzipped responses from RhodeCode (static files/application)
 gzip_responses = false
 
 ; Auto-generate javascript routes file on startup
 generate_js_files = false
 
 ; System global default language.
 ; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ; Perform a full repository scan and import on each server start.
 ; Settings this to true could lead to very long startup time.
 startup.import_repos = false
 
 ; URL at which the application is running. This is used for Bootstrapping
 ; requests in context when no web request is available. Used in ishell, or
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
+; Host at which the Service API is running.
+app.service_api.host = http://rhodecode.local:10020
+
+; Secret for Service API authentication.
+app.service_api.token =
+
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
 ; Cut off limit for large diffs (size in bytes). If overall diff size on
 ; commit, or pull request exceeds this limit this diff will be displayed
 ; partially. E.g 512000 == 512Kb
 cut_off_limit_diff = 512000
 
 ; Cut off limit for large files inside diffs (size in bytes). Each individual
 ; file inside diff which exceeds this limit will be displayed partially.
 ;  E.g 128000 == 128Kb
 cut_off_limit_file = 128000
 
 ; Use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = true
 
 ; Force https in RhodeCode, fixes https redirects, assumes it's always https.
 ; Normally this is controlled by proper flags sent from http server such as Nginx or Apache
 force_https = false
 
 ; use Strict-Transport-Security headers
 use_htsts = false
 
 ; Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ; RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ; gist URL alias, used to create nicer urls for gist. This should be an
 ; url that does rewrites to _admin/gists/{gistid}.
 ; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ; List of views (using glob pattern syntax) that AUTH TOKENS could be
 ; used for access.
 ; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ; came from the the logged in user who own this authentication token.
 ; Additionally @TOKEN syntax can be used to bound the view to specific
 ; authentication token. Such view would be only accessible when used together
 ; with this authentication token
 ; list of all views can be found under `/_admin/permissions/auth_token_access`
 ; The list should be "," separated and on a single line.
 ; Most common views to enable:
 
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
 ; Default encoding used to convert from and to unicode
 ; can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ; instance-id prefix
 ; a prefix key for this instance used for cache invalidation when running
 ; multiple instances of RhodeCode, make sure it's globally unique for
 ; all running RhodeCode instances. Leave empty if you don't use it
 instance_id =
 
 ; Fallback authentication plugin. Set this to a plugin ID to force the usage
 ; of an authentication plugin also if it is disabled by it's settings.
 ; This could be useful if you are unable to log in to the system due to broken
 ; authentication settings. Then you can enable e.g. the internal RhodeCode auth
 ; module to log in again and fix the settings.
 ; Available builtin plugin IDs (hash is part of the ID):
 ; egg:rhodecode-enterprise-ce#rhodecode
 ; egg:rhodecode-enterprise-ce#pam
 ; egg:rhodecode-enterprise-ce#ldap
 ; egg:rhodecode-enterprise-ce#jasig_cas
 ; egg:rhodecode-enterprise-ce#headers
 ; egg:rhodecode-enterprise-ce#crowd
 
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
 ; Flag to control loading of legacy plugins in py:/path format
 auth_plugin.import_legacy_plugins = true
 
 ; alternative return HTTP header for failed authentication. Default HTTP
 ; response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ; handling that causing a series of failed authentication calls.
 ; Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ; This will be served instead of default 401 on bad authentication
 auth_ret_code =
 
 ; use special detection method when serving auth_ret_code, instead of serving
 ; ret_code directly, use 401 initially (Which triggers credentials prompt)
 ; and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ; locking return code. When repository is locked return this HTTP code. 2XX
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ; allows to change the repository location in settings page
 allow_repo_location_change = true
 
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ; Generated license token required for EE edition license.
 ; New generated token value can be found in Admin > settings > license page.
 license_token =
 
 ; This flag hides sensitive information on the license page such as token, and license data
 license.hide_license_info = false
 
 ; supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 
 ; supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = dev
 
 ; Display extended labs settings
 labs_settings_active = true
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ; Send email with exception details when it happens
 #exception_tracker.send_email = false
 
 ; Comma separated list of recipients for exception emails,
 ; e.g [e-mail],[e-mail]
 ; Can be left empty, then emails will be sent to ALL super-admins
 #exception_tracker.send_email_recipients =
 
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
 ; File store configuration. This is used to store and serve uploaded files
 file_store.enabled = true
 
 ; Storage backend, available options are: local
 file_store.backend = local
 
 ; path to store the uploaded binaries
 file_store.storage_path = %(here)s/data/file_store
 
 ; Uncomment and set this path to control settings for archive download cache.
 ; Generated repo archives will be cached at this location
 ; and served from the cache during subsequent requests for the same archive of
 ; the repository. This path is important to be shared across filesystems and with
 ; RhodeCode and vcsserver
 
 ; Default is $cache_dir/archive_cache if not set
 archive_cache.store_dir = %(here)s/data/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
 archive_cache.cache_size_gb = 10
 
 ; By default cache uses sharding technique, this specifies how many shards are there
 archive_cache.cache_shards = 10
 
 ; #############
 ; CELERY CONFIG
 ; #############
 
 ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
 
 use_celery = false
 
 ; path to store schedule database
 #celerybeat-schedule.path =
 
 ; connection url to the message broker (default redis)
 celery.broker_url = redis://redis:6379/8
 
 ; results backend to get results for (default redis)
 celery.result_backend = redis://redis:6379/8
 
 ; rabbitmq example
 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
 ; maximum tasks to execute before worker restart
 celery.max_tasks_per_child = 20
 
 ; tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = false
 
 ; #############
 ; DOGPILE CACHE
 ; #############
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
 cache_dir = %(here)s/data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
 ; Only supported backend is `memory_lru`
 ; *********************************************
 rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
 rc_cache.sql_cache_short.expiration_time = 30
 
 
 ; *****************************************************
 ; `cache_repo_longterm` cache for repo object instances
 ; Only supported backend is `memory_lru`
 ; *****************************************************
 rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
 ; by default we use 30 Days, cache is still invalidated on push
 rc_cache.cache_repo_longterm.expiration_time = 2592000
 ; max items in LRU cache, set to smaller number to save memory, and expire last used caches
 rc_cache.cache_repo_longterm.max_size = 10000
 
 
 ; *********************************************
 ; `cache_general` cache for general purpose use
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *********************************************
 rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_general.expiration_time = 43200
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_general.arguments.filename = /tmp/cache_general_db
 
 ; alternative `cache_general` redis backend with distributed lock
 #rc_cache.cache_general.backend = dogpile.cache.rc.redis
 #rc_cache.cache_general.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_general.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_general.arguments.host = localhost
 #rc_cache.cache_general.arguments.port = 6379
 #rc_cache.cache_general.arguments.db = 0
 #rc_cache.cache_general.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_general.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_general.arguments.lock_auto_renewal = true
 
 ; *************************************************
 ; `cache_perms` cache for permission tree, auth TTL
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *************************************************
 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_perms.expiration_time = 3600
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_perms.arguments.filename = /tmp/cache_perms_db
 
 ; alternative `cache_perms` redis backend with distributed lock
 #rc_cache.cache_perms.backend = dogpile.cache.rc.redis
 #rc_cache.cache_perms.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_perms.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_perms.arguments.host = localhost
 #rc_cache.cache_perms.arguments.port = 6379
 #rc_cache.cache_perms.arguments.db = 0
 #rc_cache.cache_perms.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_perms.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_perms.arguments.lock_auto_renewal = true
 
 ; ***************************************************
 ; `cache_repo` cache for file tree, Readme, RSS FEEDS
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; ***************************************************
 rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_repo.expiration_time = 2592000
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_repo.arguments.filename = /tmp/cache_repo_db
 
 ; alternative `cache_repo` redis backend with distributed lock
 #rc_cache.cache_repo.backend = dogpile.cache.rc.redis
 #rc_cache.cache_repo.expiration_time = 2592000
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
 
 #rc_cache.cache_repo.arguments.host = localhost
 #rc_cache.cache_repo.arguments.port = 6379
 #rc_cache.cache_repo.arguments.db = 1
 #rc_cache.cache_repo.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_repo.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_repo.arguments.lock_auto_renewal = true
 
 ; ##############
 ; BEAKER SESSION
 ; ##############
 
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
 ; types are file, ext:redis, ext:database, ext:memcached
 ; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session
 beaker.session.type = file
 beaker.session.data_dir = %(here)s/data/sessions
 
 ; Redis based sessions
 #beaker.session.type = ext:redis
 #beaker.session.url = redis://127.0.0.1:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = develop-rc-uytcxaz
 beaker.session.lock_dir = %(here)s/data/sessions/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
 ; Sets session as invalid (also logging out user) if it haven not been
 ; accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 
 ; Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
 ; Set https secure cookie
 beaker.session.secure = false
 
 ; default cookie expiration time in seconds, set to `true` to set expire
 ; at browser close
 #beaker.session.cookie_expires = 3600
 
 ; #############################
 ; SEARCH INDEXING CONFIGURATION
 ; #############################
 
 ; Full text search indexer is available in rhodecode-tools under
 ; `rhodecode-tools index` command
 
 ; WHOOSH Backend, doesn't require additional services to run
 ; it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
 ; ####################
 ; CHANNELSTREAM CONFIG
 ; ####################
 
 ; channelstream enables persistent connections and live notification
 ; in the system. It's also used by the chat system
 
 channelstream.enabled = false
 
 ; server address for channelstream server on the backend
 channelstream.server = 127.0.0.1:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = secret
 channelstream.history.location = %(here)s/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
 ; ##############################
 ; MAIN RHODECODE DATABASE CONFIG
 ; ##############################
 
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8
 ; pymysql is an alternative driver for MySQL, use in case of problems with default one
 #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
 sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 
 ; see sqlalchemy docs for other advanced settings
 ; print the sql statements to output
 sqlalchemy.db1.echo = false
 
 ; recycle the connections after this amount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 
 ; the number of connections to keep open inside the connection pool.
 ; 0 indicates no limit
 ; the general calculus with gevent is:
 ; if your system allows 500 concurrent greenlets (max_connections) that all do database access,
 ; then increase pool size + max overflow so that they add up to 500.
 #sqlalchemy.db1.pool_size = 5
 
 ; The number of connections to allow in connection pool "overflow", that is
 ; connections that can be opened above and beyond the pool_size setting,
 ; which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 ; Connection check ping, used to detect broken database connections
 ; could be enabled to better handle cases if MySQL has gone away errors
 #sqlalchemy.db1.ping_connection = true
 
 ; ##########
 ; VCS CONFIG
 ; ##########
 vcs.server.enable = true
 vcs.server = localhost:9900
 
 ; Web server connectivity protocol, responsible for web based VCS operations
 ; Available protocols are:
 ; `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
 ; Push/Pull operations protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
 ; accessible via network.
 ; Use vcs.hooks.host = "*" to bind to current hostname (for Docker)
 vcs.hooks.host = *
 
 ; Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
 ; List of enabled VCS backends, available options are:
 ; `hg`  - mercurial
 ; `git` - git
 ; `svn` - subversion
 vcs.backends = hg, git, svn
 
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
 ; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ; Set a numeric version for your current SVN e.g 1.8, or 1.12
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
 ; Cache flag to cache vcsserver remote calls locally
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = true
 
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
 ; Maps RhodeCode repo groups into SVN paths for Apache
 ; ####################################################
 
 ; Enable or disable the config file generation.
 svn.proxy.generate_config = false
 
 ; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
 svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
 ; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako
 #svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako
 
 ; Used as a prefix to the `Location` block in the generated config file.
 ; In most cases it should be set to `/`.
 svn.proxy.location_root = /
 
 ; Command to reload the mod dav svn configuration on change.
 ; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh
 ; Make sure user who runs RhodeCode process is allowed to reload Apache
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
 
 ; If the timeout expires before the reload command finishes, the command will
 ; be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
 ; ####################
 ; SSH Support Settings
 ; ####################
 
 ; Defines if a custom authorized_keys file should be created and written on
 ; any change user ssh keys. Setting this to false also disables possibility
 ; of adding SSH keys by users from web interface. Super admins can still
 ; manage SSH Keys.
 ssh.generate_authorized_keyfile = false
 
 ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 ssh.authorized_keys_file_path = ~/.ssh/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
 ; e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper
 ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
 ; Enables logging, and detailed output send back to the client during SSH
 ; operations. Useful for debugging, shouldn't be used in production.
 ssh.enable_debug_logging = true
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
 ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg
 ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git
 ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.
 ssh.enable_ui_key_generator = true
 
 
 ; #################
 ; APPENLIGHT CONFIG
 ; #################
 
 ; Appenlight is tailored to work with RhodeCode, see
 ; http://appenlight.rhodecode.com for details how to obtain an account
 
 ; Appenlight integration enabled
 #appenlight = false
 
 #appenlight.server_url = https://api.appenlight.com
 #appenlight.api_key = YOUR_API_KEY
 #appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
 
 ; used for JS client
 #appenlight.api_public_key = YOUR_API_PUBLIC_KEY
 
 ; TWEAK AMOUNT OF INFO SENT HERE
 
 ; enables 404 error logging (default False)
 #appenlight.report_404 = false
 
 ; time in seconds after request is considered being slow (default 1)
 #appenlight.slow_request_time = 1
 
 ; record slow requests in application
 ; (needs to be enabled for slow datastore recording and time tracking)
 #appenlight.slow_requests = true
 
 ; enable hooking to application loggers
 #appenlight.logging = true
 
 ; minimum log level for log capture
 #ppenlight.logging.level = WARNING
 
 ; send logs only from erroneous/slow requests
 ; (saves API quota for intensive logging)
 #appenlight.logging_on_error = false
 
 ; list of additional keywords that should be grabbed from environ object
 ; can be string with comma separated list of words in lowercase
 ; (by default client will always send following info:
 ; 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
 ; start with HTTP* this list be extended with additional keywords here
 #appenlight.environ_keys_whitelist =
 
 ; list of keywords that should be blanked from request object
 ; can be string with comma separated list of words in lowercase
 ; (by default client will always blank keys that contain following words
 ; 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
 ; this list be extended with additional keywords set here
 #appenlight.request_keys_blacklist =
 
 ; list of namespaces that should be ignores when gathering log entries
 ; can be string with comma separated list of namespaces
 ; (by default the client ignores own entries: appenlight_client.client)
 #appenlight.log_namespace_blacklist =
 
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Prometheus
 #statsd.enabled = false
 #statsd.statsd_host = 0.0.0.0
 #statsd.statsd_port = 8125
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
 ; configure logging automatically at server startup set to false
 ; to use the below custom logging config.
 ; RC_LOGGING_FORMATTER
 ; RC_LOGGING_LEVEL
 ; env variables can control the settings for logging in case of autoconfigure
 
 #logging.autoconfigure = true
 
 ; specify your own custom logging config file to configure logging
 #logging.logging_conf_file = /path/to/custom_logging.ini
 
 ; Dummy marker to add new entries after.
 ; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1
 
 
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
 
 [loggers]
 keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, json, color_formatter, color_formatter_sql
 
 ; #######
 ; LOGGERS
 ; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
 [logger_celery]
 level = DEBUG
 handlers =
 qualname = celery
 
 
 ; ########
 ; HANDLERS
 ; ########
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr, )
 level = DEBUG
 ; To enable JSON formatted logs replace 'generic/color_formatter' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = color_formatter
 
 [handler_console_sql]
 ; "level = DEBUG" logs SQL queries and results.
 ; "level = INFO" logs SQL queries.
 ; "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
 args = (sys.stderr, )
 level = WARN
 ; To enable JSON formatted logs replace 'generic/color_formatter_sql' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = color_formatter_sql
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_json]
 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
 class = rhodecode.lib._vendor.jsonlogger.JsonFormatter

configs/production.ini

@@ -1,816 +1,822 b''
 
 ; #########################################
 ; RHODECODE COMMUNITY EDITION CONFIGURATION
 ; #########################################
 
 [DEFAULT]
 ; Debug flag sets all loggers to debug, and enables request tracking
 debug = false
 
 ; ########################################################################
 ; EMAIL CONFIGURATION
 ; These settings will be used by the RhodeCode mailing system
 ; ########################################################################
 
 ; prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
 ; email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 
 [server:main]
 ; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
 ; Host port for gunicorn are controlled by gunicorn_conf.py
 host = 127.0.0.1
 port = 10020
 
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
 ; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
 
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.
 ; allows to set RhodeCode under a prefix in server.
 ; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ; And set your prefix like: `prefix = /custom_prefix`
 ; be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ; to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
 ; Each option in the app:main can be override by an environmental variable
 ;
 ;To override an option:
 ;
 ;RC_<KeyName>
 ;Everything should be uppercase, . and - should be replaced by _.
 ;For example, if you have these configuration settings:
 ;rc_cache.repo_object.backend = foo
 ;can be overridden by
 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
 
 use = egg:rhodecode-enterprise-ce
 
 ; enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 ; encryption key used to encrypt social plugin tokens,
 ; remote_urls with credentials etc, if not set it defaults to
 ; `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
 ; decryption strict mode (enabled by default). It controls if decryption raises
 ; `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
 ; Pick algorithm for encryption. Either fernet (more secure) or aes (default)
 ; fernet is safer, and we strongly recommend switching to it.
 ; Due to backward compatibility aes is used as default.
 #rhodecode.encrypted_values.algorithm = fernet
 
 ; Return gzipped responses from RhodeCode (static files/application)
 gzip_responses = false
 
 ; Auto-generate javascript routes file on startup
 generate_js_files = false
 
 ; System global default language.
 ; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ; Perform a full repository scan and import on each server start.
 ; Settings this to true could lead to very long startup time.
 startup.import_repos = false
 
 ; URL at which the application is running. This is used for Bootstrapping
 ; requests in context when no web request is available. Used in ishell, or
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
+; Host at which the Service API is running.
+app.service_api.host= http://rhodecode.local:10020
+
+; Secret for Service API authentication.
+app.service_api.token =
+
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
 ; Cut off limit for large diffs (size in bytes). If overall diff size on
 ; commit, or pull request exceeds this limit this diff will be displayed
 ; partially. E.g 512000 == 512Kb
 cut_off_limit_diff = 512000
 
 ; Cut off limit for large files inside diffs (size in bytes). Each individual
 ; file inside diff which exceeds this limit will be displayed partially.
 ;  E.g 128000 == 128Kb
 cut_off_limit_file = 128000
 
 ; Use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = true
 
 ; Force https in RhodeCode, fixes https redirects, assumes it's always https.
 ; Normally this is controlled by proper flags sent from http server such as Nginx or Apache
 force_https = false
 
 ; use Strict-Transport-Security headers
 use_htsts = false
 
 ; Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ; RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ; gist URL alias, used to create nicer urls for gist. This should be an
 ; url that does rewrites to _admin/gists/{gistid}.
 ; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ; List of views (using glob pattern syntax) that AUTH TOKENS could be
 ; used for access.
 ; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ; came from the the logged in user who own this authentication token.
 ; Additionally @TOKEN syntax can be used to bound the view to specific
 ; authentication token. Such view would be only accessible when used together
 ; with this authentication token
 ; list of all views can be found under `/_admin/permissions/auth_token_access`
 ; The list should be "," separated and on a single line.
 ; Most common views to enable:
 
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
 ; Default encoding used to convert from and to unicode
 ; can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ; instance-id prefix
 ; a prefix key for this instance used for cache invalidation when running
 ; multiple instances of RhodeCode, make sure it's globally unique for
 ; all running RhodeCode instances. Leave empty if you don't use it
 instance_id =
 
 ; Fallback authentication plugin. Set this to a plugin ID to force the usage
 ; of an authentication plugin also if it is disabled by it's settings.
 ; This could be useful if you are unable to log in to the system due to broken
 ; authentication settings. Then you can enable e.g. the internal RhodeCode auth
 ; module to log in again and fix the settings.
 ; Available builtin plugin IDs (hash is part of the ID):
 ; egg:rhodecode-enterprise-ce#rhodecode
 ; egg:rhodecode-enterprise-ce#pam
 ; egg:rhodecode-enterprise-ce#ldap
 ; egg:rhodecode-enterprise-ce#jasig_cas
 ; egg:rhodecode-enterprise-ce#headers
 ; egg:rhodecode-enterprise-ce#crowd
 
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
 ; Flag to control loading of legacy plugins in py:/path format
 auth_plugin.import_legacy_plugins = true
 
 ; alternative return HTTP header for failed authentication. Default HTTP
 ; response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ; handling that causing a series of failed authentication calls.
 ; Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ; This will be served instead of default 401 on bad authentication
 auth_ret_code =
 
 ; use special detection method when serving auth_ret_code, instead of serving
 ; ret_code directly, use 401 initially (Which triggers credentials prompt)
 ; and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ; locking return code. When repository is locked return this HTTP code. 2XX
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ; allows to change the repository location in settings page
 allow_repo_location_change = true
 
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ; Generated license token required for EE edition license.
 ; New generated token value can be found in Admin > settings > license page.
 license_token =
 
 ; This flag hides sensitive information on the license page such as token, and license data
 license.hide_license_info = false
 
 ; supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 
 ; supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = prod
 
 ; Display extended labs settings
 labs_settings_active = true
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ; Send email with exception details when it happens
 #exception_tracker.send_email = false
 
 ; Comma separated list of recipients for exception emails,
 ; e.g [e-mail],[e-mail]
 ; Can be left empty, then emails will be sent to ALL super-admins
 #exception_tracker.send_email_recipients =
 
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
 ; File store configuration. This is used to store and serve uploaded files
 file_store.enabled = true
 
 ; Storage backend, available options are: local
 file_store.backend = local
 
 ; path to store the uploaded binaries
 file_store.storage_path = %(here)s/data/file_store
 
 ; Uncomment and set this path to control settings for archive download cache.
 ; Generated repo archives will be cached at this location
 ; and served from the cache during subsequent requests for the same archive of
 ; the repository. This path is important to be shared across filesystems and with
 ; RhodeCode and vcsserver
 
 ; Default is $cache_dir/archive_cache if not set
 archive_cache.store_dir = %(here)s/data/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
 archive_cache.cache_size_gb = 40
 
 ; By default cache uses sharding technique, this specifies how many shards are there
 archive_cache.cache_shards = 4
 
 ; #############
 ; CELERY CONFIG
 ; #############
 
 ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
 
 use_celery = false
 
 ; path to store schedule database
 #celerybeat-schedule.path =
 
 ; connection url to the message broker (default redis)
 celery.broker_url = redis://redis:6379/8
 
 ; results backend to get results for (default redis)
 celery.result_backend = redis://redis:6379/8
 
 ; rabbitmq example
 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
 ; maximum tasks to execute before worker restart
 celery.max_tasks_per_child = 20
 
 ; tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = false
 
 ; #############
 ; DOGPILE CACHE
 ; #############
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
 cache_dir = %(here)s/data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
 ; Only supported backend is `memory_lru`
 ; *********************************************
 rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
 rc_cache.sql_cache_short.expiration_time = 30
 
 
 ; *****************************************************
 ; `cache_repo_longterm` cache for repo object instances
 ; Only supported backend is `memory_lru`
 ; *****************************************************
 rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
 ; by default we use 30 Days, cache is still invalidated on push
 rc_cache.cache_repo_longterm.expiration_time = 2592000
 ; max items in LRU cache, set to smaller number to save memory, and expire last used caches
 rc_cache.cache_repo_longterm.max_size = 10000
 
 
 ; *********************************************
 ; `cache_general` cache for general purpose use
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *********************************************
 rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_general.expiration_time = 43200
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_general.arguments.filename = /tmp/cache_general_db
 
 ; alternative `cache_general` redis backend with distributed lock
 #rc_cache.cache_general.backend = dogpile.cache.rc.redis
 #rc_cache.cache_general.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_general.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_general.arguments.host = localhost
 #rc_cache.cache_general.arguments.port = 6379
 #rc_cache.cache_general.arguments.db = 0
 #rc_cache.cache_general.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_general.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_general.arguments.lock_auto_renewal = true
 
 ; *************************************************
 ; `cache_perms` cache for permission tree, auth TTL
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *************************************************
 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_perms.expiration_time = 3600
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_perms.arguments.filename = /tmp/cache_perms_db
 
 ; alternative `cache_perms` redis backend with distributed lock
 #rc_cache.cache_perms.backend = dogpile.cache.rc.redis
 #rc_cache.cache_perms.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_perms.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_perms.arguments.host = localhost
 #rc_cache.cache_perms.arguments.port = 6379
 #rc_cache.cache_perms.arguments.db = 0
 #rc_cache.cache_perms.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_perms.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_perms.arguments.lock_auto_renewal = true
 
 ; ***************************************************
 ; `cache_repo` cache for file tree, Readme, RSS FEEDS
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; ***************************************************
 rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_repo.expiration_time = 2592000
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_repo.arguments.filename = /tmp/cache_repo_db
 
 ; alternative `cache_repo` redis backend with distributed lock
 #rc_cache.cache_repo.backend = dogpile.cache.rc.redis
 #rc_cache.cache_repo.expiration_time = 2592000
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
 
 #rc_cache.cache_repo.arguments.host = localhost
 #rc_cache.cache_repo.arguments.port = 6379
 #rc_cache.cache_repo.arguments.db = 1
 #rc_cache.cache_repo.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_repo.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_repo.arguments.lock_auto_renewal = true
 
 ; ##############
 ; BEAKER SESSION
 ; ##############
 
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
 ; types are file, ext:redis, ext:database, ext:memcached
 ; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session
 beaker.session.type = file
 beaker.session.data_dir = %(here)s/data/sessions
 
 ; Redis based sessions
 #beaker.session.type = ext:redis
 #beaker.session.url = redis://127.0.0.1:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = production-rc-uytcxaz
 beaker.session.lock_dir = %(here)s/data/sessions/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
 ; Sets session as invalid (also logging out user) if it haven not been
 ; accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 
 ; Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
 ; Set https secure cookie
 beaker.session.secure = false
 
 ; default cookie expiration time in seconds, set to `true` to set expire
 ; at browser close
 #beaker.session.cookie_expires = 3600
 
 ; #############################
 ; SEARCH INDEXING CONFIGURATION
 ; #############################
 
 ; Full text search indexer is available in rhodecode-tools under
 ; `rhodecode-tools index` command
 
 ; WHOOSH Backend, doesn't require additional services to run
 ; it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
 ; ####################
 ; CHANNELSTREAM CONFIG
 ; ####################
 
 ; channelstream enables persistent connections and live notification
 ; in the system. It's also used by the chat system
 
 channelstream.enabled = false
 
 ; server address for channelstream server on the backend
 channelstream.server = 127.0.0.1:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = secret
 channelstream.history.location = %(here)s/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
 ; ##############################
 ; MAIN RHODECODE DATABASE CONFIG
 ; ##############################
 
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8
 ; pymysql is an alternative driver for MySQL, use in case of problems with default one
 #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
 sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 
 ; see sqlalchemy docs for other advanced settings
 ; print the sql statements to output
 sqlalchemy.db1.echo = false
 
 ; recycle the connections after this amount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 
 ; the number of connections to keep open inside the connection pool.
 ; 0 indicates no limit
 ; the general calculus with gevent is:
 ; if your system allows 500 concurrent greenlets (max_connections) that all do database access,
 ; then increase pool size + max overflow so that they add up to 500.
 #sqlalchemy.db1.pool_size = 5
 
 ; The number of connections to allow in connection pool "overflow", that is
 ; connections that can be opened above and beyond the pool_size setting,
 ; which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 ; Connection check ping, used to detect broken database connections
 ; could be enabled to better handle cases if MySQL has gone away errors
 #sqlalchemy.db1.ping_connection = true
 
 ; ##########
 ; VCS CONFIG
 ; ##########
 vcs.server.enable = true
 vcs.server = localhost:9900
 
 ; Web server connectivity protocol, responsible for web based VCS operations
 ; Available protocols are:
 ; `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
 ; Push/Pull operations protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
 ; accessible via network.
 ; Use vcs.hooks.host = "*" to bind to current hostname (for Docker)
 vcs.hooks.host = *
 
 ; Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
 ; List of enabled VCS backends, available options are:
 ; `hg`  - mercurial
 ; `git` - git
 ; `svn` - subversion
 vcs.backends = hg, git, svn
 
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
 ; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ; Set a numeric version for your current SVN e.g 1.8, or 1.12
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
 ; Cache flag to cache vcsserver remote calls locally
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = true
 
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
 ; Maps RhodeCode repo groups into SVN paths for Apache
 ; ####################################################
 
 ; Enable or disable the config file generation.
 svn.proxy.generate_config = false
 
 ; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
 svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
 ; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako
 #svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako
 
 ; Used as a prefix to the `Location` block in the generated config file.
 ; In most cases it should be set to `/`.
 svn.proxy.location_root = /
 
 ; Command to reload the mod dav svn configuration on change.
 ; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh
 ; Make sure user who runs RhodeCode process is allowed to reload Apache
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
 
 ; If the timeout expires before the reload command finishes, the command will
 ; be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
 ; ####################
 ; SSH Support Settings
 ; ####################
 
 ; Defines if a custom authorized_keys file should be created and written on
 ; any change user ssh keys. Setting this to false also disables possibility
 ; of adding SSH keys by users from web interface. Super admins can still
 ; manage SSH Keys.
 ssh.generate_authorized_keyfile = false
 
 ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 ssh.authorized_keys_file_path = ~/.ssh/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
 ; e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper
 ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
 ; Enables logging, and detailed output send back to the client during SSH
 ; operations. Useful for debugging, shouldn't be used in production.
 ssh.enable_debug_logging = false
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
 ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg
 ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git
 ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.
 ssh.enable_ui_key_generator = true
 
 
 ; #################
 ; APPENLIGHT CONFIG
 ; #################
 
 ; Appenlight is tailored to work with RhodeCode, see
 ; http://appenlight.rhodecode.com for details how to obtain an account
 
 ; Appenlight integration enabled
 #appenlight = false
 
 #appenlight.server_url = https://api.appenlight.com
 #appenlight.api_key = YOUR_API_KEY
 #appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
 
 ; used for JS client
 #appenlight.api_public_key = YOUR_API_PUBLIC_KEY
 
 ; TWEAK AMOUNT OF INFO SENT HERE
 
 ; enables 404 error logging (default False)
 #appenlight.report_404 = false
 
 ; time in seconds after request is considered being slow (default 1)
 #appenlight.slow_request_time = 1
 
 ; record slow requests in application
 ; (needs to be enabled for slow datastore recording and time tracking)
 #appenlight.slow_requests = true
 
 ; enable hooking to application loggers
 #appenlight.logging = true
 
 ; minimum log level for log capture
 #ppenlight.logging.level = WARNING
 
 ; send logs only from erroneous/slow requests
 ; (saves API quota for intensive logging)
 #appenlight.logging_on_error = false
 
 ; list of additional keywords that should be grabbed from environ object
 ; can be string with comma separated list of words in lowercase
 ; (by default client will always send following info:
 ; 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
 ; start with HTTP* this list be extended with additional keywords here
 #appenlight.environ_keys_whitelist =
 
 ; list of keywords that should be blanked from request object
 ; can be string with comma separated list of words in lowercase
 ; (by default client will always blank keys that contain following words
 ; 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
 ; this list be extended with additional keywords set here
 #appenlight.request_keys_blacklist =
 
 ; list of namespaces that should be ignores when gathering log entries
 ; can be string with comma separated list of namespaces
 ; (by default the client ignores own entries: appenlight_client.client)
 #appenlight.log_namespace_blacklist =
 
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Prometheus
 #statsd.enabled = false
 #statsd.statsd_host = 0.0.0.0
 #statsd.statsd_port = 8125
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
 ; configure logging automatically at server startup set to false
 ; to use the below custom logging config.
 ; RC_LOGGING_FORMATTER
 ; RC_LOGGING_LEVEL
 ; env variables can control the settings for logging in case of autoconfigure
 
 #logging.autoconfigure = true
 
 ; specify your own custom logging config file to configure logging
 #logging.logging_conf_file = /path/to/custom_logging.ini
 
 ; Dummy marker to add new entries after.
 ; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1
 
 
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
 
 [loggers]
 keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, json, color_formatter, color_formatter_sql
 
 ; #######
 ; LOGGERS
 ; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
 [logger_celery]
 level = DEBUG
 handlers =
 qualname = celery
 
 
 ; ########
 ; HANDLERS
 ; ########
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr, )
 level = INFO
 ; To enable JSON formatted logs replace 'generic/color_formatter' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 [handler_console_sql]
 ; "level = DEBUG" logs SQL queries and results.
 ; "level = INFO" logs SQL queries.
 ; "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
 args = (sys.stderr, )
 level = WARN
 ; To enable JSON formatted logs replace 'generic/color_formatter_sql' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_json]
 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
 class = rhodecode.lib._vendor.jsonlogger.JsonFormatter

rhodecode/api/__init__.py

@@ -1,573 +1,582 b''
 # Copyright (C) 2011-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import itertools
 import logging
 import sys
 import fnmatch
 
 import decorator
 import typing
 import venusian
 from collections import OrderedDict
 
 from pyramid.exceptions import ConfigurationError
 from pyramid.renderers import render
 from pyramid.response import Response
 from pyramid.httpexceptions import HTTPNotFound
 
 from rhodecode.api.exc import (
     JSONRPCBaseError, JSONRPCError, JSONRPCForbidden, JSONRPCValidationError)
 from rhodecode.apps._base import TemplateArgs
 from rhodecode.lib.auth import AuthUser
 from rhodecode.lib.base import get_ip_addr, attach_context_attributes
 from rhodecode.lib.exc_tracking import store_exception
 from rhodecode.lib import ext_json
 from rhodecode.lib.utils2 import safe_str
 from rhodecode.lib.plugins.utils import get_plugin_settings
 from rhodecode.model.db import User, UserApiKeys
 
 log = logging.getLogger(__name__)
 
 DEFAULT_RENDERER = 'jsonrpc_renderer'
 DEFAULT_URL = '/_admin/apiv2'
+SERVICE_API_IDENTIFIER = 'service_'
 
 
 def find_methods(jsonrpc_methods, pattern):
     matches = OrderedDict()
     if not isinstance(pattern, (list, tuple)):
         pattern = [pattern]
 
     for single_pattern in pattern:
-        for method_name, method in jsonrpc_methods.items():
+        for method_name, method in filter(
+                lambda x: not x[0].startswith(SERVICE_API_IDENTIFIER), jsonrpc_methods.items()
+        ):
             if fnmatch.fnmatch(method_name, single_pattern):
                 matches[method_name] = method
     return matches
 
 
 class ExtJsonRenderer(object):
     """
     Custom renderer that makes use of our ext_json lib
 
     """
 
     def __init__(self):
         self.serializer = ext_json.formatted_json
 
     def __call__(self, info):
         """ Returns a plain JSON-encoded string with content-type
         ``application/json``. The content-type may be overridden by
         setting ``request.response.content_type``."""
 
         def _render(value, system):
             request = system.get('request')
             if request is not None:
                 response = request.response
                 ct = response.content_type
                 if ct == response.default_content_type:
                     response.content_type = 'application/json'
 
             return self.serializer(value)
 
         return _render
 
 
 def jsonrpc_response(request, result):
     rpc_id = getattr(request, 'rpc_id', None)
 
     ret_value = ''
     if rpc_id:
         ret_value = {'id': rpc_id, 'result': result, 'error': None}
 
         # fetch deprecation warnings, and store it inside results
         deprecation = getattr(request, 'rpc_deprecation', None)
         if deprecation:
             ret_value['DEPRECATION_WARNING'] = deprecation
 
     raw_body = render(DEFAULT_RENDERER, ret_value, request=request)
     content_type = 'application/json'
     content_type_header = 'Content-Type'
     headers = {
         content_type_header: content_type
     }
     return Response(
         body=raw_body,
         content_type=content_type,
         headerlist=[(k, v) for k, v in headers.items()]
     )
 
 
 def jsonrpc_error(request, message, retid=None, code: int | None = None, headers: dict | None = None):
     """
     Generate a Response object with a JSON-RPC error body
     """
     headers = headers or {}
     content_type = 'application/json'
     content_type_header = 'Content-Type'
     if content_type_header not in headers:
         headers[content_type_header] = content_type
 
     err_dict = {'id': retid, 'result': None, 'error': message}
     raw_body = render(DEFAULT_RENDERER, err_dict, request=request)
 
     return Response(
         body=raw_body,
         status=code,
         content_type=content_type,
         headerlist=[(k, v) for k, v in headers.items()]
     )
 
 
 def exception_view(exc, request):
     rpc_id = getattr(request, 'rpc_id', None)
 
     if isinstance(exc, JSONRPCError):
         fault_message = safe_str(exc)
         log.debug('json-rpc error rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, JSONRPCValidationError):
         colander_exc = exc.colander_exception
         # TODO(marcink): think maybe of nicer way to serialize errors ?
         fault_message = colander_exc.asdict()
         log.debug('json-rpc colander error rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, JSONRPCForbidden):
         fault_message = 'Access was denied to this resource.'
         log.warning('json-rpc forbidden call rpc_id:%s "%s"', rpc_id, fault_message)
     elif isinstance(exc, HTTPNotFound):
         method = request.rpc_method
         log.debug('json-rpc method `%s` not found in list of '
                   'api calls: %s, rpc_id:%s',
                   method, list(request.registry.jsonrpc_methods.keys()), rpc_id)
 
         similar = 'none'
         try:
             similar_paterns = [f'*{x}*' for x in method.split('_')]
             similar_found = find_methods(
                 request.registry.jsonrpc_methods, similar_paterns)
             similar = ', '.join(similar_found.keys()) or similar
         except Exception:
             # make the whole above block safe
             pass
 
         fault_message = f"No such method: {method}. Similar methods: {similar}"
     else:
         fault_message = 'undefined error'
         exc_info = exc.exc_info()
         store_exception(id(exc_info), exc_info, prefix='rhodecode-api')
 
         statsd = request.registry.statsd
         if statsd:
             exc_type = f"{exc.__class__.__module__}.{exc.__class__.__name__}"
             statsd.incr('rhodecode_exception_total',
                         tags=["exc_source:api", f"type:{exc_type}"])
 
     return jsonrpc_error(request, fault_message, rpc_id)
 
 
 def request_view(request):
     """
     Main request handling method. It handles all logic to call a specific
     exposed method
     """
     # cython compatible inspect
     from rhodecode.config.patches import inspect_getargspec
     inspect = inspect_getargspec()
 
     # check if we can find this session using api_key, get_by_auth_token
     # search not expired tokens only
     try:
-        api_user = User.get_by_auth_token(request.rpc_api_key)
+        if not request.rpc_method.startswith(SERVICE_API_IDENTIFIER):
+            api_user = User.get_by_auth_token(request.rpc_api_key)
 
-        if api_user is None:
-            return jsonrpc_error(
-                request, retid=request.rpc_id, message='Invalid API KEY')
+            if api_user is None:
+                return jsonrpc_error(
+                    request, retid=request.rpc_id, message='Invalid API KEY')
 
-        if not api_user.active:
-            return jsonrpc_error(
-                request, retid=request.rpc_id,
-                message='Request from this user not allowed')
+            if not api_user.active:
+                return jsonrpc_error(
+                    request, retid=request.rpc_id,
+                    message='Request from this user not allowed')
 
-        # check if we are allowed to use this IP
-        auth_u = AuthUser(
-            api_user.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr)
-        if not auth_u.ip_allowed:
-            return jsonrpc_error(
-                request, retid=request.rpc_id,
-                message='Request from IP:{} not allowed'.format(
-                    request.rpc_ip_addr))
-        else:
-            log.info('Access for IP:%s allowed', request.rpc_ip_addr)
+            # check if we are allowed to use this IP
+            auth_u = AuthUser(
+                api_user.user_id, request.rpc_api_key, ip_addr=request.rpc_ip_addr)
+            if not auth_u.ip_allowed:
+                return jsonrpc_error(
+                    request, retid=request.rpc_id,
+                    message='Request from IP:{} not allowed'.format(
+                        request.rpc_ip_addr))
+            else:
+                log.info('Access for IP:%s allowed', request.rpc_ip_addr)
+
+            # register our auth-user
+            request.rpc_user = auth_u
+            request.environ['rc_auth_user_id'] = str(auth_u.user_id)
 
-        # register our auth-user
-        request.rpc_user = auth_u
-        request.environ['rc_auth_user_id'] = str(auth_u.user_id)
+            # now check if token is valid for API
+            auth_token = request.rpc_api_key
+            token_match = api_user.authenticate_by_token(
+                auth_token, roles=[UserApiKeys.ROLE_API])
+            invalid_token = not token_match
 
-        # now check if token is valid for API
-        auth_token = request.rpc_api_key
-        token_match = api_user.authenticate_by_token(
-            auth_token, roles=[UserApiKeys.ROLE_API])
-        invalid_token = not token_match
-
-        log.debug('Checking if API KEY is valid with proper role')
-        if invalid_token:
-            return jsonrpc_error(
-                request, retid=request.rpc_id,
-                message='API KEY invalid or, has bad role for an API call')
+            log.debug('Checking if API KEY is valid with proper role')
+            if invalid_token:
+                return jsonrpc_error(
+                    request, retid=request.rpc_id,
+                    message='API KEY invalid or, has bad role for an API call')
+        else:
+            auth_u = 'service'
+            if request.rpc_api_key != request.registry.settings['app.service_api.token']:
+                raise Exception("Provided service secret is not recognized!")
 
     except Exception:
         log.exception('Error on API AUTH')
         return jsonrpc_error(
             request, retid=request.rpc_id, message='Invalid API KEY')
 
     method = request.rpc_method
     func = request.registry.jsonrpc_methods[method]
 
     # now that we have a method, add request._req_params to
     # self.kargs and dispatch control to WGIController
 
     argspec = inspect.getargspec(func)
     arglist = argspec[0]
     defs = argspec[3] or []
     defaults = [type(a) for a in defs]
     default_empty = type(NotImplemented)
 
     # kw arguments required by this method
     func_kwargs = dict(itertools.zip_longest(
         reversed(arglist), reversed(defaults), fillvalue=default_empty))
 
     # This attribute will need to be first param of a method that uses
     # api_key, which is translated to instance of user at that name
     user_var = 'apiuser'
     request_var = 'request'
 
     for arg in [user_var, request_var]:
         if arg not in arglist:
             return jsonrpc_error(
                 request,
                 retid=request.rpc_id,
                 message='This method [%s] does not support '
                         'required parameter `%s`' % (func.__name__, arg))
 
     # get our arglist and check if we provided them as args
     for arg, default in func_kwargs.items():
         if arg in [user_var, request_var]:
             # user_var and request_var are pre-hardcoded parameters and we
             # don't need to do any translation
             continue
 
         # skip the required param check if it's default value is
         # NotImplementedType (default_empty)
         if default == default_empty and arg not in request.rpc_params:
             return jsonrpc_error(
                 request,
                 retid=request.rpc_id,
                 message=('Missing non optional `%s` arg in JSON DATA' % arg)
             )
 
     # sanitize extra passed arguments
     for k in list(request.rpc_params.keys()):
         if k not in func_kwargs:
             del request.rpc_params[k]
 
     call_params = request.rpc_params
     call_params.update({
         'request': request,
         'apiuser': auth_u
     })
 
     # register some common functions for usage
-    attach_context_attributes(TemplateArgs(), request, request.rpc_user.user_id)
+    rpc_user = request.rpc_user.user_id if hasattr(request, 'rpc_user') else None
+    attach_context_attributes(TemplateArgs(), request, rpc_user)
 
     statsd = request.registry.statsd
 
     try:
         ret_value = func(**call_params)
         resp = jsonrpc_response(request, ret_value)
         if statsd:
             statsd.incr('rhodecode_api_call_success_total')
         return resp
     except JSONRPCBaseError:
         raise
     except Exception:
         log.exception('Unhandled exception occurred on api call: %s', func)
         exc_info = sys.exc_info()
         exc_id, exc_type_name = store_exception(
             id(exc_info), exc_info, prefix='rhodecode-api')
         error_headers = {
             'RhodeCode-Exception-Id': str(exc_id),
             'RhodeCode-Exception-Type': str(exc_type_name)
         }
         err_resp = jsonrpc_error(
             request, retid=request.rpc_id, message='Internal server error',
             headers=error_headers)
         if statsd:
             statsd.incr('rhodecode_api_call_fail_total')
         return err_resp
 
 
 def setup_request(request):
     """
     Parse a JSON-RPC request body. It's used inside the predicates method
     to validate and bootstrap requests for usage in rpc calls.
 
     We need to raise JSONRPCError here if we want to return some errors back to
     user.
     """
 
     log.debug('Executing setup request: %r', request)
     request.rpc_ip_addr = get_ip_addr(request.environ)
     # TODO(marcink): deprecate GET at some point
     if request.method not in ['POST', 'GET']:
         log.debug('unsupported request method "%s"', request.method)
         raise JSONRPCError(
             'unsupported request method "%s". Please use POST' % request.method)
 
     if 'CONTENT_LENGTH' not in request.environ:
         log.debug("No Content-Length")
         raise JSONRPCError("Empty body, No Content-Length in request")
 
     else:
         length = request.environ['CONTENT_LENGTH']
         log.debug('Content-Length: %s', length)
 
         if length == 0:
             log.debug("Content-Length is 0")
             raise JSONRPCError("Content-Length is 0")
 
     raw_body = request.body
     log.debug("Loading JSON body now")
     try:
         json_body = ext_json.json.loads(raw_body)
     except ValueError as e:
         # catch JSON errors Here
         raise JSONRPCError(f"JSON parse error ERR:{e} RAW:{raw_body!r}")
 
     request.rpc_id = json_body.get('id')
     request.rpc_method = json_body.get('method')
 
     # check required base parameters
     try:
         api_key = json_body.get('api_key')
         if not api_key:
             api_key = json_body.get('auth_token')
 
         if not api_key:
             raise KeyError('api_key or auth_token')
 
         # TODO(marcink): support passing in token in request header
 
         request.rpc_api_key = api_key
         request.rpc_id = json_body['id']
         request.rpc_method = json_body['method']
         request.rpc_params = json_body['args'] \
             if isinstance(json_body['args'], dict) else {}
 
         log.debug('method: %s, params: %.10240r', request.rpc_method, request.rpc_params)
     except KeyError as e:
         raise JSONRPCError(f'Incorrect JSON data. Missing {e}')
 
     log.debug('setup complete, now handling method:%s rpcid:%s',
               request.rpc_method, request.rpc_id, )
 
 
 class RoutePredicate(object):
     def __init__(self, val, config):
         self.val = val
 
     def text(self):
         return f'jsonrpc route = {self.val}'
 
     phash = text
 
     def __call__(self, info, request):
         if self.val:
             # potentially setup and bootstrap our call
             setup_request(request)
 
             # Always return True so that even if it isn't a valid RPC it
             # will fall through to the underlaying handlers like notfound_view
             return True
 
 
 class NotFoundPredicate(object):
     def __init__(self, val, config):
         self.val = val
         self.methods = config.registry.jsonrpc_methods
 
     def text(self):
         return f'jsonrpc method not found = {self.val}'
 
     phash = text
 
     def __call__(self, info, request):
         return hasattr(request, 'rpc_method')
 
 
 class MethodPredicate(object):
     def __init__(self, val, config):
         self.method = val
 
     def text(self):
         return f'jsonrpc method = {self.method}'
 
     phash = text
 
     def __call__(self, context, request):
         # we need to explicitly return False here, so pyramid doesn't try to
         # execute our view directly. We need our main handler to execute things
         return getattr(request, 'rpc_method') == self.method
 
 
 def add_jsonrpc_method(config, view, **kwargs):
     # pop the method name
     method = kwargs.pop('method', None)
 
     if method is None:
         raise ConfigurationError(
             'Cannot register a JSON-RPC method without specifying the "method"')
 
     # we define custom predicate, to enable to detect conflicting methods,
     # those predicates are kind of "translation" from the decorator variables
     # to internal predicates names
 
     kwargs['jsonrpc_method'] = method
 
     # register our view into global view store for validation
     config.registry.jsonrpc_methods[method] = view
 
     # we're using our main request_view handler, here, so each method
     # has a unified handler for itself
     config.add_view(request_view, route_name='apiv2', **kwargs)
 
 
 class jsonrpc_method(object):
     """
     decorator that works similar to @add_view_config decorator,
     but tailored for our JSON RPC
     """
 
     venusian = venusian  # for testing injection
 
     def __init__(self, method=None, **kwargs):
         self.method = method
         self.kwargs = kwargs
 
     def __call__(self, wrapped):
         kwargs = self.kwargs.copy()
         kwargs['method'] = self.method or wrapped.__name__
         depth = kwargs.pop('_depth', 0)
 
         def callback(context, name, ob):
             config = context.config.with_package(info.module)
             config.add_jsonrpc_method(view=ob, **kwargs)
 
         info = venusian.attach(wrapped, callback, category='pyramid',
                                depth=depth + 1)
         if info.scope == 'class':
             # ensure that attr is set if decorating a class method
             kwargs.setdefault('attr', wrapped.__name__)
 
         kwargs['_info'] = info.codeinfo  # fbo action_method
         return wrapped
 
 
 class jsonrpc_deprecated_method(object):
     """
     Marks method as deprecated, adds log.warning, and inject special key to
     the request variable to mark method as deprecated.
     Also injects special docstring that extract_docs will catch to mark
     method as deprecated.
 
     :param use_method: specify which method should be used instead of
         the decorated one
 
     Use like::
 
         @jsonrpc_method()
         @jsonrpc_deprecated_method(use_method='new_func', deprecated_at_version='3.0.0')
         def old_func(request, apiuser, arg1, arg2):
             ...
     """
 
     def __init__(self, use_method, deprecated_at_version):
         self.use_method = use_method
         self.deprecated_at_version = deprecated_at_version
         self.deprecated_msg = ''
 
     def __call__(self, func):
         self.deprecated_msg = 'Please use method `{method}` instead.'.format(
             method=self.use_method)
 
         docstring = """\n
         .. deprecated:: {version}
 
            {deprecation_message}
 
         {original_docstring}
         """
         func.__doc__ = docstring.format(
             version=self.deprecated_at_version,
             deprecation_message=self.deprecated_msg,
             original_docstring=func.__doc__)
         return decorator.decorator(self.__wrapper, func)
 
     def __wrapper(self, func, *fargs, **fkwargs):
         log.warning('DEPRECATED API CALL on function %s, please '
                     'use `%s` instead', func, self.use_method)
         # alter function docstring to mark as deprecated, this is picked up
         # via fabric file that generates API DOC.
         result = func(*fargs, **fkwargs)
 
         request = fargs[0]
         request.rpc_deprecation = 'DEPRECATED METHOD ' + self.deprecated_msg
         return result
 
 
 def add_api_methods(config):
     from rhodecode.api.views import (
         deprecated_api, gist_api, pull_request_api, repo_api, repo_group_api,
         server_api, search_api, testing_api, user_api, user_group_api)
 
     config.scan('rhodecode.api.views')
 
 
 def includeme(config):
     plugin_module = 'rhodecode.api'
     plugin_settings = get_plugin_settings(
         plugin_module, config.registry.settings)
 
     if not hasattr(config.registry, 'jsonrpc_methods'):
         config.registry.jsonrpc_methods = OrderedDict()
 
     # match filter by given method only
     config.add_view_predicate('jsonrpc_method', MethodPredicate)
     config.add_view_predicate('jsonrpc_method_not_found', NotFoundPredicate)
 
     config.add_renderer(DEFAULT_RENDERER, ExtJsonRenderer())
     config.add_directive('add_jsonrpc_method', add_jsonrpc_method)
 
     config.add_route_predicate(
         'jsonrpc_call', RoutePredicate)
 
     config.add_route(
         'apiv2', plugin_settings.get('url', DEFAULT_URL), jsonrpc_call=True)
 
     # register some exception handling view
     config.add_view(exception_view, context=JSONRPCBaseError)
     config.add_notfound_view(exception_view, jsonrpc_method_not_found=True)
 
     add_api_methods(config)

rhodecode/apps/ssh_support/lib/backends/__init__.py

@@ -1,263 +1,392 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import re
 import logging
 import datetime
 import configparser
 from sqlalchemy import Table
 
+from rhodecode.lib.utils import call_service_api
 from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.model.scm import ScmModel
 
 from .hg import MercurialServer
 from .git import GitServer
 from .svn import SubversionServer
 log = logging.getLogger(__name__)
 
 
 class SshWrapper(object):
     hg_cmd_pat  = re.compile(r'^hg\s+\-R\s+(\S+)\s+serve\s+\-\-stdio$')
     git_cmd_pat = re.compile(r'^git-(receive-pack|upload-pack)\s\'[/]?(\S+?)(|\.git)\'$')
     svn_cmd_pat = re.compile(r'^svnserve -t')
 
     def __init__(self, command, connection_info, mode,
                  user, user_id, key_id: int, shell, ini_path: str, env):
         self.command = command
         self.connection_info = connection_info
         self.mode = mode
         self.username = user
         self.user_id = user_id
         self.key_id = key_id
         self.shell = shell
         self.ini_path = ini_path
         self.env = env
 
         self.config = self.parse_config(ini_path)
         self.server_impl = None
 
     def parse_config(self, config_path):
         parser = configparser.ConfigParser()
         parser.read(config_path)
         return parser
 
     def update_key_access_time(self, key_id):
         from rhodecode.model.meta import raw_query_executor, Base
 
         table = Table('user_ssh_keys', Base.metadata, autoload=False)
         atime = datetime.datetime.utcnow()
         stmt = (
             table.update()
             .where(table.c.ssh_key_id == key_id)
             .values(accessed_on=atime)
             # no MySQL Support for .returning :((
             #.returning(table.c.accessed_on, table.c.ssh_key_fingerprint)
         )
 
         res_count = None
         with raw_query_executor() as session:
             result = session.execute(stmt)
             if result.rowcount:
                 res_count = result.rowcount
 
         if res_count:
             log.debug('Update key id:`%s` access time', key_id)
 
     def get_user(self, user_id):
         user = AttributeDict()
         # lazy load db imports
         from rhodecode.model.db import User
         dbuser = User.get(user_id)
         if not dbuser:
             return None
         user.user_id = dbuser.user_id
         user.username = dbuser.username
         user.auth_user = dbuser.AuthUser()
         return user
 
     def get_connection_info(self):
         """
         connection_info
 
         Identifies the client and server ends of the connection.
         The variable contains four space-separated values: client IP address,
         client port number, server IP address, and server port number.
         """
         conn = dict(
             client_ip=None,
             client_port=None,
             server_ip=None,
             server_port=None,
         )
 
         info = self.connection_info.split(' ')
         if len(info) == 4:
             conn['client_ip'] = info[0]
             conn['client_port'] = info[1]
             conn['server_ip'] = info[2]
             conn['server_port'] = info[3]
 
         return conn
 
     def maybe_translate_repo_uid(self, repo_name):
         _org_name = repo_name
         if _org_name.startswith('_'):
             # remove format of _ID/subrepo
             _org_name = _org_name.split('/', 1)[0]
 
         if repo_name.startswith('_'):
             from rhodecode.model.repo import RepoModel
             org_repo_name = repo_name
             log.debug('translating UID repo %s', org_repo_name)
             by_id_match = RepoModel().get_repo_by_id(repo_name)
             if by_id_match:
                 repo_name = by_id_match.repo_name
                 log.debug('translation of UID repo %s got `%s`', org_repo_name, repo_name)
 
         return repo_name, _org_name
 
     def get_repo_details(self, mode):
         vcs_type = mode if mode in ['svn', 'hg', 'git'] else None
         repo_name = None
 
         hg_match = self.hg_cmd_pat.match(self.command)
         if hg_match is not None:
             vcs_type = 'hg'
             repo_id = hg_match.group(1).strip('/')
             repo_name, org_name = self.maybe_translate_repo_uid(repo_id)
             return vcs_type, repo_name, mode
 
         git_match = self.git_cmd_pat.match(self.command)
         if git_match is not None:
             mode = git_match.group(1)
             vcs_type = 'git'
             repo_id = git_match.group(2).strip('/')
             repo_name, org_name = self.maybe_translate_repo_uid(repo_id)
             return vcs_type, repo_name, mode
 
         svn_match = self.svn_cmd_pat.match(self.command)
         if svn_match is not None:
             vcs_type = 'svn'
             # Repo name should be extracted from the input stream, we're unable to
             # extract it at this point in execution
             return vcs_type, repo_name, mode
 
         return vcs_type, repo_name, mode
 
     def serve(self, vcs, repo, mode, user, permissions, branch_permissions):
         store = ScmModel().repos_path
 
         check_branch_perms = False
         detect_force_push = False
 
         if branch_permissions:
             check_branch_perms = True
             detect_force_push = True
 
         log.debug(
             'VCS detected:`%s` mode: `%s` repo_name: %s, branch_permission_checks:%s',
             vcs, mode, repo, check_branch_perms)
 
         # detect if we have to check branch permissions
         extras = {
             'detect_force_push': detect_force_push,
             'check_branch_perms': check_branch_perms,
             'config': self.ini_path
         }
 
         if vcs == 'hg':
             server = MercurialServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=repo, user=user,
                 user_permissions=permissions, config=self.config, env=self.env)
             self.server_impl = server
             return server.run(tunnel_extras=extras)
 
         elif vcs == 'git':
             server = GitServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=repo, repo_mode=mode, user=user,
                 user_permissions=permissions, config=self.config, env=self.env)
             self.server_impl = server
             return server.run(tunnel_extras=extras)
 
         elif vcs == 'svn':
             server = SubversionServer(
                 store=store, ini_path=self.ini_path,
                 repo_name=None, user=user,
                 user_permissions=permissions, config=self.config, env=self.env)
             self.server_impl = server
             return server.run(tunnel_extras=extras)
 
         else:
             raise Exception(f'Unrecognised VCS: {vcs}')
 
     def wrap(self):
         mode = self.mode
         username = self.username
         user_id = self.user_id
         key_id = self.key_id
         shell = self.shell
 
         scm_detected, scm_repo, scm_mode = self.get_repo_details(mode)
 
         log.debug(
             'Mode: `%s` User: `name:%s : id:%s` Shell: `%s` SSH Command: `\"%s\"` '
             'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`',
             mode, username, user_id, shell, self.command,
             scm_detected, scm_mode, scm_repo)
 
         log.debug('SSH Connection info %s', self.get_connection_info())
 
         # update last access time for this key
         if key_id:
             self.update_key_access_time(key_id)
 
         if shell and self.command is None:
             log.info('Dropping to shell, no command given and shell is allowed')
             os.execl('/bin/bash', '-l')
             exit_code = 1
 
         elif scm_detected:
             user = self.get_user(user_id)
             if not user:
                 log.warning('User with id %s not found', user_id)
                 exit_code = -1
                 return exit_code
 
             auth_user = user.auth_user
             permissions = auth_user.permissions['repositories']
             repo_branch_permissions = auth_user.get_branch_permissions(scm_repo)
             try:
                 exit_code, is_updated = self.serve(
                     scm_detected, scm_repo, scm_mode, user, permissions,
                     repo_branch_permissions)
             except Exception:
                 log.exception('Error occurred during execution of SshWrapper')
                 exit_code = -1
 
         elif self.command is None and shell is False:
             log.error('No Command given.')
             exit_code = -1
 
         else:
             log.error('Unhandled Command: "%s" Aborting.', self.command)
             exit_code = -1
 
         return exit_code
+
+
+class SshWrapperStandalone(SshWrapper):
+    """
+    New version of SshWrapper designed to be depended only on service API
+    """
+    repos_path = None
+
+    @staticmethod
+    def parse_user_related_data(user_data):
+        user = AttributeDict()
+        user.user_id = user_data['user_id']
+        user.username = user_data['username']
+        user.repo_permissions = user_data['repo_permissions']
+        user.branch_permissions = user_data['branch_permissions']
+        return user
+
+    def wrap(self):
+        mode = self.mode
+        username = self.username
+        user_id = self.user_id
+        shell = self.shell
+
+        scm_detected, scm_repo, scm_mode = self.get_repo_details(mode)
+
+        log.debug(
+            'Mode: `%s` User: `name:%s : id:%s` Shell: `%s` SSH Command: `\"%s\"` '
+            'SCM_DETECTED: `%s` SCM Mode: `%s` SCM Repo: `%s`',
+            mode, username, user_id, shell, self.command,
+            scm_detected, scm_mode, scm_repo)
+
+        log.debug('SSH Connection info %s', self.get_connection_info())
+
+        if shell and self.command is None:
+            log.info('Dropping to shell, no command given and shell is allowed')
+            os.execl('/bin/bash', '-l')
+            exit_code = 1
+
+        elif scm_detected:
+            data = call_service_api(self.ini_path, {
+                "method": "service_get_data_for_ssh_wrapper",
+                "args": {"user_id": user_id, "repo_name": scm_repo, "key_id": self.key_id}
+            })
+            user = self.parse_user_related_data(data)
+            if not user:
+                log.warning('User with id %s not found', user_id)
+                exit_code = -1
+                return exit_code
+            self.repos_path = data['repos_path']
+            permissions = user.repo_permissions
+            repo_branch_permissions = user.branch_permissions
+            try:
+                exit_code, is_updated = self.serve(
+                    scm_detected, scm_repo, scm_mode, user, permissions,
+                    repo_branch_permissions)
+            except Exception:
+                log.exception('Error occurred during execution of SshWrapper')
+                exit_code = -1
+
+        elif self.command is None and shell is False:
+            log.error('No Command given.')
+            exit_code = -1
+
+        else:
+            log.error('Unhandled Command: "%s" Aborting.', self.command)
+            exit_code = -1
+
+        return exit_code
+
+    def maybe_translate_repo_uid(self, repo_name):
+        _org_name = repo_name
+        if _org_name.startswith('_'):
+            _org_name = _org_name.split('/', 1)[0]
+
+        if repo_name.startswith('_'):
+            org_repo_name = repo_name
+            log.debug('translating UID repo %s', org_repo_name)
+            by_id_match = call_service_api(self.ini_path, {
+                'method': 'service_get_repo_name_by_id',
+                "args": {"repo_id": repo_name}
+            })
+            if by_id_match:
+                repo_name = by_id_match['repo_name']
+                log.debug('translation of UID repo %s got `%s`', org_repo_name, repo_name)
+
+        return repo_name, _org_name
+
+    def serve(self, vcs, repo, mode, user, permissions, branch_permissions):
+        store = self.repos_path
+
+        check_branch_perms = False
+        detect_force_push = False
+
+        if branch_permissions:
+            check_branch_perms = True
+            detect_force_push = True
+
+        log.debug(
+            'VCS detected:`%s` mode: `%s` repo_name: %s, branch_permission_checks:%s',
+            vcs, mode, repo, check_branch_perms)
+
+        # detect if we have to check branch permissions
+        extras = {
+            'detect_force_push': detect_force_push,
+            'check_branch_perms': check_branch_perms,
+            'config': self.ini_path
+        }
+
+        match vcs:
+            case 'hg':
+                server = MercurialServer(
+                    store=store, ini_path=self.ini_path,
+                    repo_name=repo, user=user,
+                    user_permissions=permissions, config=self.config, env=self.env)
+            case 'git':
+                server = GitServer(
+                    store=store, ini_path=self.ini_path,
+                    repo_name=repo, repo_mode=mode, user=user,
+                    user_permissions=permissions, config=self.config, env=self.env)
+            case 'svn':
+                server = SubversionServer(
+                    store=store, ini_path=self.ini_path,
+                    repo_name=None, user=user,
+                    user_permissions=permissions, config=self.config, env=self.env)
+            case _:
+                raise Exception(f'Unrecognised VCS: {vcs}')
+        self.server_impl = server
+        return server.run(tunnel_extras=extras)

rhodecode/apps/ssh_support/lib/backends/base.py

@@ -1,161 +1,174 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import sys
 import logging
 
 from rhodecode.lib.hooks_daemon import prepare_callback_daemon
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.lib.vcs.conf import settings as vcs_settings
+from rhodecode.lib.utils import call_service_api
 from rhodecode.model.scm import ScmModel
 
 log = logging.getLogger(__name__)
 
 
 class VcsServer(object):
     repo_user_agent = None  # set in child classes
     _path = None  # set executable path for hg/git/svn binary
     backend = None  # set in child classes
     tunnel = None  # subprocess handling tunnel
     write_perms = ['repository.admin', 'repository.write']
     read_perms = ['repository.read', 'repository.admin', 'repository.write']
 
     def __init__(self, user, user_permissions, config, env):
         self.user = user
         self.user_permissions = user_permissions
         self.config = config
         self.env = env
         self.stdin = sys.stdin
 
         self.repo_name = None
         self.repo_mode = None
         self.store = ''
         self.ini_path = ''
+        self.hooks_protocol = None
 
     def _invalidate_cache(self, repo_name):
         """
         Set's cache for this repository for invalidation on next access
 
         :param repo_name: full repo name, also a cache key
         """
-        ScmModel().mark_for_invalidation(repo_name)
+        # Todo: Leave only "celery" case after transition.
+        match self.hooks_protocol:
+            case 'http':
+                ScmModel().mark_for_invalidation(repo_name)
+            case 'celery':
+                call_service_api(self.ini_path, {
+                    "method": "service_mark_for_invalidation",
+                    "args": {"repo_name": repo_name}
+                })
 
     def has_write_perm(self):
         permission = self.user_permissions.get(self.repo_name)
         if permission in ['repository.write', 'repository.admin']:
             return True
 
         return False
 
     def _check_permissions(self, action):
         permission = self.user_permissions.get(self.repo_name)
+        user_info = f'{self.user["user_id"]}:{self.user["username"]}'
         log.debug('permission for %s on %s are: %s',
-                  self.user, self.repo_name, permission)
+                  user_info, self.repo_name, permission)
 
         if not permission:
             log.error('user `%s` permissions to repo:%s are empty. Forbidding access.',
-                      self.user, self.repo_name)
+                      user_info, self.repo_name)
             return -2
 
         if action == 'pull':
             if permission in self.read_perms:
                 log.info(
                     'READ Permissions for User "%s" detected to repo "%s"!',
-                    self.user, self.repo_name)
+                    user_info, self.repo_name)
                 return 0
         else:
             if permission in self.write_perms:
                 log.info(
                     'WRITE, or Higher Permissions for User "%s" detected to repo "%s"!',
-                    self.user, self.repo_name)
+                    user_info, self.repo_name)
                 return 0
 
         log.error('Cannot properly fetch or verify user `%s` permissions. '
                   'Permissions: %s, vcs action: %s',
-                  self.user, permission, action)
+                  user_info, permission, action)
         return -2
 
     def update_environment(self, action, extras=None):
 
         scm_data = {
             'ip': os.environ['SSH_CLIENT'].split()[0],
             'username': self.user.username,
             'user_id': self.user.user_id,
             'action': action,
             'repository': self.repo_name,
             'scm': self.backend,
             'config': self.ini_path,
             'repo_store': self.store,
             'make_lock': None,
             'locked_by': [None, None],
             'server_url': None,
             'user_agent': f'{self.repo_user_agent}/ssh-user-agent',
             'hooks': ['push', 'pull'],
             'hooks_module': 'rhodecode.lib.hooks_daemon',
             'is_shadow_repo': False,
             'detect_force_push': False,
             'check_branch_perms': False,
 
             'SSH': True,
             'SSH_PERMISSIONS': self.user_permissions.get(self.repo_name),
         }
         if extras:
             scm_data.update(extras)
         os.putenv("RC_SCM_DATA", json.dumps(scm_data))
         return scm_data
 
     def get_root_store(self):
         root_store = self.store
         if not root_store.endswith('/'):
             # always append trailing slash
             root_store = root_store + '/'
         return root_store
 
     def _handle_tunnel(self, extras):
         # pre-auth
         action = 'pull'
         exit_code = self._check_permissions(action)
         if exit_code:
             return exit_code, False
 
-        req = self.env['request']
-        server_url = req.host_url + req.script_name
-        extras['server_url'] = server_url
+        req = self.env.get('request')
+        if req:
+            server_url = req.host_url + req.script_name
+            extras['server_url'] = server_url
 
         log.debug('Using %s binaries from path %s', self.backend, self._path)
         exit_code = self.tunnel.run(extras)
 
         return exit_code, action == "push"
 
     def run(self, tunnel_extras=None):
+        self.hooks_protocol = self.config.get('app:main', 'vcs.hooks.protocol')
         tunnel_extras = tunnel_extras or {}
         extras = {}
         extras.update(tunnel_extras)
 
         callback_daemon, extras = prepare_callback_daemon(
-            extras, protocol=vcs_settings.HOOKS_PROTOCOL,
+            extras, protocol=self.hooks_protocol,
             host=vcs_settings.HOOKS_HOST)
 
         with callback_daemon:
             try:
                 return self._handle_tunnel(extras)
             finally:
                 log.debug('Running cleanup with cache invalidation')
                 if self.repo_name:
                     self._invalidate_cache(self.repo_name)

rhodecode/apps/ssh_support/lib/backends/hg.py

@@ -1,146 +1,155 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import sys
 import logging
 import tempfile
 import textwrap
 import collections
 from .base import VcsServer
+from rhodecode.lib.utils import call_service_api
 from rhodecode.model.db import RhodeCodeUi
 from rhodecode.model.settings import VcsSettingsModel
 
 log = logging.getLogger(__name__)
 
 
 class MercurialTunnelWrapper(object):
     process = None
 
     def __init__(self, server):
         self.server = server
         self.stdin = sys.stdin
         self.stdout = sys.stdout
         self.hooks_env_fd, self.hooks_env_path = tempfile.mkstemp(prefix='hgrc_rhodecode_')
 
     def create_hooks_env(self):
         repo_name = self.server.repo_name
         hg_flags = self.server.config_to_hgrc(repo_name)
 
         content = textwrap.dedent(
             '''
             # RhodeCode SSH hooks version=2.0.0
             {custom}
             '''
         ).format(custom='\n'.join(hg_flags))
 
         root = self.server.get_root_store()
         hgrc_custom = os.path.join(root, repo_name, '.hg', 'hgrc_rhodecode')
         hgrc_main = os.path.join(root, repo_name, '.hg', 'hgrc')
 
         # cleanup custom hgrc file
         if os.path.isfile(hgrc_custom):
             with open(hgrc_custom, 'wb') as f:
                 f.write('')
             log.debug('Cleanup custom hgrc file under %s', hgrc_custom)
 
         # write temp
         with os.fdopen(self.hooks_env_fd, 'w') as hooks_env_file:
             hooks_env_file.write(content)
 
         return self.hooks_env_path
 
     def remove_configs(self):
         os.remove(self.hooks_env_path)
 
     def command(self, hgrc_path):
         root = self.server.get_root_store()
 
         command = (
             "cd {root}; HGRCPATH={hgrc} {hg_path} -R {root}{repo_name} "
             "serve --stdio".format(
                 root=root, hg_path=self.server.hg_path,
                 repo_name=self.server.repo_name, hgrc=hgrc_path))
         log.debug("Final CMD: %s", command)
         return command
 
     def run(self, extras):
         # at this point we cannot tell, we do further ACL checks
         # inside the hooks
         action = '?'
         # permissions are check via `pre_push_ssh_auth` hook
         self.server.update_environment(action=action, extras=extras)
         custom_hgrc_file = self.create_hooks_env()
 
         try:
             return os.system(self.command(custom_hgrc_file))
         finally:
             self.remove_configs()
 
 
 class MercurialServer(VcsServer):
     backend = 'hg'
     repo_user_agent = 'mercurial'
     cli_flags = ['phases', 'largefiles', 'extensions', 'experimental', 'hooks']
 
     def __init__(self, store, ini_path, repo_name, user, user_permissions, config, env):
         super().__init__(user, user_permissions, config, env)
 
         self.store = store
         self.ini_path = ini_path
         self.repo_name = repo_name
         self._path = self.hg_path = config.get('app:main', 'ssh.executable.hg')
         self.tunnel = MercurialTunnelWrapper(server=self)
 
     def config_to_hgrc(self, repo_name):
+        # Todo: once transition is done only call to service api should exist
+        if self.hooks_protocol == 'celery':
+            data = call_service_api(self.ini_path, {
+                "method": "service_config_to_hgrc",
+                "args": {"cli_flags": self.cli_flags, "repo_name": repo_name}
+            })
+            return data['flags']
+
         ui_sections = collections.defaultdict(list)
         ui = VcsSettingsModel(repo=repo_name).get_ui_settings(section=None, key=None)
 
         # write default hooks
         default_hooks = [
             ('pretxnchangegroup.ssh_auth', 'python:vcsserver.hooks.pre_push_ssh_auth'),
             ('pretxnchangegroup.ssh', 'python:vcsserver.hooks.pre_push_ssh'),
             ('changegroup.ssh', 'python:vcsserver.hooks.post_push_ssh'),
 
             ('preoutgoing.ssh', 'python:vcsserver.hooks.pre_pull_ssh'),
             ('outgoing.ssh', 'python:vcsserver.hooks.post_pull_ssh'),
         ]
 
         for k, v in default_hooks:
             ui_sections['hooks'].append((k, v))
 
         for entry in ui:
             if not entry.active:
                 continue
             sec = entry.section
             key = entry.key
 
             if sec in self.cli_flags:
                 # we want only custom hooks, so we skip builtins
                 if sec == 'hooks' and key in RhodeCodeUi.HOOKS_BUILTIN:
                     continue
 
                 ui_sections[sec].append([key, entry.value])
 
         flags = []
         for _sec, key_val in ui_sections.items():
             flags.append(' ')
             flags.append(f'[{_sec}]')
             for key, val in key_val:
                 flags.append(f'{key}= {val}')
         return flags

rhodecode/apps/ssh_support/tests/conftest.py

@@ -1,68 +1,68 b''
 # Copyright (C) 2016-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import pytest
 import configparser
 
-from rhodecode.apps.ssh_support.lib.ssh_wrapper import SshWrapper
+from rhodecode.apps.ssh_support.lib.ssh_wrapper_v1 import SshWrapper
 from rhodecode.lib.utils2 import AttributeDict
 
 
 @pytest.fixture()
 def dummy_conf_file(tmpdir):
     conf = configparser.ConfigParser()
     conf.add_section('app:main')
     conf.set('app:main', 'ssh.executable.hg', '/usr/bin/hg')
     conf.set('app:main', 'ssh.executable.git', '/usr/bin/git')
     conf.set('app:main', 'ssh.executable.svn', '/usr/bin/svnserve')
 
     f_path = os.path.join(str(tmpdir), 'ssh_wrapper_test.ini')
     with open(f_path, 'wt') as f:
         conf.write(f)
 
     return os.path.join(f_path)
 
 
 def plain_dummy_env():
     return {
         'request':
             AttributeDict(host_url='http://localhost', script_name='/')
     }
 
 
 @pytest.fixture()
 def dummy_env():
     return plain_dummy_env()
 
 
 def plain_dummy_user():
     return AttributeDict(username='test_user')
 
 
 @pytest.fixture()
 def dummy_user():
     return plain_dummy_user()
 
 
 @pytest.fixture()
 def ssh_wrapper(app, dummy_conf_file, dummy_env):
     conn_info = '127.0.0.1 22 10.0.0.1 443'
     return SshWrapper(
         'random command', conn_info, 'auto', 'admin', '1', key_id='1',
         shell=False, ini_path=dummy_conf_file, env=dummy_env)

rhodecode/lib/utils.py

@@ -1,823 +1,851 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Utilities library for RhodeCode
 """
 
 import datetime
 import decorator
 import logging
 import os
 import re
 import sys
 import shutil
 import socket
 import tempfile
 import traceback
 import tarfile
 import warnings
 from functools import wraps
 from os.path import join as jn
+from configparser import NoOptionError
 
 import paste
 import pkg_resources
 from webhelpers2.text import collapse, strip_tags, convert_accented_entities, convert_misc_entities
 
 from mako import exceptions
 
 from rhodecode.lib.hash_utils import sha256_safe, md5, sha1
 from rhodecode.lib.type_utils import AttributeDict
 from rhodecode.lib.str_utils import safe_bytes, safe_str
 from rhodecode.lib.vcs.backends.base import Config
 from rhodecode.lib.vcs.exceptions import VCSError
 from rhodecode.lib.vcs.utils.helpers import get_scm, get_scm_backend
 from rhodecode.lib.ext_json import sjson as json
 from rhodecode.model import meta
 from rhodecode.model.db import (
     Repository, User, RhodeCodeUi, UserLog, RepoGroup, UserGroup)
 from rhodecode.model.meta import Session
+from rhodecode.lib.pyramid_utils import get_config
+from rhodecode.lib.vcs import CurlSession
+from rhodecode.lib.vcs.exceptions import ImproperlyConfiguredError
 
 
 log = logging.getLogger(__name__)
 
 REMOVED_REPO_PAT = re.compile(r'rm__\d{8}_\d{6}_\d{6}__.*')
 
 # String which contains characters that are not allowed in slug names for
 # repositories or repository groups. It is properly escaped to use it in
 # regular expressions.
 SLUG_BAD_CHARS = re.escape(r'`?=[]\;\'"<>,/~!@#$%^&*()+{}|:')
 
 # Regex that matches forbidden characters in repo/group slugs.
 SLUG_BAD_CHAR_RE = re.compile(r'[{}\x00-\x08\x0b-\x0c\x0e-\x1f]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches allowed characters in repo/group slugs.
 SLUG_GOOD_CHAR_RE = re.compile(r'[^{}]'.format(SLUG_BAD_CHARS))
 
 # Regex that matches whole repo/group slugs.
 SLUG_RE = re.compile(r'[^{}]+'.format(SLUG_BAD_CHARS))
 
 _license_cache = None
 
 
 def adopt_for_celery(func):
     """
     Decorator designed to adopt hooks (from rhodecode.lib.hooks_base)
     for further usage as a celery tasks.
     """
     @wraps(func)
     def wrapper(extras):
         extras = AttributeDict(extras)
         # HooksResponse implements to_json method which must be used there.
         return func(extras).to_json()
     return wrapper
 
 
 def repo_name_slug(value):
     """
     Return slug of name of repository
     This function is called on each creation/modification
     of repository to prevent bad names in repo
     """
 
     replacement_char = '-'
 
     slug = strip_tags(value)
     slug = convert_accented_entities(slug)
     slug = convert_misc_entities(slug)
 
     slug = SLUG_BAD_CHAR_RE.sub('', slug)
     slug = re.sub(r'[\s]+', '-', slug)
     slug = collapse(slug, replacement_char)
 
     return slug
 
 
 #==============================================================================
 # PERM DECORATOR HELPERS FOR EXTRACTING NAMES FOR PERM CHECKS
 #==============================================================================
 def get_repo_slug(request):
     _repo = ''
 
     if hasattr(request, 'db_repo_name'):
         # if our requests has set db reference use it for name, this
         # translates the example.com/_<id> into proper repo names
         _repo = request.db_repo_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _repo = request.matchdict.get('repo_name')
 
     if _repo:
         _repo = _repo.rstrip('/')
     return _repo
 
 
 def get_repo_group_slug(request):
     _group = ''
     if hasattr(request, 'db_repo_group'):
         # if our requests has set db reference use it for name, this
         # translates the example.com/_<id> into proper repo group names
         _group = request.db_repo_group.group_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _group = request.matchdict.get('repo_group_name')
 
     if _group:
         _group = _group.rstrip('/')
     return _group
 
 
 def get_user_group_slug(request):
     _user_group = ''
 
     if hasattr(request, 'db_user_group'):
         _user_group = request.db_user_group.users_group_name
     elif getattr(request, 'matchdict', None):
         # pyramid
         _user_group = request.matchdict.get('user_group_id')
         _user_group_name = request.matchdict.get('user_group_name')
         try:
             if _user_group:
                 _user_group = UserGroup.get(_user_group)
             elif _user_group_name:
                 _user_group = UserGroup.get_by_group_name(_user_group_name)
 
             if _user_group:
                 _user_group = _user_group.users_group_name
         except Exception:
             log.exception('Failed to get user group by id and name')
             # catch all failures here
             return None
 
     return _user_group
 
 
 def get_filesystem_repos(path, recursive=False, skip_removed_repos=True):
     """
     Scans given path for repos and return (name,(type,path)) tuple
 
     :param path: path to scan for repositories
     :param recursive: recursive search and return names with subdirs in front
     """
 
     # remove ending slash for better results
     path = path.rstrip(os.sep)
     log.debug('now scanning in %s location recursive:%s...', path, recursive)
 
     def _get_repos(p):
         dirpaths = get_dirpaths(p)
         if not _is_dir_writable(p):
             log.warning('repo path without write access: %s', p)
 
         for dirpath in dirpaths:
             if os.path.isfile(os.path.join(p, dirpath)):
                 continue
             cur_path = os.path.join(p, dirpath)
 
             # skip removed repos
             if skip_removed_repos and REMOVED_REPO_PAT.match(dirpath):
                 continue
 
             #skip .<somethin> dirs
             if dirpath.startswith('.'):
                 continue
 
             try:
                 scm_info = get_scm(cur_path)
                 yield scm_info[1].split(path, 1)[-1].lstrip(os.sep), scm_info
             except VCSError:
                 if not recursive:
                     continue
                 #check if this dir containts other repos for recursive scan
                 rec_path = os.path.join(p, dirpath)
                 if os.path.isdir(rec_path):
                     yield from _get_repos(rec_path)
 
     return _get_repos(path)
 
 
 def get_dirpaths(p: str) -> list:
     try:
         # OS-independable way of checking if we have at least read-only
         # access or not.
         dirpaths = os.listdir(p)
     except OSError:
         log.warning('ignoring repo path without read access: %s', p)
         return []
 
     # os.listpath has a tweak: If a unicode is passed into it, then it tries to
     # decode paths and suddenly returns unicode objects itself. The items it
     # cannot decode are returned as strings and cause issues.
     #
     # Those paths are ignored here until a solid solution for path handling has
     # been built.
     expected_type = type(p)
 
     def _has_correct_type(item):
         if type(item) is not expected_type:
             log.error(
                 "Ignoring path %s since it cannot be decoded into str.",
                 # Using "repr" to make sure that we see the byte value in case
                 # of support.
                 repr(item))
             return False
         return True
 
     dirpaths = [item for item in dirpaths if _has_correct_type(item)]
 
     return dirpaths
 
 
 def _is_dir_writable(path):
     """
     Probe if `path` is writable.
 
     Due to trouble on Cygwin / Windows, this is actually probing if it is
     possible to create a file inside of `path`, stat does not produce reliable
     results in this case.
     """
     try:
         with tempfile.TemporaryFile(dir=path):
             pass
     except OSError:
         return False
     return True
 
 
 def is_valid_repo(repo_name, base_path, expect_scm=None, explicit_scm=None, config=None):
     """
     Returns True if given path is a valid repository False otherwise.
     If expect_scm param is given also, compare if given scm is the same
     as expected from scm parameter. If explicit_scm is given don't try to
     detect the scm, just use the given one to check if repo is valid
 
     :param repo_name:
     :param base_path:
     :param expect_scm:
     :param explicit_scm:
     :param config:
 
     :return True: if given path is a valid repository
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_name))
     log.debug('Checking if `%s` is a valid path for repository. '
               'Explicit type: %s', repo_name, explicit_scm)
 
     try:
         if explicit_scm:
             detected_scms = [get_scm_backend(explicit_scm)(
                 full_path, config=config).alias]
         else:
             detected_scms = get_scm(full_path)
 
         if expect_scm:
             return detected_scms[0] == expect_scm
         log.debug('path: %s is an vcs object:%s', full_path, detected_scms)
         return True
     except VCSError:
         log.debug('path: %s is not a valid repo !', full_path)
         return False
 
 
 def is_valid_repo_group(repo_group_name, base_path, skip_path_check=False):
     """
     Returns True if a given path is a repository group, False otherwise
 
     :param repo_group_name:
     :param base_path:
     """
     full_path = os.path.join(safe_str(base_path), safe_str(repo_group_name))
     log.debug('Checking if `%s` is a valid path for repository group',
               repo_group_name)
 
     # check if it's not a repo
     if is_valid_repo(repo_group_name, base_path):
         log.debug('Repo called %s exist, it is not a valid repo group', repo_group_name)
         return False
 
     try:
         # we need to check bare git repos at higher level
         # since we might match branches/hooks/info/objects or possible
         # other things inside bare git repo
         maybe_repo = os.path.dirname(full_path)
         if maybe_repo == base_path:
             # skip root level repo check; we know root location CANNOT BE a repo group
             return False
 
         scm_ = get_scm(maybe_repo)
         log.debug('path: %s is a vcs object:%s, not valid repo group', full_path, scm_)
         return False
     except VCSError:
         pass
 
     # check if it's a valid path
     if skip_path_check or os.path.isdir(full_path):
         log.debug('path: %s is a valid repo group !', full_path)
         return True
 
     log.debug('path: %s is not a valid repo group !', full_path)
     return False
 
 
 def ask_ok(prompt, retries=4, complaint='[y]es or [n]o please!'):
     while True:
         ok = input(prompt)
         if ok.lower() in ('y', 'ye', 'yes'):
             return True
         if ok.lower() in ('n', 'no', 'nop', 'nope'):
             return False
         retries = retries - 1
         if retries < 0:
             raise OSError
         print(complaint)
 
 # propagated from mercurial documentation
 ui_sections = [
     'alias', 'auth',
     'decode/encode', 'defaults',
     'diff', 'email',
     'extensions', 'format',
     'merge-patterns', 'merge-tools',
     'hooks', 'http_proxy',
     'smtp', 'patch',
     'paths', 'profiling',
     'server', 'trusted',
     'ui', 'web', ]
 
 
 def config_data_from_db(clear_session=True, repo=None):
     """
     Read the configuration data from the database and return configuration
     tuples.
     """
     from rhodecode.model.settings import VcsSettingsModel
 
     config = []
 
     sa = meta.Session()
     settings_model = VcsSettingsModel(repo=repo, sa=sa)
 
     ui_settings = settings_model.get_ui_settings()
 
     ui_data = []
     for setting in ui_settings:
         if setting.active:
             ui_data.append((setting.section, setting.key, setting.value))
             config.append((
                 safe_str(setting.section), safe_str(setting.key),
                 safe_str(setting.value)))
         if setting.key == 'push_ssl':
             # force set push_ssl requirement to False, rhodecode
             # handles that
             config.append((
                 safe_str(setting.section), safe_str(setting.key), False))
     log.debug(
         'settings ui from db@repo[%s]: %s',
         repo,
         ','.join(['[{}] {}={}'.format(*s) for s in ui_data]))
     if clear_session:
         meta.Session.remove()
 
     # TODO: mikhail: probably it makes no sense to re-read hooks information.
     # It's already there and activated/deactivated
     skip_entries = []
     enabled_hook_classes = get_enabled_hook_classes(ui_settings)
     if 'pull' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PULL))
     if 'push' not in enabled_hook_classes:
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRE_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PRETX_PUSH))
         skip_entries.append(('hooks', RhodeCodeUi.HOOK_PUSH_KEY))
 
     config = [entry for entry in config if entry[:2] not in skip_entries]
 
     return config
 
 
 def make_db_config(clear_session=True, repo=None):
     """
     Create a :class:`Config` instance based on the values in the database.
     """
     config = Config()
     config_data = config_data_from_db(clear_session=clear_session, repo=repo)
     for section, option, value in config_data:
         config.set(section, option, value)
     return config
 
 
 def get_enabled_hook_classes(ui_settings):
     """
     Return the enabled hook classes.
 
     :param ui_settings: List of ui_settings as returned
         by :meth:`VcsSettingsModel.get_ui_settings`
 
     :return: a list with the enabled hook classes. The order is not guaranteed.
     :rtype: list
     """
     enabled_hooks = []
     active_hook_keys = [
         key for section, key, value, active in ui_settings
         if section == 'hooks' and active]
 
     hook_names = {
         RhodeCodeUi.HOOK_PUSH: 'push',
         RhodeCodeUi.HOOK_PULL: 'pull',
         RhodeCodeUi.HOOK_REPO_SIZE: 'repo_size'
     }
 
     for key in active_hook_keys:
         hook = hook_names.get(key)
         if hook:
             enabled_hooks.append(hook)
 
     return enabled_hooks
 
 
 def set_rhodecode_config(config):
     """
     Updates pyramid config with new settings from database
 
     :param config:
     """
     from rhodecode.model.settings import SettingsModel
     app_settings = SettingsModel().get_all_settings()
 
     for k, v in list(app_settings.items()):
         config[k] = v
 
 
 def get_rhodecode_realm():
     """
     Return the rhodecode realm from database.
     """
     from rhodecode.model.settings import SettingsModel
     realm = SettingsModel().get_setting_by_name('realm')
     return safe_str(realm.app_settings_value)
 
 
 def get_rhodecode_base_path():
     """
     Returns the base path. The base path is the filesystem path which points
     to the repository store.
     """
 
     import rhodecode
     return rhodecode.CONFIG['default_base_path']
 
 
 def map_groups(path):
     """
     Given a full path to a repository, create all nested groups that this
     repo is inside. This function creates parent-child relationships between
     groups and creates default perms for all new groups.
 
     :param paths: full path to repository
     """
     from rhodecode.model.repo_group import RepoGroupModel
     sa = meta.Session()
     groups = path.split(Repository.NAME_SEP)
     parent = None
     group = None
 
     # last element is repo in nested groups structure
     groups = groups[:-1]
     rgm = RepoGroupModel(sa)
     owner = User.get_first_super_admin()
     for lvl, group_name in enumerate(groups):
         group_name = '/'.join(groups[:lvl] + [group_name])
         group = RepoGroup.get_by_group_name(group_name)
         desc = '%s group' % group_name
 
         # skip folders that are now removed repos
         if REMOVED_REPO_PAT.match(group_name):
             break
 
         if group is None:
             log.debug('creating group level: %s group_name: %s',
                        lvl, group_name)
             group = RepoGroup(group_name, parent)
             group.group_description = desc
             group.user = owner
             sa.add(group)
             perm_obj = rgm._create_default_perms(group)
             sa.add(perm_obj)
             sa.flush()
 
         parent = group
     return group
 
 
 def repo2db_mapper(initial_repo_list, remove_obsolete=False, force_hooks_rebuild=False):
     """
     maps all repos given in initial_repo_list, non existing repositories
     are created, if remove_obsolete is True it also checks for db entries
     that are not in initial_repo_list and removes them.
 
     :param initial_repo_list: list of repositories found by scanning methods
     :param remove_obsolete: check for obsolete entries in database
     """
     from rhodecode.model.repo import RepoModel
     from rhodecode.model.repo_group import RepoGroupModel
     from rhodecode.model.settings import SettingsModel
 
     sa = meta.Session()
     repo_model = RepoModel()
     user = User.get_first_super_admin()
     added = []
 
     # creation defaults
     defs = SettingsModel().get_default_repo_settings(strip_prefix=True)
     enable_statistics = defs.get('repo_enable_statistics')
     enable_locking = defs.get('repo_enable_locking')
     enable_downloads = defs.get('repo_enable_downloads')
     private = defs.get('repo_private')
 
     for name, repo in list(initial_repo_list.items()):
         group = map_groups(name)
         str_name = safe_str(name)
         db_repo = repo_model.get_by_repo_name(str_name)
 
         # found repo that is on filesystem not in RhodeCode database
         if not db_repo:
             log.info('repository `%s` not found in the database, creating now', name)
             added.append(name)
             desc = (repo.description
                     if repo.description != 'unknown'
                     else '%s repository' % name)
 
             db_repo = repo_model._create_repo(
                 repo_name=name,
                 repo_type=repo.alias,
                 description=desc,
                 repo_group=getattr(group, 'group_id', None),
                 owner=user,
                 enable_locking=enable_locking,
                 enable_downloads=enable_downloads,
                 enable_statistics=enable_statistics,
                 private=private,
                 state=Repository.STATE_CREATED
             )
             sa.commit()
             # we added that repo just now, and make sure we updated server info
             if db_repo.repo_type == 'git':
                 git_repo = db_repo.scm_instance()
                 # update repository server-info
                 log.debug('Running update server info')
                 git_repo._update_server_info(force=True)
 
             db_repo.update_commit_cache()
 
         config = db_repo._config
         config.set('extensions', 'largefiles', '')
         repo = db_repo.scm_instance(config=config)
         repo.install_hooks(force=force_hooks_rebuild)
 
     removed = []
     if remove_obsolete:
         # remove from database those repositories that are not in the filesystem
         for repo in sa.query(Repository).all():
             if repo.repo_name not in list(initial_repo_list.keys()):
                 log.debug("Removing non-existing repository found in db `%s`",
                           repo.repo_name)
                 try:
                     RepoModel(sa).delete(repo, forks='detach', fs_remove=False)
                     sa.commit()
                     removed.append(repo.repo_name)
                 except Exception:
                     # don't hold further removals on error
                     log.error(traceback.format_exc())
                     sa.rollback()
 
         def splitter(full_repo_name):
             _parts = full_repo_name.rsplit(RepoGroup.url_sep(), 1)
             gr_name = None
             if len(_parts) == 2:
                 gr_name = _parts[0]
             return gr_name
 
         initial_repo_group_list = [splitter(x) for x in
                                    list(initial_repo_list.keys()) if splitter(x)]
 
         # remove from database those repository groups that are not in the
         # filesystem due to parent child relationships we need to delete them
         # in a specific order of most nested first
         all_groups = [x.group_name for x in sa.query(RepoGroup).all()]
         def nested_sort(gr):
             return len(gr.split('/'))
         for group_name in sorted(all_groups, key=nested_sort, reverse=True):
             if group_name not in initial_repo_group_list:
                 repo_group = RepoGroup.get_by_group_name(group_name)
                 if (repo_group.children.all() or
                     not RepoGroupModel().check_exist_filesystem(
                         group_name=group_name, exc_on_failure=False)):
                     continue
 
                 log.info(
                     'Removing non-existing repository group found in db `%s`',
                     group_name)
                 try:
                     RepoGroupModel(sa).delete(group_name, fs_remove=False)
                     sa.commit()
                     removed.append(group_name)
                 except Exception:
                     # don't hold further removals on error
                     log.exception(
                         'Unable to remove repository group `%s`',
                         group_name)
                     sa.rollback()
                     raise
 
     return added, removed
 
 
 def load_rcextensions(root_path):
     import rhodecode
     from rhodecode.config import conf
 
     path = os.path.join(root_path)
     sys.path.append(path)
 
     try:
         rcextensions = __import__('rcextensions')
     except ImportError:
         if os.path.isdir(os.path.join(path, 'rcextensions')):
             log.warning('Unable to load rcextensions from %s', path)
         rcextensions = None
 
     if rcextensions:
         log.info('Loaded rcextensions from %s...', rcextensions)
         rhodecode.EXTENSIONS = rcextensions
 
         # Additional mappings that are not present in the pygments lexers
         conf.LANGUAGES_EXTENSIONS_MAP.update(
             getattr(rhodecode.EXTENSIONS, 'EXTRA_MAPPINGS', {}))
 
 
 def get_custom_lexer(extension):
     """
     returns a custom lexer if it is defined in rcextensions module, or None
     if there's no custom lexer defined
     """
     import rhodecode
     from pygments import lexers
 
     # custom override made by RhodeCode
     if extension in ['mako']:
         return lexers.get_lexer_by_name('html+mako')
 
     # check if we didn't define this extension as other lexer
     extensions = rhodecode.EXTENSIONS and getattr(rhodecode.EXTENSIONS, 'EXTRA_LEXERS', None)
     if extensions and extension in rhodecode.EXTENSIONS.EXTRA_LEXERS:
         _lexer_name = rhodecode.EXTENSIONS.EXTRA_LEXERS[extension]
         return lexers.get_lexer_by_name(_lexer_name)
 
 
 #==============================================================================
 # TEST FUNCTIONS AND CREATORS
 #==============================================================================
 def create_test_index(repo_location, config):
     """
     Makes default test index.
     """
     try:
         import rc_testdata
     except ImportError:
         raise ImportError('Failed to import rc_testdata, '
                           'please make sure this package is installed from requirements_test.txt')
     rc_testdata.extract_search_index(
         'vcs_search_index', os.path.dirname(config['search.location']))
 
 
 def create_test_directory(test_path):
     """
     Create test directory if it doesn't exist.
     """
     if not os.path.isdir(test_path):
         log.debug('Creating testdir %s', test_path)
         os.makedirs(test_path)
 
 
 def create_test_database(test_path, config):
     """
     Makes a fresh database.
     """
     from rhodecode.lib.db_manage import DbManage
     from rhodecode.lib.utils2 import get_encryption_key
 
     # PART ONE create db
     dbconf = config['sqlalchemy.db1.url']
     enc_key = get_encryption_key(config)
 
     log.debug('making test db %s', dbconf)
 
     dbmanage = DbManage(log_sql=False, dbconf=dbconf, root=config['here'],
                         tests=True, cli_args={'force_ask': True}, enc_key=enc_key)
     dbmanage.create_tables(override=True)
     dbmanage.set_db_version()
     # for tests dynamically set new root paths based on generated content
     dbmanage.create_settings(dbmanage.config_prompt(test_path))
     dbmanage.create_default_user()
     dbmanage.create_test_admin_and_users()
     dbmanage.create_permissions()
     dbmanage.populate_default_permissions()
     Session().commit()
 
 
 def create_test_repositories(test_path, config):
     """
     Creates test repositories in the temporary directory. Repositories are
     extracted from archives within the rc_testdata package.
     """
     import rc_testdata
     from rhodecode.tests import HG_REPO, GIT_REPO, SVN_REPO
 
     log.debug('making test vcs repositories')
 
     idx_path = config['search.location']
     data_path = config['cache_dir']
 
     # clean index and data
     if idx_path and os.path.exists(idx_path):
         log.debug('remove %s', idx_path)
         shutil.rmtree(idx_path)
 
     if data_path and os.path.exists(data_path):
         log.debug('remove %s', data_path)
         shutil.rmtree(data_path)
 
     rc_testdata.extract_hg_dump('vcs_test_hg', jn(test_path, HG_REPO))
     rc_testdata.extract_git_dump('vcs_test_git', jn(test_path, GIT_REPO))
 
     # Note: Subversion is in the process of being integrated with the system,
     # until we have a properly packed version of the test svn repository, this
     # tries to copy over the repo from a package "rc_testdata"
     svn_repo_path = rc_testdata.get_svn_repo_archive()
     with tarfile.open(svn_repo_path) as tar:
         tar.extractall(jn(test_path, SVN_REPO))
 
 
 def password_changed(auth_user, session):
     # Never report password change in case of default user or anonymous user.
     if auth_user.username == User.DEFAULT_USER or auth_user.user_id is None:
         return False
 
     password_hash = md5(safe_bytes(auth_user.password)) if auth_user.password else None
     rhodecode_user = session.get('rhodecode_user', {})
     session_password_hash = rhodecode_user.get('password', '')
     return password_hash != session_password_hash
 
 
 def read_opensource_licenses():
     global _license_cache
 
     if not _license_cache:
         licenses = pkg_resources.resource_string(
             'rhodecode', 'config/licenses.json')
         _license_cache = json.loads(licenses)
 
     return _license_cache
 
 
 def generate_platform_uuid():
     """
     Generates platform UUID based on it's name
     """
     import platform
 
     try:
         uuid_list = [platform.platform()]
         return sha256_safe(':'.join(uuid_list))
     except Exception as e:
         log.error('Failed to generate host uuid: %s', e)
         return 'UNDEFINED'
 
 
 def send_test_email(recipients, email_body='TEST EMAIL'):
     """
     Simple code for generating test emails.
     Usage::
 
         from rhodecode.lib import utils
         utils.send_test_email()
     """
     from rhodecode.lib.celerylib import tasks, run_task
 
     email_body = email_body_plaintext = email_body
     subject = f'SUBJECT FROM: {socket.gethostname()}'
     tasks.send_email(recipients, subject, email_body_plaintext, email_body)
+
+
+def call_service_api(ini_path, payload):
+    config = get_config(ini_path)
+    try:
+        host = config.get('app:main', 'app.service_api.host')
+    except NoOptionError:
+        raise ImproperlyConfiguredError(
+            "app.service_api.host is missing. "
+            "Please ensure that app.service_api.host and app.service_api.token are "
+            "defined inside of .ini configuration file."
+        )
+    api_url = config.get('app:main', 'rhodecode.api.url')
+    payload.update({
+        'id': 'service',
+        'auth_token': config.get('app:main', 'app.service_api.token')
+    })
+
+    response = CurlSession().post(f'{host}{api_url}', json.dumps(payload))
+
+    if response.status_code != 200:
+        raise Exception("Service API responded with error")
+
+    return json.loads(response.content)['result']

rhodecode/lib/vcs/exceptions.py

@@ -1,232 +1,236 b''
 # Copyright (C) 2014-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 Custom vcs exceptions module.
 """
 import logging
 import functools
 import urllib.error
 import urllib.parse
 import rhodecode
 
 log = logging.getLogger(__name__)
 
 
 class VCSCommunicationError(Exception):
     pass
 
 
 class HttpVCSCommunicationError(VCSCommunicationError):
     pass
 
 
 class VCSError(Exception):
     pass
 
 
 class RepositoryError(VCSError):
     pass
 
 
 class RepositoryRequirementError(RepositoryError):
     pass
 
 
 class UnresolvedFilesInRepo(RepositoryError):
     pass
 
 
 class VCSBackendNotSupportedError(VCSError):
     """
     Exception raised when VCSServer does not support requested backend
     """
 
 
 class EmptyRepositoryError(RepositoryError):
     pass
 
 
 class TagAlreadyExistError(RepositoryError):
     pass
 
 
 class TagDoesNotExistError(RepositoryError):
     pass
 
 
 class BranchAlreadyExistError(RepositoryError):
     pass
 
 
 class BranchDoesNotExistError(RepositoryError):
     pass
 
 
 class CommitError(RepositoryError):
     """
     Exceptions related to an existing commit
     """
 
 
 class CommitDoesNotExistError(CommitError):
     pass
 
 
 class CommittingError(RepositoryError):
     """
     Exceptions happening while creating a new commit
     """
 
 
 class NothingChangedError(CommittingError):
     pass
 
 
 class NodeError(VCSError):
     pass
 
 
 class RemovedFileNodeError(NodeError):
     pass
 
 
 class NodeAlreadyExistsError(CommittingError):
     pass
 
 
 class NodeAlreadyChangedError(CommittingError):
     pass
 
 
 class NodeDoesNotExistError(CommittingError):
     pass
 
 
 class NodeNotChangedError(CommittingError):
     pass
 
 
 class NodeAlreadyAddedError(CommittingError):
     pass
 
 
 class NodeAlreadyRemovedError(CommittingError):
     pass
 
 
 class SubrepoMergeError(RepositoryError):
     """
     This happens if we try to merge a repository which contains subrepos and
     the subrepos cannot be merged. The subrepos are not merged itself but
     their references in the root repo are merged.
     """
 
 
 class ImproperArchiveTypeError(VCSError):
     pass
 
 
 class CommandError(VCSError):
     pass
 
 
+class ImproperlyConfiguredError(Exception):
+    pass
+
+
 class UnhandledException(VCSError):
     """
     Signals that something unexpected went wrong.
 
     This usually means we have a programming error on the side of the VCSServer
     and should inspect the logfile of the VCSServer to find more details.
     """
 
 
 _EXCEPTION_MAP = {
     'abort': RepositoryError,
     'archive': ImproperArchiveTypeError,
     'error': RepositoryError,
     'lookup': CommitDoesNotExistError,
     'repo_locked': RepositoryError,
     'requirement': RepositoryRequirementError,
     'unhandled': UnhandledException,
     # TODO: johbo: Define our own exception for this and stop abusing
     # urllib's exception class.
     'url_error': urllib.error.URLError,
     'subrepo_merge_error': SubrepoMergeError,
 }
 
 
 def map_vcs_exceptions(func):
     """
     Utility to decorate functions so that plain exceptions are translated.
 
     The translation is based on `exc_map` which maps a `str` indicating
     the error type into an exception class representing this error inside
     of the vcs layer.
     """
 
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         try:
             return func(*args, **kwargs)
         except Exception as e:
             debug = rhodecode.ConfigGet().get_bool('debug')
 
             # The error middleware adds information if it finds
             # __traceback_info__ in a frame object. This way the remote
             # traceback information is made available in error reports.
 
             remote_tb = getattr(e, '_vcs_server_traceback', None)
             org_remote_tb = getattr(e, '_vcs_server_org_exc_tb', '')
             __traceback_info__ = None
             if remote_tb:
                 if isinstance(remote_tb, str):
                     remote_tb = [remote_tb]
                 __traceback_info__ = (
                     'Found VCSServer remote traceback information:\n'
                     '{}\n'
                     '+++ BEG SOURCE EXCEPTION +++\n\n'
                     '{}\n'
                     '+++ END SOURCE EXCEPTION +++\n'
                     ''.format('\n'.join(remote_tb), org_remote_tb)
                 )
 
                 # Avoid that remote_tb also appears in the frame
                 del remote_tb
 
             # Special vcs errors had an attribute "_vcs_kind" which is used
             # to translate them to the proper exception class in the vcs
             # client layer.
             kind = getattr(e, '_vcs_kind', None)
             exc_name = getattr(e, '_vcs_server_org_exc_name', None)
 
             if kind:
                 if any(e.args):
                     _args = [a for a in e.args]
                     # replace the first argument with a prefix exc name
                     args = ['{}:{}'.format(exc_name, _args[0] if _args else '?')] + _args[1:]
                 else:
                     args = [__traceback_info__ or f'{exc_name}: UnhandledException']
                 if debug or __traceback_info__ and kind not in ['unhandled', 'lookup']:
                     # for other than unhandled errors also log the traceback
                     # can be useful for debugging
                     log.error(__traceback_info__)
 
                 raise _EXCEPTION_MAP[kind](*args)
             else:
                 raise
     return wrapper

rhodecode/tests/fixture_mods/fixture_pyramid.py

@@ -1,222 +1,223 b''
 
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import pytest
 
 from rhodecode.lib.pyramid_utils import get_app_config
 from rhodecode.tests.fixture import TestINI
 from rhodecode.tests.server_utils import RcVCSServer
 
 
 @pytest.fixture(scope='session')
 def vcsserver(request, vcsserver_port, vcsserver_factory):
     """
     Session scope VCSServer.
 
     Tests which need the VCSServer have to rely on this fixture in order
     to ensure it will be running.
 
     For specific needs, the fixture vcsserver_factory can be used. It allows to
     adjust the configuration file for the test run.
 
     Command line args:
 
     --without-vcsserver: Allows to switch this fixture off. You have to
     manually start the server.
 
     --vcsserver-port: Will expect the VCSServer to listen on this port.
     """
 
     if not request.config.getoption('with_vcsserver'):
         return None
 
     return vcsserver_factory(
         request, vcsserver_port=vcsserver_port)
 
 
 @pytest.fixture(scope='session')
 def vcsserver_factory(tmpdir_factory):
     """
     Use this if you need a running vcsserver with a special configuration.
     """
 
     def factory(request, overrides=(), vcsserver_port=None,
                 log_file=None, workers='2'):
 
         if vcsserver_port is None:
             vcsserver_port = get_available_port()
 
         overrides = list(overrides)
         overrides.append({'server:main': {'port': vcsserver_port}})
 
         option_name = 'vcsserver_config_http'
         override_option_name = 'vcsserver_config_override'
         config_file = get_config(
             request.config, option_name=option_name,
             override_option_name=override_option_name, overrides=overrides,
             basetemp=tmpdir_factory.getbasetemp().strpath,
             prefix='test_vcs_')
 
         server = RcVCSServer(config_file, log_file, workers)
         server.start()
 
         @request.addfinalizer
         def cleanup():
             server.shutdown()
 
         server.wait_until_ready()
         return server
 
     return factory
 
 
 def _use_log_level(config):
     level = config.getoption('test_loglevel') or 'critical'
     return level.upper()
 
 
 @pytest.fixture(scope='session')
 def ini_config(request, tmpdir_factory, rcserver_port, vcsserver_port):
     option_name = 'pyramid_config'
     log_level = _use_log_level(request.config)
 
     overrides = [
         {'server:main': {'port': rcserver_port}},
         {'app:main': {
             'cache_dir': '%(here)s/rc_data',
             'vcs.server': f'localhost:{vcsserver_port}',
             # johbo: We will always start the VCSServer on our own based on the
             # fixtures of the test cases. For the test run it must always be
             # off in the INI file.
             'vcs.start_server': 'false',
 
             'vcs.server.protocol': 'http',
             'vcs.scm_app_implementation': 'http',
             'vcs.hooks.protocol': 'http',
             'vcs.hooks.host': '*',
+            'app.service_api.token': 'service_secret_token',
         }},
 
         {'handler_console': {
             'class': 'StreamHandler',
             'args': '(sys.stderr,)',
             'level': log_level,
         }},
 
     ]
 
     filename = get_config(
         request.config, option_name=option_name,
         override_option_name='{}_override'.format(option_name),
         overrides=overrides,
         basetemp=tmpdir_factory.getbasetemp().strpath,
         prefix='test_rce_')
     return filename
 
 
 @pytest.fixture(scope='session')
 def ini_settings(ini_config):
     ini_path = ini_config
     return get_app_config(ini_path)
 
 
 def get_available_port(min_port=40000, max_port=55555):
     from rhodecode.lib.utils2 import get_available_port as _get_port
     return _get_port(min_port, max_port)
 
 
 @pytest.fixture(scope='session')
 def rcserver_port(request):
     port = get_available_port()
     print(f'Using rhodecode port {port}')
     return port
 
 
 @pytest.fixture(scope='session')
 def vcsserver_port(request):
     port = request.config.getoption('--vcsserver-port')
     if port is None:
         port = get_available_port()
         print(f'Using vcsserver port {port}')
     return port
 
 
 @pytest.fixture(scope='session')
 def available_port_factory():
     """
     Returns a callable which returns free port numbers.
     """
     return get_available_port
 
 
 @pytest.fixture()
 def available_port(available_port_factory):
     """
     Gives you one free port for the current test.
 
     Uses "available_port_factory" to retrieve the port.
     """
     return available_port_factory()
 
 
 @pytest.fixture(scope='session')
 def testini_factory(tmpdir_factory, ini_config):
     """
     Factory to create an INI file based on TestINI.
 
     It will make sure to place the INI file in the correct directory.
     """
     basetemp = tmpdir_factory.getbasetemp().strpath
     return TestIniFactory(basetemp, ini_config)
 
 
 class TestIniFactory(object):
 
     def __init__(self, basetemp, template_ini):
         self._basetemp = basetemp
         self._template_ini = template_ini
 
     def __call__(self, ini_params, new_file_prefix='test'):
         ini_file = TestINI(
             self._template_ini, ini_params=ini_params,
             new_file_prefix=new_file_prefix, dir=self._basetemp)
         result = ini_file.create()
         return result
 
 
 def get_config(
         config, option_name, override_option_name, overrides=None,
         basetemp=None, prefix='test'):
     """
     Find a configuration file and apply overrides for the given `prefix`.
     """
     config_file = (
         config.getoption(option_name) or config.getini(option_name))
     if not config_file:
         pytest.exit(
             "Configuration error, could not extract {}.".format(option_name))
 
     overrides = overrides or []
     config_override = config.getoption(override_option_name)
     if config_override:
         overrides.append(config_override)
     temp_ini_file = TestINI(
         config_file, ini_params=overrides, new_file_prefix=prefix,
         dir=basetemp)
 
     return temp_ini_file.create()

setup.py

@@ -1,206 +1,207 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 # Import early to make sure things are patched up properly
 from setuptools import setup, find_packages, Extension
 
 import os
 import re
 import sys
 import pkgutil
 import platform
 import codecs
 
 import pip
 
 pip_major_version = int(pip.__version__.split(".")[0])
 if pip_major_version >= 20:
     from pip._internal.req import parse_requirements
     from pip._internal.network.session import PipSession
 elif pip_major_version >= 10:
     from pip._internal.req import parse_requirements
     from pip._internal.download import PipSession
 else:
     from pip.req import parse_requirements
     from pip.download import PipSession
 
 
 def get_package_name(req_object):
     package_name = None
     try:
         from pip._internal.req.constructors import install_req_from_parsed_requirement
     except ImportError:
         install_req_from_parsed_requirement = None
 
     # In 20.1 of pip, the requirements object changed
     if hasattr(req_object, 'req'):
         package_name = req_object.req.name
 
     if package_name is None:
         if install_req_from_parsed_requirement:
             package = install_req_from_parsed_requirement(req_object)
             package_name = package.req.name
 
     if package_name is None:
         # fallback for older pip
         package_name = re.split('===|<=|!=|==|>=|~=|<|>', req_object.requirement)[0]
 
     return package_name
 
 
 if sys.version_info < (3, 10):
     raise Exception('RhodeCode requires Python 3.10 or later')
 
 here = os.path.abspath(os.path.dirname(__file__))
 
 # defines current platform
 __platform__ = platform.system()
 __license__ = 'AGPLv3, and Commercial License'
 __author__ = 'RhodeCode GmbH'
 __url__ = 'https://code.rhodecode.com'
 is_windows = __platform__ in ('Windows',)
 
 
 def _get_requirements(req_filename, exclude=None, extras=None):
     extras = extras or []
     exclude = exclude or []
 
     try:
         parsed = parse_requirements(
             os.path.join(here, req_filename), session=PipSession())
     except TypeError:
         # try pip < 6.0.0, that doesn't support session
         parsed = parse_requirements(os.path.join(here, req_filename))
 
     requirements = []
     for int_req in parsed:
         req_name = get_package_name(int_req)
         if req_name not in exclude:
             requirements.append(req_name)
     return requirements + extras
 
 
 # requirements extract
 setup_requirements = ['PasteScript']
 install_requirements = _get_requirements(
     'requirements.txt', exclude=['setuptools', 'entrypoints'])
 test_requirements = _get_requirements(
     'requirements_test.txt')
 
 
 def get_version():
     here = os.path.abspath(os.path.dirname(__file__))
     ver_file = os.path.join(here, "rhodecode", "VERSION")
     with open(ver_file, "rt") as f:
         version = f.read().strip()
 
     return version
 
 
 # additional files that goes into package itself
 package_data = {
     '': ['*.txt', '*.rst'],
     'configs': ['*.ini'],
     'rhodecode': ['VERSION', 'i18n/*/LC_MESSAGES/*.mo', ],
 }
 
 description = 'Source Code Management Platform'
 keywords = ' '.join([
     'rhodecode', 'mercurial', 'git', 'svn',
     'code review',
     'repo groups', 'ldap', 'repository management', 'hgweb',
     'hgwebdir', 'gitweb', 'serving hgweb',
 ])
 
 
 # README/DESCRIPTION generation
 readme_file = 'README.rst'
 changelog_file = 'CHANGES.rst'
 try:
     long_description = codecs.open(readme_file).read() + '\n\n' + \
                        codecs.open(changelog_file).read()
 except IOError as err:
     sys.stderr.write(
         "[WARNING] Cannot find file specified as long_description (%s)\n "
         "or changelog (%s) skipping that file" % (readme_file, changelog_file))
     long_description = description
 
 
 setup(
     name='rhodecode-enterprise-ce',
     version=get_version(),
     description=description,
     long_description=long_description,
     keywords=keywords,
     license=__license__,
     author=__author__,
     author_email='[e-mail]',
     url=__url__,
     setup_requires=setup_requirements,
     install_requires=install_requirements,
     tests_require=test_requirements,
     zip_safe=False,
     packages=find_packages(exclude=["docs", "tests*"]),
     package_data=package_data,
     include_package_data=True,
     classifiers=[
         'Development Status :: 6 - Mature',
         'Environment :: Web Environment',
         'Intended Audience :: Developers',
         'Operating System :: OS Independent',
         'Topic :: Software Development :: Version Control',
         'License :: OSI Approved :: Affero GNU General Public License v3 or later (AGPLv3+)',
         'Programming Language :: Python :: 3.10',
     ],
     message_extractors={
         'rhodecode': [
             ('**.py', 'python', None),
             ('**.js', 'javascript', None),
             ('templates/**.mako', 'mako', {'input_encoding': 'utf-8'}),
             ('templates/**.html', 'mako', {'input_encoding': 'utf-8'}),
             ('public/**', 'ignore', None),
         ]
     },
     paster_plugins=['PasteScript'],
     entry_points={
         'paste.app_factory': [
             'main=rhodecode.config.middleware:make_pyramid_app',
         ],
         'paste.global_paster_command': [
             'ishell=rhodecode.lib.paster_commands.ishell:Command',
             'upgrade-db=rhodecode.lib.paster_commands.upgrade_db:UpgradeDb',
 
             'setup-rhodecode=rhodecode.lib.paster_commands.deprecated.setup_rhodecode:Command',
             'celeryd=rhodecode.lib.paster_commands.deprecated.celeryd:Command',
         ],
         'pyramid.pshell_runner': [
             'ipython = rhodecode.lib.pyramid_shell:ipython_shell_runner',
         ],
         'console_scripts': [
             'rc-setup-app=rhodecode.lib.rc_commands.setup_rc:main',
             'rc-upgrade-db=rhodecode.lib.rc_commands.upgrade_db:main',
             'rc-ishell=rhodecode.lib.rc_commands.ishell:main',
             'rc-add-artifact=rhodecode.lib.rc_commands.add_artifact:main',
-            'rc-ssh-wrapper=rhodecode.apps.ssh_support.lib.ssh_wrapper:main',
+            'rc-ssh-wrapper=rhodecode.apps.ssh_support.lib.ssh_wrapper_v1:main',
+            'rc-ssh-wrapper-v2=rhodecode.apps.ssh_support.lib.ssh_wrapper_v2:main',
         ],
         'beaker.backends': [
             'memorylru_base=rhodecode.lib.memory_lru_dict:MemoryLRUNamespaceManagerBase',
             'memorylru_debug=rhodecode.lib.memory_lru_dict:MemoryLRUNamespaceManagerDebug'
         ]
     },
 )