feat(archive-cache): objectstore now makes bucket required, and use shards as folders inside it

configs/development.ini

@@ -1,850 +1,849 b''
 
 ; #########################################
 ; RHODECODE COMMUNITY EDITION CONFIGURATION
 ; #########################################
 
 [DEFAULT]
 ; Debug flag sets all loggers to debug, and enables request tracking
 debug = true
 
 ; ########################################################################
 ; EMAIL CONFIGURATION
 ; These settings will be used by the RhodeCode mailing system
 ; ########################################################################
 
 ; prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
 ; email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 
 [server:main]
 ; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
 ; Host port for gunicorn are controlled by gunicorn_conf.py
 host = 127.0.0.1
 port = 10020
 
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
 ; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
 
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.
 ; allows to set RhodeCode under a prefix in server.
 ; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ; And set your prefix like: `prefix = /custom_prefix`
 ; be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ; to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
 ; Each option in the app:main can be override by an environmental variable
 ;
 ;To override an option:
 ;
 ;RC_<KeyName>
 ;Everything should be uppercase, . and - should be replaced by _.
 ;For example, if you have these configuration settings:
 ;rc_cache.repo_object.backend = foo
 ;can be overridden by
 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
 
 use = egg:rhodecode-enterprise-ce
 
 ; enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 ; #############
 ; DEBUG OPTIONS
 ; #############
 
 pyramid.reload_templates = true
 
 # During development the we want to have the debug toolbar enabled
 pyramid.includes =
     pyramid_debugtoolbar
 
 debugtoolbar.hosts = 0.0.0.0/0
 debugtoolbar.exclude_prefixes =
     /css
     /fonts
     /images
     /js
 
 ## RHODECODE PLUGINS ##
 rhodecode.includes =
     rhodecode.api
 
 
 # api prefix url
 rhodecode.api.url = /_admin/api
 
 ; enable debug style page
 debug_style = true
 
 ; #################
 ; END DEBUG OPTIONS
 ; #################
 
 ; encryption key used to encrypt social plugin tokens,
 ; remote_urls with credentials etc, if not set it defaults to
 ; `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
 ; decryption strict mode (enabled by default). It controls if decryption raises
 ; `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
 ; Pick algorithm for encryption. Either fernet (more secure) or aes (default)
 ; fernet is safer, and we strongly recommend switching to it.
 ; Due to backward compatibility aes is used as default.
 #rhodecode.encrypted_values.algorithm = fernet
 
 ; Return gzipped responses from RhodeCode (static files/application)
 gzip_responses = false
 
 ; Auto-generate javascript routes file on startup
 generate_js_files = false
 
 ; System global default language.
 ; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ; Perform a full repository scan and import on each server start.
 ; Settings this to true could lead to very long startup time.
 startup.import_repos = false
 
 ; URL at which the application is running. This is used for Bootstrapping
 ; requests in context when no web request is available. Used in ishell, or
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
 ; Host at which the Service API is running.
 app.service_api.host = http://rhodecode.local:10020
 
 ; Secret for Service API authentication.
 app.service_api.token =
 
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
 ; Cut off limit for large diffs (size in bytes). If overall diff size on
 ; commit, or pull request exceeds this limit this diff will be displayed
 ; partially. E.g 512000 == 512Kb
 cut_off_limit_diff = 512000
 
 ; Cut off limit for large files inside diffs (size in bytes). Each individual
 ; file inside diff which exceeds this limit will be displayed partially.
 ;  E.g 128000 == 128Kb
 cut_off_limit_file = 128000
 
 ; Use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = true
 
 ; Force https in RhodeCode, fixes https redirects, assumes it's always https.
 ; Normally this is controlled by proper flags sent from http server such as Nginx or Apache
 force_https = false
 
 ; use Strict-Transport-Security headers
 use_htsts = false
 
 ; Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ; RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ; gist URL alias, used to create nicer urls for gist. This should be an
 ; url that does rewrites to _admin/gists/{gistid}.
 ; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ; List of views (using glob pattern syntax) that AUTH TOKENS could be
 ; used for access.
 ; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ; came from the the logged in user who own this authentication token.
 ; Additionally @TOKEN syntax can be used to bound the view to specific
 ; authentication token. Such view would be only accessible when used together
 ; with this authentication token
 ; list of all views can be found under `/_admin/permissions/auth_token_access`
 ; The list should be "," separated and on a single line.
 ; Most common views to enable:
 
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
 ; Default encoding used to convert from and to unicode
 ; can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ; instance-id prefix
 ; a prefix key for this instance used for cache invalidation when running
 ; multiple instances of RhodeCode, make sure it's globally unique for
 ; all running RhodeCode instances. Leave empty if you don't use it
 instance_id =
 
 ; Fallback authentication plugin. Set this to a plugin ID to force the usage
 ; of an authentication plugin also if it is disabled by it's settings.
 ; This could be useful if you are unable to log in to the system due to broken
 ; authentication settings. Then you can enable e.g. the internal RhodeCode auth
 ; module to log in again and fix the settings.
 ; Available builtin plugin IDs (hash is part of the ID):
 ; egg:rhodecode-enterprise-ce#rhodecode
 ; egg:rhodecode-enterprise-ce#pam
 ; egg:rhodecode-enterprise-ce#ldap
 ; egg:rhodecode-enterprise-ce#jasig_cas
 ; egg:rhodecode-enterprise-ce#headers
 ; egg:rhodecode-enterprise-ce#crowd
 
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
 ; Flag to control loading of legacy plugins in py:/path format
 auth_plugin.import_legacy_plugins = true
 
 ; alternative return HTTP header for failed authentication. Default HTTP
 ; response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ; handling that causing a series of failed authentication calls.
 ; Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ; This will be served instead of default 401 on bad authentication
 auth_ret_code =
 
 ; use special detection method when serving auth_ret_code, instead of serving
 ; ret_code directly, use 401 initially (Which triggers credentials prompt)
 ; and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ; locking return code. When repository is locked return this HTTP code. 2XX
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ; Filesystem location were repositories should be stored
 repo_store.path = /var/opt/rhodecode_repo_store
 
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ; Generated license token required for EE edition license.
 ; New generated token value can be found in Admin > settings > license page.
 license_token =
 
 ; This flag hides sensitive information on the license page such as token, and license data
 license.hide_license_info = false
 
 ; supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 
 ; supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = dev
 
 ; Display extended labs settings
 labs_settings_active = true
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ; Send email with exception details when it happens
 #exception_tracker.send_email = false
 
 ; Comma separated list of recipients for exception emails,
 ; e.g [e-mail],[e-mail]
 ; Can be left empty, then emails will be sent to ALL super-admins
 #exception_tracker.send_email_recipients =
 
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
 ; File store configuration. This is used to store and serve uploaded files
 file_store.enabled = true
 
 ; Storage backend, available options are: local
 file_store.backend = local
 
 ; path to store the uploaded binaries and artifacts
 file_store.storage_path = /var/opt/rhodecode_data/file_store
 
 
 ; Redis url to acquire/check generation of archives locks
 archive_cache.locking.url = redis://redis:6379/1
 
 ; Storage backend, only 'filesystem' and 'objectstore' are available now
 archive_cache.backend.type = filesystem
 
 ; url for s3 compatible storage that allows to upload artifacts
 ; e.g http://minio:9000
 archive_cache.objectstore.url = http://s3-minio:9000
 
 ; key for s3 auth
 archive_cache.objectstore.key = key
 
 ; secret for s3 auth
 archive_cache.objectstore.secret = secret
 
 ; number of sharded buckets to create to distribute archives across
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8
 
-; a top-level bucket to put all other sharded buckets in
-; in case it's empty all buckets will be created in top-level (not recommended)
-; objects will be stored in rhodecode-archive-cache/shard-bucket-N based on the bucket_shards number
-archive_cache.objectstore.bucket_root = rhodecode-archive-cache
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-archive-cache/shard-N based on the bucket_shards number
+archive_cache.objectstore.bucket = rhodecode-archive-cache
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.objectstore.retry = false
 
 ; number of seconds to wait for next try using retry
 archive_cache.objectstore.retry_backoff = 1
 
 ; how many tries do do a retry fetch from this backend
 archive_cache.objectstore.retry_attempts = 10
 
 ; Default is $cache_dir/archive_cache if not set
 ; Generated repo archives will be cached at this location
 ; and served from the cache during subsequent requests for the same archive of
 ; the repository. This path is important to be shared across filesystems and with
 ; RhodeCode and vcsserver
 archive_cache.filesystem.store_dir = /var/opt/rhodecode_data/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
 archive_cache.filesystem.cache_size_gb = 1
 
 ; Eviction policy used to clear out after cache_size_gb limit is reached
 archive_cache.filesystem.eviction_policy = least-recently-stored
 
 ; By default cache uses sharding technique, this specifies how many shards are there
 ; default is 8 shards
 archive_cache.filesystem.cache_shards = 8
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.filesystem.retry = false
 
 ; number of seconds to wait for next try using retry
 archive_cache.filesystem.retry_backoff = 1
 
 ; how many tries do do a retry fetch from this backend
 archive_cache.filesystem.retry_attempts = 10
 
 
 ; #############
 ; CELERY CONFIG
 ; #############
 
 ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
 
 use_celery = true
 
 ; path to store schedule database
 #celerybeat-schedule.path =
 
 ; connection url to the message broker (default redis)
 celery.broker_url = redis://redis:6379/8
 
 ; results backend to get results for (default redis)
 celery.result_backend = redis://redis:6379/8
 
 ; rabbitmq example
 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
 ; maximum tasks to execute before worker restart
 celery.max_tasks_per_child = 20
 
 ; tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = false
 
 ; #############
 ; DOGPILE CACHE
 ; #############
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
 cache_dir = /var/opt/rhodecode_data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
 ; Only supported backend is `memory_lru`
 ; *********************************************
 rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
 rc_cache.sql_cache_short.expiration_time = 30
 
 
 ; *****************************************************
 ; `cache_repo_longterm` cache for repo object instances
 ; Only supported backend is `memory_lru`
 ; *****************************************************
 rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
 ; by default we use 30 Days, cache is still invalidated on push
 rc_cache.cache_repo_longterm.expiration_time = 2592000
 ; max items in LRU cache, set to smaller number to save memory, and expire last used caches
 rc_cache.cache_repo_longterm.max_size = 10000
 
 
 ; *********************************************
 ; `cache_general` cache for general purpose use
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *********************************************
 rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_general.expiration_time = 43200
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_general.arguments.filename = /tmp/cache_general_db
 
 ; alternative `cache_general` redis backend with distributed lock
 #rc_cache.cache_general.backend = dogpile.cache.rc.redis
 #rc_cache.cache_general.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_general.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_general.arguments.host = localhost
 #rc_cache.cache_general.arguments.port = 6379
 #rc_cache.cache_general.arguments.db = 0
 #rc_cache.cache_general.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_general.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_general.arguments.lock_auto_renewal = true
 
 ; *************************************************
 ; `cache_perms` cache for permission tree, auth TTL
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *************************************************
 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_perms.expiration_time = 3600
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_perms.arguments.filename = /tmp/cache_perms_db
 
 ; alternative `cache_perms` redis backend with distributed lock
 #rc_cache.cache_perms.backend = dogpile.cache.rc.redis
 #rc_cache.cache_perms.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_perms.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_perms.arguments.host = localhost
 #rc_cache.cache_perms.arguments.port = 6379
 #rc_cache.cache_perms.arguments.db = 0
 #rc_cache.cache_perms.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_perms.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_perms.arguments.lock_auto_renewal = true
 
 ; ***************************************************
 ; `cache_repo` cache for file tree, Readme, RSS FEEDS
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; ***************************************************
 rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_repo.expiration_time = 2592000
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_repo.arguments.filename = /tmp/cache_repo_db
 
 ; alternative `cache_repo` redis backend with distributed lock
 #rc_cache.cache_repo.backend = dogpile.cache.rc.redis
 #rc_cache.cache_repo.expiration_time = 2592000
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
 
 #rc_cache.cache_repo.arguments.host = localhost
 #rc_cache.cache_repo.arguments.port = 6379
 #rc_cache.cache_repo.arguments.db = 1
 #rc_cache.cache_repo.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_repo.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_repo.arguments.lock_auto_renewal = true
 
 ; ##############
 ; BEAKER SESSION
 ; ##############
 
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
 ; types are file, ext:redis, ext:database, ext:memcached
 ; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session
 #beaker.session.type = file
 #beaker.session.data_dir = %(here)s/data/sessions
 
 ; Redis based sessions
 beaker.session.type = ext:redis
 beaker.session.url = redis://redis:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = develop-rc-uytcxaz
 beaker.session.lock_dir = /data_ramdisk/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
 ; Sets session as invalid (also logging out user) if it haven not been
 ; accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 
 ; Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
 ; Set https secure cookie
 beaker.session.secure = false
 
 ; default cookie expiration time in seconds, set to `true` to set expire
 ; at browser close
 #beaker.session.cookie_expires = 3600
 
 ; #############################
 ; SEARCH INDEXING CONFIGURATION
 ; #############################
 
 ; Full text search indexer is available in rhodecode-tools under
 ; `rhodecode-tools index` command
 
 ; WHOOSH Backend, doesn't require additional services to run
 ; it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
 ; ####################
 ; CHANNELSTREAM CONFIG
 ; ####################
 
 ; channelstream enables persistent connections and live notification
 ; in the system. It's also used by the chat system
 
 channelstream.enabled = true
 
 ; server address for channelstream server on the backend
 channelstream.server = channelstream:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = ENV_GENERATED
 channelstream.history.location = /var/opt/rhodecode_data/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
 ; ##############################
 ; MAIN RHODECODE DATABASE CONFIG
 ; ##############################
 
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8
 ; pymysql is an alternative driver for MySQL, use in case of problems with default one
 #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
 sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 
 ; see sqlalchemy docs for other advanced settings
 ; print the sql statements to output
 sqlalchemy.db1.echo = false
 
 ; recycle the connections after this amount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 
 ; the number of connections to keep open inside the connection pool.
 ; 0 indicates no limit
 ; the general calculus with gevent is:
 ; if your system allows 500 concurrent greenlets (max_connections) that all do database access,
 ; then increase pool size + max overflow so that they add up to 500.
 #sqlalchemy.db1.pool_size = 5
 
 ; The number of connections to allow in connection pool "overflow", that is
 ; connections that can be opened above and beyond the pool_size setting,
 ; which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 ; Connection check ping, used to detect broken database connections
 ; could be enabled to better handle cases if MySQL has gone away errors
 #sqlalchemy.db1.ping_connection = true
 
 ; ##########
 ; VCS CONFIG
 ; ##########
 vcs.server.enable = true
 vcs.server = vcsserver:10010
 
 ; Web server connectivity protocol, responsible for web based VCS operations
 ; Available protocols are:
 ; `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
 ; Push/Pull operations protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
 ; `celery` - use celery based hooks
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
 ; accessible via network.
 ; Use vcs.hooks.host = "*" to bind to current hostname (for Docker)
 vcs.hooks.host = *
 
 ; Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
 ; List of enabled VCS backends, available options are:
 ; `hg`  - mercurial
 ; `git` - git
 ; `svn` - subversion
 vcs.backends = hg, git, svn
 
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
 ; Cache flag to cache vcsserver remote calls locally
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = true
 
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
 ; Maps RhodeCode repo groups into SVN paths for Apache
 ; ####################################################
 
 ; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ; Set a numeric version for your current SVN e.g 1.8, or 1.12
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
 ; Enable SVN proxy of requests over HTTP
 vcs.svn.proxy.enabled = true
 
 ; host to connect to running SVN subsystem
 vcs.svn.proxy.host = http://svn:8090
 
 ; Enable or disable the config file generation.
 svn.proxy.generate_config = true
 
 ; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
 svn.proxy.config_file_path = /etc/rhodecode/conf/svn/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
 ; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako
 #svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako
 
 ; Used as a prefix to the `Location` block in the generated config file.
 ; In most cases it should be set to `/`.
 svn.proxy.location_root = /
 
 ; Command to reload the mod dav svn configuration on change.
 ; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh
 ; Make sure user who runs RhodeCode process is allowed to reload Apache
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
 
 ; If the timeout expires before the reload command finishes, the command will
 ; be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
 ; ####################
 ; SSH Support Settings
 ; ####################
 
 ; Defines if a custom authorized_keys file should be created and written on
 ; any change user ssh keys. Setting this to false also disables possibility
 ; of adding SSH keys by users from web interface. Super admins can still
 ; manage SSH Keys.
 ssh.generate_authorized_keyfile = true
 
 ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
 ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
 ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
 ; Enables logging, and detailed output send back to the client during SSH
 ; operations. Useful for debugging, shouldn't be used in production.
 ssh.enable_debug_logging = true
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
 ssh.executable.hg = /usr/local/bin/rhodecode_bin/vcs_bin/hg
 ssh.executable.git = /usr/local/bin/rhodecode_bin/vcs_bin/git
 ssh.executable.svn = /usr/local/bin/rhodecode_bin/vcs_bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.
 ssh.enable_ui_key_generator = true
 
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Prometheus
 #statsd.enabled = false
 #statsd.statsd_host = 0.0.0.0
 #statsd.statsd_port = 8125
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
 ; configure logging automatically at server startup set to false
 ; to use the below custom logging config.
 ; RC_LOGGING_FORMATTER
 ; RC_LOGGING_LEVEL
 ; env variables can control the settings for logging in case of autoconfigure
 
 #logging.autoconfigure = true
 
 ; specify your own custom logging config file to configure logging
 #logging.logging_conf_file = /path/to/custom_logging.ini
 
 ; Dummy marker to add new entries after.
 ; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1
 
 
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
 
 [loggers]
 keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, json, color_formatter, color_formatter_sql
 
 ; #######
 ; LOGGERS
 ; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
 [logger_celery]
 level = DEBUG
 handlers =
 qualname = celery
 
 
 ; ########
 ; HANDLERS
 ; ########
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr, )
 level = DEBUG
 ; To enable JSON formatted logs replace 'generic/color_formatter' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = color_formatter
 
 [handler_console_sql]
 ; "level = DEBUG" logs SQL queries and results.
 ; "level = INFO" logs SQL queries.
 ; "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
 args = (sys.stderr, )
 level = WARN
 ; To enable JSON formatted logs replace 'generic/color_formatter_sql' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = color_formatter_sql
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_json]
 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
 class = rhodecode.lib._vendor.jsonlogger.JsonFormatter

configs/production.ini

@@ -1,818 +1,817 b''
 
 ; #########################################
 ; RHODECODE COMMUNITY EDITION CONFIGURATION
 ; #########################################
 
 [DEFAULT]
 ; Debug flag sets all loggers to debug, and enables request tracking
 debug = false
 
 ; ########################################################################
 ; EMAIL CONFIGURATION
 ; These settings will be used by the RhodeCode mailing system
 ; ########################################################################
 
 ; prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
 ; email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 
 [server:main]
 ; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
 ; Host port for gunicorn are controlled by gunicorn_conf.py
 host = 127.0.0.1
 port = 10020
 
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
 ; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
 
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.
 ; allows to set RhodeCode under a prefix in server.
 ; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ; And set your prefix like: `prefix = /custom_prefix`
 ; be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ; to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
 ; Each option in the app:main can be override by an environmental variable
 ;
 ;To override an option:
 ;
 ;RC_<KeyName>
 ;Everything should be uppercase, . and - should be replaced by _.
 ;For example, if you have these configuration settings:
 ;rc_cache.repo_object.backend = foo
 ;can be overridden by
 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
 
 use = egg:rhodecode-enterprise-ce
 
 ; enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 ; encryption key used to encrypt social plugin tokens,
 ; remote_urls with credentials etc, if not set it defaults to
 ; `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
 ; decryption strict mode (enabled by default). It controls if decryption raises
 ; `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
 ; Pick algorithm for encryption. Either fernet (more secure) or aes (default)
 ; fernet is safer, and we strongly recommend switching to it.
 ; Due to backward compatibility aes is used as default.
 #rhodecode.encrypted_values.algorithm = fernet
 
 ; Return gzipped responses from RhodeCode (static files/application)
 gzip_responses = false
 
 ; Auto-generate javascript routes file on startup
 generate_js_files = false
 
 ; System global default language.
 ; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ; Perform a full repository scan and import on each server start.
 ; Settings this to true could lead to very long startup time.
 startup.import_repos = false
 
 ; URL at which the application is running. This is used for Bootstrapping
 ; requests in context when no web request is available. Used in ishell, or
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
 ; Host at which the Service API is running.
 app.service_api.host = http://rhodecode.local:10020
 
 ; Secret for Service API authentication.
 app.service_api.token =
 
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
 ; Cut off limit for large diffs (size in bytes). If overall diff size on
 ; commit, or pull request exceeds this limit this diff will be displayed
 ; partially. E.g 512000 == 512Kb
 cut_off_limit_diff = 512000
 
 ; Cut off limit for large files inside diffs (size in bytes). Each individual
 ; file inside diff which exceeds this limit will be displayed partially.
 ;  E.g 128000 == 128Kb
 cut_off_limit_file = 128000
 
 ; Use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = true
 
 ; Force https in RhodeCode, fixes https redirects, assumes it's always https.
 ; Normally this is controlled by proper flags sent from http server such as Nginx or Apache
 force_https = false
 
 ; use Strict-Transport-Security headers
 use_htsts = false
 
 ; Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ; RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ; gist URL alias, used to create nicer urls for gist. This should be an
 ; url that does rewrites to _admin/gists/{gistid}.
 ; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ; List of views (using glob pattern syntax) that AUTH TOKENS could be
 ; used for access.
 ; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ; came from the the logged in user who own this authentication token.
 ; Additionally @TOKEN syntax can be used to bound the view to specific
 ; authentication token. Such view would be only accessible when used together
 ; with this authentication token
 ; list of all views can be found under `/_admin/permissions/auth_token_access`
 ; The list should be "," separated and on a single line.
 ; Most common views to enable:
 
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
 ; Default encoding used to convert from and to unicode
 ; can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ; instance-id prefix
 ; a prefix key for this instance used for cache invalidation when running
 ; multiple instances of RhodeCode, make sure it's globally unique for
 ; all running RhodeCode instances. Leave empty if you don't use it
 instance_id =
 
 ; Fallback authentication plugin. Set this to a plugin ID to force the usage
 ; of an authentication plugin also if it is disabled by it's settings.
 ; This could be useful if you are unable to log in to the system due to broken
 ; authentication settings. Then you can enable e.g. the internal RhodeCode auth
 ; module to log in again and fix the settings.
 ; Available builtin plugin IDs (hash is part of the ID):
 ; egg:rhodecode-enterprise-ce#rhodecode
 ; egg:rhodecode-enterprise-ce#pam
 ; egg:rhodecode-enterprise-ce#ldap
 ; egg:rhodecode-enterprise-ce#jasig_cas
 ; egg:rhodecode-enterprise-ce#headers
 ; egg:rhodecode-enterprise-ce#crowd
 
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
 ; Flag to control loading of legacy plugins in py:/path format
 auth_plugin.import_legacy_plugins = true
 
 ; alternative return HTTP header for failed authentication. Default HTTP
 ; response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ; handling that causing a series of failed authentication calls.
 ; Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ; This will be served instead of default 401 on bad authentication
 auth_ret_code =
 
 ; use special detection method when serving auth_ret_code, instead of serving
 ; ret_code directly, use 401 initially (Which triggers credentials prompt)
 ; and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ; locking return code. When repository is locked return this HTTP code. 2XX
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ; Filesystem location were repositories should be stored
 repo_store.path = /var/opt/rhodecode_repo_store
 
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ; Generated license token required for EE edition license.
 ; New generated token value can be found in Admin > settings > license page.
 license_token =
 
 ; This flag hides sensitive information on the license page such as token, and license data
 license.hide_license_info = false
 
 ; supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 
 ; supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = prod
 
 ; Display extended labs settings
 labs_settings_active = true
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ; Send email with exception details when it happens
 #exception_tracker.send_email = false
 
 ; Comma separated list of recipients for exception emails,
 ; e.g [e-mail],[e-mail]
 ; Can be left empty, then emails will be sent to ALL super-admins
 #exception_tracker.send_email_recipients =
 
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
 ; File store configuration. This is used to store and serve uploaded files
 file_store.enabled = true
 
 ; Storage backend, available options are: local
 file_store.backend = local
 
 ; path to store the uploaded binaries and artifacts
 file_store.storage_path = /var/opt/rhodecode_data/file_store
 
 
 ; Redis url to acquire/check generation of archives locks
 archive_cache.locking.url = redis://redis:6379/1
 
 ; Storage backend, only 'filesystem' and 'objectstore' are available now
 archive_cache.backend.type = filesystem
 
 ; url for s3 compatible storage that allows to upload artifacts
 ; e.g http://minio:9000
 archive_cache.objectstore.url = http://s3-minio:9000
 
 ; key for s3 auth
 archive_cache.objectstore.key = key
 
 ; secret for s3 auth
 archive_cache.objectstore.secret = secret
 
 ; number of sharded buckets to create to distribute archives across
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8
 
-; a top-level bucket to put all other sharded buckets in
-; in case it's empty all buckets will be created in top-level (not recommended)
-; objects will be stored in rhodecode-archive-cache/shard-bucket-N based on the bucket_shards number
-archive_cache.objectstore.bucket_root = rhodecode-archive-cache
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-archive-cache/shard-N based on the bucket_shards number
+archive_cache.objectstore.bucket = rhodecode-archive-cache
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.objectstore.retry = false
 
 ; number of seconds to wait for next try using retry
 archive_cache.objectstore.retry_backoff = 1
 
 ; how many tries do do a retry fetch from this backend
 archive_cache.objectstore.retry_attempts = 10
 
 ; Default is $cache_dir/archive_cache if not set
 ; Generated repo archives will be cached at this location
 ; and served from the cache during subsequent requests for the same archive of
 ; the repository. This path is important to be shared across filesystems and with
 ; RhodeCode and vcsserver
 archive_cache.filesystem.store_dir = /var/opt/rhodecode_data/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
 archive_cache.filesystem.cache_size_gb = 40
 
 ; Eviction policy used to clear out after cache_size_gb limit is reached
 archive_cache.filesystem.eviction_policy = least-recently-stored
 
 ; By default cache uses sharding technique, this specifies how many shards are there
 ; default is 8 shards
 archive_cache.filesystem.cache_shards = 8
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.filesystem.retry = false
 
 ; number of seconds to wait for next try using retry
 archive_cache.filesystem.retry_backoff = 1
 
 ; how many tries do do a retry fetch from this backend
 archive_cache.filesystem.retry_attempts = 10
 
 
 ; #############
 ; CELERY CONFIG
 ; #############
 
 ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
 
 use_celery = true
 
 ; path to store schedule database
 #celerybeat-schedule.path =
 
 ; connection url to the message broker (default redis)
 celery.broker_url = redis://redis:6379/8
 
 ; results backend to get results for (default redis)
 celery.result_backend = redis://redis:6379/8
 
 ; rabbitmq example
 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
 ; maximum tasks to execute before worker restart
 celery.max_tasks_per_child = 20
 
 ; tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = false
 
 ; #############
 ; DOGPILE CACHE
 ; #############
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
 cache_dir = /var/opt/rhodecode_data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
 ; Only supported backend is `memory_lru`
 ; *********************************************
 rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
 rc_cache.sql_cache_short.expiration_time = 30
 
 
 ; *****************************************************
 ; `cache_repo_longterm` cache for repo object instances
 ; Only supported backend is `memory_lru`
 ; *****************************************************
 rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
 ; by default we use 30 Days, cache is still invalidated on push
 rc_cache.cache_repo_longterm.expiration_time = 2592000
 ; max items in LRU cache, set to smaller number to save memory, and expire last used caches
 rc_cache.cache_repo_longterm.max_size = 10000
 
 
 ; *********************************************
 ; `cache_general` cache for general purpose use
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *********************************************
 rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_general.expiration_time = 43200
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_general.arguments.filename = /tmp/cache_general_db
 
 ; alternative `cache_general` redis backend with distributed lock
 #rc_cache.cache_general.backend = dogpile.cache.rc.redis
 #rc_cache.cache_general.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_general.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_general.arguments.host = localhost
 #rc_cache.cache_general.arguments.port = 6379
 #rc_cache.cache_general.arguments.db = 0
 #rc_cache.cache_general.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_general.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_general.arguments.lock_auto_renewal = true
 
 ; *************************************************
 ; `cache_perms` cache for permission tree, auth TTL
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *************************************************
 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_perms.expiration_time = 3600
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_perms.arguments.filename = /tmp/cache_perms_db
 
 ; alternative `cache_perms` redis backend with distributed lock
 #rc_cache.cache_perms.backend = dogpile.cache.rc.redis
 #rc_cache.cache_perms.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_perms.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_perms.arguments.host = localhost
 #rc_cache.cache_perms.arguments.port = 6379
 #rc_cache.cache_perms.arguments.db = 0
 #rc_cache.cache_perms.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_perms.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_perms.arguments.lock_auto_renewal = true
 
 ; ***************************************************
 ; `cache_repo` cache for file tree, Readme, RSS FEEDS
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; ***************************************************
 rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_repo.expiration_time = 2592000
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_repo.arguments.filename = /tmp/cache_repo_db
 
 ; alternative `cache_repo` redis backend with distributed lock
 #rc_cache.cache_repo.backend = dogpile.cache.rc.redis
 #rc_cache.cache_repo.expiration_time = 2592000
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
 
 #rc_cache.cache_repo.arguments.host = localhost
 #rc_cache.cache_repo.arguments.port = 6379
 #rc_cache.cache_repo.arguments.db = 1
 #rc_cache.cache_repo.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_repo.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_repo.arguments.lock_auto_renewal = true
 
 ; ##############
 ; BEAKER SESSION
 ; ##############
 
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
 ; types are file, ext:redis, ext:database, ext:memcached
 ; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session
 #beaker.session.type = file
 #beaker.session.data_dir = %(here)s/data/sessions
 
 ; Redis based sessions
 beaker.session.type = ext:redis
 beaker.session.url = redis://redis:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = production-rc-uytcxaz
 beaker.session.lock_dir = /data_ramdisk/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
 ; Sets session as invalid (also logging out user) if it haven not been
 ; accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 
 ; Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
 ; Set https secure cookie
 beaker.session.secure = false
 
 ; default cookie expiration time in seconds, set to `true` to set expire
 ; at browser close
 #beaker.session.cookie_expires = 3600
 
 ; #############################
 ; SEARCH INDEXING CONFIGURATION
 ; #############################
 
 ; Full text search indexer is available in rhodecode-tools under
 ; `rhodecode-tools index` command
 
 ; WHOOSH Backend, doesn't require additional services to run
 ; it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
 ; ####################
 ; CHANNELSTREAM CONFIG
 ; ####################
 
 ; channelstream enables persistent connections and live notification
 ; in the system. It's also used by the chat system
 
 channelstream.enabled = true
 
 ; server address for channelstream server on the backend
 channelstream.server = channelstream:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = ENV_GENERATED
 channelstream.history.location = /var/opt/rhodecode_data/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
 ; ##############################
 ; MAIN RHODECODE DATABASE CONFIG
 ; ##############################
 
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8
 ; pymysql is an alternative driver for MySQL, use in case of problems with default one
 #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
 sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 
 ; see sqlalchemy docs for other advanced settings
 ; print the sql statements to output
 sqlalchemy.db1.echo = false
 
 ; recycle the connections after this amount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 
 ; the number of connections to keep open inside the connection pool.
 ; 0 indicates no limit
 ; the general calculus with gevent is:
 ; if your system allows 500 concurrent greenlets (max_connections) that all do database access,
 ; then increase pool size + max overflow so that they add up to 500.
 #sqlalchemy.db1.pool_size = 5
 
 ; The number of connections to allow in connection pool "overflow", that is
 ; connections that can be opened above and beyond the pool_size setting,
 ; which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 ; Connection check ping, used to detect broken database connections
 ; could be enabled to better handle cases if MySQL has gone away errors
 #sqlalchemy.db1.ping_connection = true
 
 ; ##########
 ; VCS CONFIG
 ; ##########
 vcs.server.enable = true
 vcs.server = vcsserver:10010
 
 ; Web server connectivity protocol, responsible for web based VCS operations
 ; Available protocols are:
 ; `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
 ; Push/Pull operations protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
 ; `celery` - use celery based hooks
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
 ; accessible via network.
 ; Use vcs.hooks.host = "*" to bind to current hostname (for Docker)
 vcs.hooks.host = *
 
 ; Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
 ; List of enabled VCS backends, available options are:
 ; `hg`  - mercurial
 ; `git` - git
 ; `svn` - subversion
 vcs.backends = hg, git, svn
 
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
 ; Cache flag to cache vcsserver remote calls locally
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = true
 
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
 ; Maps RhodeCode repo groups into SVN paths for Apache
 ; ####################################################
 
 ; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ; Set a numeric version for your current SVN e.g 1.8, or 1.12
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
 ; Enable SVN proxy of requests over HTTP
 vcs.svn.proxy.enabled = true
 
 ; host to connect to running SVN subsystem
 vcs.svn.proxy.host = http://svn:8090
 
 ; Enable or disable the config file generation.
 svn.proxy.generate_config = true
 
 ; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
 svn.proxy.config_file_path = /etc/rhodecode/conf/svn/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
 ; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako
 #svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako
 
 ; Used as a prefix to the `Location` block in the generated config file.
 ; In most cases it should be set to `/`.
 svn.proxy.location_root = /
 
 ; Command to reload the mod dav svn configuration on change.
 ; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh
 ; Make sure user who runs RhodeCode process is allowed to reload Apache
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
 
 ; If the timeout expires before the reload command finishes, the command will
 ; be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
 ; ####################
 ; SSH Support Settings
 ; ####################
 
 ; Defines if a custom authorized_keys file should be created and written on
 ; any change user ssh keys. Setting this to false also disables possibility
 ; of adding SSH keys by users from web interface. Super admins can still
 ; manage SSH Keys.
 ssh.generate_authorized_keyfile = true
 
 ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
 ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
 ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
 ; Enables logging, and detailed output send back to the client during SSH
 ; operations. Useful for debugging, shouldn't be used in production.
 ssh.enable_debug_logging = false
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
 ssh.executable.hg = /usr/local/bin/rhodecode_bin/vcs_bin/hg
 ssh.executable.git = /usr/local/bin/rhodecode_bin/vcs_bin/git
 ssh.executable.svn = /usr/local/bin/rhodecode_bin/vcs_bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.
 ssh.enable_ui_key_generator = true
 
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Prometheus
 #statsd.enabled = false
 #statsd.statsd_host = 0.0.0.0
 #statsd.statsd_port = 8125
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
 ; configure logging automatically at server startup set to false
 ; to use the below custom logging config.
 ; RC_LOGGING_FORMATTER
 ; RC_LOGGING_LEVEL
 ; env variables can control the settings for logging in case of autoconfigure
 
 #logging.autoconfigure = true
 
 ; specify your own custom logging config file to configure logging
 #logging.logging_conf_file = /path/to/custom_logging.ini
 
 ; Dummy marker to add new entries after.
 ; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1
 
 
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
 
 [loggers]
 keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, json, color_formatter, color_formatter_sql
 
 ; #######
 ; LOGGERS
 ; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
 [logger_celery]
 level = DEBUG
 handlers =
 qualname = celery
 
 
 ; ########
 ; HANDLERS
 ; ########
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr, )
 level = INFO
 ; To enable JSON formatted logs replace 'generic/color_formatter' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 [handler_console_sql]
 ; "level = DEBUG" logs SQL queries and results.
 ; "level = INFO" logs SQL queries.
 ; "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
 args = (sys.stderr, )
 level = WARN
 ; To enable JSON formatted logs replace 'generic/color_formatter_sql' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_json]
 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
 class = rhodecode.lib._vendor.jsonlogger.JsonFormatter

rhodecode/config/config_maker.py

@@ -1,222 +1,222 b''
 # Copyright (C) 2010-2023 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import tempfile
 import logging
 
 from pyramid.settings import asbool
 
 from rhodecode.config.settings_maker import SettingsMaker
 from rhodecode.config import utils as config_utils
 
 log = logging.getLogger(__name__)
 
 
 def sanitize_settings_and_apply_defaults(global_config, settings):
     """
     Applies settings defaults and does all type conversion.
 
     We would move all settings parsing and preparation into this place, so that
     we have only one place left which deals with this part. The remaining parts
     of the application would start to rely fully on well-prepared settings.
 
     This piece would later be split up per topic to avoid a big fat monster
     function.
     """
     jn = os.path.join
 
     global_settings_maker = SettingsMaker(global_config)
     global_settings_maker.make_setting('debug', default=False, parser='bool')
     debug_enabled = asbool(global_config.get('debug'))
 
     settings_maker = SettingsMaker(settings)
 
     settings_maker.make_setting(
         'logging.autoconfigure',
         default=False,
         parser='bool')
 
     logging_conf = jn(os.path.dirname(global_config.get('__file__')), 'logging.ini')
     settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
 
     # Default includes, possible to change as a user
     pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
     log.debug(
         "Using the following pyramid.includes: %s",
         pyramid_includes)
 
     settings_maker.make_setting('rhodecode.edition', 'Community Edition')
     settings_maker.make_setting('rhodecode.edition_id', 'CE')
 
     if 'mako.default_filters' not in settings:
         # set custom default filters if we don't have it defined
         settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
         settings['mako.default_filters'] = 'h_filter'
 
     if 'mako.directories' not in settings:
         mako_directories = settings.setdefault('mako.directories', [
             # Base templates of the original application
             'rhodecode:templates',
         ])
         log.debug(
             "Using the following Mako template directories: %s",
             mako_directories)
 
     # NOTE(marcink): fix redis requirement for schema of connection since 3.X
     if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
         raw_url = settings['beaker.session.url']
         if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
             settings['beaker.session.url'] = 'redis://' + raw_url
 
     settings_maker.make_setting('__file__', global_config.get('__file__'))
 
     # TODO: johbo: Re-think this, usually the call to config.include
     # should allow to pass in a prefix.
     settings_maker.make_setting('rhodecode.api.url', '/_admin/api')
 
     # Sanitize generic settings.
     settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
     settings_maker.make_setting('gzip_responses', False, parser='bool')
     settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
 
     # statsd
     settings_maker.make_setting('statsd.enabled', False, parser='bool')
     settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
     settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
     settings_maker.make_setting('statsd.statsd_prefix', '')
     settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
 
     settings_maker.make_setting('vcs.svn.compatible_version', '')
     settings_maker.make_setting('vcs.svn.proxy.enabled', True, parser='bool')
     settings_maker.make_setting('vcs.svn.proxy.host', 'http://svn:8090', parser='string')
     settings_maker.make_setting('vcs.hooks.protocol', 'http')
     settings_maker.make_setting('vcs.hooks.host', '*')
     settings_maker.make_setting('vcs.scm_app_implementation', 'http')
     settings_maker.make_setting('vcs.server', '')
     settings_maker.make_setting('vcs.server.protocol', 'http')
     settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
     settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
     settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
     settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
     settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
 
     settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
 
     # repo_store path
     settings_maker.make_setting('repo_store.path', '/var/opt/rhodecode_repo_store')
     # Support legacy values of vcs.scm_app_implementation. Legacy
     # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
     # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
     scm_app_impl = settings['vcs.scm_app_implementation']
     if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
         settings['vcs.scm_app_implementation'] = 'http'
 
     settings_maker.make_setting('appenlight', False, parser='bool')
 
     temp_store = tempfile.gettempdir()
     tmp_cache_dir = jn(temp_store, 'rc_cache')
 
     # save default, cache dir, and use it for all backends later.
     default_cache_dir = settings_maker.make_setting(
         'cache_dir',
         default=tmp_cache_dir, default_when_empty=True,
         parser='dir:ensured')
 
     # exception store cache
     settings_maker.make_setting(
         'exception_tracker.store_path',
         default=jn(default_cache_dir, 'exc_store'), default_when_empty=True,
         parser='dir:ensured'
     )
 
     settings_maker.make_setting(
         'celerybeat-schedule.path',
         default=jn(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
         parser='file:ensured'
     )
 
     settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
     settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
 
     # sessions, ensure file since no-value is memory
     settings_maker.make_setting('beaker.session.type', 'file')
     settings_maker.make_setting('beaker.session.data_dir',  jn(default_cache_dir, 'session_data'))
 
     # cache_general
     settings_maker.make_setting('rc_cache.cache_general.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_general.expiration_time', 60 * 60 * 12, parser='int')
     settings_maker.make_setting('rc_cache.cache_general.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_general.db'))
 
     # cache_perms
     settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60 * 60, parser='int')
     settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_perms_db'))
 
     # cache_repo
     settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60 * 60 * 24 * 30, parser='int')
     settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_repo_db'))
 
     # cache_license
     settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
     settings_maker.make_setting('rc_cache.cache_license.expiration_time', 60 * 5, parser='int')
     settings_maker.make_setting('rc_cache.cache_license.arguments.filename', jn(default_cache_dir, 'rhodecode_cache_license_db'))
 
     # cache_repo_longterm memory, 96H
     settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
     settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
     settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
 
     # sql_cache_short
     settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
     settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
     settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')
 
     # archive_cache
     settings_maker.make_setting('archive_cache.locking.url', 'redis://redis:6379/1')
     settings_maker.make_setting('archive_cache.backend.type', 'filesystem')
 
     settings_maker.make_setting('archive_cache.filesystem.store_dir', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
     settings_maker.make_setting('archive_cache.filesystem.cache_shards', 8, parser='int')
     settings_maker.make_setting('archive_cache.filesystem.cache_size_gb', 10, parser='float')
     settings_maker.make_setting('archive_cache.filesystem.eviction_policy', 'least-recently-stored')
 
     settings_maker.make_setting('archive_cache.filesystem.retry', False, parser='bool')
     settings_maker.make_setting('archive_cache.filesystem.retry_backoff', 1, parser='int')
     settings_maker.make_setting('archive_cache.filesystem.retry_attempts', 10, parser='int')
 
     settings_maker.make_setting('archive_cache.objectstore.url', jn(default_cache_dir, 'archive_cache'), default_when_empty=True,)
     settings_maker.make_setting('archive_cache.objectstore.key', '')
     settings_maker.make_setting('archive_cache.objectstore.secret', '')
-    settings_maker.make_setting('archive_cache.objectstore.bucket_root', 'rhodecode-archive-cache')
+    settings_maker.make_setting('archive_cache.objectstore.bucket', 'rhodecode-archive-cache', default_when_empty=True,)
     settings_maker.make_setting('archive_cache.objectstore.bucket_shards', 8, parser='int')
 
     settings_maker.make_setting('archive_cache.objectstore.cache_size_gb', 10, parser='float')
     settings_maker.make_setting('archive_cache.objectstore.eviction_policy', 'least-recently-stored')
 
     settings_maker.make_setting('archive_cache.objectstore.retry', False, parser='bool')
     settings_maker.make_setting('archive_cache.objectstore.retry_backoff', 1, parser='int')
     settings_maker.make_setting('archive_cache.objectstore.retry_attempts', 10, parser='int')
 
     settings_maker.env_expand()
 
     # configure instance id
     config_utils.set_instance_id(settings)
 
     return settings

rhodecode/lib/archive_cache/backends/base.py

@@ -1,352 +1,355 b''
 # Copyright (C) 2015-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import functools
 import logging
 import typing
 import time
 import zlib
 
 from ...ext_json import json
 from ..utils import StatsDB, NOT_GIVEN, ShardFileReader, EVICTION_POLICY, format_size
 from ..lock import GenerationLock
 
 log = logging.getLogger(__name__)
 
 
 class BaseShard:
     storage_type: str = ''
     fs = None
 
     @classmethod
     def hash(cls, key):
         """Compute portable hash for `key`.
 
         :param key: key to hash
         :return: hash value
 
         """
         mask = 0xFFFFFFFF
         return zlib.adler32(key.encode('utf-8')) & mask  # noqa
 
     def _write_file(self, full_path, read_iterator, mode):
         raise NotImplementedError
 
     def _get_keyfile(self, key):
         raise NotImplementedError
 
     def random_filename(self):
         raise NotImplementedError
 
     def _store(self, key, value_reader, metadata, mode):
         (filename,  # hash-name
          full_path  # full-path/hash-name
          ) = self.random_filename()
 
         key_file, key_file_path = self._get_keyfile(key)
 
         # STORE METADATA
         _metadata = {
             "version": "v1",
 
             "key_file": key_file,  # this is the .key.json file storing meta
             "key_file_path": key_file_path,  # full path to key_file
             "archive_key": key,  # original name we stored archive under, e.g my-archive.zip
             "archive_filename": filename,  # the actual filename we stored that file under
             "archive_full_path": full_path,
 
             "store_time": time.time(),
             "access_count": 0,
             "access_time": 0,
 
             "size": 0
         }
         if metadata:
             _metadata.update(metadata)
 
         read_iterator = iter(functools.partial(value_reader.read, 2**22), b'')
         size, sha256 = self._write_file(full_path, read_iterator, mode)
         _metadata['size'] = size
         _metadata['sha256'] = sha256
 
         # after archive is finished, we create a key to save the presence of the binary file
         with self.fs.open(key_file_path, 'wb') as f:
             f.write(json.dumps(_metadata))
 
         return key, filename, size, _metadata
 
     def _fetch(self,  key, retry, retry_attempts, retry_backoff):
         if retry is NOT_GIVEN:
             retry = False
         if retry_attempts is NOT_GIVEN:
             retry_attempts = 0
 
         if retry and retry_attempts > 0:
             for attempt in range(1, retry_attempts + 1):
                 if key in self:
                     break
                 # we didn't find the key, wait retry_backoff N seconds, and re-check
                 time.sleep(retry_backoff)
 
         if key not in self:
             log.exception(f'requested key={key} not found in {self} retry={retry}, attempts={retry_attempts}')
             raise KeyError(key)
 
         key_file, key_file_path = self._get_keyfile(key)
         with self.fs.open(key_file_path, 'rb') as f:
             metadata = json.loads(f.read())
 
         archive_path = metadata['archive_full_path']
 
         try:
             return ShardFileReader(self.fs.open(archive_path, 'rb')), metadata
         finally:
             # update usage stats, count and accessed
             metadata["access_count"] = metadata.get("access_count", 0) + 1
             metadata["access_time"] = time.time()
             log.debug('Updated %s with access snapshot, access_count=%s access_time=%s',
                       key_file, metadata['access_count'], metadata['access_time'])
             with self.fs.open(key_file_path, 'wb') as f:
                 f.write(json.dumps(metadata))
 
     def _remove(self, key):
         if key not in self:
             log.exception(f'requested key={key} not found in {self}')
             raise KeyError(key)
 
         key_file, key_file_path = self._get_keyfile(key)
         with self.fs.open(key_file_path, 'rb') as f:
             metadata = json.loads(f.read())
 
         archive_path = metadata['archive_full_path']
         self.fs.rm(archive_path)
         self.fs.rm(key_file_path)
         return 1
 
     @property
     def storage_medium(self):
         return getattr(self, self.storage_type)
 
     @property
     def key_suffix(self):
         return 'key.json'
 
     def __contains__(self, key):
         """Return `True` if `key` matching item is found in cache.
 
         :param key: key matching item
         :return: True if key matching item
 
         """
         key_file, key_file_path = self._get_keyfile(key)
         return self.fs.exists(key_file_path)
 
 
 class BaseCache:
     _locking_url: str = ''
     _storage_path: str = ''
     _config = {}
     retry = False
     retry_attempts = 0
     retry_backoff = 1
     _shards = tuple()
+    shard_cls = BaseShard
 
     def __contains__(self, key):
         """Return `True` if `key` matching item is found in cache.
 
         :param key: key matching item
         :return: True if key matching item
 
         """
         return self.has_key(key)
 
     def __repr__(self):
         return f'<{self.__class__.__name__}(storage={self._storage_path})>'
 
     @classmethod
     def gb_to_bytes(cls, gb):
         return gb * (1024 ** 3)
 
     @property
     def storage_path(self):
         return self._storage_path
 
     @classmethod
     def get_stats_db(cls):
         return StatsDB()
 
     def get_conf(self, key, pop=False):
         if key not in self._config:
             raise ValueError(f"No configuration key '{key}', please make sure it exists in archive_cache config")
         val = self._config[key]
         if pop:
             del self._config[key]
         return val
 
-    def _get_shard(self, key):
-        raise NotImplementedError
+    def _get_shard(self, key) -> shard_cls:
+        index = self._hash(key) % self._shard_count
+        shard = self._shards[index]
+        return shard
 
     def _get_size(self, shard, archive_path):
         raise NotImplementedError
 
     def store(self, key, value_reader, metadata=None):
         shard = self._get_shard(key)
         return shard.store(key, value_reader, metadata)
 
     def fetch(self, key, retry=NOT_GIVEN, retry_attempts=NOT_GIVEN) -> tuple[typing.BinaryIO, dict]:
         """
         Return file handle corresponding to `key` from specific shard cache.
         """
         if retry is NOT_GIVEN:
             retry = self.retry
         if retry_attempts is NOT_GIVEN:
             retry_attempts = self.retry_attempts
         retry_backoff = self.retry_backoff
 
         shard = self._get_shard(key)
         return shard.fetch(key, retry=retry, retry_attempts=retry_attempts, retry_backoff=retry_backoff)
 
     def remove(self, key):
         shard = self._get_shard(key)
         return shard.remove(key)
 
     def has_key(self, archive_key):
         """Return `True` if `key` matching item is found in cache.
 
         :param archive_key: key for item, this is a unique archive name we want to store data under. e.g my-archive-svn.zip
         :return: True if key is found
 
         """
         shard = self._get_shard(archive_key)
         return archive_key in shard
 
     def iter_keys(self):
         for shard in self._shards:
             if shard.fs.exists(shard.storage_medium):
                 for path, _dirs, _files in shard.fs.walk(shard.storage_medium):
                     for key_file_path in _files:
                         if key_file_path.endswith(shard.key_suffix):
                             yield shard, key_file_path
 
     def get_lock(self, lock_key):
         return GenerationLock(lock_key, self._locking_url)
 
     def evict(self, policy=None, size_limit=None) -> dict:
         """
         Remove old items based on the conditions
 
 
         explanation of this algo:
         iterate over each shard, then for each shard iterate over the .key files
         read the key files metadata stored. This gives us a full list of keys, cached_archived, their size and
         access data, time creation, and access counts.
 
         Store that into a memory DB in order we can run different sorting strategies easily.
         Summing the size is a sum sql query.
 
         Then we run a sorting strategy based on eviction policy.
         We iterate over sorted keys, and remove each checking if we hit the overall limit.
         """
         removal_info = {
             "removed_items": 0,
             "removed_size": 0
         }
         policy = policy or self._eviction_policy
         size_limit = size_limit or self._cache_size_limit
 
         select_policy = EVICTION_POLICY[policy]['evict']
 
         log.debug('Running eviction policy \'%s\', and checking for size limit: %s',
                   policy, format_size(size_limit))
 
         if select_policy is None:
             return removal_info
 
         db = self.get_stats_db()
 
         data = []
         cnt = 1
 
         for shard, key_file in self.iter_keys():
             with shard.fs.open(os.path.join(shard.storage_medium, key_file), 'rb') as f:
                 metadata = json.loads(f.read())
 
             key_file_path = os.path.join(shard.storage_medium, key_file)
 
             archive_key = metadata['archive_key']
             archive_path = metadata['archive_full_path']
 
             size = metadata.get('size')
             if not size:
                 # in case we don't have size re-calc it...
                 size = self._get_size(shard, archive_path)
 
             data.append([
                 cnt,
                 key_file,
                 key_file_path,
                 archive_key,
                 archive_path,
                 metadata.get('store_time', 0),
                 metadata.get('access_time', 0),
                 metadata.get('access_count', 0),
                 size,
             ])
             cnt += 1
 
         # Insert bulk data using executemany
         db.bulk_insert(data)
 
         total_size = db.get_total_size()
         log.debug('Analyzed %s keys, occupying: %s, running eviction to match %s',
                   len(data), format_size(total_size), format_size(size_limit))
 
         removed_items = 0
         removed_size = 0
         for key_file, archive_key, size in db.get_sorted_keys(select_policy):
             # simulate removal impact BEFORE removal
             total_size -= size
 
             if total_size <= size_limit:
                 # we obtained what we wanted...
                 break
 
             self.remove(archive_key)
             removed_items += 1
             removed_size += size
         removal_info['removed_items'] = removed_items
         removal_info['removed_size'] = removed_size
         log.debug('Removed %s cache archives, and reduced size by: %s',
                   removed_items, format_size(removed_size))
         return removal_info
 
     def get_statistics(self):
         total_files = 0
         total_size = 0
         meta = {}
 
         for shard, key_file in self.iter_keys():
             json_key = f"{shard.storage_medium}/{key_file}"
             with shard.fs.open(json_key, 'rb') as f:
                 total_files += 1
                 metadata = json.loads(f.read())
                 total_size += metadata['size']
 
         return total_files, total_size, meta
 

rhodecode/lib/archive_cache/backends/fanout_cache.py

@@ -1,167 +1,174 b''
 # Copyright (C) 2015-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import codecs
 import hashlib
 import logging
 import os
 
 import fsspec
 
 from .base import BaseCache, BaseShard
 from ..utils import ShardFileReader, NOT_GIVEN
 from ...type_utils import str2bool
 
 log = logging.getLogger(__name__)
 
 
 class FileSystemShard(BaseShard):
 
-    def __init__(self, index, directory, **settings):
+    def __init__(self, index, directory, directory_folder, fs, **settings):
         self._index = index
         self._directory = directory
+        self._directory_folder = directory_folder
         self.storage_type = 'directory'
-        self.fs = fsspec.filesystem('file')
+
+        self.fs = fs
 
     @property
     def directory(self):
-        """Cache directory."""
-        return self._directory
+        """Cache directory final path."""
+        return os.path.join(self._directory, self._directory_folder)
 
     def _get_keyfile(self, archive_key) -> tuple[str, str]:
         key_file = f'{archive_key}.{self.key_suffix}'
         return key_file, os.path.join(self.directory, key_file)
 
     def _get_writer(self, path, mode):
         for count in range(1, 11):
             try:
                 # Another cache may have deleted the directory before
                 # the file could be opened.
                 return self.fs.open(path, mode)
             except OSError:
                 if count == 10:
                     # Give up after 10 tries to open the file.
                     raise
                 continue
 
     def _write_file(self, full_path, iterator, mode):
         # ensure dir exists
         destination, _ = os.path.split(full_path)
         if not self.fs.exists(destination):
             self.fs.makedirs(destination)
 
         writer = self._get_writer(full_path, mode)
 
         digest = hashlib.sha256()
         with writer:
             size = 0
             for chunk in iterator:
                 size += len(chunk)
                 digest.update(chunk)
                 writer.write(chunk)
             writer.flush()
             # Get the file descriptor
             fd = writer.fileno()
 
             # Sync the file descriptor to disk, helps with NFS cases...
             os.fsync(fd)
             sha256 = digest.hexdigest()
             log.debug('written new archive cache under %s, sha256: %s', full_path, sha256)
             return size, sha256
 
     def store(self, key, value_reader, metadata: dict | None = None):
         return self._store(key, value_reader, metadata, mode='xb')
 
     def fetch(self, key, retry=NOT_GIVEN, retry_attempts=NOT_GIVEN, retry_backoff=1) -> tuple[ShardFileReader, dict]:
         return self._fetch(key, retry, retry_attempts, retry_backoff)
 
     def remove(self, key):
         return self._remove(key)
 
     def random_filename(self):
         """Return filename and full-path tuple for file storage.
 
         Filename will be a randomly generated 28 character hexadecimal string
         with ".archive_cache" suffixed. Two levels of sub-directories will be used to
         reduce the size of directories. On older filesystems, lookups in
         directories with many files may be slow.
         """
 
         hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8')
 
         archive_name = hex_name[4:] + '.archive_cache'
         filename = f"{hex_name[:2]}/{hex_name[2:4]}/{archive_name}"
 
         full_path = os.path.join(self.directory, filename)
         return archive_name, full_path
 
     def __repr__(self):
         return f'{self.__class__.__name__}(index={self._index}, dir={self.directory})'
 
 
 class FileSystemFanoutCache(BaseCache):
     shard_name = 'shard_%03d'
+    shard_cls = FileSystemShard
 
     def __init__(self, locking_url, **settings):
         """
         Initialize file system cache instance.
 
         :param str locking_url: redis url for a lock
         :param settings: settings dict
 
         """
         self._locking_url = locking_url
         self._config = settings
         cache_dir = self.get_conf('archive_cache.filesystem.store_dir')
         directory = str(cache_dir)
         directory = os.path.expanduser(directory)
         directory = os.path.expandvars(directory)
         self._directory = directory
-        self._storage_path = directory
+        self._storage_path = directory  # common path for all from BaseCache
 
-        # check if it's ok to write, and re-create the archive cache
-        if not os.path.isdir(self._directory):
-            os.makedirs(self._directory, exist_ok=True)
-
-        self._count = int(self.get_conf('archive_cache.filesystem.cache_shards', pop=True))
+        self._shard_count = int(self.get_conf('archive_cache.filesystem.cache_shards', pop=True))
+        if self._shard_count < 1:
+            raise ValueError('cache_shards must be 1 or more')
 
         self._eviction_policy = self.get_conf('archive_cache.filesystem.eviction_policy', pop=True)
         self._cache_size_limit = self.gb_to_bytes(int(self.get_conf('archive_cache.filesystem.cache_size_gb')))
 
         self.retry = str2bool(self.get_conf('archive_cache.filesystem.retry', pop=True))
         self.retry_attempts = int(self.get_conf('archive_cache.filesystem.retry_attempts', pop=True))
         self.retry_backoff = int(self.get_conf('archive_cache.filesystem.retry_backoff', pop=True))
 
-        log.debug('Initializing archival cache instance under %s', self._directory)
+        log.debug('Initializing %s archival cache instance under %s', self)
+        fs = fsspec.filesystem('file')
+        # check if it's ok to write, and re-create the archive cache main dir
+        # A directory is the virtual equivalent of a physical file cabinet.
+        # In other words, it's a container for organizing digital data.
+        # Unlike a folder, which can only store files, a directory can store files,
+        # subdirectories, and other directories.
+        if not fs.exists(self._directory):
+            fs.makedirs(self._directory, exist_ok=True)
+
         self._shards = tuple(
-            FileSystemShard(
+            self.shard_cls(
                 index=num,
-                directory=os.path.join(directory, self.shard_name % num),
+                directory=directory,
+                directory_folder=self.shard_name % num,
+                fs=fs,
                 **settings,
             )
-            for num in range(self._count)
+            for num in range(self._shard_count)
         )
         self._hash = self._shards[0].hash
 
-    def _get_shard(self, key) -> FileSystemShard:
-        index = self._hash(key) % self._count
-        shard = self._shards[index]
-        return shard
-
     def _get_size(self, shard, archive_path):
         return os.stat(archive_path).st_size

rhodecode/lib/archive_cache/backends/objectstore_cache.py

@@ -1,158 +1,164 b''
 # Copyright (C) 2015-2024 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import codecs
 import hashlib
 import logging
 import os
 
 import fsspec
 
 from .base import BaseCache, BaseShard
 from ..utils import ShardFileReader, NOT_GIVEN
 from ...type_utils import str2bool
 
 log = logging.getLogger(__name__)
 
 
 class S3Shard(BaseShard):
 
-    def __init__(self, index, bucket, **settings):
+    def __init__(self, index, bucket, bucket_folder, fs, **settings):
         self._index = index
-        self._bucket = bucket
+        self._bucket_folder = bucket_folder
         self.storage_type = 'bucket'
+        self._bucket_main = bucket
 
-        endpoint_url = settings.pop('archive_cache.objectstore.url')
-        key = settings.pop('archive_cache.objectstore.key')
-        secret = settings.pop('archive_cache.objectstore.secret')
-
-        # TODO: Add it all over the place...
-        self._bucket_root = settings.pop('archive_cache.objectstore.bucket_root')
-
-        self.fs = fsspec.filesystem('s3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret)
+        self.fs = fs
 
     @property
     def bucket(self):
-        """Cache bucket."""
-        return os.path.join(self._bucket_root, self._bucket)
+        """Cache bucket final path."""
+        return os.path.join(self._bucket_main, self._bucket_folder)
 
     def _get_keyfile(self, archive_key) -> tuple[str, str]:
         key_file = f'{archive_key}-{self.key_suffix}'
         return key_file, os.path.join(self.bucket, key_file)
 
     def _get_writer(self, path, mode):
         return self.fs.open(path, 'wb')
 
     def _write_file(self, full_path, iterator, mode):
-        if self._bucket_root:
-            if not self.fs.exists(self._bucket_root):
-                self.fs.mkdir(self._bucket_root)
 
-        # ensure bucket exists
+        # ensure folder in bucket exists
         destination = self.bucket
         if not self.fs.exists(destination):
             self.fs.mkdir(destination, s3_additional_kwargs={})
 
         writer = self._get_writer(full_path, mode)
 
         digest = hashlib.sha256()
         with writer:
             size = 0
             for chunk in iterator:
                 size += len(chunk)
                 digest.update(chunk)
                 writer.write(chunk)
 
             sha256 = digest.hexdigest()
             log.debug('written new archive cache under %s, sha256: %s', full_path, sha256)
             return size, sha256
 
     def store(self, key, value_reader, metadata: dict | None = None):
         return self._store(key, value_reader, metadata, mode='wb')
 
     def fetch(self, key, retry=NOT_GIVEN, retry_attempts=NOT_GIVEN, retry_backoff=1) -> tuple[ShardFileReader, dict]:
         return self._fetch(key, retry, retry_attempts, retry_backoff)
 
     def remove(self, key):
         return self._remove(key)
 
     def random_filename(self):
         """Return filename and full-path tuple for file storage.
 
         Filename will be a randomly generated 28 character hexadecimal string
         with ".archive_cache" suffixed. Two levels of sub-directories will be used to
         reduce the size of directories. On older filesystems, lookups in
         directories with many files may be slow.
         """
 
         hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8')
 
         archive_name = hex_name[4:] + '.archive_cache'
         filename = f"{hex_name[:2]}-{hex_name[2:4]}-{archive_name}"
 
         full_path = os.path.join(self.bucket, filename)
         return archive_name, full_path
 
     def __repr__(self):
         return f'{self.__class__.__name__}(index={self._index}, bucket={self.bucket})'
 
 
 class ObjectStoreCache(BaseCache):
-    shard_name = 'shard-bucket-%03d'
+    shard_name = 'shard-%03d'
+    shard_cls = S3Shard
 
     def __init__(self, locking_url, **settings):
         """
         Initialize objectstore cache instance.
 
         :param str locking_url: redis url for a lock
         :param settings: settings dict
 
         """
         self._locking_url = locking_url
         self._config = settings
 
         objectstore_url = self.get_conf('archive_cache.objectstore.url')
-        self._storage_path = objectstore_url
+        self._storage_path = objectstore_url  # common path for all from BaseCache
 
-        self._count = int(self.get_conf('archive_cache.objectstore.bucket_shards', pop=True))
+        self._shard_count = int(self.get_conf('archive_cache.objectstore.bucket_shards', pop=True))
+        if self._shard_count < 1:
+            raise ValueError('cache_shards must be 1 or more')
+
+        self._bucket = settings.pop('archive_cache.objectstore.bucket')
+        if not self._bucket:
+            raise ValueError('archive_cache.objectstore.bucket needs to have a value')
 
         self._eviction_policy = self.get_conf('archive_cache.objectstore.eviction_policy', pop=True)
         self._cache_size_limit = self.gb_to_bytes(int(self.get_conf('archive_cache.objectstore.cache_size_gb')))
 
         self.retry = str2bool(self.get_conf('archive_cache.objectstore.retry', pop=True))
         self.retry_attempts = int(self.get_conf('archive_cache.objectstore.retry_attempts', pop=True))
         self.retry_backoff = int(self.get_conf('archive_cache.objectstore.retry_backoff', pop=True))
 
-        log.debug('Initializing archival cache instance under %s', objectstore_url)
+        endpoint_url = settings.pop('archive_cache.objectstore.url')
+        key = settings.pop('archive_cache.objectstore.key')
+        secret = settings.pop('archive_cache.objectstore.secret')
+
+        log.debug('Initializing %s archival cache instance under %s', self)
+
+        fs = fsspec.filesystem('s3', anon=False, endpoint_url=endpoint_url, key=key, secret=secret)
+
+        # init main bucket
+        if not fs.exists(self._bucket):
+            fs.mkdir(self._bucket)
+
         self._shards = tuple(
-            S3Shard(
+            self.shard_cls(
                 index=num,
-                bucket=self.shard_name % num,
+                bucket=self._bucket,
+                bucket_folder=self.shard_name % num,
+                fs=fs,
                 **settings,
             )
-            for num in range(self._count)
+            for num in range(self._shard_count)
         )
         self._hash = self._shards[0].hash
 
-    def _get_shard(self, key) -> S3Shard:
-        index = self._hash(key) % self._count
-        shard = self._shards[index]
-        return shard
-
     def _get_size(self, shard, archive_path):
         return shard.fs.info(archive_path)['size']

rhodecode/tests/rhodecode.ini

@@ -1,832 +1,831 b''
 
 ; #########################################
 ; RHODECODE COMMUNITY EDITION CONFIGURATION
 ; #########################################
 
 [DEFAULT]
 ; Debug flag sets all loggers to debug, and enables request tracking
 debug = true
 
 ; ########################################################################
 ; EMAIL CONFIGURATION
 ; These settings will be used by the RhodeCode mailing system
 ; ########################################################################
 
 ; prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
 ; email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 
 [server:main]
 ; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
 ; Host port for gunicorn are controlled by gunicorn_conf.py
 host = 127.0.0.1
 port = 10020
 
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
 ; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
 
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.
 ; allows to set RhodeCode under a prefix in server.
 ; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ; And set your prefix like: `prefix = /custom_prefix`
 ; be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ; to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
 ; Each option in the app:main can be override by an environmental variable
 ;
 ;To override an option:
 ;
 ;RC_<KeyName>
 ;Everything should be uppercase, . and - should be replaced by _.
 ;For example, if you have these configuration settings:
 ;rc_cache.repo_object.backend = foo
 ;can be overridden by
 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
 
 use = egg:rhodecode-enterprise-ce
 
 ; enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 ; encryption key used to encrypt social plugin tokens,
 ; remote_urls with credentials etc, if not set it defaults to
 ; `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
 ; decryption strict mode (enabled by default). It controls if decryption raises
 ; `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
 ; Pick algorithm for encryption. Either fernet (more secure) or aes (default)
 ; fernet is safer, and we strongly recommend switching to it.
 ; Due to backward compatibility aes is used as default.
 #rhodecode.encrypted_values.algorithm = fernet
 
 ; Return gzipped responses from RhodeCode (static files/application)
 gzip_responses = false
 
 ; Auto-generate javascript routes file on startup
 generate_js_files = false
 
 ; System global default language.
 ; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ; Perform a full repository scan and import on each server start.
 ; Settings this to true could lead to very long startup time.
 startup.import_repos = true
 
 ; URL at which the application is running. This is used for Bootstrapping
 ; requests in context when no web request is available. Used in ishell, or
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
 ; Host at which the Service API is running.
 app.service_api.host = http://rhodecode.local:10020
 
 ; Secret for Service API authentication.
 app.service_api.token =
 
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
 ; Cut off limit for large diffs (size in bytes). If overall diff size on
 ; commit, or pull request exceeds this limit this diff will be displayed
 ; partially. E.g 512000 == 512Kb
 cut_off_limit_diff = 1024000
 
 ; Cut off limit for large files inside diffs (size in bytes). Each individual
 ; file inside diff which exceeds this limit will be displayed partially.
 ;  E.g 128000 == 128Kb
 cut_off_limit_file = 256000
 
 ; Use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = false
 
 ; Force https in RhodeCode, fixes https redirects, assumes it's always https.
 ; Normally this is controlled by proper flags sent from http server such as Nginx or Apache
 force_https = false
 
 ; use Strict-Transport-Security headers
 use_htsts = false
 
 ; Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ; RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ; gist URL alias, used to create nicer urls for gist. This should be an
 ; url that does rewrites to _admin/gists/{gistid}.
 ; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ; List of views (using glob pattern syntax) that AUTH TOKENS could be
 ; used for access.
 ; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ; came from the the logged in user who own this authentication token.
 ; Additionally @TOKEN syntax can be used to bound the view to specific
 ; authentication token. Such view would be only accessible when used together
 ; with this authentication token
 ; list of all views can be found under `/_admin/permissions/auth_token_access`
 ; The list should be "," separated and on a single line.
 ; Most common views to enable:
 
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
 ; Default encoding used to convert from and to unicode
 ; can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ; instance-id prefix
 ; a prefix key for this instance used for cache invalidation when running
 ; multiple instances of RhodeCode, make sure it's globally unique for
 ; all running RhodeCode instances. Leave empty if you don't use it
 instance_id =
 
 ; Fallback authentication plugin. Set this to a plugin ID to force the usage
 ; of an authentication plugin also if it is disabled by it's settings.
 ; This could be useful if you are unable to log in to the system due to broken
 ; authentication settings. Then you can enable e.g. the internal RhodeCode auth
 ; module to log in again and fix the settings.
 ; Available builtin plugin IDs (hash is part of the ID):
 ; egg:rhodecode-enterprise-ce#rhodecode
 ; egg:rhodecode-enterprise-ce#pam
 ; egg:rhodecode-enterprise-ce#ldap
 ; egg:rhodecode-enterprise-ce#jasig_cas
 ; egg:rhodecode-enterprise-ce#headers
 ; egg:rhodecode-enterprise-ce#crowd
 
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
 ; Flag to control loading of legacy plugins in py:/path format
 auth_plugin.import_legacy_plugins = true
 
 ; alternative return HTTP header for failed authentication. Default HTTP
 ; response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ; handling that causing a series of failed authentication calls.
 ; Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ; This will be served instead of default 401 on bad authentication
 auth_ret_code =
 
 ; use special detection method when serving auth_ret_code, instead of serving
 ; ret_code directly, use 401 initially (Which triggers credentials prompt)
 ; and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ; locking return code. When repository is locked return this HTTP code. 2XX
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ; Filesystem location were repositories should be stored
 repo_store.path = /var/opt/rhodecode_repo_store
 
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ; Generated license token required for EE edition license.
 ; New generated token value can be found in Admin > settings > license page.
 license_token = abra-cada-bra1-rce3
 
 ; This flag hides sensitive information on the license page such as token, and license data
 license.hide_license_info = false
 
 ; supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 
 ; supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = dev
 
 ; Display extended labs settings
 labs_settings_active = true
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ; Send email with exception details when it happens
 #exception_tracker.send_email = false
 
 ; Comma separated list of recipients for exception emails,
 ; e.g [e-mail],[e-mail]
 ; Can be left empty, then emails will be sent to ALL super-admins
 #exception_tracker.send_email_recipients =
 
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
 ; File store configuration. This is used to store and serve uploaded files
 file_store.enabled = true
 
 ; Storage backend, available options are: local
 file_store.backend = local
 
 ; path to store the uploaded binaries and artifacts
 file_store.storage_path = /var/opt/rhodecode_data/file_store
 
 
 ; Redis url to acquire/check generation of archives locks
 archive_cache.locking.url = redis://redis:6379/1
 
 ; Storage backend, only 'filesystem' and 'objectstore' are available now
 archive_cache.backend.type = filesystem
 
 ; url for s3 compatible storage that allows to upload artifacts
 ; e.g http://minio:9000
 archive_cache.objectstore.url = http://s3-minio:9000
 
 ; key for s3 auth
 archive_cache.objectstore.key = key
 
 ; secret for s3 auth
 archive_cache.objectstore.secret = secret
 
 ; number of sharded buckets to create to distribute archives across
 ; default is 8 shards
 archive_cache.objectstore.bucket_shards = 8
 
-; a top-level bucket to put all other sharded buckets in
-; in case it's empty all buckets will be created in top-level (not recommended)
-; objects will be stored in rhodecode-archive-cache/shard-bucket-N based on the bucket_shards number
-archive_cache.objectstore.bucket_root = rhodecode-archive-cache
+; a top-level bucket to put all other shards in
+; objects will be stored in rhodecode-archive-cache/shard-N based on the bucket_shards number
+archive_cache.objectstore.bucket = rhodecode-archive-cache
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.objectstore.retry = false
 
 ; number of seconds to wait for next try using retry
 archive_cache.objectstore.retry_backoff = 1
 
 ; how many tries do do a retry fetch from this backend
 archive_cache.objectstore.retry_attempts = 10
 
 ; Default is $cache_dir/archive_cache if not set
 ; Generated repo archives will be cached at this location
 ; and served from the cache during subsequent requests for the same archive of
 ; the repository. This path is important to be shared across filesystems and with
 ; RhodeCode and vcsserver
 archive_cache.filesystem.store_dir = %(here)s/rc-tests/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
 archive_cache.filesystem.cache_size_gb = 2
 
 ; Eviction policy used to clear out after cache_size_gb limit is reached
 archive_cache.filesystem.eviction_policy = least-recently-stored
 
 ; By default cache uses sharding technique, this specifies how many shards are there
 ; default is 8 shards
 archive_cache.filesystem.cache_shards = 8
 
 ; if true, this cache will try to retry with retry_attempts=N times waiting retry_backoff time
 archive_cache.filesystem.retry = false
 
 ; number of seconds to wait for next try using retry
 archive_cache.filesystem.retry_backoff = 1
 
 ; how many tries do do a retry fetch from this backend
 archive_cache.filesystem.retry_attempts = 10
 
 
 ; #############
 ; CELERY CONFIG
 ; #############
 
 ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
 
 use_celery = false
 
 ; path to store schedule database
 #celerybeat-schedule.path =
 
 ; connection url to the message broker (default redis)
 celery.broker_url = redis://redis:6379/8
 
 ; results backend to get results for (default redis)
 celery.result_backend = redis://redis:6379/8
 
 ; rabbitmq example
 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
 ; maximum tasks to execute before worker restart
 celery.max_tasks_per_child = 20
 
 ; tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = true
 celery.task_store_eager_result = true
 
 ; #############
 ; DOGPILE CACHE
 ; #############
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
 cache_dir = %(here)s/rc-test-data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
 ; Only supported backend is `memory_lru`
 ; *********************************************
 rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
 rc_cache.sql_cache_short.expiration_time = 0
 
 
 ; *****************************************************
 ; `cache_repo_longterm` cache for repo object instances
 ; Only supported backend is `memory_lru`
 ; *****************************************************
 rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
 ; by default we use 30 Days, cache is still invalidated on push
 rc_cache.cache_repo_longterm.expiration_time = 2592000
 ; max items in LRU cache, set to smaller number to save memory, and expire last used caches
 rc_cache.cache_repo_longterm.max_size = 10000
 
 
 ; *********************************************
 ; `cache_general` cache for general purpose use
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *********************************************
 rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_general.expiration_time = 43200
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 rc_cache.cache_general.arguments.filename = %(here)s/rc-tests/cache-backend/cache_general_db
 
 ; alternative `cache_general` redis backend with distributed lock
 #rc_cache.cache_general.backend = dogpile.cache.rc.redis
 #rc_cache.cache_general.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_general.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_general.arguments.host = localhost
 #rc_cache.cache_general.arguments.port = 6379
 #rc_cache.cache_general.arguments.db = 0
 #rc_cache.cache_general.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_general.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_general.arguments.lock_auto_renewal = true
 
 ; *************************************************
 ; `cache_perms` cache for permission tree, auth TTL
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *************************************************
 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_perms.expiration_time = 0
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 rc_cache.cache_perms.arguments.filename = %(here)s/rc-tests/cache-backend/cache_perms_db
 
 ; alternative `cache_perms` redis backend with distributed lock
 #rc_cache.cache_perms.backend = dogpile.cache.rc.redis
 #rc_cache.cache_perms.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_perms.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_perms.arguments.host = localhost
 #rc_cache.cache_perms.arguments.port = 6379
 #rc_cache.cache_perms.arguments.db = 0
 #rc_cache.cache_perms.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_perms.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_perms.arguments.lock_auto_renewal = true
 
 ; ***************************************************
 ; `cache_repo` cache for file tree, Readme, RSS FEEDS
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; ***************************************************
 rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_repo.expiration_time = 2592000
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 rc_cache.cache_repo.arguments.filename = %(here)s/rc-tests/cache-backend/cache_repo_db
 
 ; alternative `cache_repo` redis backend with distributed lock
 #rc_cache.cache_repo.backend = dogpile.cache.rc.redis
 #rc_cache.cache_repo.expiration_time = 2592000
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
 
 #rc_cache.cache_repo.arguments.host = localhost
 #rc_cache.cache_repo.arguments.port = 6379
 #rc_cache.cache_repo.arguments.db = 1
 #rc_cache.cache_repo.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_repo.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_repo.arguments.lock_auto_renewal = true
 
 ; ##############
 ; BEAKER SESSION
 ; ##############
 
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
 ; types are file, ext:redis, ext:database, ext:memcached
 ; Fastest ones are ext:redis and ext:database, DO NOT use memory type for session
 beaker.session.type = file
 beaker.session.data_dir = %(here)s/rc-tests/data/sessions
 
 ; Redis based sessions
 #beaker.session.type = ext:redis
 #beaker.session.url = redis://redis:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = test-rc-uytcxaz
 beaker.session.lock_dir = %(here)s/rc-tests/data/sessions/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
 ; Sets session as invalid (also logging out user) if it haven not been
 ; accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 
 ; Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
 ; Set https secure cookie
 beaker.session.secure = false
 
 ; default cookie expiration time in seconds, set to `true` to set expire
 ; at browser close
 #beaker.session.cookie_expires = 3600
 
 ; #############################
 ; SEARCH INDEXING CONFIGURATION
 ; #############################
 
 ; Full text search indexer is available in rhodecode-tools under
 ; `rhodecode-tools index` command
 
 ; WHOOSH Backend, doesn't require additional services to run
 ; it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/rc-tests/data/index
 
 ; ####################
 ; CHANNELSTREAM CONFIG
 ; ####################
 
 ; channelstream enables persistent connections and live notification
 ; in the system. It's also used by the chat system
 
 channelstream.enabled = false
 
 ; server address for channelstream server on the backend
 channelstream.server = channelstream:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = ENV_GENERATED
 channelstream.history.location = %(here)s/rc-tests/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
 ; ##############################
 ; MAIN RHODECODE DATABASE CONFIG
 ; ##############################
 
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8
 ; pymysql is an alternative driver for MySQL, use in case of problems with default one
 #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
 sqlalchemy.db1.url = sqlite:///%(here)s/rc-tests/rhodecode_test.db?timeout=30
 
 ; see sqlalchemy docs for other advanced settings
 ; print the sql statements to output
 sqlalchemy.db1.echo = false
 
 ; recycle the connections after this amount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 
 ; the number of connections to keep open inside the connection pool.
 ; 0 indicates no limit
 ; the general calculus with gevent is:
 ; if your system allows 500 concurrent greenlets (max_connections) that all do database access,
 ; then increase pool size + max overflow so that they add up to 500.
 #sqlalchemy.db1.pool_size = 5
 
 ; The number of connections to allow in connection pool "overflow", that is
 ; connections that can be opened above and beyond the pool_size setting,
 ; which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 ; Connection check ping, used to detect broken database connections
 ; could be enabled to better handle cases if MySQL has gone away errors
 #sqlalchemy.db1.ping_connection = true
 
 ; ##########
 ; VCS CONFIG
 ; ##########
 vcs.server.enable = true
 vcs.server = vcsserver:10010
 
 ; Web server connectivity protocol, responsible for web based VCS operations
 ; Available protocols are:
 ; `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
 ; Push/Pull operations protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
 ; `celery` - use celery based hooks
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
 ; accessible via network.
 ; Use vcs.hooks.host = "*" to bind to current hostname (for Docker)
 vcs.hooks.host = *
 
 ; Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
 ; List of enabled VCS backends, available options are:
 ; `hg`  - mercurial
 ; `git` - git
 ; `svn` - subversion
 vcs.backends = hg, git, svn
 
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
 ; Cache flag to cache vcsserver remote calls locally
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = false
 
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
 ; Maps RhodeCode repo groups into SVN paths for Apache
 ; ####################################################
 
 ; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ; Set a numeric version for your current SVN e.g 1.8, or 1.12
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
 ; Enable SVN proxy of requests over HTTP
 vcs.svn.proxy.enabled = true
 
 ; host to connect to running SVN subsystem
 vcs.svn.proxy.host = http://svn:8090
 
 ; Enable or disable the config file generation.
 svn.proxy.generate_config = false
 
 ; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
 svn.proxy.config_file_path = %(here)s/rc-tests/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
 ; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako
 #svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako
 
 ; Used as a prefix to the `Location` block in the generated config file.
 ; In most cases it should be set to `/`.
 svn.proxy.location_root = /
 
 ; Command to reload the mod dav svn configuration on change.
 ; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh
 ; Make sure user who runs RhodeCode process is allowed to reload Apache
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
 
 ; If the timeout expires before the reload command finishes, the command will
 ; be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
 ; ####################
 ; SSH Support Settings
 ; ####################
 
 ; Defines if a custom authorized_keys file should be created and written on
 ; any change user ssh keys. Setting this to false also disables possibility
 ; of adding SSH keys by users from web interface. Super admins can still
 ; manage SSH Keys.
 ssh.generate_authorized_keyfile = true
 
 ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 ssh.authorized_keys_file_path = %(here)s/rc-tests/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
 ; legacy: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 ; new rewrite: /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper-v2
 ssh.wrapper_cmd = /usr/local/bin/rhodecode_bin/bin/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
 ; Enables logging, and detailed output send back to the client during SSH
 ; operations. Useful for debugging, shouldn't be used in production.
 ssh.enable_debug_logging = true
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
 ssh.executable.hg = /usr/local/bin/rhodecode_bin/vcs_bin/hg
 ssh.executable.git = /usr/local/bin/rhodecode_bin/vcs_bin/git
 ssh.executable.svn = /usr/local/bin/rhodecode_bin/vcs_bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.
 ssh.enable_ui_key_generator = true
 
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Prometheus
 #statsd.enabled = false
 #statsd.statsd_host = 0.0.0.0
 #statsd.statsd_port = 8125
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
 ; configure logging automatically at server startup set to false
 ; to use the below custom logging config.
 ; RC_LOGGING_FORMATTER
 ; RC_LOGGING_LEVEL
 ; env variables can control the settings for logging in case of autoconfigure
 
 logging.autoconfigure = false
 
 ; specify your own custom logging config file to configure logging
 #logging.logging_conf_file = /path/to/custom_logging.ini
 
 ; Dummy marker to add new entries after.
 ; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1
 
 
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
 
 [loggers]
 keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper, dogpile
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, json, color_formatter, color_formatter_sql
 
 ; #######
 ; LOGGERS
 ; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_routes]
 level = DEBUG
 handlers =
 qualname = routes.middleware
 ## "level = DEBUG" logs the route matched and routing variables.
 propagate = 1
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_dogpile]
 level = INFO
 handlers = console
 qualname = dogpile
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
 [logger_celery]
 level = DEBUG
 handlers =
 qualname = celery
 
 
 ; ########
 ; HANDLERS
 ; ########
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr, )
 level = DEBUG
 ; To enable JSON formatted logs replace 'generic/color_formatter' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 [handler_console_sql]
 ; "level = DEBUG" logs SQL queries and results.
 ; "level = INFO" logs SQL queries.
 ; "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
 args = (sys.stderr, )
 level = WARN
 ; To enable JSON formatted logs replace 'generic/color_formatter_sql' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_json]
 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
 class = rhodecode.lib._vendor.jsonlogger.JsonFormatter