From ffcea5dc8a5265e65aeb514007ca1735e91eed4c 2022-11-29 21:11:47 From: RhodeCode Admin Date: 2022-11-29 21:11:47 Subject: [PATCH] release of new installer/docker stack updates Tons of changes to move towards new rccontrol docker stack based installer --- diff --git a/.bootstrap/download-artifacts.sh b/.bootstrap/download-artifacts.sh deleted file mode 100755 index ffb9702..0000000 --- a/.bootstrap/download-artifacts.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -IFS=$'\n\t' -# e.g 4.24.1 -source ../.env -VER=$RC_VERSION -AUTH="" - -INSTALLER_URL=https://dls.rhodecode.com/dls/N2E2ZTY1NzA3NjYxNDA2NTc1NjI3MTcyNzA2MjcxNzIyZTcwNjI3YQ==/rhodecode-control/latest-linux-ee - -MANIFEST=https://dls.rhodecode.com/linux/MANIFEST - -CACHE_DIR=../.cache -VER_REGEX="$VER+x86_64" - -echo "Downloading Artifacts for version: $VER" - -echo "Downloading locale-archive" -curl -L https://dls.rhodecode.com/assets/locale-archive -J -O -mv -v locale-archive $CACHE_DIR - -ARTS=$(curl -s $MANIFEST | grep --ignore-case "$VER_REGEX" | cut -d ' ' -f 2) - -# vcsserver/ce/ee -echo "Found following $ARTS" - -for url in $ARTS; do - echo "Downloading $url with $AUTH" - curl $AUTH --fail-early -L ${url} -J -O - -done - -## rhodecode control -#for url in $(curl -s $MANIFEST | grep --ignore-case -E 'control.+\+x86_64' | cut -d ' ' -f 2); do -# echo "Downloading $url" -# curl -L ${url} -J -O -#done - -## installer -echo "Downloading installer from $INSTALLER_URL" -curl -L $INSTALLER_URL -J -O - -INSTALLER=$(ls -Art RhodeCode-installer-* | tail -n 1) -if [[ -n $INSTALLER ]]; then - chmod +x "${INSTALLER}" -fi - -mv -v "${INSTALLER}" $CACHE_DIR -mv -v *.bz2 $CACHE_DIR -ls -lh $CACHE_DIR diff --git a/.bootstrap/download-source.sh b/.bootstrap/download-source.sh deleted file mode 100755 index d05a55a..0000000 --- a/.bootstrap/download-source.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -IFS=$'\n\t' -# e.g :) -source ../.env -SERVER_URL="https://code.rhodecode.com" - -AUTH_TOKEN="" - -SOURCE_DIR=../.source - -RHODECODE_VCS_HASH="default" -RHODECODE_CE_HASH="default" -RHODECODE_EE_HASH="default" - -# download sources -echo "** download rhodecode source for build **" - -curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-vcsserver/archive/$RHODECODE_VCS_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR -curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-ce/archive/$RHODECODE_CE_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR -curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-ee/archive/$RHODECODE_EE_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR - -rm -rf rhodecode-vcsserver && mv $SOURCE_DIR/rhodecode-vcsserver-plain $SOURCE_DIR/rhodecode-vcsserver -rm -rf rhodecode-enterprise-ce && mv $SOURCE_DIR/rhodecode-enterprise-ce-plain $SOURCE_DIR/rhodecode-enterprise-ce -rm -rf rhodecode-enterprise-ee && cp -r $SOURCE_DIR/rhodecode-enterprise-ce $SOURCE_DIR/rhodecode-enterprise-ee diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..44a4c51 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.git/* + +#.source/* +#.cache/* \ No newline at end of file diff --git a/.env b/.env index b9335a9..3decc97 100644 --- a/.env +++ b/.env @@ -1,31 +1,56 @@ -COMPOSE_PROJECT_NAME=rc_cluster +#COMPOSE_PROJECT_NAME=rc_cluster TZ="UTC" + # Version to deploy and run -RC_VERSION="4.28.0" +## VERSION_INFO +RC_VERSION="4.28.0.REL20221120_SOURCE" +#RC_CLI_VERSION_NAME="4.27.0" + +# ce, or ee +RC_EDITION=ee +RC_IMAGE_CE="rhodecode/rhodecode-ee:$RC_VERSION" +RC_IMAGE_EE="rhodecode/rhodecode-ce:$RC_VERSION" -# Database access credentials +# Database bootstrap/access credentials DB_NAME=rhodecode DB_USER=rhodecode -DB_PASSWORD=hUc1adS7oDd6Oj3in3 -# base url for running app -RHODECODE_BASE_URL=http://localhost:8888 +# Bootstrap container with specific password +# This password will be used to generate new account and password for the database +DB_PASSWORD=[GENERATED_AT_BOOTSTRAP] -# HTTP and HTTPS ports for running app -RC_HTTP_PORT=8888 -RC_HTTPS_PORT=8443 +# set limit of redis in-memory cache before keys get evicted using LRU cache +RC_REDIS_MAXMEMORY=1024mb + +# set the size of ramdisk used for locking files +RC_DATA_RAMDISK_SIZE=1G + +# base url for running app, this is used for traefik hasname matching, so accessing this host will redirect +# traffic to the running instance +# replace this with your name of host. e.g http://yourdomain.com or https://rc.company.local +RHODECODE_HOSTNAME=docker-dev -# SSH Port exposed, increased security is to not used default 22 -RC_SSH_PORT=9022 # user/password for first admin user created for access -RHODECODE_USER_EMAIL=admin@rhodecode.com +# this is a bootstrap super-admin account to access the web interface +RHODECODE_USER_EMAIL=admin@server-com RHODECODE_USER_NAME=admin RHODECODE_USER_PASS=secret4 + + + +# HTTP and HTTPS ports for running app +RC_HTTP_PORT=8888 + +# SSH Port exposed, increased security is to not used default 22 +RC_SSH_PORT=3022 + + +## RHODECODE RUNTIME VARS + ## New since 4.27.0, default GIT branch name can be changed GIT_DEFAULT_BRANCH_NAME=master -## Source build version for image -SOURCE_VER=4.28.0.REL1 + diff --git a/.gitignore b/.gitignore index e4facd7..7f1b3a7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,12 @@ .cache/*.tar.bz2 .cache/locale-archive +# Source code downloaded +.source/* + + # Custom / override wont be tracked +.custom/* + # LOGS logs/*.log logs/nginx/*.log @@ -13,9 +19,5 @@ logs/postgres/*.log .idea config/_shared/.rcmetadata.json -.source/* -scripts/download-artifacts-secret.sh - -scripts/download-source-secret.sh - -.rccontrol-bootstrap \ No newline at end of file +.rccontrol-bootstrap +.rccontrol.ini \ No newline at end of file diff --git a/README.md b/README.md index 15de768..fc1c24b 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,45 @@ # RhodeCode Cluster -RhodeCode Cluster is a multi-node highly-scalable setup to run RhodeCode and +RhodeCode Cluster is a multi-node highly-scalable setup to run +RhodeCode, Edge Router(Traefik + SSL) and Metrics Stack (Prometheus, Loki, Grafana) and all its additional components in single environment using Docker. Using a docker-compose this setup creates following services for RhodeCode: -- Nginx HTTP Server for load balancing and reverse proxy -- RhodeCode HTTP -- VCSServer for GIT/SVN/HG support -- SSH Server for cloning over SSH -- SVN webserver for HTTP support over SVN -- Celery workers for asynchronous tasks -- Celery beat for automation tasks -- Redis Database for caching -- Postgres database for persistent storage -- Channelstream websocket server for live components +Edge-Router: + - Traefik, Edge Router, SSL termination etc + +Core Services: + - Database (defaults to PostgreSQL) + - Redis, acts as cache and queue exchange + - ChannelStream - live websocket communications + - Nginx (static/channelstream) proxy serving RhodeCode APPS and channelstream communication + - Elasticsearch (full text search backend) + +RhodeCode + - RhodeCode CE/EE + - VCSServer for GIT/SVN/HG support + - SSH Server for cloning over SSH + - SVN webserver for HTTP support over SVN + - Celery workers for asynchronous tasks + - Celery beat for automation tasks + +Metrics + - Loki, logs aggregation + - Grafana, Metrics Dashboard + - Promethues, Metrics time-series + - statsd-exporter - statsd to promethues bridge + - node-exporter - machine stats + - promtail - log scraping ## Pre requisites +To Run this stack Docker engine and Docker Compose needs to be installed on the host machine. Visit docker site and install docker (min version 20.10) and docker compose: -- https://docs.docker.com/engine/install/ -- https://docs.docker.com/compose/install/ +- https://docs.docker.com/engine/install/ +- https://docs.docker.com/compose/install/ The instructions below outline installation of the current stable release (v1.29.2) of Compose. @@ -33,19 +50,19 @@ There are 4 volumes defined: - `/etc/rhodecode/conf` - Used for configuration files for rhodecode, vcsserver and supervisord, and some cache data + Shared volume used for configuration files for rhodecode, vcsserver and supervisord, and some cache data - `/var/opt/rhodecode_repo_store` - Used for main repository storage where repositories would be stored + Used for main repository storage where repositories would be stored - `/var/opt/rhodecode_data` - data dir for rhodecode cache/lock files, or user sessions (for file backend) + Data dir for rhodecode cache/lock files, or user sessions (for file backend) - `/var/log/rhodecode` - Used to store all logs from RhodeCode + Used to store all logs from RhodeCode ## Setup/Build options @@ -54,7 +71,8 @@ There are 3 options to run the docker stack. - [Standalone cluster build from installer](#standalone-cluster-build-from-installer) - [Standalone cluster build from source](#standalone-cluster-build-from-source) -- [dev enviroment setup](#like-this-one) +- [dev environment setup](#dev-env-build) + # Standalone cluster build from installer @@ -91,10 +109,12 @@ drwxr-xr-x 14 rcdev rcdev 448B Feb 8 10:40 .. ## Set License for EE version -In order to install EE edition a valid license is required. +In order to install EE edition a license file is required to be present. +It can contain your current license, or when empty license can be applied via Web interface. + To apply it during build phase save your raw license data into a file -`config/compose/rhodecode_enterprise.license` +`config/rhodecode_enterprise.license` If this file is present build phase will read it and license will be applied at creation. This file can also be empty and license can be applied via a WEB interface after first login. @@ -111,7 +131,10 @@ _Disk space problems?_ ```shell docker system df docker builder prune +# optionally +docker image prune -a ``` +### Creating & building images 1) start with running the required database for the build stage in the background. @@ -120,15 +143,18 @@ docker builder prune ``` This will start our postgres database, and expose it to the network. + Postgres DB is configured to output logs into a file `/var/log/rhodecode/postgres` 2) We can now run the full installation. Database needs to be running for the next build command. - + This will build the rhodecode base image used for rhodecode, vcsserver, celery, ssh, svn + Then it will build all other components required. + ```shell docker-compose build rhodecode docker-compose build ``` -3) Once we build the rhodecode app, we can run the whole stack using `docker-compose up` +4) Once we build all required containers, we can run the whole stack using `docker-compose up` ```shell docker-compose up @@ -146,21 +172,31 @@ In case for bigger setups docker-compose can scale more rhodecode/vcsserver work docker-compose up --scale vcsserver=3 rhodecode=3 ``` +Logging is pushed to stdout from all services. + -Upgrade: +### Upgrade procedure: - pull the latest rhodecode-docker repo - check .env file for correct update version - re-build rhodecode - docker-compose build rhodecode +- docker-compose build - docker-compose stop - docker-compose up # Standalone cluster build from source -There's an option to build the latest branches as a docker installation. +There's an option to build the latest release from the source code as a docker installation. + +If you have the 3 required projects source code already, the next step can be omitted, and the +sources can be copied to the `.source/` directory. note: symlinks don't work. -Download the source: + - https://code.rhodecode.com/rhodecode-vcsserver + - https://code.rhodecode.com/rhodecode-enterprise-ce + - https://code.rhodecode.com/rhodecode-enterprise-ee (assuming access is granted to this) + +Download the source using bootstrap script: ```shell cd .boostrap/ && ./download-source.sh && cd ../ @@ -174,20 +210,16 @@ drwxr-xr-x 1 docker docker 1120 Nov 25 12:27 rhodecode-enterprise-ee drwxr-xr-x 1 docker docker 800 Nov 25 12:27 rhodecode-vcsserver ``` -If you have the source already, this step can be omitted, and the -sources can be linked or copied to this directory - +With this done, you can now proceed with every step of normal source installation (Creating & building images), but instead of using +just `docker-compose` command it needs to be replaced with `docker-compose -f docker-compose.yaml -f docker-compose.source.yaml` -Build the source based image +For example to override the installer build with the source `rhodecode` based image, and also setting proper version, run: ```shell -docker-compose -f docker-compose.yaml -f docker-compose.source.yaml build --no-cache --progress=plain rhodecode +RC_VERSION="4.27.0.REL202221020" docker-compose -f docker-compose-apps.yaml -f docker-compose.source.yaml build --no-cache --progress=plain rhodecode ``` -to create a source install override and build based on the downloaded sources. - - -Logging is pushed to stdout from all services. +NOTE THAT it's recommended to keep rc_version and source_Ver the same ## Simple build @@ -196,7 +228,7 @@ simple sqlite database and file based caches. This is a fully running instance good for small use with 3-5 users. ```shell -docker build -t rhodecode:4.23.2 -f rhodecode.dockerfile \ +docker build -t rhodecode/rhodecode-ce:4.23.2 -f rhodecode.dockerfile \ -e RHODECODE_TYPE=Community \ -e RHODECODE_VERSION=4.23.2 \ -e RHODECODE_DB=sqlite \ @@ -229,6 +261,7 @@ password: secret4 ``` We've not built our image using specific version. It's time to run it: +We specify the run.ini by selecting config option we have locally ```shell docker run \ @@ -236,8 +269,9 @@ docker run \ --publish 10020:10020 \ --restart unless-stopped \ --volume $PWD/config:/etc/rhodecode/conf \ + --volume $PWD/config/rhodecode.optimized.ini:/etc/rhodecode/conf_build/run.ini \ --volume $PWD/logs:/var/log/rhodecode \ - 'rhodecode:4.23.2' + 'rhodecode/rhodecode-ee:4.23.2' ``` Enter container @@ -249,10 +283,16 @@ docker exec -it rhodecode-container /bin/bash Enter interactive shell ```shell -docker exec -it rhodecode-container /var/opt/rhodecode_bin/bin/rc-ishell /etc/rhodecode/conf/rhodecode.ini +docker exec -it rhodecode-container /usr/local/bin/rhodecode_bin/bin/rc-ishell /etc/rhodecode/conf/rhodecode.ini ``` Run Database migrations ```shell -docker exec -it rhodecode-container /var/opt/rhodecode_bin/bin/rc-upgrade-db /etc/rhodecode/conf/rhodecode.ini --force-yes -``` \ No newline at end of file +docker exec -it rhodecode-container /usr/local/bin/rhodecode_bin/bin/rc-upgrade-db /etc/rhodecode/conf/rhodecode.ini --force-yes +``` + + + +### Registry for docker swarm + +docker run -d -p 5000:5000 --restart always --name registry registry:2 \ No newline at end of file diff --git a/config/compose/rhodecode_enterprise.license b/config/_shared/rhodecode_enterprise.license similarity index 100% rename from config/compose/rhodecode_enterprise.license rename to config/_shared/rhodecode_enterprise.license diff --git a/config/compose/channelstream.ini b/config/compose/channelstream.ini deleted file mode 100644 index d07436d..0000000 --- a/config/compose/channelstream.ini +++ /dev/null @@ -1,22 +0,0 @@ -[channelstream] -host = 0.0.0.0 -port = 9800 - -admin_user = admin -# admin panel password -admin_secret = b39acb28b2304a27a6a0e911500bf7d1 -# auth cookie secret, leave empty for random string generated at server start -# fill in if you need to have multiple servers and want to keep admin session between them -cookie_secret = - -# API password -secret = b39acb28b2304a27a6a0e911500bf7d1 - -demo = false -allow_posting_from = 127.0.0.1, - 0.0.0.0 -log_level = INFO -# should require SSL connections? -enforce_https = -# enforce https:// in links -http_scheme = diff --git a/config/compose/rhodecode.optimized.ini b/config/compose/rhodecode.optimized.ini deleted file mode 100644 index a048200..0000000 --- a/config/compose/rhodecode.optimized.ini +++ /dev/null @@ -1,746 +0,0 @@ - -; ########################################## -; RHODECODE ENTERPRISE EDITION CONFIGURATION -; ########################################## - -[DEFAULT] -; Debug flag sets all loggers to debug, and enables request tracking -debug = false - -; ######################################################################## -; EMAIL CONFIGURATION -; These settings will be used by the RhodeCode mailing system -; ######################################################################## - -; prefix all emails subjects with given prefix, helps filtering out emails -#email_prefix = [RhodeCode] - -; email FROM address all mails will be sent -#app_email_from = rhodecode-noreply@localhost - -#smtp_server = mail.server.com -#smtp_username = -#smtp_password = -#smtp_port = -#smtp_use_tls = false -#smtp_use_ssl = true - -[server:main] -; COMMON HOST/IP CONFIG -host = 0.0.0.0 -port = 10020 - - -; ########################### -; GUNICORN APPLICATION SERVER -; ########################### - -; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini - -; Module to use, this setting shouldn't be changed -use = egg:gunicorn#main - -; Sets the number of process workers. More workers means more concurrent connections -; RhodeCode can handle at the same time. Each additional worker also it increases -; memory usage as each has it's own set of caches. -; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more -; than 8-10 unless for really big deployments .e.g 700-1000 users. -; `instance_id = *` must be set in the [app:main] section below (which is the default) -; when using more than 1 worker. -workers = 2 - -; Gunicorn access log level -loglevel = info - -; Process name visible in process list -proc_name = gunicorn-web-1 - -; Type of worker class, one of `sync`, `gevent` -; Recommended type is `gevent` -worker_class = gevent - -; The maximum number of simultaneous clients per worker. Valid only for gevent -worker_connections = 10 - -; Max number of requests that worker will handle before being gracefully restarted. -; Prevents memory leaks, jitter adds variability so not all workers are restarted at once. -max_requests = 2000 -max_requests_jitter = 100 - -; Amount of time a worker can spend with handling a request before it -; gets killed and restarted. By default set to 21600 (6hrs) -; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h) -timeout = 21600 - -; The maximum size of HTTP request line in bytes. -; 0 for unlimited -limit_request_line = 0 - -; Limit the number of HTTP headers fields in a request. -; By default this value is 100 and can't be larger than 32768. -limit_request_fields = 32768 - -; Limit the allowed size of an HTTP request header field. -; Value is a positive number or 0. -; Setting it to 0 will allow unlimited header field sizes. -limit_request_field_size = 0 - -; Timeout for graceful workers restart. -; After receiving a restart signal, workers have this much time to finish -; serving requests. Workers still alive after the timeout (starting from the -; receipt of the restart signal) are force killed. -; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h) -graceful_timeout = 3600 - -# The number of seconds to wait for requests on a Keep-Alive connection. -# Generally set in the 1-5 seconds range. -keepalive = 2 - -; Maximum memory usage that each worker can use before it will receive a -; graceful restart signal 0 = memory monitoring is disabled -; Examples: 268435456 (256MB), 536870912 (512MB) -; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB) -memory_max_usage = 2147483648 - -; How often in seconds to check for memory usage for each gunicorn worker -memory_usage_check_interval = 60 - -; Threshold value for which we don't recycle worker if GarbageCollection -; frees up enough resources. Before each restart we try to run GC on worker -; in case we get enough free memory after that, restart will not happen. -memory_usage_recovery_threshold = 0.8 - - -; Prefix middleware for RhodeCode. -; recommended when using proxy setup. -; allows to set RhodeCode under a prefix in server. -; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well. -; And set your prefix like: `prefix = /custom_prefix` -; be sure to also set beaker.session.cookie_path = /custom_prefix if you need -; to make your cookies only work on prefix url -[filter:proxy-prefix] -use = egg:PasteDeploy#prefix -prefix = / - -[app:main] -; The %(here)s variable will be replaced with the absolute path of parent directory -; of this file -; In addition ENVIRONMENT variables usage is possible, e.g -; sqlalchemy.db1.url = {ENV_RC_DB_URL} - -use = egg:rhodecode-enterprise-ee - -; enable proxy prefix middleware, defined above -#filter-with = proxy-prefix - -; encryption key used to encrypt social plugin tokens, -; remote_urls with credentials etc, if not set it defaults to -; `beaker.session.secret` -#rhodecode.encrypted_values.secret = - -; decryption strict mode (enabled by default). It controls if decryption raises -; `SignatureVerificationError` in case of wrong key, or damaged encryption data. -#rhodecode.encrypted_values.strict = false - -; Pick algorithm for encryption. Either fernet (more secure) or aes (default) -; fernet is safer, and we strongly recommend switching to it. -; Due to backward compatibility aes is used as default. -#rhodecode.encrypted_values.algorithm = fernet - -; Return gzipped responses from RhodeCode (static files/application) -gzip_responses = false - -; Auto-generate javascript routes file on startup -generate_js_files = false - -; System global default language. -; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh -lang = en - -; Perform a full repository scan and import on each server start. -; Settings this to true could lead to very long startup time. -startup.import_repos = false - -; Uncomment and set this path to use archive download cache. -; Once enabled, generated archives will be cached at this location -; and served from the cache during subsequent requests for the same archive of -; the repository. -archive_cache_dir = /etc/rhodecode/conf/data/tarballcache - -; URL at which the application is running. This is used for Bootstrapping -; requests in context when no web request is available. Used in ishell, or -; SSH calls. Set this for events to receive proper url for SSH calls. -app.base_url = {ENV_RC_BASE_URL} - -; Unique application ID. Should be a random unique string for security. -app_instance_uuid = 4442f2dac4dc4fb982f781546735bb99 - -; Cut off limit for large diffs (size in bytes). If overall diff size on -; commit, or pull request exceeds this limit this diff will be displayed -; partially. E.g 512000 == 512Kb -cut_off_limit_diff = 512000 - -; Cut off limit for large files inside diffs (size in bytes). Each individual -; file inside diff which exceeds this limit will be displayed partially. -; E.g 128000 == 128Kb -cut_off_limit_file = 128000 - -; Use cached version of vcs repositories everywhere. Recommended to be `true` -vcs_full_cache = true - -; Force https in RhodeCode, fixes https redirects, assumes it's always https. -; Normally this is controlled by proper flags sent from http server such as Nginx or Apache -force_https = false - -; use Strict-Transport-Security headers -use_htsts = false - -; Set to true if your repos are exposed using the dumb protocol -git_update_server_info = false - -; RSS/ATOM feed options -rss_cut_off_limit = 256000 -rss_items_per_page = 10 -rss_include_diff = false - -; gist URL alias, used to create nicer urls for gist. This should be an -; url that does rewrites to _admin/gists/{gistid}. -; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal -; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid} -gist_alias_url = - -; List of views (using glob pattern syntax) that AUTH TOKENS could be -; used for access. -; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it -; came from the the logged in user who own this authentication token. -; Additionally @TOKEN syntax can be used to bound the view to specific -; authentication token. Such view would be only accessible when used together -; with this authentication token -; list of all views can be found under `/_admin/permissions/auth_token_access` -; The list should be "," separated and on a single line. -; Most common views to enable: - -# RepoCommitsView:repo_commit_download -# RepoCommitsView:repo_commit_patch -# RepoCommitsView:repo_commit_raw -# RepoCommitsView:repo_commit_raw@TOKEN -# RepoFilesView:repo_files_diff -# RepoFilesView:repo_archivefile -# RepoFilesView:repo_file_raw -# GistView:* -api_access_controllers_whitelist = - -; Default encoding used to convert from and to unicode -; can be also a comma separated list of encoding in case of mixed encodings -default_encoding = UTF-8 - -; instance-id prefix -; a prefix key for this instance used for cache invalidation when running -; multiple instances of RhodeCode, make sure it's globally unique for -; all running RhodeCode instances. Leave empty if you don't use it -instance_id = - -; Fallback authentication plugin. Set this to a plugin ID to force the usage -; of an authentication plugin also if it is disabled by it's settings. -; This could be useful if you are unable to log in to the system due to broken -; authentication settings. Then you can enable e.g. the internal RhodeCode auth -; module to log in again and fix the settings. -; Available builtin plugin IDs (hash is part of the ID): -; egg:rhodecode-enterprise-ce#rhodecode -; egg:rhodecode-enterprise-ce#pam -; egg:rhodecode-enterprise-ce#ldap -; egg:rhodecode-enterprise-ce#jasig_cas -; egg:rhodecode-enterprise-ce#headers -; egg:rhodecode-enterprise-ce#crowd - -#rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode - -; Flag to control loading of legacy plugins in py:/path format -auth_plugin.import_legacy_plugins = true - -; alternative return HTTP header for failed authentication. Default HTTP -; response is 401 HTTPUnauthorized. Currently HG clients have troubles with -; handling that causing a series of failed authentication calls. -; Set this variable to 403 to return HTTPForbidden, or any other HTTP code -; This will be served instead of default 401 on bad authentication -auth_ret_code = - -; use special detection method when serving auth_ret_code, instead of serving -; ret_code directly, use 401 initially (Which triggers credentials prompt) -; and then serve auth_ret_code to clients -auth_ret_code_detection = false - -; locking return code. When repository is locked return this HTTP code. 2XX -; codes don't break the transactions while 4XX codes do -lock_ret_code = 423 - -; allows to change the repository location in settings page -allow_repo_location_change = true - -; allows to setup custom hooks in settings page -allow_custom_hooks_settings = true - -; Generated license token required for EE edition license. -; New generated token value can be found in Admin > settings > license page. -license_token = abra-cada-bra1-rce3 - -; This flag hides sensitive information on the license page such as token, and license data -license.hide_license_info = false - -; supervisor connection uri, for managing supervisor and logs. -supervisor.uri = 127.0.0.1:10001 - -; supervisord group name/id we only want this RC instance to handle -supervisor.group_id = web-1 - -; Display extended labs settings -labs_settings_active = true - -; Custom exception store path, defaults to TMPDIR -; This is used to store exception from RhodeCode in shared directory -#exception_tracker.store_path = - -; Send email with exception details when it happens -#exception_tracker.send_email = false - -; Comma separated list of recipients for exception emails, -; e.g admin@rhodecode.com,devops@rhodecode.com -; Can be left empty, then emails will be sent to ALL super-admins -#exception_tracker.send_email_recipients = - -; optional prefix to Add to email Subject -#exception_tracker.email_prefix = [RHODECODE ERROR] - -; File store configuration. This is used to store and serve uploaded files -file_store.enabled = true - -; Storage backend, available options are: local -file_store.backend = local - -; path to store the uploaded binaries -file_store.storage_path = /var/opt/rhodecode_data/file_store - - -; ############# -; CELERY CONFIG -; ############# - -; manually run celery: /path/to/celery worker -E --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini - -use_celery = true - -; connection url to the message broker (default redis) -celery.broker_url = redis://redis:6379/8 - -; rabbitmq example -#celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost - -; maximum tasks to execute before worker restart -celery.max_tasks_per_child = 20 - -; tasks will never be sent to the queue, but executed locally instead. -celery.task_always_eager = false - -; ############# -; DOGPILE CACHE -; ############# - -; Default cache dir for caches. Putting this into a ramdisk can boost performance. -; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space -cache_dir = /var/opt/rhodecode_data - -; ********************************************* -; `sql_cache_short` cache for heavy SQL queries -; Only supported backend is `memory_lru` -; ********************************************* -rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru -rc_cache.sql_cache_short.expiration_time = 30 - - -; ***************************************************** -; `cache_repo_longterm` cache for repo object instances -; Only supported backend is `memory_lru` -; ***************************************************** -rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru -; by default we use 30 Days, cache is still invalidated on push -rc_cache.cache_repo_longterm.expiration_time = 2592000 -; max items in LRU cache, set to smaller number to save memory, and expire last used caches -rc_cache.cache_repo_longterm.max_size = 10000 - - -; ************************************************* -; `cache_perms` cache for permission tree, auth TTL -; ************************************************* -#rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace -#rc_cache.cache_perms.expiration_time = 300 -; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set -#rc_cache.cache_perms.arguments.filename = /tmp/cache_perms.db - -; alternative `cache_perms` redis backend with distributed lock -rc_cache.cache_perms.backend = dogpile.cache.rc.redis -rc_cache.cache_perms.expiration_time = 300 - -; redis_expiration_time needs to be greater then expiration_time -rc_cache.cache_perms.arguments.redis_expiration_time = 7200 - -rc_cache.cache_perms.arguments.host = redis -rc_cache.cache_perms.arguments.port = 6379 -rc_cache.cache_perms.arguments.db = 0 -rc_cache.cache_perms.arguments.socket_timeout = 30 -; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends -#rc_cache.cache_perms.arguments.distributed_lock = true - - -; *************************************************** -; `cache_repo` cache for file tree, Readme, RSS FEEDS -; *************************************************** -#rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace -#rc_cache.cache_repo.expiration_time = 2592000 -; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set -#rc_cache.cache_repo.arguments.filename = /tmp/cache_repo.db - -; alternative `cache_repo` redis backend with distributed lock -rc_cache.cache_repo.backend = dogpile.cache.rc.redis -rc_cache.cache_repo.expiration_time = 2592000 - -; redis_expiration_time needs to be greater then expiration_time -#rc_cache.cache_repo.arguments.redis_expiration_time = 2678400 - -rc_cache.cache_repo.arguments.host = redis -rc_cache.cache_repo.arguments.port = 6379 -rc_cache.cache_repo.arguments.db = 1 -rc_cache.cache_repo.arguments.socket_timeout = 30 -; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends -#rc_cache.cache_repo.arguments.distributed_lock = true - - -; ############## -; BEAKER SESSION -; ############## - -; beaker.session.type is type of storage options for the logged users sessions. Current allowed -; types are file, ext:redis, ext:database, ext:memcached, and memory (default if not specified). -; Fastest ones are Redis and ext:database -#beaker.session.type = file -#beaker.session.data_dir = /var/opt/rhodecode_data/sessions - -; Redis based sessions -beaker.session.type = ext:redis -beaker.session.url = redis://redis:6379/2 - -; DB based session, fast, and allows easy management over logged in users -#beaker.session.type = ext:database -#beaker.session.table_name = db_session -#beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode -#beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode -#beaker.session.sa.pool_recycle = 3600 -#beaker.session.sa.echo = false - -beaker.session.key = http_app -beaker.session.secret = b39acb28b2304a27a6a0e911500bf7d1 -beaker.session.lock_dir = /data_ramdisk/lock - -; Secure encrypted cookie. Requires AES and AES python libraries -; you must disable beaker.session.secret to use this -#beaker.session.encrypt_key = key_for_encryption -#beaker.session.validate_key = validation_key - -; Sets session as invalid (also logging out user) if it haven not been -; accessed for given amount of time in seconds -beaker.session.timeout = 2592000 -beaker.session.httponly = true - -; Path to use for the cookie. Set to prefix if you use prefix middleware -#beaker.session.cookie_path = /custom_prefix - -; Set https secure cookie -beaker.session.secure = false - -; default cookie expiration time in seconds, set to `true` to set expire -; at browser close -#beaker.session.cookie_expires = 3600 - -; ############################# -; SEARCH INDEXING CONFIGURATION -; ############################# - -; Full text search indexer is available in rhodecode-tools under -; `rhodecode-tools index` command - -; WHOOSH Backend, doesn't require additional services to run -; it works good with few dozen repos -#search.module = rhodecode.lib.index.whoosh -#search.location = /var/opt/rhodecode_data/index - -; ElasticSearch (EE edition only). Requires Elastic Search cluster -; to be installed, and running. Recommended for large amount of repositories -search.module = rc_elasticsearch -search.location = http://elasticsearch:9200 -; specify Elastic Search version, 6 for latest or 2 for legacy -search.es_version = 6 - -; #################### -; CHANNELSTREAM CONFIG -; #################### - -; channelstream enables persistent connections and live notification -; in the system. It's also used by the chat system - -channelstream.enabled = true - -; server address for channelstream server on the backend -channelstream.server = channelstream:9800 - -; location of the channelstream server from outside world -; use ws:// for http or wss:// for https. This address needs to be handled -; by external HTTP server such as Nginx or Apache -; see Nginx/Apache configuration examples in our docs -channelstream.ws_url = ws:/localhost:8888/_channelstream -channelstream.secret = b39acb28b2304a27a6a0e911500bf7d1 -channelstream.history.location = /var/opt/rhodecode_data/channelstream_history - -; Internal application path that Javascript uses to connect into. -; If you use proxy-prefix the prefix should be added before /_channelstream -channelstream.proxy_path = /_channelstream - -; Live chat for commits/pull requests. Requires CHANNELSTREAM to be enabled -; and configured. (EE edition only) -chat.enabled = false - - -; ############################## -; MAIN RHODECODE DATABASE CONFIG -; ############################## - -#sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30 -#sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode -#sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8 -; pymysql is an alternative driver for MySQL, use in case of problems with default one -#sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode - -sqlalchemy.db1.url = postgresql://rhodecode:hUc1adS7oDd6Oj3in3@database/rhodecode - -; see sqlalchemy docs for other advanced settings -; print the sql statements to output -sqlalchemy.db1.echo = false - -; recycle the connections after this amount of seconds -sqlalchemy.db1.pool_recycle = 3600 -sqlalchemy.db1.convert_unicode = true - -; the number of connections to keep open inside the connection pool. -; 0 indicates no limit -#sqlalchemy.db1.pool_size = 5 - -; The number of connections to allow in connection pool "overflow", that is -; connections that can be opened above and beyond the pool_size setting, -; which defaults to five. -#sqlalchemy.db1.max_overflow = 10 - -; Connection check ping, used to detect broken database connections -; could be enabled to better handle cases if MySQL has gone away errors -#sqlalchemy.db1.ping_connection = true - -; ########## -; VCS CONFIG -; ########## -vcs.server.enable = true -vcs.server = vcsserver:10010 - -; Web server connectivity protocol, responsible for web based VCS operations -; Available protocols are: -; `http` - use http-rpc backend (default) -vcs.server.protocol = http - -; Push/Pull operations protocol, available options are: -; `http` - use http-rpc backend (default) -vcs.scm_app_implementation = http - -; Push/Pull operations hooks protocol, available options are: -; `http` - use http-rpc backend (default) -vcs.hooks.protocol = http - -; Host on which this instance is listening for hooks. If vcsserver is in other location -; this should be adjusted. -vcs.hooks.host = rhodecode - -; Start VCSServer with this instance as a subprocess, useful for development -vcs.start_server = false - -; List of enabled VCS backends, available options are: -; `hg` - mercurial -; `git` - git -; `svn` - subversion -vcs.backends = hg, git, svn - -; Wait this number of seconds before killing connection to the vcsserver -vcs.connection_timeout = 3600 - -; Compatibility version when creating SVN repositories. Defaults to newest version when commented out. -; Available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible -#vcs.svn.compatible_version = pre-1.8-compatible - - -; #################################################### -; Subversion proxy support (mod_dav_svn) -; Maps RhodeCode repo groups into SVN paths for Apache -; #################################################### - -; Enable or disable the config file generation. -svn.proxy.generate_config = true - -; Generate config file with `SVNListParentPath` set to `On`. -svn.proxy.list_parent_path = true - -; Set location and file name of generated config file. -svn.proxy.config_file_path = /etc/rhodecode/conf/svn/mod_dav_svn.conf - -; alternative mod_dav config template. This needs to be a valid mako template -; Example template can be found in the source code: -; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako -#svn.proxy.config_template = ~/.rccontrol/community-1/custom_svn_conf.mako - -; Used as a prefix to the `Location` block in the generated config file. -; In most cases it should be set to `/`. -svn.proxy.location_root = / - -; Command to reload the mod dav svn configuration on change. -; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh -; Make sure user who runs RhodeCode process is allowed to reload Apache -#svn.proxy.reload_cmd = /etc/init.d/apache2 reload - -; If the timeout expires before the reload command finishes, the command will -; be killed. Setting it to zero means no timeout. Defaults to 10 seconds. -#svn.proxy.reload_timeout = 10 - -; #################### -; SSH Support Settings -; #################### - -; Defines if a custom authorized_keys file should be created and written on -; any change user ssh keys. Setting this to false also disables possibility -; of adding SSH keys by users from web interface. Super admins can still -; manage SSH Keys. -ssh.generate_authorized_keyfile = true - -; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding` -# ssh.authorized_keys_ssh_opts = - -; Path to the authorized_keys file where the generate entries are placed. -; It is possible to have multiple key files specified in `sshd_config` e.g. -; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode -ssh.authorized_keys_file_path = /etc/rhodecode/conf/ssh/authorized_keys_rhodecode - -; Command to execute the SSH wrapper. The binary is available in the -; RhodeCode installation directory. -; e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper -ssh.wrapper_cmd = /var/opt/rhodecode_bin/bin/rc-ssh-wrapper - -; Allow shell when executing the ssh-wrapper command -ssh.wrapper_cmd_allow_shell = false - -; Enables logging, and detailed output send back to the client during SSH -; operations. Useful for debugging, shouldn't be used in production. -ssh.enable_debug_logging = false - -; Paths to binary executable, by default they are the names, but we can -; override them if we want to use a custom one -ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg -ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git -ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve - -; Enables SSH key generator web interface. Disabling this still allows users -; to add their own keys. -ssh.enable_ui_key_generator = true - -; Dummy marker to add new entries after. -; Add any custom entries below. Please don't remove this marker. -custom.conf = 1 - - -; ##################### -; LOGGING CONFIGURATION -; ##################### -[loggers] -keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper - -[handlers] -keys = console, console_sql - -[formatters] -keys = generic, color_formatter, color_formatter_sql - -; ####### -; LOGGERS -; ####### -[logger_root] -level = NOTSET -handlers = console - -[logger_sqlalchemy] -level = INFO -handlers = console_sql -qualname = sqlalchemy.engine -propagate = 0 - -[logger_beaker] -level = DEBUG -handlers = -qualname = beaker.container -propagate = 1 - -[logger_rhodecode] -level = DEBUG -handlers = -qualname = rhodecode -propagate = 1 - -[logger_ssh_wrapper] -level = DEBUG -handlers = -qualname = ssh_wrapper -propagate = 1 - -[logger_celery] -level = DEBUG -handlers = -qualname = celery - - -; ######## -; HANDLERS -; ######## - -[handler_console] -class = StreamHandler -args = (sys.stderr, ) -level = INFO -formatter = generic - -[handler_console_sql] -; "level = DEBUG" logs SQL queries and results. -; "level = INFO" logs SQL queries. -; "level = WARN" logs neither. (Recommended for production systems.) -class = StreamHandler -args = (sys.stderr, ) -level = WARN -formatter = generic - -; ########## -; FORMATTERS -; ########## - -[formatter_generic] -class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter -format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %Y-%m-%d %H:%M:%S - -[formatter_color_formatter] -class = rhodecode.lib.logging_formatter.ColorFormatter -format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %Y-%m-%d %H:%M:%S - -[formatter_color_formatter_sql] -class = rhodecode.lib.logging_formatter.ColorFormatterSql -format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %Y-%m-%d %H:%M:%S diff --git a/config/compose/vcsserver.optimized.ini b/config/compose/vcsserver.optimized.ini deleted file mode 100644 index 6b4b04c..0000000 --- a/config/compose/vcsserver.optimized.ini +++ /dev/null @@ -1,199 +0,0 @@ - -; ################################# -; RHODECODE VCSSERVER CONFIGURATION -; ################################# - -[server:main] -; COMMON HOST/IP CONFIG -host = 0.0.0.0 -port = 10010 - - -; ########################### -; GUNICORN APPLICATION SERVER -; ########################### - -; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini - -; Module to use, this setting shouldn't be changed -use = egg:gunicorn#main - -; Sets the number of process workers. More workers means more concurrent connections -; RhodeCode can handle at the same time. Each additional worker also it increases -; memory usage as each has it's own set of caches. -; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more -; than 8-10 unless for really big deployments .e.g 700-1000 users. -; `instance_id = *` must be set in the [app:main] section below (which is the default) -; when using more than 1 worker. -workers = 3 - -; Gunicorn access log level -loglevel = info - -; Process name visible in process list -proc_name = gunicorn-vcsserver-1 - -; Type of worker class, one of `sync`, `gevent` -; currently `sync` is the only option allowed. -worker_class = sync - -; The maximum number of simultaneous clients. Valid only for gevent -worker_connections = 10 - -; Max number of requests that worker will handle before being gracefully restarted. -; Prevents memory leaks, jitter adds variability so not all workers are restarted at once. -max_requests = 3000 -max_requests_jitter = 100 - -; Amount of time a worker can spend with handling a request before it -; gets killed and restarted. By default set to 21600 (6hrs) -; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h) -timeout = 21600 - -; The maximum size of HTTP request line in bytes. -; 0 for unlimited -limit_request_line = 0 - -; Limit the number of HTTP headers fields in a request. -; By default this value is 100 and can't be larger than 32768. -limit_request_fields = 32768 - -; Limit the allowed size of an HTTP request header field. -; Value is a positive number or 0. -; Setting it to 0 will allow unlimited header field sizes. -limit_request_field_size = 0 - -; Timeout for graceful workers restart. -; After receiving a restart signal, workers have this much time to finish -; serving requests. Workers still alive after the timeout (starting from the -; receipt of the restart signal) are force killed. -; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h) -graceful_timeout = 3600 - -# The number of seconds to wait for requests on a Keep-Alive connection. -# Generally set in the 1-5 seconds range. -keepalive = 2 - -; Maximum memory usage that each worker can use before it will receive a -; graceful restart signal 0 = memory monitoring is disabled -; Examples: 268435456 (256MB), 536870912 (512MB) -; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB) -memory_max_usage = 2147483648 - -; How often in seconds to check for memory usage for each gunicorn worker -memory_usage_check_interval = 60 - -; Threshold value for which we don't recycle worker if GarbageCollection -; frees up enough resources. Before each restart we try to run GC on worker -; in case we get enough free memory after that, restart will not happen. -memory_usage_recovery_threshold = 0.8 - - -[app:main] -; The %(here)s variable will be replaced with the absolute path of parent directory -; of this file -use = egg:rhodecode-vcsserver - -; Pyramid default locales, we need this to be set -pyramid.default_locale_name = en - -; default locale used by VCS systems -locale = en_US.UTF-8 - -; path to binaries for vcsserver, it should be set by the installer -; at installation time, e.g /home/user/vcsserver-1/profile/bin -; it can also be a path to nix-build output in case of development -core.binary_dir = /home/rhodecode/.rccontrol/vcsserver-1/profile/bin - -; Custom exception store path, defaults to TMPDIR -; This is used to store exception from RhodeCode in shared directory -#exception_tracker.store_path = - -; ############# -; DOGPILE CACHE -; ############# - -; Default cache dir for caches. Putting this into a ramdisk can boost performance. -; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space -cache_dir = /var/opt/rhodecode_data - -; *************************************** -; `repo_object` cache, default file based -; *************************************** - -; `repo_object` cache settings for vcs methods for repositories -#rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace - -; cache auto-expires after N seconds -; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days) -#rc_cache.repo_object.expiration_time = 2592000 - -; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set -#rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db - -; *********************************************************** -; `repo_object` cache with redis backend -; recommended for larger instance, and for better performance -; *********************************************************** - -; `repo_object` cache settings for vcs methods for repositories -rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack - -; cache auto-expires after N seconds -; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days) -rc_cache.repo_object.expiration_time = 2592000 - -; redis_expiration_time needs to be greater then expiration_time -rc_cache.repo_object.arguments.redis_expiration_time = 3592000 - -rc_cache.repo_object.arguments.host = redis -rc_cache.repo_object.arguments.port = 6379 -rc_cache.repo_object.arguments.db = 5 -rc_cache.repo_object.arguments.socket_timeout = 30 -; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends -#rc_cache.repo_object.arguments.distributed_lock = true - - -; ##################### -; LOGGING CONFIGURATION -; ##################### -[loggers] -keys = root, vcsserver - -[handlers] -keys = console - -[formatters] -keys = generic - -; ####### -; LOGGERS -; ####### -[logger_root] -level = NOTSET -handlers = console - -[logger_vcsserver] -level = DEBUG -handlers = -qualname = vcsserver -propagate = 1 - - -; ######## -; HANDLERS -; ######## - -[handler_console] -class = StreamHandler -args = (sys.stderr, ) -level = INFO -formatter = generic - -; ########## -; FORMATTERS -; ########## - -[formatter_generic] -format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %Y-%m-%d %H:%M:%S diff --git a/service/database/mysql_customized.conf b/config/database/mysql_customized.conf similarity index 100% rename from service/database/mysql_customized.conf rename to config/database/mysql_customized.conf diff --git a/service/database/pg_customized.conf b/config/database/pg_customized.conf similarity index 92% rename from service/database/pg_customized.conf rename to config/database/pg_customized.conf index 120fc7a..437fb0e 100644 --- a/service/database/pg_customized.conf +++ b/config/database/pg_customized.conf @@ -1,3 +1,4 @@ +## docker run -i --rm postgres cat /usr/share/postgresql/postgresql.conf.sample > my-postgres.conf # ----------------------------- # PostgreSQL configuration file # ----------------------------- @@ -93,7 +94,8 @@ listen_addresses = '*' # - Authentication - #authentication_timeout = 1min # 1s-600s -#password_encryption = scram-sha-256 # scram-sha-256 or md5 +#note; Rhodecode needs to use old md5 for backward compat +password_encryption = md5 # scram-sha-256 or md5 #db_user_namespace = off # GSSAPI using Kerberos @@ -436,21 +438,21 @@ log_destination = 'stderr' # Valid values are combinations of # requires logging_collector to be on. # This is used when logging to stderr: -logging_collector = on # Enable capturing of stderr and csvlog +#logging_collector = on # Enable capturing of stderr and csvlog # into log files. Required to be on for # csvlogs. # (change requires restart) # These are only used if logging_collector is on: -log_directory = '/var/log/rhodecode/postgres' # directory where log files are written, +#log_directory = '/var/log/rhodecode/postgres' # directory where log files are written, # can be absolute or relative to PGDATA -log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, +#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, # can include strftime() escapes #log_file_mode = 0600 # creation mode for log files, # begin with 0 to use octal notation #log_rotation_age = 1d # Automatic rotation of logfiles will # happen after that time. 0 disables. -log_rotation_size = 512MB # Automatic rotation of logfiles will +#log_rotation_size = 512MB # Automatic rotation of logfiles will # happen after that much log output. # 0 disables. #log_truncate_on_rotation = off # If on, an existing log file with the @@ -534,7 +536,7 @@ log_min_duration_statement = 1000 # -1 is disabled, 0 logs all statements # actions running at least this number # of milliseconds. #log_checkpoints = off -#log_connections = off +log_connections = off #log_disconnections = off #log_duration = off #log_error_verbosity = default # terse, default, or verbose messages diff --git a/config/grafana/grafana.env b/config/grafana/grafana.env deleted file mode 100644 index fc4b961..0000000 --- a/config/grafana/grafana.env +++ /dev/null @@ -1,10 +0,0 @@ -GF_SECURITY_ADMIN_USER=admin -GF_SECURITY_ADMIN_PASSWORD=admin -GF_USERS_ALLOW_SIGN_UP=false - -GF_STMP_ENABLED=true -GF_STMP_HOST= -GF_STMP_USER= -GF_STMP_PASSWORD= -GF_STMP_FROM_ADDRESS=alerting-grafana@grafana.localhost -GF_STMP_FROM_NAME=[Grafana] Alerting \ No newline at end of file diff --git a/config/grafana/grafana.ini b/config/grafana/grafana.ini new file mode 100644 index 0000000..2995c20 --- /dev/null +++ b/config/grafana/grafana.ini @@ -0,0 +1,19 @@ +[security] +admin_user=admin +admin_password=admin + +[users] +allow_sign_up=false + +[smtp] +enabled=true +host= +user= +password= +from_address=alerting-grafana@grafana.localhost +from_name=[Grafana] Alerting + +# subpath /_grafana +[server] +serve_from_sub_path=true +root_url=%(protocol)s://%(domain)s:%(http_port)s/_grafana/ diff --git a/service/nginx/http.conf b/config/nginx/http.conf similarity index 86% rename from service/nginx/http.conf rename to config/nginx/http.conf index df58b77..703c8da 100644 --- a/service/nginx/http.conf +++ b/config/nginx/http.conf @@ -58,10 +58,10 @@ server { # ensure we get the proper Docker DNS resolver for load balancing. resolver 127.0.0.11 ipv6=off; server_name localhost 127.0.0.1; - access_log /var/log/rhodecode/nginx/rhodecode.http.access.log json_log_custom; - error_log /var/log/rhodecode/nginx/rhodecode.http.error.log; + access_log /dev/stdout json_log_custom; + error_log /dev/stdout; - # uncomment to redirect http traffic to https + # uncomment to redirect http traffic to https (not applicable in Docker setup) #return 301 https://$host$request_uri; client_body_buffer_size 128k; @@ -103,30 +103,33 @@ server { proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; - proxy_pass http://channelstream:9800; + set $upstream_channelstream http://channelstream:8000; + proxy_pass $upstream_channelstream; } - ## rate limit this endpoint to prevent login page brute-force attacks - location /_admin/login { - limit_req zone=http_req_limit burst=10 nodelay; - try_files $uri @rhodecode_http; - } +# ## rate limit this endpoint to prevent login page brute-force attacks +# location /_admin/login { +# limit_req zone=http_req_limit burst=10 nodelay; +# try_files $uri @rhodecode_http; +# } location / { - include /etc/nginx/proxy.conf; - try_files $uri @rhodecode_http; + include /etc/nginx/proxy.conf; + try_files $uri @rhodecode_http; } location @rhodecode_http { - include /etc/nginx/proxy.conf; - proxy_pass http://rhodecode:10020; + set $upstream http://rhodecode:10020; + include /etc/nginx/proxy.conf; + proxy_pass $upstream; } ## Custom 502 error page. ## Will be displayed while RhodeCode server is turned off error_page 502 /502.html; location = /502.html { - root /var/opt/static; + root /var/opt/rhodecode_data/static; + internal; } } diff --git a/service/nginx/nginx.conf b/config/nginx/nginx.conf similarity index 85% rename from service/nginx/nginx.conf rename to config/nginx/nginx.conf index f5e4f6f..8656036 100644 --- a/service/nginx/nginx.conf +++ b/config/nginx/nginx.conf @@ -2,7 +2,7 @@ user root; worker_processes 2; pid /var/run/nginx.pid; -error_log /var/log/rhodecode/nginx/default.error.log warn; +error_log /dev/stdout info; events { worker_connections 1024; @@ -12,13 +12,14 @@ events { http { include /etc/nginx/mime.types; default_type application/octet-stream; + server_tokens off; log_format main '$remote_addr - $remote_user [$time_local] ' '"$request" $status $body_bytes_sent ' '"$http_referer" "$http_user_agent" ' '$request_time $upstream_response_time $pipe'; - access_log /var/log/rhodecode/nginx/default.access.log main; + access_log /dev/stdout main; sendfile on; tcp_nopush on; @@ -26,5 +27,5 @@ http { keepalive_timeout 65; types_hash_max_size 2048; - include /etc/nginx/sites-enabled/*.conf; + include /etc/nginx/http.conf; } diff --git a/service/nginx/proxy.conf b/config/nginx/proxy.conf similarity index 100% rename from service/nginx/proxy.conf rename to config/nginx/proxy.conf diff --git a/config/prometheus/prometheus.yml b/config/prometheus/prometheus.yml index 42e11b4..3ffac78 100644 --- a/config/prometheus/prometheus.yml +++ b/config/prometheus/prometheus.yml @@ -15,3 +15,9 @@ scrape_configs: metrics_path: "/metrics" static_configs: - targets: ["node-exporter:9100"] + + # scrape traefik node + - job_name: traefik_metrics + metrics_path: "/metrics" + static_configs: + - targets: ["traefik:7000"] diff --git a/config/promtail/promtail-config.yaml b/config/promtail/promtail-config.yaml index 6ee78d0..06389ae 100644 --- a/config/promtail/promtail-config.yaml +++ b/config/promtail/promtail-config.yaml @@ -13,26 +13,26 @@ clients: ## Definition to what to scrape scrape_configs: -- job_name: scraping_nginx - static_configs: - - targets: - - localhost - labels: - __path__: /var/log_volume/nginx/rhodecode* - job: nginx - -- job_name: scraping_vcsserver - static_configs: - - targets: - - localhost - labels: - __path__: /var/log_volume/apps/rhodecode* - job: rhodecode - -- job_name: scraping_rhodecode - static_configs: - - targets: - - localhost - labels: - __path__: /var/log_volume/apps/vcsserver* - job: vcsserver \ No newline at end of file +#- job_name: scraping_nginx +# static_configs: +# - targets: +# - localhost +# labels: +# __path__: /var/log_volume/nginx/rhodecode* +# job: nginx +# +#- job_name: scraping_vcsserver +# static_configs: +# - targets: +# - localhost +# labels: +# __path__: /var/log_volume/apps/rhodecode* +# job: rhodecode +# +#- job_name: scraping_rhodecode +# static_configs: +# - targets: +# - localhost +# labels: +# __path__: /var/log_volume/apps/vcsserver* +# job: vcsserver \ No newline at end of file diff --git a/service/redis/redis.conf b/config/redis/redis.conf similarity index 100% rename from service/redis/redis.conf rename to config/redis/redis.conf index 3adfeb5..410f842 100644 --- a/service/redis/redis.conf +++ b/config/redis/redis.conf @@ -280,7 +280,7 @@ databases 16 # # However it is possible to force the pre-4.0 behavior and always show a # ASCII art logo in startup logs by setting the following option to yes. -always-show-logo yes +always-show-logo no ################################ SNAPSHOTTING ################################ # @@ -362,7 +362,7 @@ rdb-del-sync-files no # The Append Only File will also be created inside this directory. # # Note that you must specify a directory here, not a file name. -dir /var/opt/rhodecode_data/ +dir /data ################################# REPLICATION ################################# @@ -857,7 +857,7 @@ acllog-max-len 128 # limit for maxmemory so that there is some free RAM on the system for replica # output buffers (but this is not needed if the policy is 'noeviction'). # -maxmemory 8192mb +#maxmemory 8192mb # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory # is reached. You can select one from the following behaviors: @@ -888,7 +888,7 @@ maxmemory 8192mb # # The default is: # -maxmemory-policy volatile-lru +#maxmemory-policy volatile-lru # LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated # algorithms (in order to save memory), so you can tune it for speed or @@ -899,7 +899,7 @@ maxmemory-policy volatile-lru # The default of 5 produces good enough results. 10 Approximates very closely # true LRU but costs more CPU. 3 is faster but not very accurate. # -maxmemory-samples 5 +maxmemory-samples 10 # Starting from Redis 5, by default a replica will ignore its maxmemory setting # (unless it is promoted to master after a failover or manually). It means diff --git a/config/traefik/certs/.keepdir b/config/traefik/certs/.keepdir new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/config/traefik/certs/.keepdir diff --git a/config/traefik/certs/README.md b/config/traefik/certs/README.md new file mode 100644 index 0000000..141a0b2 --- /dev/null +++ b/config/traefik/certs/README.md @@ -0,0 +1,3 @@ +Store your certificate files here. +your-domain.crt +your-domain.key \ No newline at end of file diff --git a/config/traefik/traefik.yaml b/config/traefik/traefik.yaml new file mode 100644 index 0000000..4a223e7 --- /dev/null +++ b/config/traefik/traefik.yaml @@ -0,0 +1,42 @@ +## Static configuration, needs traefik reload + +entryPoints: + http: + address: ":80" + + https: + address: ":443" + +# ssh: +# address: ":${RC_SSH_PORT:?must-specify-ssh-port}" + + traefik: + address: ":7000" + +api: + dashboard: false + insecure: false # change to true to expose dashboard + +accessLog: {} + +log: + # DEBUG / INFO + level: INFO + +metrics: + prometheus: {} + + +providers: + docker: + exposedByDefault: false + swarmMode: false + + file: + filename: "/etc/traefik/traefik_dynamic.yaml" + watch: true + + +serversTransport: + #this disables SSL certificate verification for legacy own cert services like NextCloud etc + insecureSkipVerify: true diff --git a/config/traefik/traefik_dynamic.yaml b/config/traefik/traefik_dynamic.yaml new file mode 100644 index 0000000..36a09de --- /dev/null +++ b/config/traefik/traefik_dynamic.yaml @@ -0,0 +1,24 @@ + +## Setting up the middleware for redirect to https ## +http: + + middlewares: + + https-redirect: + redirectScheme: + scheme: https + # permanent: true + + middlewares-rate-limit: + rateLimit: + average: 100 + burst: 50 + +# Dynamic configuration for standard file based SSL cert +# create a custom copy of the traefik into .custom/ and remount this +# to enable custom certificates +#tls: +# certificates: +# # first certificate +# - certFile: /etc/traefik/certs/rhodecode-com.cert +# keyFile: /etc/traefik/certs/rhodecode-com.key diff --git a/docker-compose.dev.yaml b/docker-compose-apps.dev.yaml similarity index 94% rename from docker-compose.dev.yaml rename to docker-compose-apps.dev.yaml index 6cc27f4..88807a5 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose-apps.dev.yaml @@ -13,7 +13,7 @@ services: - nixstore:/opt/nix/store ports: - - "8080:8080" + - "10020:10020" vcsserver: volumes: @@ -30,7 +30,7 @@ services: - /media/psf/Home/workspace/rhodecode-enterprise-ee:/home/rhodecode/rhodecode-enterprise-ee:delegated - nixstore:/opt/nix/store - beat: + celery-beat: volumes: - /media/psf/Home/workspace/rhodecode-vcsserver:/home/rhodecode/rhodecode-vcsserver:delegated - /media/psf/Home/workspace/rhodecode-enterprise-ce:/home/rhodecode/rhodecode-enterprise-ce:delegated @@ -50,7 +50,3 @@ services: - /media/psf/Home/workspace/rhodecode-enterprise-ce:/home/rhodecode/rhodecode-enterprise-ce:delegated - /media/psf/Home/workspace/rhodecode-enterprise-ee:/home/rhodecode/rhodecode-enterprise-ee:delegated - nixstore:/opt/nix/store - - database: - volumes: - - /tmp:/usr-tmp diff --git a/docker-compose.source.yaml b/docker-compose-apps.source.yaml similarity index 63% rename from docker-compose.source.yaml rename to docker-compose-apps.source.yaml index 47d99ec..70f313c 100644 --- a/docker-compose.source.yaml +++ b/docker-compose-apps.source.yaml @@ -10,42 +10,26 @@ services: DB_UPGRADE: 0 # run the DB upgrade volumes: - - bashhistory:/home/rhodecode + - bashhistory:/home/rhodecode/.bash_history_docker build: context: . dockerfile: service/rhodecode/rhodecode_source.dockerfile - image: rhodecode/app_source:${SOURCE_VER:?specify-SOURCE_VER-env-var} - - command: [ - "/home/rhodecode/rhodecode-enterprise-ee/result/bin/gunicorn", - "--name=gunicorn-rhodecode-1", - "--error-logfile=-", - "--paster=/etc/rhodecode/conf/compose/rhodecode.optimized.ini", - "--config=/etc/rhodecode/conf/gunicorn_conf.py" - ] + image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var}_SOURCE vcsserver: environment: HISTFILE: /home/rhodecode/.bash_history_docker volumes: - - bashhistory:/home/rhodecode + - bashhistory:/home/rhodecode/.bash_history_docker build: context: . dockerfile: service/rhodecode/rhodecode_source.dockerfile - image: rhodecode/app_source:${SOURCE_VER:?specify-SOURCE_VER-env-var} - - command: [ - "/home/rhodecode/rhodecode-vcsserver/result/bin/gunicorn", - "--name=gunicorn-vcsserver-1", - "--error-logfile=-", - "--paster=/etc/rhodecode/conf/compose/vcsserver.optimized.ini", - "--config=/etc/rhodecode/conf/gunicorn_conf.py" - ] + image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var}_SOURCE celery: @@ -53,15 +37,15 @@ services: context: . dockerfile: service/rhodecode/rhodecode_source.dockerfile - image: rhodecode/app_source:${SOURCE_VER:?specify-SOURCE_VER-env-var} + image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var}_SOURCE - beat: + celery-beat: build: context: . dockerfile: service/rhodecode/rhodecode_source.dockerfile - image: rhodecode/app_source:${SOURCE_VER:?specify-SOURCE_VER-env-var} + image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var}_SOURCE svn: @@ -69,7 +53,7 @@ services: context: . dockerfile: service/rhodecode/rhodecode_source.dockerfile - image: rhodecode/app_source:${SOURCE_VER:?specify-SOURCE_VER-env-var} + image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var}_SOURCE sshd: @@ -77,4 +61,4 @@ services: context: . dockerfile: service/rhodecode/rhodecode_source.dockerfile - image: rhodecode/app_source:${SOURCE_VER:?specify-SOURCE_VER-env-var} \ No newline at end of file + image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var}_SOURCE \ No newline at end of file diff --git a/docker-compose-apps.yaml b/docker-compose-apps.yaml new file mode 100644 index 0000000..7ec066b --- /dev/null +++ b/docker-compose-apps.yaml @@ -0,0 +1,312 @@ +version: '3.9' + +x-logging: &custom-logging + # docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions + # NOTE: loki logging driver ONLY works for host type networks... + driver: loki + options: + loki-url: "http://127.0.0.1:3100/loki/api/v1/push" + #loki-url: "http://loki:3100/loki/api/v1/push" + loki-retries: "5" + loki-timeout: "1s" + loki-max-backoff: "800ms" + +volumes: + + # volume for RhodeCode repo-store, it's where the repositories will be stored + rhodecode_repos: + labels: + "keep": 1 + + +services: + + rhodecode: + networks: + - rhodecode_network + image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var} + env_file: + - .custom/.runtime.env + stdin_open: true + tty: true + restart: always + command: [ + "/usr/local/bin/rhodecode_bin/bin/gunicorn", + "--name=gunicorn-rhodecode-1", + "--error-logfile=-", + "--paster=/etc/rhodecode/conf/rhodecode.optimized.ini", + "--config=/etc/rhodecode/conf/gunicorn_conf.py" + ] + +# ports: +# - "127.0.0.1::10020" + + build: + context: . + dockerfile: service/rhodecode/rhodecode.dockerfile + args: + TZ: ${TZ} + RHODECODE_VERSION: ${RC_VERSION:?specify-RC_VERSION-env-var} + RHODECODE_DB: postgresql://rhodecode:${DB_PASSWORD:?must-specify-db-password}@database/${DB_NAME:?must-specify-db-name} + RHODECODE_USER_NAME: ${RHODECODE_USER_NAME} + RHODECODE_USER_PASS: ${RHODECODE_USER_PASS} + RHODECODE_USER_EMAIL: ${RHODECODE_USER_EMAIL} + + environment: + RC_APP_TYPE: rhodecode_http + RC_APP_PROC: 1 + SSL_CERT_FILE: "/etc/rhodecode/conf/ca-bundle.crt" + REQUESTS_CA_BUNDLE: "/etc/rhodecode/conf/ca-bundle.crt" + GIT_SSL_CAINFO: "/etc/rhodecode/conf/ca-bundle.crt" + GEVENT_RESOLVER: "ares" + + DB_UPGRADE: 1 # run the DB upgrade + SETUP_APP: 1 # run the application default settings setup, can be turned off after initial run + #FORCE_DB_INIT_FILE: 1 # force the database init, warning: destroys old DB + #FORCE_RC_SETUP_APP: 1 # force running setup scripts for configuration/license application + MAIN_INI_PATH: /etc/rhodecode/conf/rhodecode.optimized.ini + + # SVN Specific + MOD_DAV_SVN_PORT: 8090 + APACHE_LOG_DIR: /var/log/rhodecode/svn + MOD_DAV_SVN_CONF_FILE: /etc/rhodecode/conf/svn/mod_dav_svn.conf + + healthcheck: + test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:10020/_admin/ops/ping" ] + timeout: 30s + interval: 60s + retries: 10 + +# depends_on: +# - database +# - redis +# - channelstream + + volumes: + - confvolume:/etc/rhodecode/conf + - logvolume:/var/log/rhodecode + - rhodecode_repos:/var/opt/rhodecode_repo_store + - rc_datavolume:/var/opt/rhodecode_data + + tmpfs: + - /data_ramdisk:size=${RC_DATA_RAMDISK_SIZE:?specify-RC_DATA_RAMDISK_SIZE-env-var} + + logging: + *custom-logging + + labels: + - "traefik.enable=false" + + vcsserver: + networks: + - rhodecode_network + image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var} + env_file: + - .custom/.runtime.env + stdin_open: true + tty: true + restart: always + command: [ + "/home/rhodecode/.rccontrol/vcsserver-1/profile/bin/gunicorn", + "--name=gunicorn-vcsserver-1", + "--error-logfile=-", + "--paster=/etc/rhodecode/conf/vcsserver.optimized.ini", + "--config=/etc/rhodecode/conf/gunicorn_conf_vcs.py" + ] + +# ports: +# - "127.0.0.1::10010" + + healthcheck: + test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:10010/status" ] + timeout: 30s + interval: 60s + retries: 10 + + environment: + RC_APP_TYPE: rhodecode_vcsserver + RC_APP_PROC: 1 + MAIN_INI_PATH: /etc/rhodecode/conf/vcsserver.optimized.ini + SSL_CERT_FILE: "/etc/rhodecode/conf/ca-bundle.crt" + REQUESTS_CA_BUNDLE: "/etc/rhodecode/conf/ca-bundle.crt" + GIT_SSL_CAINFO: "/etc/rhodecode/conf/ca-bundle.crt" + +# depends_on: +# - redis + + volumes: + - confvolume:/etc/rhodecode/conf + - logvolume:/var/log/rhodecode + - rhodecode_repos:/var/opt/rhodecode_repo_store + - rc_datavolume:/var/opt/rhodecode_data + + logging: + *custom-logging + + celery: + networks: + - rhodecode_network + image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var} + env_file: + - .custom/.runtime.env + stdin_open: true + tty: true + restart: always + command: [ + "/usr/local/bin/rhodecode_bin/bin/celery", + "worker", + "--task-events", + "--autoscale=10,2", + "--no-color", + "--app=rhodecode.lib.celerylib.loader", + "--loglevel=DEBUG", + "--ini=/etc/rhodecode/conf/rhodecode.optimized.ini" + ] + + environment: + RC_APP_TYPE: rhodecode_celery + RC_APP_PROC: 1 + MAIN_INI_PATH: /etc/rhodecode/conf/rhodecode.optimized.ini + SSL_CERT_FILE: "/etc/rhodecode/conf/ca-bundle.crt" + REQUESTS_CA_BUNDLE: "/etc/rhodecode/conf/ca-bundle.crt" + GIT_SSL_CAINFO: "/etc/rhodecode/conf/ca-bundle.crt" + +# depends_on: +# - database +# - redis + + volumes: + - confvolume:/etc/rhodecode/conf + - logvolume:/var/log/rhodecode + - rhodecode_repos:/var/opt/rhodecode_repo_store + - rc_datavolume:/var/opt/rhodecode_data + + logging: + *custom-logging + + labels: + - "traefik.enable=false" + + celery-beat: + # This service is not scalable + networks: + - rhodecode_network + image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var} + env_file: + - .custom/.runtime.env + stdin_open: true + tty: true + restart: always + command: [ + "/usr/local/bin/rhodecode_bin/bin/celery", + "beat", + "--no-color", + "--app=rhodecode.lib.celerylib.loader", + "--scheduler=rhodecode.lib.celerylib.scheduler.RcScheduler", + "--loglevel=DEBUG", + "--ini=/etc/rhodecode/conf/rhodecode.optimized.ini" + ] + + environment: + RC_APP_TYPE: rhodecode_beat + RC_APP_PROC: 1 + MAIN_INI_PATH: /etc/rhodecode/conf/rhodecode.optimized.ini + SSL_CERT_FILE: "/etc/rhodecode/conf/ca-bundle.crt" + REQUESTS_CA_BUNDLE: "/etc/rhodecode/conf/ca-bundle.crt" + GIT_SSL_CAINFO: "/etc/rhodecode/conf/ca-bundle.crt" + +# depends_on: +# - database +# - redis + + volumes: + - confvolume:/etc/rhodecode/conf + - logvolume:/var/log/rhodecode + - rhodecode_repos:/var/opt/rhodecode_repo_store + - rc_datavolume:/var/opt/rhodecode_data + + logging: + *custom-logging + + labels: + - "traefik.enable=false" + + svn: + networks: + - rhodecode_network + image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var} + env_file: + - .custom/.runtime.env + +# build: +# context: . +# dockerfile: service/svn/rhodecode_svn.dockerfile +# args: +# APACHE_VER: 1.3 + + stdin_open: true + tty: true + restart: always + command: ["apachectl", "-D", "FOREGROUND"] + + environment: + RC_APP_TYPE: rhodecode_svn + + # SVN Specific + MOD_DAV_SVN_PORT: 8090 + APACHE_LOG_DIR: /var/log/rhodecode/svn + MOD_DAV_SVN_CONF_FILE: /etc/rhodecode/conf/svn/mod_dav_svn.conf + +# ports: +# - "127.0.0.1::8090" + + healthcheck: + test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:8090/_server_status" ] + timeout: 30s + interval: 60s + retries: 10 + + volumes: + - confvolume:/etc/rhodecode/conf + - logvolume:/var/log/rhodecode + - rhodecode_repos:/var/opt/rhodecode_repo_store + + logging: + *custom-logging + + sshd: + networks: + - rhodecode_network + image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var} + env_file: + - .custom/.runtime.env + + stdin_open: true + tty: true + restart: always + command: ["/usr/sbin/sshd", "-f", "/etc/rhodecode/sshd_config", "-D", "-e"] + + environment: + RC_APP_TYPE: rhodecode_sshd + SSH_BOOTSTRAP: 1 + +# ports: +# # set from .env file +# - "${RC_SSH_PORT:?must-specify-ssh-port}:22" + + volumes: + - confvolume:/etc/rhodecode/conf + - logvolume:/var/log/rhodecode + - rhodecode_repos:/var/opt/rhodecode_repo_store + - rc_datavolume:/var/opt/rhodecode_data + + logging: + *custom-logging + + labels: + - "traefik.enable=true" + - "traefik.http.routers.sshd.entrypoints=ssh" + - "traefik.http.routers.sshd.rule=Host(`*`)" + - "traefik.http.services.sshd.loadbalancer.server.port=${RC_SSH_PORT:?must-specify-ssh-port}" + + diff --git a/docker-compose-base.yaml b/docker-compose-base.yaml new file mode 100644 index 0000000..a95e1c7 --- /dev/null +++ b/docker-compose-base.yaml @@ -0,0 +1,46 @@ +version: '3.9' + +## Shared base stuff for all compose files in stack + +x-logging: &custom-logging + # docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions + # NOTE: loki logging driver ONLY works for host type networks... + driver: loki + options: + loki-url: "http://127.0.0.1:3100/loki/api/v1/push" + #loki-url: "http://loki:3100/loki/api/v1/push" + loki-retries: "5" + loki-timeout: "1s" + loki-max-backoff: "800ms" + +volumes: + # shared logvolume + logvolume: + driver: local + driver_opts: + type: none + o: bind + device: $PWD/logs + + # bind-mount with configs + confvolume: + driver: local + driver_opts: + type: none + o: bind + device: $PWD/config/_shared + + # SHARED volume for rhodecode caches, archive caches, nginx static, + # must be run via: docker volume create --name=rc_datavolume + rc_datavolume: + external: true + +networks: + + # SHARED network for all containers + # must be run via: docker network create --name=rhodecode_network + rhodecode_network: + name: rhodecode_network + driver: bridge + external: true + diff --git a/docker-compose-grafana.yaml b/docker-compose-metrics.yaml similarity index 62% rename from docker-compose-grafana.yaml rename to docker-compose-metrics.yaml index 65ca8c2..7e9ec40 100644 --- a/docker-compose-grafana.yaml +++ b/docker-compose-metrics.yaml @@ -1,25 +1,20 @@ version: '3.9' volumes: - prometheus_data: {} - grafana_data: {} - logvolume: - driver: local - driver_opts: - type: none - o: bind - device: $PWD/logs -networks: - rhodecode_network: - name: rhodecode_network + prometheus_data: + labels: + "keep": 1 + grafana_data: + labels: + "keep": 1 services: statsd-exporter: image: prom/statsd-exporter:v0.22.8 - restart: unless-stopped + restart: always command: [ '--statsd.mapping-config=/etc/statsd/statsd.yaml', '--statsd.listen-udp=:9125', @@ -27,11 +22,16 @@ services: ] networks: - rhodecode_network - ports: - - "127.0.0.1:9125:9125/udp" - - "127.0.0.1:9102:9102" +# ports: +# - "9125:9125/udp" +# - "9102:9102" + volumes: - ./config/statsd-exporter:/etc/statsd + labels: + - "traefik.enable=false" + - "traefik.http.services.statsd-exporter.loadbalancer.server.port=9125" + - "traefik.http.services.statsd-exporter.loadbalancer.server.port=9102" node-exporter: image: prom/node-exporter:v1.4.0 @@ -45,17 +45,20 @@ services: networks: - rhodecode_network pid: host - restart: unless-stopped - ports: - - "127.0.0.1:9100:9100" + restart: always +# ports: +# - "9100:9100" volumes: - "/proc:/host/proc:ro" - "/sys:/host/sys:ro" - "/:/rootfs:ro" + labels: + - "traefik.enable=false" + - "traefik.http.services.node-exporter.loadbalancer.server.port=9100" prometheus: - image: prom/prometheus:v2.39.1 - restart: unless-stopped + image: prom/prometheus:v2.40.2 + restart: always command: - "--config.file=/etc/prometheus/prometheus.yml" - "--storage.tsdb.path=/prometheus" @@ -63,32 +66,40 @@ services: - "--web.console.templates=/etc/prometheus/consoles" - "--storage.tsdb.retention.time=24h" - "--web.enable-lifecycle" - ports: - - "127.0.0.1:9090:9090" +# ports: +# - "9090:9090" networks: - rhodecode_network volumes: - ./config/prometheus:/etc/prometheus - prometheus_data:/prometheus + labels: + - "traefik.enable=false" + - "traefik.http.services.prometheus.loadbalancer.server.port=9090" grafana: - image: grafana/grafana:9.2.1-ubuntu - restart: unless-stopped - env_file: - - ./config/grafana/grafana.env - ports: - - "4000:3000" + image: grafana/grafana:9.2.5 + restart: always +# ports: +# - "3000:3000" volumes: - grafana_data:/var/lib/grafana + - ./config/grafana:/etc/grafana networks: - rhodecode_network + labels: + - "traefik.enable=true" + - "traefik.http.routers.grafana.entrypoints=http" + - "traefik.http.routers.grafana.rule=PathPrefix(`/_grafana`)" + - "traefik.http.routers.grafana.service=grafana-http" + - "traefik.http.services.grafana-http.loadbalancer.server.port=3000" loki: - image: grafana/loki:2.6.1 - restart: unless-stopped - ports: - - "127.0.0.1:3100:3100" - - "127.0.0.1:9095:9095" + image: grafana/loki:2.7.0 + restart: always +# ports: +# - "3100:3100" +# - "9095:9095" command: [ "-config.file=/etc/loki/loki-config.yaml" ] @@ -97,6 +108,11 @@ services: volumes: - ./config/loki:/etc/loki + labels: + - "traefik.enable=false" + - "traefik.http.services.loki.loadbalancer.server.port=3100" + - "traefik.http.services.loki.loadbalancer.server.port=9095" + promtail: image: grafana/promtail:latest command: [ @@ -108,3 +124,5 @@ services: - /var/log:/var/log - logvolume:/var/log_volume - ./config/promtail:/etc/promtail + labels: + - "traefik.enable=false" diff --git a/docker-compose-router.yaml b/docker-compose-router.yaml new file mode 100644 index 0000000..74e46c6 --- /dev/null +++ b/docker-compose-router.yaml @@ -0,0 +1,78 @@ +version: '3.9' + +x-logging: &custom-logging + # docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions + # NOTE: loki logging driver ONLY works for host type networks... + driver: loki + options: + loki-url: "http://127.0.0.1:3100/loki/api/v1/push" + #loki-url: "http://loki:3100/loki/api/v1/push" + loki-retries: "5" + loki-timeout: "1s" + loki-max-backoff: "800ms" + +## docker network create -d overlay lb-net +services: + + traefik: + + image: traefik:v2.9.5 + + ports: + # The HTTP port, exposed as http entrypoint + - "80:80" + # The HTTPS port, exposed as https entrypoint + - "443:443" + # The SSH port + - "${RC_SSH_PORT}:${RC_SSH_PORT}" + # The Web UI (enabled by --api.insecure=true) + # uncomment to expose dashboard at port :7000 + #- "7000:7000" + volumes: + # So that Traefik can listen to the Docker events + - /var/run/docker.sock:/var/run/docker.sock + - ./config/traefik:/etc/traefik + deploy: + placement: + constraints: + # limit swarm deploy to MANAGER only + - node.role == manager + networks: + - rhodecode_network + + labels: + - "traefik.enable=true" + + logging: + *custom-logging + + portainer: + # Run with COMPOSE_PROFILES=portainer + # to access portainer set HEADER `X-Docker-Host=portainer` + image: portainer/portainer-ce:latest + restart: always + volumes: + - portainer_data:/data + - /var/run/docker.sock:/var/run/docker.sock + deploy: + mode: replicated + replicas: 1 + placement: + constraints: + # limit swarm deploy to MANAGER only + - node.role == manager + + networks: + - rhodecode_network + labels: + - "traefik.enable=true" + - "traefik.http.services.portainer.loadbalancer.server.port=9000" + - "traefik.http.routers.portainer.entrypoints=https" + - "traefik.http.routers.portainer.rule=Headers(`X-Docker-Host`, `portainer`)" + + profiles: + ["portainer"] + +volumes: + portainer_data: + external: true \ No newline at end of file diff --git a/docker-compose-services.yaml b/docker-compose-services.yaml new file mode 100644 index 0000000..219a1fb --- /dev/null +++ b/docker-compose-services.yaml @@ -0,0 +1,240 @@ +version: '3.9' + +x-logging: &custom-logging + # docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions + # NOTE: loki logging driver ONLY works for host type networks... + driver: loki + options: + loki-url: "http://127.0.0.1:3100/loki/api/v1/push" + #loki-url: "http://loki:3100/loki/api/v1/push" + loki-retries: "5" + loki-timeout: "1s" + loki-max-backoff: "800ms" + +volumes: + + # volume for redis data store + redis_data: + labels: + "keep": 1 + + # volume for Postgres db store + + # volume for Postgres Data + pg_data: + labels: + "keep": 1 + + # volume for rhodecode elasticsearch + es_data: + labels: + "keep": 1 + + +services: + + channelstream: + networks: + - rhodecode_network + image: channelstream/channelstream:0.7.1 + + restart: always + +# ports: +# - "127.0.0.1:9800:9800" + + command: ["channelstream", "-i", "/etc/rhodecode/conf/channelstream.ini"] + + environment: + CHANNELSTREAM_ALLOW_POSTING_FROM: 0.0.0.0 + + healthcheck: + test: [ "CMD", "curl", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://channelstream:8000/admin/sign_in" ] + timeout: 5s + interval: 60s + retries: 10 + + volumes: + - confvolume:/etc/rhodecode/conf + - logvolume:/var/log/rhodecode + + logging: + *custom-logging + + labels: + - "traefik.enable=false" + #- "traefik.http.routers.channelstream.entrypoints=http" + #- "traefik.http.services.channelstream.loadbalancer.server.port=9800" + + profiles: + ["channelstream"] + + nginx: + networks: + - rhodecode_network + image: library/nginx:1.23.2 + + restart: always + + environment: + NGINX_ENTRYPOINT_QUIET_LOGS: 1 + + healthcheck: + # change port 80 to 443 when only using SSL + test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:80/_admin/ops/ping" ] + timeout: 30s + interval: 60s + retries: 10 + +# depends_on: +# - channelstream + + volumes: + - ./config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./config/nginx/http.conf:/etc/nginx/http.conf:ro + - ./config/nginx/proxy.conf:/etc/nginx/proxy.conf:ro + - logvolume:/var/log/rhodecode + - rc_datavolume:/var/opt/rhodecode_data + + logging: + *custom-logging + + labels: + - "traefik.enable=true" + - "traefik.http.routers.nginx.entrypoints=http" + - "traefik.http.services.nginx.loadbalancer.server.port=80" + - "traefik.http.routers.nginx.rule=Host(`${RHODECODE_HOSTNAME:?must-specify-rhodecode-hostname}`)" + + elasticsearch: + networks: + - rhodecode_network + image: elasticsearch:6.8.23 + + environment: + - cluster.name=elasticsearch-cluster + - network.host=0.0.0.0 + - bootstrap.memory_lock=true + - discovery.type=single-node + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + healthcheck: + # change port 80 to 443 when only using SSL + test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "elasticsearch:9200/_cat/health" ] + timeout: 30s + interval: 60s + retries: 10 + + ulimits: + memlock: + soft: -1 + hard: -1 + + volumes: + - es_data:/usr/share/elasticsearch/data + + logging: + *custom-logging + + profiles: + ["elasticsearch"] + + redis: + networks: + - rhodecode_network + image: library/redis:7.0.5 + + restart: always + command: + - "redis-server" + - "/etc/redis/redis.conf" + - "--maxmemory-policy allkeys-lru" + - "--maxmemory ${RC_REDIS_MAXMEMORY}" + +# ports: +# - "127.0.0.1::6379" + + healthcheck: + test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ] + interval: 60s + + volumes: + - ./config/redis/redis.conf:/etc/redis/redis.conf:ro + - logvolume:/var/log/rhodecode + - redis_data:/data + + profiles: + ["redis"] + + logging: + *custom-logging + + database: + networks: + - rhodecode_network + image: library/postgres:14.6 + + environment: + POSTGRES_DB: ${DB_NAME:?must-specify-db-name} + POSTGRES_USER: ${DB_USER:?must-specify-db-user} + PGUSER: ${DB_USER:?must-specify-db-user} + POSTGRES_PASSWORD: ${DB_PASSWORD:?must-specify-db-password} + POSTGRES_HOST_AUTH_METHOD: md5 + POSTGRES_INITDB_ARGS: "--auth-host=md5 --auth-local=md5" + + restart: unless-stopped + command: + - "postgres" + - "-c" + - "log_statement=ddl" + - "-c" + - "config_file=/etc/conf.d/pg_customized.conf" + +# ports: +# - "127.0.0.1::5432" + + healthcheck: + test: ["CMD-SHELL", "pg_isready", '-U', "$DB_USER"] + interval: 10s + timeout: 5s + retries: 5 + + volumes: + - $PWD/config/database/pg_customized.conf:/etc/conf.d/pg_customized.conf:ro + # db dumps reverse mount + - $PWD/.custom/db_dump:/var/rc-data-dump + # save the pg_data volume + - pg_data:/var/lib/postgresql/data + - logvolume:/var/log/rhodecode + + profiles: + ["postgres"] + + logging: + *custom-logging + + database-mysql: + networks: + - rhodecode_network + image: library/mysql:8.0.31 + + environment: + MYSQL_DATABASE: ${DB_NAME:?must-specify-db-name} + MYSQL_USER: ${DB_USER:?must-specify-db-user} + MYSQL_PASSWORD: ${DB_PASSWORD:?must-specify-db-password} + MYSQL_ROOT_PASSWORD: ${DB_PASSWORD:?must-specify-db-password} + + restart: unless-stopped + +# ports: +# - "127.0.0.1::3306" + + volumes: + - ./config/database/mysql_customized.conf:/etc/mysql/conf.d/config-file.cnf:ro + # save the mysql_data volume + - $PWD/mysql_dir:/var/lib/mysql + - logvolume:/var/log/rhodecode + + profiles: + ["mysql"] + + logging: + *custom-logging diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 161806a..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,441 +0,0 @@ -version: '3.9' - -volumes: - - # bind volume with logs - # note this is used as well to scrape logs using promtail - # any changes here should be made as well into docker-compose-grafana.yaml file - logvolume: - driver: local - driver_opts: - type: none - o: bind - device: $PWD/logs - - # bind-mount with configs - confvolume: - driver: local - driver_opts: - type: none - o: bind - device: $PWD/config - - # volume for rhodecode caches, archive caches, elasticsearch etc - datavolume: {} - - # volume for RhodeCode repo-store, it's where the repositories will be stored - rhodecode_repos: - labels: - "keep": 1 - - # volume for Postgres db store - pg_data: - labels: - "keep": 1 - - # volume for MySQL db store - mysql_data: - labels: - "keep": 1 - - # volume for rhodecode elasticsearch - es_data: - labels: - "keep": 1 - -networks: - rhodecode_network: - name: rhodecode_network - driver: bridge - -services: - - rhodecode: - networks: - - rhodecode_network - image: rhodecode/app:${RC_VERSION:?specify-RC_VERSION-env-var} - stdin_open: true - tty: true - restart: unless-stopped - command: [ - "/var/opt/rhodecode_bin/bin/gunicorn", - "--name=gunicorn-rhodecode-1", - "--error-logfile=-", - "--paster=/etc/rhodecode/conf/compose/rhodecode.optimized.ini", - "--config=/etc/rhodecode/conf/gunicorn_conf.py" - ] - ports: - - "127.0.0.1::10020" - - build: - context: . - dockerfile: service/rhodecode/rhodecode.dockerfile - args: - TZ: ${TZ} - RHODECODE_VERSION: ${RC_VERSION:?specify-RC_VERSION-env-var} - RHODECODE_DB: postgresql://rhodecode:${DB_PASSWORD:?must-specify-db-password}@database/${DB_NAME:?must-specify-db-name} - RHODECODE_USER_NAME: ${RHODECODE_USER_NAME} - RHODECODE_USER_PASS: ${RHODECODE_USER_PASS} - RHODECODE_USER_EMAIL: ${RHODECODE_USER_EMAIL} - - environment: - RC_APP_TYPE: rhodecode_http - RC_APP_PROC: 1 - ENV_RC_BASE_URL: ${RHODECODE_BASE_URL} - SSL_CERT_FILE: "/etc/rhodecode/conf/ca-bundle.crt" - REQUESTS_CA_BUNDLE: "/etc/rhodecode/conf/ca-bundle.crt" - GIT_SSL_CAINFO: "/etc/rhodecode/conf/ca-bundle.crt" - GEVENT_RESOLVER: "ares" - - DB_UPGRADE: 1 # run the DB upgrade - SETUP_APP: 1 # run the application default settings setup, can be turned off after initial run - #FORCE_DB_INIT_FILE: 1 # force the database init, warning: destroys old DB - #FORCE_RC_SETUP_APP: 1 # force running setup scripts for configuration/license application - MAIN_INI_PATH: /etc/rhodecode/conf/compose/rhodecode.optimized.ini - - # SVN Specific - MOD_DAV_SVN_PORT: 8090 - APACHE_LOG_DIR: /var/log/rhodecode/svn - MOD_DAV_SVN_CONF_FILE: /etc/rhodecode/conf/svn/mod_dav_svn.conf - - healthcheck: - test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:10020/_admin/ops/ping" ] - timeout: 30s - interval: 60s - retries: 10 - - depends_on: - - database - - redis - - channelstream - - volumes: - - confvolume:/etc/rhodecode/conf - - logvolume:/var/log/rhodecode - - rhodecode_repos:/var/opt/rhodecode_repo_store - - datavolume:/var/opt/rhodecode_data - - tmpfs: - - /data_ramdisk:size=1G - - vcsserver: - networks: - - rhodecode_network - image: rhodecode/app:${RC_VERSION:?specify-RC_VERSION-env-var} - stdin_open: true - tty: true - restart: unless-stopped - command: [ - "/home/rhodecode/.rccontrol/vcsserver-1/profile/bin/gunicorn", - "--name=gunicorn-vcsserver-1", - "--error-logfile=-", - "--paster=/etc/rhodecode/conf/compose/vcsserver.optimized.ini", - "--config=/etc/rhodecode/conf/gunicorn_conf.py" - ] - ports: - - "127.0.0.1::10010" - - healthcheck: - test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:10010/status" ] - timeout: 30s - interval: 60s - retries: 10 - - environment: - RC_APP_TYPE: rhodecode_vcsserver - RC_APP_PROC: 1 - MAIN_INI_PATH: /etc/rhodecode/conf/compose/vcsserver.optimized.ini - ENV_RC_BASE_URL: ${RHODECODE_BASE_URL} - SSL_CERT_FILE: "/etc/rhodecode/conf/ca-bundle.crt" - REQUESTS_CA_BUNDLE: "/etc/rhodecode/conf/ca-bundle.crt" - GIT_SSL_CAINFO: "/etc/rhodecode/conf/ca-bundle.crt" - - depends_on: - - redis - - volumes: - - confvolume:/etc/rhodecode/conf - - logvolume:/var/log/rhodecode - - rhodecode_repos:/var/opt/rhodecode_repo_store - - datavolume:/var/opt/rhodecode_data - - celery: - networks: - - rhodecode_network - image: rhodecode/app:${RC_VERSION:?specify-RC_VERSION-env-var} - stdin_open: true - tty: true - restart: unless-stopped - command: [ - "/var/opt/rhodecode_bin/bin/celery", - "worker", - "--task-events", - "--autoscale=10,2", - "--no-color", - "--app=rhodecode.lib.celerylib.loader", - "--loglevel=DEBUG", - "--ini=/etc/rhodecode/conf/compose/rhodecode.optimized.ini" - ] - - environment: - RC_APP_TYPE: rhodecode_celery - RC_APP_PROC: 1 - MAIN_INI_PATH: /etc/rhodecode/conf/compose/rhodecode.optimized.ini - ENV_RC_BASE_URL: ${RHODECODE_BASE_URL} - SSL_CERT_FILE: "/etc/rhodecode/conf/ca-bundle.crt" - REQUESTS_CA_BUNDLE: "/etc/rhodecode/conf/ca-bundle.crt" - GIT_SSL_CAINFO: "/etc/rhodecode/conf/ca-bundle.crt" - - depends_on: - - database - - redis - - volumes: - - confvolume:/etc/rhodecode/conf - - logvolume:/var/log/rhodecode - - rhodecode_repos:/var/opt/rhodecode_repo_store - - datavolume:/var/opt/rhodecode_data - - beat: - # This service is not scalable - networks: - - rhodecode_network - image: rhodecode/app:${RC_VERSION:?specify-RC_VERSION-env-var} - stdin_open: true - tty: true - restart: unless-stopped - command: [ - "/var/opt/rhodecode_bin/bin/celery", - "beat", - "--no-color", - "--app=rhodecode.lib.celerylib.loader", - "--scheduler=rhodecode.lib.celerylib.scheduler.RcScheduler", - "--loglevel=DEBUG", - "--ini=/etc/rhodecode/conf/compose/rhodecode.optimized.ini" - ] - - environment: - RC_APP_TYPE: rhodecode_beat - RC_APP_PROC: 1 - MAIN_INI_PATH: /etc/rhodecode/conf/compose/rhodecode.optimized.ini - ENV_RC_BASE_URL: ${RHODECODE_BASE_URL} - SSL_CERT_FILE: "/etc/rhodecode/conf/ca-bundle.crt" - REQUESTS_CA_BUNDLE: "/etc/rhodecode/conf/ca-bundle.crt" - GIT_SSL_CAINFO: "/etc/rhodecode/conf/ca-bundle.crt" - - depends_on: - - database - - redis - - volumes: - - confvolume:/etc/rhodecode/conf - - logvolume:/var/log/rhodecode - - rhodecode_repos:/var/opt/rhodecode_repo_store - - datavolume:/var/opt/rhodecode_data - - svn: - networks: - - rhodecode_network - image: rhodecode/app:${RC_VERSION:?specify-RC_VERSION-env-var} - stdin_open: true - tty: true - restart: unless-stopped - command: ["apachectl", "-D", "FOREGROUND"] - - environment: - RC_APP_TYPE: rhodecode_svn - - # SVN Specific - MOD_DAV_SVN_PORT: 8090 - APACHE_LOG_DIR: /var/log/rhodecode/svn - MOD_DAV_SVN_CONF_FILE: /etc/rhodecode/conf/svn/mod_dav_svn.conf - - ports: - - "127.0.0.1::8090" - - healthcheck: - test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:8090/_server_status" ] - timeout: 30s - interval: 60s - retries: 10 - - volumes: - - confvolume:/etc/rhodecode/conf - - logvolume:/var/log/rhodecode - - rhodecode_repos:/var/opt/rhodecode_repo_store - - sshd: - networks: - - rhodecode_network - image: rhodecode/app:${RC_VERSION:?specify-RC_VERSION-env-var} - stdin_open: true - tty: true - restart: unless-stopped - command: ["/usr/sbin/sshd", "-f", "/etc/rhodecode/sshd_config", "-D", "-e"] - - environment: - RC_APP_TYPE: rhodecode_sshd - SSH_BOOTSTRAP: 1 - - ports: - # set from .env file - - "${RC_SSH_PORT:?must-specify-ssh-port}:22" - - volumes: - - confvolume:/etc/rhodecode/conf - - logvolume:/var/log/rhodecode - - rhodecode_repos:/var/opt/rhodecode_repo_store - - datavolume:/var/opt/rhodecode_data - - elasticsearch: - networks: - - rhodecode_network - image: docker.elastic.co/elasticsearch/elasticsearch:6.8.23 - - environment: - - cluster.name=elasticsearch-cluster - - network.host=0.0.0.0 - - bootstrap.memory_lock=true - - discovery.type=single-node - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - - ulimits: - memlock: - soft: -1 - hard: -1 - - volumes: - - es_data:/usr/share/elasticsearch/data - - channelstream: - networks: - - rhodecode_network - image: channelstream/channelstream:0.7.1 - - restart: unless-stopped - - ports: - - "127.0.0.1:9800:9800" - - command: ["channelstream", "-i", "/etc/rhodecode/conf/compose/channelstream.ini"] - - environment: - CHANNELSTREAM_ALLOW_POSTING_FROM: 0.0.0.0 - - healthcheck: - test: [ "CMD", "curl", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:9800/admin/sign_in" ] - timeout: 30s - interval: 60s - retries: 10 - - volumes: - - confvolume:/etc/rhodecode/conf - - logvolume:/var/log/rhodecode - - redis: - networks: - - rhodecode_network - image: rhodecode/redis:7.0.5 - - build: - context: . - dockerfile: service/redis/rhodecode_redis.dockerfile - args: - REDIS_BUILD: 7.0.5 - - restart: unless-stopped - - ports: - - "127.0.0.1::6379" - - volumes: - - logvolume:/var/log/rhodecode - - datavolume:/var/opt/rhodecode_data - - database: - networks: - - rhodecode_network - image: rhodecode/database:13.8 - - build: - context: . - dockerfile: service/database/rhodecode_database.dockerfile - args: - POSTGRES_BUILD: 13.8 - - environment: - POSTGRES_DB: ${DB_NAME:?must-specify-db-name} - POSTGRES_USER: ${DB_USER:?must-specify-db-user} - POSTGRES_PASSWORD: ${DB_PASSWORD:?must-specify-db-password} - POSTGRES_HOST_AUTH_METHOD: md5 - - restart: unless-stopped - - ports: - - "127.0.0.1::5432" - - volumes: - # save the pg_data volume - - pg_data:/var/lib/postgresql/data - - logvolume:/var/log/rhodecode - - database-mysql: - networks: - - rhodecode_network - - image: rhodecode/database_mysql:8.0.30 - - build: - context: . - dockerfile: service/database/rhodecode_database_mysql.dockerfile - args: - MYSQL_BUILD: 8.0.30 - - environment: - MYSQL_DATABASE: ${DB_NAME:?must-specify-db-name} - MYSQL_USER: ${DB_USER:?must-specify-db-user} - MYSQL_PASSWORD: ${DB_PASSWORD:?must-specify-db-password} - MYSQL_ROOT_PASSWORD: ${DB_PASSWORD:?must-specify-db-password} - - restart: unless-stopped - - ports: - - "127.0.0.1::3306" - - volumes: - # save the mysql_data volume - - mysql_data:/var/lib/mysql - - logvolume:/var/log/rhodecode - - nginx: - networks: - - rhodecode_network - image: rhodecode/nginx:1.23.2 - - build: - context: . - dockerfile: service/nginx/rhodecode_nginx.dockerfile - args: - NGINX_BUILD: 1.23.2 - - restart: unless-stopped - - ports: - # set from .env file - - "${RC_HTTP_PORT:?must-specify-http-port}:80" - - "${RC_HTTPS_PORT:?must-specify-https-port}:443" - - healthcheck: - # change port 80 to 443 when only using SSL - test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:80/_admin/ops/ping" ] - timeout: 30s - interval: 60s - retries: 10 - - depends_on: - - channelstream - - volumes: - - confvolume:/etc/rhodecode/conf - - logvolume:/var/log/rhodecode - - datavolume:/var/opt/rhodecode_data diff --git a/entrypoints.d/entrypoint.sh b/entrypoints.d/entrypoint.sh index f0b08ff..bc6f670 100755 --- a/entrypoints.d/entrypoint.sh +++ b/entrypoints.d/entrypoint.sh @@ -15,13 +15,13 @@ function config_copy() { function db_upgrade() { echo 'ENTRYPOINT: Upgrading database.' - /var/opt/rhodecode_bin/bin/rc-upgrade-db $MAIN_INI_PATH --force-yes + /usr/local/bin/rhodecode_bin/bin/rc-upgrade-db $MAIN_INI_PATH --force-yes } function db_init() { gosu $RC_USER \ - /home/$RC_USER/.rccontrol/$RC_TYPE_ID/profile/bin/rc-setup-app \ + /usr/local/bin/rhodecode_bin/bin/rc-setup-app \ $MAIN_INI_PATH \ --force-yes \ --skip-existing-db \ @@ -36,7 +36,7 @@ function rhodecode_setup() { fname=${f##*/} echo "Running script $fname on $RC_TYPE_ID" - gosu $RC_USER /home/$RC_USER/.rccontrol/$RC_TYPE_ID/profile/bin/rc-ishell $MAIN_INI_PATH <<< "%run $f" + gosu $RC_USER /usr/local/bin/rhodecode_bin/bin/rc-ishell $MAIN_INI_PATH <<< "%run $f" done } @@ -131,6 +131,7 @@ if [[ $RC_APP_TYPE = "rhodecode_http" ]]; then if [ "$DB_UPGRADE" = 1 ]; then # run DB migrate + echo "Found DB_UPGRADE flag, running DB upgrade" db_upgrade fi diff --git a/rccontrol b/rccontrol new file mode 120000 index 0000000..6cbddf4 --- /dev/null +++ b/rccontrol @@ -0,0 +1 @@ +scripts/rccontrol/rccontrol \ No newline at end of file diff --git a/scripts/rccontrol/rccontrol b/scripts/rccontrol/rccontrol new file mode 100755 index 0000000..0d28326 --- /dev/null +++ b/scripts/rccontrol/rccontrol @@ -0,0 +1,2889 @@ +#!/usr/bin/env bash +# This script was generated by bashly 0.8.10 (https://bashly.dannyb.co) +# Modifying it manually is not recommended + +# :wrapper.bash3_bouncer +if [[ "${BASH_VERSINFO:-0}" -lt 4 ]]; then + printf "bash version 4 or higher is required\n" >&2 + exit 1 +fi + +# :command.master_script + +# :command.version_command +version_command() { + echo "$version" +} + +# :command.usage +rccontrol_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol - RhodeCode Control - CLI for manaing RhodeCode Cluster Stack\n" + echo + + else + printf "rccontrol - RhodeCode Control - CLI for manaing RhodeCode Cluster Stack\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol [OPTIONS] COMMAND\n" + printf " rccontrol [COMMAND] --help | -h\n" + printf " rccontrol --version | -v\n" + echo + # :command.usage_commands + printf "Commands:\n" + echo " self-update update rccontrol and it's docker definitions" + echo " bootstrap Bootstrap this machine, check docker version and install rhodecode-network" + echo " stack run one of available cluster stacks, use -h for more details" + echo " stack-status show stack status" + echo " stack-upgrade upgrade ALL stack status" + echo + printf "Build Commands:\n" + echo " get-build-artifacts Fetch Artifacts to run installer based build" + echo " build Build RhodeCode image from installer" + echo " get-build-source Fetch RhodeCode sources, store in .source dir to run a source-based builds" + echo " build-source Build RhodeCode image from source, requires upgrade-source initially" + echo + printf "CLI Commands:\n" + echo " cli-redis CLI" + echo " cli-db CLI" + echo " cli-db-upgrade CLI" + echo + printf "Backup Commands:\n" + echo " backup-db CLI" + echo " backup-data CLI" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + echo " --version, -v" + printf " Show version number\n" + echo + + # :command.usage_flags + # :flag.usage + echo " --debug" + printf " Enable debug and detailed output\n" + echo + + # :command.usage_environment_variables + printf "Environment Variables:\n" + + # :environment_variable.usage + echo " AUTH_TOKEN" + printf " Set your RhodeCode AUTH Token\n" + echo + + # :environment_variable.usage + echo " RC_CLI_VERSION_NAME" + printf " default version to build and install\n" + printf " Default: 4.27.0\n" + echo + + # :environment_variable.usage + echo " RC_STACK_ROUTER_EXT" + printf "\n" + printf " Default: .custom/docker-compose-router.override.yaml\n" + echo + + # :environment_variable.usage + echo " RC_STACK_METRICS_EXT" + printf "\n" + printf " Default: .custom/docker-compose-metrics.override.yaml\n" + echo + + # :environment_variable.usage + echo " RC_STACK_SERVICES_EXT" + printf "\n" + printf " Default: .custom/docker-compose-services.override.yaml\n" + echo + + # :environment_variable.usage + echo " RC_STACK_RHODECODE_EXT" + printf "\n" + printf " Default: .custom/docker-compose-apps.override.yaml\n" + echo + + # :command.footer + printf "RhodeCode Inc 2022\n\n" + echo + + fi +} + +# :command.usage +rccontrol_self_update_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol self-update - update rccontrol and it's docker definitions\n" + echo + + else + printf "rccontrol self-update - update rccontrol and it's docker definitions\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol self-update [OPTIONS]\n" + printf " rccontrol self-update --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_flags + # :flag.usage + echo " --auth-token AUTH_TOKEN" + printf " Optionally specify AUTH TOKEN to obtain sources\n" + echo + + # :flag.usage + echo " --server-url SERVER_URL" + printf " Specify RhodeCode server location where projects should be downloaded\n" + printf " Default: https://code.rhodecode.com\n" + echo + + fi +} + +# :command.usage +rccontrol_bootstrap_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol bootstrap - Bootstrap this machine, check docker version and install rhodecode-network\n" + echo + + else + printf "rccontrol bootstrap - Bootstrap this machine, check docker version and install rhodecode-network\n" + echo + + fi + + printf "Alias: install\n" + echo + + printf "Usage:\n" + printf " rccontrol bootstrap [OPTIONS]\n" + printf " rccontrol bootstrap --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_flags + # :flag.usage + echo " --force, -f" + printf " Overwrite existing files\n" + echo + + # :flag.usage + echo " --auth-token AUTH_TOKEN" + printf " Optionally specify AUTH TOKEN to obtain sources\n" + echo + + # :flag.usage + echo " --server-url SERVER_URL" + printf " Specify RhodeCode server location where projects should be downloaded\n" + printf " Default: https://code.rhodecode.com\n" + echo + + # :command.usage_examples + printf "Examples:\n" + printf " rccontrol3 bootstrap\n" + printf " rccontrol3 bootstrap --force\n" + echo + + fi +} + +# :command.usage +rccontrol_get_build_artifacts_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol get-build-artifacts - Fetch Artifacts to run installer based build\n" + echo + + else + printf "rccontrol get-build-artifacts - Fetch Artifacts to run installer based build\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol get-build-artifacts [OPTIONS]\n" + printf " rccontrol get-build-artifacts --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_flags + # :flag.usage + echo " --auth AUTH" + printf " Specify custom auth for curl e.g -u admin:secret\n" + printf " Default: \n" + echo + + # :flag.usage + echo " --installer-url INSTALLER_URL" + printf " Installer Download URL\n" + printf " Default: https://dls.rhodecode.com/dls/N2E2ZTY1NzA3NjYxNDA2NTc1NjI3MTcyNzA2MjcxNzIyZTcwNjI3YQ==/rhodecode-control/latest-linux-ee\n" + echo + + # :flag.usage + echo " --manifest-url MANIFEST_URL" + printf " Manifest file url\n" + printf " Default: https://dls.rhodecode.com/linux/MANIFEST\n" + echo + + # :flag.usage + echo " --version-name VERSION_NAME" + printf " Specify custom build ver e.g $RC_CLI_VERSION_NAME\n" + printf " Default: $RC_CLI_VERSION_NAME\n" + echo + + fi +} + +# :command.usage +rccontrol_build_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol build - Build RhodeCode image from installer\n" + echo + + else + printf "rccontrol build - Build RhodeCode image from installer\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol build [OPTIONS]\n" + printf " rccontrol build --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_flags + # :flag.usage + echo " --version-name VERSION_NAME" + printf " Specify custom build ver e.g $RC_CLI_VERSION_NAME\n" + printf " Default: $RC_CLI_VERSION_NAME\n" + echo + + fi +} + +# :command.usage +rccontrol_get_build_source_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol get-build-source - Fetch RhodeCode sources, store in .source dir to run a source-based builds\n" + echo + + else + printf "rccontrol get-build-source - Fetch RhodeCode sources, store in .source dir to run a source-based builds\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol get-build-source REVISION [OPTIONS]\n" + printf " rccontrol get-build-source --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_flags + # :flag.usage + echo " --auth-token AUTH_TOKEN" + printf " Specify AUTH TOKEN to obtain sources\n" + echo + + # :flag.usage + echo " --server-url SERVER_URL" + printf " Specify RhodeCode server location where projects should be downloaded\n" + printf " Default: https://code.rhodecode.com\n" + echo + + # :command.usage_args + printf "Arguments:\n" + + # :argument.usage + echo " REVISION" + printf " revision to download\n" + echo + + # :command.usage_examples + printf "Examples:\n" + printf " rccontrol3 get-sources $RC_CLI_VERSION_NAME\n" + printf " rccontrol3 get-sources default --auth-token xyxyxyx --server-url\n https://secret.repo/\n" + echo + + fi +} + +# :command.usage +rccontrol_build_source_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol build-source - Build RhodeCode image from source, requires upgrade-source initially\n" + echo + + else + printf "rccontrol build-source - Build RhodeCode image from source, requires upgrade-source initially\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol build-source [OPTIONS]\n" + printf " rccontrol build-source --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_flags + # :flag.usage + echo " --version-name VERSION_NAME" + printf " Specify custom build ver e.g $RC_CLI_VERSION_NAME\n" + printf " Default: $RC_CLI_VERSION_NAME\n" + echo + + # :command.usage_examples + printf "Examples:\n" + printf " build foo\n" + echo + + fi +} + +# :command.usage +rccontrol_stack_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol stack - run one of available cluster stacks, use -h for more details\n" + echo + + else + printf "rccontrol stack - run one of available cluster stacks, use -h for more details\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol stack NAME [SERVICES PARAMS...]\n" + printf " rccontrol stack --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_args + printf "Arguments:\n" + + # :argument.usage + echo " NAME" + printf " Stack name\n" + printf " Allowed: router, metrics, services, rhodecode\n" + echo + + echo " SERVICES PARAMS..." + printf " Additional arguments or flags for services command\n" + echo + + # :command.usage_examples + printf "Examples:\n" + printf " - ./rccontrol3 stack router up # run router stack with output to\n console\n - ./rccontrol3 stack router up --detach # run router stack detached\n - ./rccontrol3 stack router down # stop whole router stack\n - ./rccontrol3 stack router ps # check status of router stack\n - ./rccontrol3 stack router -f docker-overrides.yaml up -d # run router stack\n with your overrides\n" + echo + + fi +} + +# :command.usage +rccontrol_stack_status_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol stack-status - show stack status\n" + echo + + else + printf "rccontrol stack-status - show stack status\n" + echo + + fi + + printf "Alias: status\n" + echo + + printf "Usage:\n" + printf " rccontrol stack-status\n" + printf " rccontrol stack-status --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + fi +} + +# :command.usage +rccontrol_stack_upgrade_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol stack-upgrade - upgrade ALL stack status\n" + echo + + else + printf "rccontrol stack-upgrade - upgrade ALL stack status\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol stack-upgrade\n" + printf " rccontrol stack-upgrade --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + fi +} + +# :command.usage +rccontrol_cli_redis_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol cli-redis - CLI\n" + echo + + else + printf "rccontrol cli-redis - CLI\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol cli-redis\n" + printf " rccontrol cli-redis --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + fi +} + +# :command.usage +rccontrol_cli_db_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol cli-db - CLI\n" + echo + + else + printf "rccontrol cli-db - CLI\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol cli-db\n" + printf " rccontrol cli-db --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + fi +} + +# :command.usage +rccontrol_cli_db_upgrade_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol cli-db-upgrade - CLI\n" + echo + + else + printf "rccontrol cli-db-upgrade - CLI\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol cli-db-upgrade\n" + printf " rccontrol cli-db-upgrade --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + fi +} + +# :command.usage +rccontrol__completions_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol _completions - Generate completions, eval "$(./rccontrol _completions)"\n" + echo + + else + printf "rccontrol _completions - Generate completions, eval "$(./rccontrol _completions)"\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol _completions\n" + printf " rccontrol _completions --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + fi +} + +# :command.usage +rccontrol_backup_db_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol backup-db - CLI\n" + echo + + else + printf "rccontrol backup-db - CLI\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol backup-db\n" + printf " rccontrol backup-db --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_examples + printf "Examples:\n" + printf " ./rccontrol backup-db\n" + echo + + fi +} + +# :command.usage +rccontrol_backup_data_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol backup-data - CLI\n" + echo + + else + printf "rccontrol backup-data - CLI\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol backup-data\n" + printf " rccontrol backup-data --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + fi +} + +# :command.normalize_input +normalize_input() { + local arg flags + + while [[ $# -gt 0 ]]; do + arg="$1" + if [[ $arg =~ ^(--[a-zA-Z0-9_\-]+)=(.+)$ ]]; then + input+=("${BASH_REMATCH[1]}") + input+=("${BASH_REMATCH[2]}") + elif [[ $arg =~ ^(-[a-zA-Z0-9])=(.+)$ ]]; then + input+=("${BASH_REMATCH[1]}") + input+=("${BASH_REMATCH[2]}") + elif [[ $arg =~ ^-([a-zA-Z0-9][a-zA-Z0-9]+)$ ]]; then + flags="${BASH_REMATCH[1]}" + for (( i=0 ; i < ${#flags} ; i++ )); do + input+=("-${flags:i:1}") + done + else + input+=("$arg") + fi + + shift + done +} +# :command.inspect_args +inspect_args() { + readarray -t sorted_keys < <(printf '%s\n' "${!args[@]}" | sort) + if (( ${#args[@]} )); then + echo args: + for k in "${sorted_keys[@]}"; do echo "- \${args[$k]} = ${args[$k]}"; done + else + echo args: none + fi + + if (( ${#other_args[@]} )); then + echo + echo other_args: + echo "- \${other_args[*]} = ${other_args[*]}" + for i in "${!other_args[@]}"; do + echo "- \${other_args[$i]} = ${other_args[$i]}" + done + fi +} + +# :command.user_lib +# src/lib/colors.sh +print_in_color() { + local color="$1" + shift + if [[ -z ${NO_COLOR+x} ]]; then + printf "$color%b\e[0m\n" "$*"; + else + printf "%b\n" "$*"; + fi +} + +red() { print_in_color "\e[31m" "$*"; } +green() { print_in_color "\e[32m" "$*"; } +yellow() { print_in_color "\e[33m" "$*"; } +blue() { print_in_color "\e[34m" "$*"; } +magenta() { print_in_color "\e[35m" "$*"; } +cyan() { print_in_color "\e[36m" "$*"; } +bold() { print_in_color "\e[1m" "$*"; } +underlined() { print_in_color "\e[4m" "$*"; } +red_bold() { print_in_color "\e[1;31m" "$*"; } +green_bold() { print_in_color "\e[1;32m" "$*"; } +yellow_bold() { print_in_color "\e[1;33m" "$*"; } +blue_bold() { print_in_color "\e[1;34m" "$*"; } +magenta_bold() { print_in_color "\e[1;35m" "$*"; } +cyan_bold() { print_in_color "\e[1;36m" "$*"; } +red_underlined() { print_in_color "\e[4;31m" "$*"; } +green_underlined() { print_in_color "\e[4;32m" "$*"; } +yellow_underlined() { print_in_color "\e[4;33m" "$*"; } +blue_underlined() { print_in_color "\e[4;34m" "$*"; } +magenta_underlined() { print_in_color "\e[4;35m" "$*"; } +cyan_underlined() { print_in_color "\e[4;36m" "$*"; } + +# src/lib/config.sh +config_init() { + CONFIG_FILE=${CONFIG_FILE:=config.ini} + [[ -f "$CONFIG_FILE" ]] || touch "$CONFIG_FILE" +} + +config_get() { + local key=$1 + local regex="^$key *= *(.+)$" + local value="" + + config_init + + while IFS= read -r line || [ -n "$line" ]; do + if [[ $line =~ $regex ]]; then + value="${BASH_REMATCH[1]}" + break + fi + done < "$CONFIG_FILE" + + echo "$value" +} + +config_set() { + local key=$1 + shift + local value="$*" + + config_init + + local regex="^($key) *= *.+$" + local output="" + local found_key="" + local newline + + while IFS= read -r line || [ -n "$line" ]; do + newline=$line + if [[ $line =~ $regex ]]; then + found_key="${BASH_REMATCH[1]}" + newline="$key = $value" + output="$output$newline\n" + elif [[ $line ]]; then + output="$output$line\n" + fi + done < "$CONFIG_FILE" + + if [[ -z $found_key ]]; then + output="$output$key = $value\n" + fi + + printf "%b\n" "$output" > "$CONFIG_FILE" +} + +config_del() { + local key=$1 + + local regex="^($key) *=" + local output="" + + config_init + + while IFS= read -r line || [ -n "$line" ]; do + if [[ $line ]] && [[ ! $line =~ $regex ]]; then + output="$output$line\n" + fi + done < "$CONFIG_FILE" + + printf "%b\n" "$output" > "$CONFIG_FILE" +} + +config_show() { + config_init + cat "$CONFIG_FILE" +} + +config_keys() { + local regex="^([a-zA-Z0-9_\-\/\.]+) *=" + + config_init + + local keys=() + local key + + while IFS= read -r line || [ -n "$line" ]; do + if [[ $line =~ $regex ]]; then + key="${BASH_REMATCH[1]}" + keys+=("$key") + fi + done < "$CONFIG_FILE" + echo "${keys[@]}" +} + +config_has_key() { + [[ $(config_get "$1") ]] +} + +# src/lib/sample_function.sh + +docker_ping_host() { + PING_HOST="$1" + docker run --network rhodecode_network --rm alpine ping "$PING_HOST" +} + +check_bootstrap() { + # Avoid destroying bootstrapping by simple start/stop + if [[ ! -e $BOOTSTRAP_FILE ]]; then + echo "$(yellow WARNING:) initial bootstrap file $BOOTSTRAP_FILE not found !" + echo "$(yellow NOTICE:) Please run ./rccontrol bootstrap first" + exit + fi +} + +# src/lib/send_completions.sh +send_completions() { + echo $'# rccontrol3 completion -*- shell-script -*-' + echo $'' + echo $'# This bash completions script was generated by' + echo $'# completely (https://github.com/dannyben/completely)' + echo $'# Modifying it manually is not recommended' + echo $'' + echo $'_rccontrol3_completions_filter() {' + echo $' local words="$1"' + echo $' local cur=${COMP_WORDS[COMP_CWORD]}' + echo $' local result=()' + echo $'' + echo $' if [[ "${cur:0:1}" == "-" ]]; then' + echo $' echo "$words"' + echo $' ' + echo $' else' + echo $' for word in $words; do' + echo $' [[ "${word:0:1}" != "-" ]] && result+=("$word")' + echo $' done' + echo $'' + echo $' echo "${result[*]}"' + echo $'' + echo $' fi' + echo $'}' + echo $'' + echo $'_rccontrol3_completions() {' + echo $' local cur=${COMP_WORDS[COMP_CWORD]}' + echo $' local compwords=("${COMP_WORDS[@]:1:$COMP_CWORD-1}")' + echo $' local compline="${compwords[*]}"' + echo $'' + echo $' case "$compline" in' + echo $' \'upgrade-source\'*)' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_rccontrol3_completions_filter "--auth-token --help --server-url -h")" -- "$cur" )' + echo $' ;;' + echo $'' + echo $' \'build-source\'*)' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_rccontrol3_completions_filter "--help --version-name -h")" -- "$cur" )' + echo $' ;;' + echo $'' + echo $' \'self-update\'*)' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_rccontrol3_completions_filter "--auth-token --help --server-url -h")" -- "$cur" )' + echo $' ;;' + echo $'' + echo $' \'bootstrap\'*)' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_rccontrol3_completions_filter "--auth-token --force --help --server-url -f -h")" -- "$cur" )' + echo $' ;;' + echo $'' + echo $' \'build\'*)' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_rccontrol3_completions_filter "--help --version-name -h")" -- "$cur" )' + echo $' ;;' + echo $'' + echo $' \'stack\'*)' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_rccontrol3_completions_filter "--help -h metrics rhodecode router services")" -- "$cur" )' + echo $' ;;' + echo $'' + echo $' *)' + echo $' while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_rccontrol3_completions_filter "--help --version -h -v bootstrap build build-source self-update stack upgrade-source")" -- "$cur" )' + echo $' ;;' + echo $'' + echo $' esac' + echo $'} &&' + echo $'complete -F _rccontrol3_completions rccontrol3' + echo $'' + echo $'# ex: filetype=sh' +} + +# src/lib/validate_stack_exists.sh + +validate_stack_exists() { + err="" + invalid="1" + + for item in $VALID_SERVICES + do + if [ "$1" == "$item" ]; then + invalid="" + break + fi + done + + if [[ -n $invalid ]]; then + err="command '$1' not in list of $VALID_SERVICES" + fi + + echo $err +} + +# :command.command_functions +# :command.function +rccontrol_self_update_command() { + # src/self_update_command.sh + check_bootstrap + + echo "# this file is located in 'src/self_update_command.sh'" + echo "# code for 'rccontrol3 self-update' goes here" + echo "# you can edit it freely and regenerate (it will not be overwritten)" + inspect_args + +} + +# :command.function +rccontrol_bootstrap_command() { + # src/bootstrap_command.sh + DEBUG=${args[--debug]} + force=${args[--force]} + + check_bash_version() { + + if [ ! "${BASH_VERSINFO:-0}" -ge 4 ]; then + echo "$(red Bash version 4 or greater is required, please update your bash version!)" + exit + fi + } + + check_docker() { + (which docker || which docker.io) &>/dev/null + } + + check_and_install_docker() { + failMsg="Failed to find docker on your PATH" + + if ! check_docker; then + echo "$failMsg" + read -p "Enter to install Docker directly from https://get.docker.com/ or Ctrl+C to exit" + curl https://get.docker.com/ | sh + + if ! check_docker; then + echo "$failMsg" + echo "Docker install failed. Quitting." + exit + fi + fi + } + + docker_bootstrap() { + check_and_install_docker + echo 'Docker: Running bootstrap.' + + echo "Docker: creating network 'rc_datavolume'" + docker volume create --name=rc_datavolume + echo "Docker: done" + echo "" + + echo "Docker: creating network 'rhodecode_network'" + docker network inspect rhodecode_network >/dev/null 2>&1 || docker network create rhodecode_network + echo "Docker: done" + echo "" + + echo "Docker: creating loki logging" + loki_driver=$(docker plugin ls --format {{.Name}} --filter enabled=true | grep loki || echo "") + if [[ $loki_driver == "" ]]; then + docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions + else + echo "Docker: loki driver already exists" + fi + } + + definitions_bootstrap() { + + SOURCE_DIR=$PWD + RHODECODE_DOCKER_HASH='master' + + AUTH_TOKEN=${args[--auth-token]} + SERVER_URL=${args[--server-url]} + + DEFINITIONS_EXIST="" + CHECK_FILES="rccontrol .env docker-compose-services.yaml docker-compose-apps.yaml" + for check_file in $CHECK_FILES; do + if [[ -f "$check_file" ]]; then + DEFINITIONS_EXIST="1" + fi + done + + if [[ -n $DEFINITIONS_EXIST && ! $force ]]; then + echo "$(yellow skipping docker defs creation, existing files found. Use --force to create them anyway)" + return + fi + + if [[ -n $DEFINITIONS_EXIST ]]; then + echo "$(yellow docker definitions exists, are you sure to force re-create them?)" + while true; do + read -p "Would you like to continue with overriding file? [yn] " yn + case $yn in + [Yy]*) return 2 ;; + [Nn]*) exit ;; + *) echo "Please answer y or n." ;; + esac + done + fi + + exit + + # download sources + echo "Files: download rhodecode docker definitions from $SERVER_URL" + echo "" + + if [ $DEBUG ]; then + echo "downloading: $SERVER_URL/rhodecode-enterprise-docker/archive/$RHODECODE_DOCKER_HASH.tgz?with_hash=0" + curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-docker/archive/$RHODECODE_DOCKER_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR + + echo "running CP $SOURCE_DIR/*rhodecode-enterprise-docker-plain/* $SOURCE_DIR" + cp -v -r -f --update --backup=numbered $SOURCE_DIR/*rhodecode-enterprise-docker-plain/* $SOURCE_DIR + + echo "removing $SOURCE_DIR/*rhodecode-enterprise-docker-plain" + rm -r $SOURCE_DIR/*rhodecode-enterprise-docker-plain + else + curl -s --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-docker/archive/$RHODECODE_DOCKER_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR + + cp -r -f --update --backup=numbered $SOURCE_DIR/*rhodecode-enterprise-docker-plain/* $SOURCE_DIR + + rm -r $SOURCE_DIR/*rhodecode-enterprise-docker-plain + fi + + echo "$(green_bold DONE: docker definitions extracted to $SOURCE_DIR)" + } + + config_bootstrap() { + + if [[ ! -f "$CONFIG_FILE" ]]; then + echo "init config at: $CONFIG_FILE" + config_init + else + echo "re-using config at: $CONFIG_FILE" + fi + + if ! config_has_key "rc_encrypted_secret" ; then + key=$(echo $RANDOM | md5sum | head -c 32) + config_set "rc_encrypted_secret" $key + fi + + if ! config_has_key "rc_db_url" ; then + key=$(echo $RANDOM | md5sum | head -c 32) + config_set "rc_db_url" "postgresql://$DB_USER:$key@database/$DB_NAME" + fi + + if ! config_has_key "rc_license_token" ; then + config_set "rc_license_token" abra-cada-bra1-rce4 + fi + + if ! config_has_key "rc_base_url" ; then + config_set "rc_base_url" http://docker-dev + fi + + if ! config_has_key "rc_log_formatter" ; then + # json is another option + config_set "rc_log_formatter" generic + fi + + if ! config_has_key "rc_use_celery" ; then + config_set "rc_use_celery" true + fi + + BOOTSTRAP_RUNTIME_ENV=$PWD/.custom/.runtime.env + if [[ ! -f "$BOOTSTRAP_RUNTIME_ENV" ]]; then + echo "init runtime env config at: $BOOTSTRAP_RUNTIME_ENV" + touch BOOTSTRAP_RUNTIME_ENV + + #ENV_EXPAND="" + for k in $(config_keys); do + k_upper=${k^^} + echo "$k_upper='$(config_get "$k")'" >> $BOOTSTRAP_RUNTIME_ENV + done + + fi + } + + cur_date=$(date '+%Y-%m-%d %H:%M:%S') + + check_bash_version + + if [[ ! -e $BOOTSTRAP_FILE ]]; then + echo "initial bootstrap file $BOOTSTRAP_FILE not found !" + + docker_bootstrap + definitions_bootstrap + config_bootstrap + + echo "$cur_date" > "$BOOTSTRAP_FILE" + + else + if [ $force ]; then + + docker_bootstrap $force + definitions_bootstrap $force ## TODO: remove old + config_bootstrap $force + + echo "$cur_date" > "$BOOTSTRAP_FILE" + + exit + fi + + echo "bootstrap file $BOOTSTRAP_FILE was found add --force to force bootstrap" + fi + + get_started +} + +# :command.function +rccontrol_get_build_artifacts_command() { + # src/get_build_artifacts_command.sh + check_bootstrap + + DEBUG=${args[--debug]} + AUTH=${args[--auth]} + INSTALLER_URL=${args[--installer-url]} + MANIFEST_URL=${args[--manifest-url]} + RC_VERSION=${args[--version-name]} + VER=$RC_VERSION + + CACHE_DIR=$PWD/.cache + VER_REGEX="$VER+x86_64" + + echo "Downloading Artifacts for version: $VER" + + echo "1/4 Checking available downloads from MANIFEST file" + + ARTS=$(curl -s $AUTH $MANIFEST_URL | grep --ignore-case "$VER_REGEX" | cut -d ' ' -f 2) + + if [[ $DEBUG ]]; then + echo "DEBUG START" + curl -s $AUTH $MANIFEST_URL | grep --ignore-case "$VER_REGEX" || echo "no regex match" + curl -s $AUTH $MANIFEST_URL | grep --ignore-case "$VER_REGEX" | cut -d ' ' -f 2 + echo "Found following artifacts:" + echo $ARTS + echo "DEBUG END" + fi + + if [[ $ARTS == "" ]]; then + MSG="Failed to found any MANIFEST entries for version $VER make sure there exists release with that version or use --version to specify different version" + echo "$(red $MSG)" + exit + fi + + echo "2/4 Downloading locale-archive" + curl -L https://dls.rhodecode.com/assets/locale-archive -J -O + mv -v locale-archive "$CACHE_DIR" + + # vcsserver/ce/ee + echo "3/4 Downloading installer artifacts" + for url in $ARTS; do + echo "Downloading $url with $AUTH" + curl $AUTH --fail-early -L ${url} -J -O + done + + #for url in $(curl -s $MANIFEST_URL | grep --ignore-case -E 'control.+\+x86_64' | cut -d ' ' -f 2); do + # echo "Downloading $url" + # curl -L ${url} -J -O + #done + + echo "4/4 Downloading installer from $INSTALLER_URL" + curl $AUTH -L $INSTALLER_URL -J -O + + INSTALLER=$(ls -Art RhodeCode-installer-* | tail -n 1) + if [[ -n $INSTALLER ]]; then + chmod +x "${INSTALLER}" + fi + + echo "Copying artifacts into $CACHE_DIR" + + mv -v "${INSTALLER}" $CACHE_DIR + mv -v *.bz2 $CACHE_DIR + ls -lh $CACHE_DIR + +} + +# :command.function +rccontrol_build_command() { + # src/build_command.sh + check_bootstrap + + rc_version=${args[--version-name]} + export RC_VERSION=$rc_version + eval "echo INSTALLER BASED BUILDING${RC_VERSION}" + + RC_VERSION=$rc_version ./rccontrol stack rhodecode build --progress plain rhodecode + +} + +# :command.function +rccontrol_get_build_source_command() { + # src/get_build_source_command.sh + check_bootstrap + + AUTH_TOKEN=${args[--auth-token]} + SERVER_URL=${args[--server-url]} + revision=${args[revision]} + + SOURCE_DIR=$PWD/.source + + RHODECODE_VCS_HASH=$revision + RHODECODE_CE_HASH=$revision + RHODECODE_EE_HASH=$revision + + # download sources + echo "** download rhodecode source for build from $SERVER_URL **" + + curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-vcsserver/archive/$RHODECODE_VCS_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR + curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-ce/archive/$RHODECODE_CE_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR + #TODO: fix just CE build... + curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-ee/archive/$RHODECODE_EE_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR + + rm -rf $SOURCE_DIR/rhodecode-vcsserver && mv $SOURCE_DIR/*rhodecode-vcsserver-plain $SOURCE_DIR/rhodecode-vcsserver + rm -rf $SOURCE_DIR/rhodecode-enterprise-ce && mv $SOURCE_DIR/*rhodecode-enterprise-ce-plain $SOURCE_DIR/rhodecode-enterprise-ce + rm -rf $SOURCE_DIR/rhodecode-enterprise-ee && cp -r $SOURCE_DIR/*rhodecode-enterprise-ee-plain $SOURCE_DIR/rhodecode-enterprise-ee + + echo "downloading sources done to $SOURCE_DIR" + +} + +# :command.function +rccontrol_build_source_command() { + # src/build_source_command.sh + check_bootstrap + + rc_version=${args[--version-name]} + export RC_VERSION=$rc_version + eval "echo SOURCE BASED BUILDING${RC_VERSION}" + + RC_VERSION=$rc_version ./rccontrol stack rhodecode -f docker-compose-apps.source.yaml build --progress plain rhodecode + +} + +# :command.function +rccontrol_stack_command() { + # src/stack_command.sh + check_bootstrap + + DEBUG=${args[--debug]} + service_name=${args[name]} + + if [[ ! -f $RC_STACK_ROUTER_EXT ]]; then + RC_STACK_ROUTER_EXT="" + else + RC_STACK_ROUTER_EXT="-f $RC_STACK_ROUTER_EXT" + fi + + CMD_ROUTER="\ + docker compose \ + --env-file $ENV_FILE \ + $ENV_EXPAND \ + -p rc_cluster_router \ + -f docker-compose-base.yaml \ + -f docker-compose-router.yaml $RC_STACK_ROUTER_EXT" + + if [[ ! -f $RC_STACK_SERVICES_EXT ]]; then + RC_STACK_SERVICES_EXT_LCL="" + else + RC_STACK_SERVICES_EXT_LCL="-f $RC_STACK_SERVICES_EXT" + fi + + RC_STACK_PROFILES="--profile postgres --profile redis --profile elasticsearch --profile channelstream" + + CMD_SERVICES="\ + docker compose \ + --env-file $ENV_FILE \ + $ENV_EXPAND \ + $RC_STACK_PROFILES \ + -p rc_cluster_services \ + -f docker-compose-base.yaml \ + -f docker-compose-services.yaml $RC_STACK_SERVICES_EXT_LCL" + + if [[ ! -f $RC_STACK_METRICS_EXT ]]; then + RC_STACK_METRICS_EXT_LCL="" + else + RC_STACK_METRICS_EXT_LCL="-f $RC_STACK_METRICS_EXT" + fi + + CMD_METRICS="\ + docker compose \ + --env-file $ENV_FILE \ + $ENV_EXPAND \ + -p rc_cluster_metrics \ + -f docker-compose-base.yaml \ + -f docker-compose-metrics.yaml $RC_STACK_METRICS_EXT_LCL" + + if [[ ! -f $RC_STACK_RHODECODE_EXT ]]; then + RC_STACK_RHODECODE_EXT_LCL="" + else + RC_STACK_RHODECODE_EXT_LCL="-f $RC_STACK_RHODECODE_EXT" + fi + + CMD_RHODECODE="\ + docker compose \ + --env-file $ENV_FILE \ + $ENV_EXPAND \ + -p rc_cluster_apps \ + -f docker-compose-base.yaml \ + -f docker-compose-apps.yaml $RC_STACK_RHODECODE_EXT_LCL" + + CMD_RHODECODE_SOURCE="\ + docker compose \ + --env-file $ENV_FILE \ + $ENV_EXPAND \ + -p rc_cluster_apps \ + -f docker-compose-base.yaml \ + -f docker-compose-apps.yaml \ + -f docker-compose-apps.source.yaml $RC_STACK_RHODECODE_EXT_LCL" + + case $service_name in + + services ) + if [[ $DEBUG ]]; then + echo "---" + echo "stacks docker: $RC_STACK_SERVICES_EXT_LCL" + echo "running command: ${CMD_SERVICES}" + echo "ARGS: ${other_args[*]}" + echo "---" + fi + + eval "${CMD_SERVICES} ${other_args[*]}" + exit + ;; + router ) + if [[ $DEBUG ]]; then + echo "---" + echo "stacks docker: $RC_STACK_ROUTER_EXT_LCL" + echo "running command: ${CMD_ROUTER}" + echo "ARGS: ${other_args[*]}" + echo "---" + fi + eval "${CMD_ROUTER} ${other_args[*]}" + exit + ;; + metrics ) + if [[ $DEBUG ]]; then + echo "---" + echo "stacks docker: $RC_STACK_METRICS_EXT_LCL" + echo "running command: ${CMD_METRICS}" + echo "ARGS: ${other_args[*]}" + echo "---" + fi + eval "${CMD_METRICS} ${other_args[*]}" + exit + ;; + rhodecode ) + if [[ $DEBUG ]]; then + echo "---" + echo "stacks docker: $RC_STACK_RHODECODE_EXT_LCL" + echo "running command: ${CMD_RHODECODE}" + echo "ARGS: ${other_args[*]}" + echo "---" + fi + eval "${CMD_RHODECODE} ${other_args[*]}" + exit + ;; + esac + +} + +# :command.function +rccontrol_stack_status_command() { + # src/stack_status_command.sh + check_bootstrap + DEBUG=${args[--debug]} + if [[ $DEBUG ]]; then + echo "---" + + ps_cmd=$(docker ps --filter=name=rc_cluster --format="{{.ID}}") + + for service in $ps_cmd; do + + servicename=`docker inspect --format '{{ .Name }}' $service` + servicename=${servicename:1} + echo $servicename + docker inspect $service --format='{{.State.Status}}: {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' + echo "" + + done + fi + echo "---" + docker ps --filter=name=rc_cluster --format="table {{.ID}}\\t{{.Image}}\\t{{.Status}}\\t{{.Names}}\\t{{.Ports}}" +} + +# :command.function +rccontrol_stack_upgrade_command() { + # src/stack_upgrade_command.sh + check_bootstrap + + echo "# this file is located in 'src/stack_upgrade_command.sh'" + echo "# code for 'rccontrol stack-upgrade' goes here" + echo "# you can edit it freely and regenerate (it will not be overwritten)" + inspect_args + +} + +# :command.function +rccontrol_cli_redis_command() { + # src/cli_redis_command.sh + check_bootstrap + + DEBUG=${args[--debug]} + + target_container=rc_cluster_services-redis-1 + docker_id=$(docker ps --filter name=$target_container -q) + + if [[ $DEBUG ]]; then + echo "container id: $docker_id, based on $target_container filter" + docker ps + fi + + if [ "$docker_id" == "" ]; then + MSG="Cannot find container ID with name $target_container" + echo "$(red $MSG)" + exit + fi + + if [[ $DEBUG ]]; then + echo "Running |docker exec --interactive --tty $docker_id $CMD|" + echo "docker exec --interactive --tty $docker_id /bin/bash -c 'redis-cli'" + fi + + eval "docker exec --interactive --tty $docker_id /bin/bash -c 'redis-cli'" + +} + +# :command.function +rccontrol_cli_db_command() { + # src/cli_db_command.sh + check_bootstrap + + DEBUG=${args[--debug]} + + target_container=rc_cluster_services-database-1 + docker_id=$(docker ps --filter name=$target_container -q) + + if [[ $DEBUG ]]; then + echo "container id: $docker_id, based on $target_container filter" + docker ps + fi + + if [ "$docker_id" == "" ]; then + MSG="Cannot find container ID with name $target_container" + echo "$(red $MSG)" + exit + fi + + if [[ $DEBUG ]]; then + echo "Running |docker exec --interactive --tty $docker_id $CMD|" + echo "docker exec --interactive --tty $docker_id /bin/bash -c 'PGPASSWORD=$DB_PASSWORD psql --username=$DB_USER --dbname=$DB_NAME'" + fi + + eval "docker exec --interactive --tty $docker_id /bin/bash -c 'PGPASSWORD=$DB_PASSWORD psql --username=$DB_USER --dbname=$DB_NAME'" + +} + +# :command.function +rccontrol_cli_db_upgrade_command() { + # src/cli_db_upgrade_command.sh + check_bootstrap + + DEBUG=${args[--debug]} + + target_container=rc_cluster_apps-rhodecode-1 + docker_id=$(docker ps --filter name=$target_container -q) + + if [[ $DEBUG ]]; then + echo "container id: $docker_id, based on $target_container filter" + docker ps + fi + + if [ "$docker_id" == "" ]; then + MSG="Cannot find container ID with name $target_container" + echo "$(red $MSG)" + exit + fi + + if [[ $DEBUG ]]; then + echo "./rccontrol stack rhodecode exec rhodecode /usr/local/bin/rhodecode_bin/bin/rc-upgrade-db /etc/rhodecode/conf/rhodecode.optimized.ini --force-yes" + fi + + ./rccontrol stack rhodecode exec rhodecode /usr/local/bin/rhodecode_bin/bin/rc-upgrade-db /etc/rhodecode/conf/rhodecode.optimized.ini --force-yes +} + +# :command.function +rccontrol__completions_command() { + # src/_completions_command.sh + send_completions +} + +# :command.function +rccontrol_backup_db_command() { + # src/backup_db_command.sh + check_bootstrap + + DEBUG=${args[--debug]} + DESTINATION=${args[destination]} + + target_container=rc_cluster_services-database-1 + + docker_id=$(docker ps --filter name=$target_container -q) + backup_name=rc_db_dump-$(date +%Y-%m-%d).tar.gz + + echo "creating backup $backup_name" + + if [[ $DEBUG ]]; then + echo "container id: $docker_id, based on $target_container filter" + docker ps + echo "---" + fi + + #image_id=$(docker inspect "$docker_id" --format {{.Image}} | cut -c 8-) + # + #if [[ $DEBUG ]]; then + # echo "image id: $image_id, based on $target_container filter" + # docker image ls + # echo "---" + #fi + + if [ "$docker_id" == "" ]; then + MSG="Cannot find container ID with name $target_container" + echo "$(red $MSG)" + exit + fi + + #if [ "image_id" == "" ]; then + # MSG="Cannot find image ID with name $target_container" + # echo "$(red $MSG)" + # exit + #fi + + if [[ $DEBUG ]]; then + echo "docker exec -e PGPASSWORD=$DB_PASSWORD $docker_id /bin/bash -c 'pg_dump --inserts -U $DB_USER -h 127.0.0.1 --dbname=$DB_NAME | gzip > /var/rc-data-dump/$backup_name'" + fi + + eval "docker exec -e PGPASSWORD=$DB_PASSWORD $docker_id /bin/bash -c 'pg_dump --inserts -U $DB_USER -h 127.0.0.1 --dbname=$DB_NAME | gzip > /var/rc-data-dump/$backup_name'" + echo "$(green Backup created in /var/rc-data-dump/ mount !)" + +} + +# :command.function +rccontrol_backup_data_command() { + # src/backup_data_command.sh + echo "# this file is located in 'src/backup_data_command.sh'" + echo "# code for 'rccontrol backup-data' goes here" + echo "# you can edit it freely and regenerate (it will not be overwritten)" + inspect_args + +} + +# :command.parse_requirements +parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --version | -v ) + version_command + exit + ;; + + --help | -h ) + long_usage=yes + rccontrol_usage + exit + ;; + + # :flag.case + --debug ) + + # :flag.case_no_arg + args[--debug]=1 + shift + ;; + + esac + + # :command.environment_variables_filter + # :command.environment_variables_default + export RC_CLI_VERSION_NAME="${RC_CLI_VERSION_NAME:-4.27.0}" + export RC_STACK_ROUTER_EXT="${RC_STACK_ROUTER_EXT:-.custom/docker-compose-router.override.yaml}" + export RC_STACK_METRICS_EXT="${RC_STACK_METRICS_EXT:-.custom/docker-compose-metrics.override.yaml}" + export RC_STACK_SERVICES_EXT="${RC_STACK_SERVICES_EXT:-.custom/docker-compose-services.override.yaml}" + export RC_STACK_RHODECODE_EXT="${RC_STACK_RHODECODE_EXT:-.custom/docker-compose-apps.override.yaml}" + + # :command.command_filter + action=${1:-} + + case $action in + -* ) + ;; + + self-update ) + action="self-update" + shift + rccontrol_self_update_parse_requirements "$@" + shift $# + ;; + + bootstrap | install ) + action="bootstrap" + shift + rccontrol_bootstrap_parse_requirements "$@" + shift $# + ;; + + get-build-artifacts ) + action="get-build-artifacts" + shift + rccontrol_get_build_artifacts_parse_requirements "$@" + shift $# + ;; + + build ) + action="build" + shift + rccontrol_build_parse_requirements "$@" + shift $# + ;; + + get-build-source ) + action="get-build-source" + shift + rccontrol_get_build_source_parse_requirements "$@" + shift $# + ;; + + build-source ) + action="build-source" + shift + rccontrol_build_source_parse_requirements "$@" + shift $# + ;; + + stack ) + action="stack" + shift + rccontrol_stack_parse_requirements "$@" + shift $# + ;; + + stack-status | status ) + action="stack-status" + shift + rccontrol_stack_status_parse_requirements "$@" + shift $# + ;; + + stack-upgrade ) + action="stack-upgrade" + shift + rccontrol_stack_upgrade_parse_requirements "$@" + shift $# + ;; + + cli-redis ) + action="cli-redis" + shift + rccontrol_cli_redis_parse_requirements "$@" + shift $# + ;; + + cli-db ) + action="cli-db" + shift + rccontrol_cli_db_parse_requirements "$@" + shift $# + ;; + + cli-db-upgrade ) + action="cli-db-upgrade" + shift + rccontrol_cli_db_upgrade_parse_requirements "$@" + shift $# + ;; + + _completions ) + action="_completions" + shift + rccontrol__completions_parse_requirements "$@" + shift $# + ;; + + backup-db ) + action="backup-db" + shift + rccontrol_backup_db_parse_requirements "$@" + shift $# + ;; + + backup-data ) + action="backup-data" + shift + rccontrol_backup_data_parse_requirements "$@" + shift $# + ;; + + # :command.command_fallback + "" ) + rccontrol_usage >&2 + exit 1 + ;; + + * ) + printf "invalid command: %s\n" "$action" >&2 + exit 1 + ;; + + esac + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + +} + +# :command.parse_requirements +rccontrol_self_update_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_self_update_usage + exit + ;; + + esac + + # :command.dependencies_filter + if ! [[ -x "$(command -v curl)" ]]; then + printf "missing dependency: curl\n" >&2 + exit 1 + fi + if ! [[ -x "$(command -v tar)" ]]; then + printf "missing dependency: tar\n" >&2 + exit 1 + fi + + # :command.command_filter + action="self-update" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + # :flag.case + --auth-token ) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args[--auth-token]="$2" + shift + shift + else + printf "%s\n" "--auth-token requires an argument: --auth-token AUTH_TOKEN" >&2 + exit 1 + fi + ;; + + # :flag.case + --server-url ) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args[--server-url]="$2" + shift + shift + else + printf "%s\n" "--server-url requires an argument: --server-url SERVER_URL" >&2 + exit 1 + fi + ;; + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + + # :command.default_assignments + [[ -n ${args[--server-url]:-} ]] || args[--server-url]="https://code.rhodecode.com" + +} + +# :command.parse_requirements +rccontrol_bootstrap_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_bootstrap_usage + exit + ;; + + esac + + # :command.dependencies_filter + if ! [[ -x "$(command -v curl)" ]]; then + printf "missing dependency: curl\n" >&2 + exit 1 + fi + if ! [[ -x "$(command -v tar)" ]]; then + printf "missing dependency: tar\n" >&2 + exit 1 + fi + if ! [[ -x "$(command -v md5sum)" ]]; then + printf "missing dependency: md5sum\n" >&2 + exit 1 + fi + + # :command.command_filter + action="bootstrap" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + # :flag.case + --force | -f ) + + # :flag.case_no_arg + args[--force]=1 + shift + ;; + + # :flag.case + --auth-token ) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args[--auth-token]="$2" + shift + shift + else + printf "%s\n" "--auth-token requires an argument: --auth-token AUTH_TOKEN" >&2 + exit 1 + fi + ;; + + # :flag.case + --server-url ) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args[--server-url]="$2" + shift + shift + else + printf "%s\n" "--server-url requires an argument: --server-url SERVER_URL" >&2 + exit 1 + fi + ;; + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + + # :command.default_assignments + [[ -n ${args[--server-url]:-} ]] || args[--server-url]="https://code.rhodecode.com" + +} + +# :command.parse_requirements +rccontrol_get_build_artifacts_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_get_build_artifacts_usage + exit + ;; + + esac + + # :command.command_filter + action="get-build-artifacts" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + # :flag.case + --auth ) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args[--auth]="$2" + shift + shift + else + printf "%s\n" "--auth requires an argument: --auth AUTH" >&2 + exit 1 + fi + ;; + + # :flag.case + --installer-url ) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args[--installer-url]="$2" + shift + shift + else + printf "%s\n" "--installer-url requires an argument: --installer-url INSTALLER_URL" >&2 + exit 1 + fi + ;; + + # :flag.case + --manifest-url ) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args[--manifest-url]="$2" + shift + shift + else + printf "%s\n" "--manifest-url requires an argument: --manifest-url MANIFEST_URL" >&2 + exit 1 + fi + ;; + + # :flag.case + --version-name ) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args[--version-name]="$2" + shift + shift + else + printf "%s\n" "--version-name requires an argument: --version-name VERSION_NAME" >&2 + exit 1 + fi + ;; + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + + # :command.default_assignments + [[ -n ${args[--auth]:-} ]] || args[--auth]="" + [[ -n ${args[--installer-url]:-} ]] || args[--installer-url]="https://dls.rhodecode.com/dls/N2E2ZTY1NzA3NjYxNDA2NTc1NjI3MTcyNzA2MjcxNzIyZTcwNjI3YQ==/rhodecode-control/latest-linux-ee" + [[ -n ${args[--manifest-url]:-} ]] || args[--manifest-url]="https://dls.rhodecode.com/linux/MANIFEST" + [[ -n ${args[--version-name]:-} ]] || args[--version-name]="$RC_CLI_VERSION_NAME" + +} + +# :command.parse_requirements +rccontrol_build_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_build_usage + exit + ;; + + esac + + # :command.command_filter + action="build" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + # :flag.case + --version-name ) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args[--version-name]="$2" + shift + shift + else + printf "%s\n" "--version-name requires an argument: --version-name VERSION_NAME" >&2 + exit 1 + fi + ;; + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + + # :command.default_assignments + [[ -n ${args[--version-name]:-} ]] || args[--version-name]="$RC_CLI_VERSION_NAME" + +} + +# :command.parse_requirements +rccontrol_get_build_source_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_get_build_source_usage + exit + ;; + + esac + + # :command.dependencies_filter + if ! [[ -x "$(command -v curl)" ]]; then + printf "missing dependency: curl\n" >&2 + exit 1 + fi + if ! [[ -x "$(command -v tar)" ]]; then + printf "missing dependency: tar\n" >&2 + exit 1 + fi + + # :command.command_filter + action="get-build-source" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + # :flag.case + --auth-token ) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args[--auth-token]="$2" + shift + shift + else + printf "%s\n" "--auth-token requires an argument: --auth-token AUTH_TOKEN" >&2 + exit 1 + fi + ;; + + # :flag.case + --server-url ) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args[--server-url]="$2" + shift + shift + else + printf "%s\n" "--server-url requires an argument: --server-url SERVER_URL" >&2 + exit 1 + fi + ;; + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + if [[ -z ${args[revision]+x} ]]; then + + args[revision]=$1 + shift + else + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + fi + + ;; + + esac + done + + # :command.required_args_filter + if [[ -z ${args[revision]+x} ]]; then + printf "missing required argument: REVISION\nusage: rccontrol get-build-source REVISION [OPTIONS]\n" >&2 + exit 1 + fi + + # :command.default_assignments + [[ -n ${args[--server-url]:-} ]] || args[--server-url]="https://code.rhodecode.com" + +} + +# :command.parse_requirements +rccontrol_build_source_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_build_source_usage + exit + ;; + + esac + + # :command.command_filter + action="build-source" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + # :flag.case + --version-name ) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args[--version-name]="$2" + shift + shift + else + printf "%s\n" "--version-name requires an argument: --version-name VERSION_NAME" >&2 + exit 1 + fi + ;; + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + + # :command.default_assignments + [[ -n ${args[--version-name]:-} ]] || args[--version-name]="$RC_CLI_VERSION_NAME" + +} + +# :command.parse_requirements +rccontrol_stack_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_stack_usage + exit + ;; + + esac + + # :command.command_filter + action="stack" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + other_args+=("$1") + shift + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_catch_all + if [[ -z ${args[name]+x} ]]; then + + args[name]=$1 + shift + else + other_args+=("$1") + shift + fi + + ;; + + esac + done + + # :command.required_args_filter + if [[ -z ${args[name]+x} ]]; then + printf "missing required argument: NAME\nusage: rccontrol stack NAME [SERVICES PARAMS...]\n" >&2 + exit 1 + fi + + # :command.whitelist_filter + if [[ ! ${args[name]} =~ ^(router|metrics|services|rhodecode)$ ]]; then + printf "%s\n" "name must be one of: router, metrics, services, rhodecode" >&2 + exit 1 + fi + +} + +# :command.parse_requirements +rccontrol_stack_status_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_stack_status_usage + exit + ;; + + esac + + # :command.command_filter + action="stack-status" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + +} + +# :command.parse_requirements +rccontrol_stack_upgrade_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_stack_upgrade_usage + exit + ;; + + esac + + # :command.command_filter + action="stack-upgrade" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + +} + +# :command.parse_requirements +rccontrol_cli_redis_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_cli_redis_usage + exit + ;; + + esac + + # :command.command_filter + action="cli-redis" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + +} + +# :command.parse_requirements +rccontrol_cli_db_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_cli_db_usage + exit + ;; + + esac + + # :command.command_filter + action="cli-db" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + +} + +# :command.parse_requirements +rccontrol_cli_db_upgrade_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_cli_db_upgrade_usage + exit + ;; + + esac + + # :command.command_filter + action="cli-db-upgrade" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + +} + +# :command.parse_requirements +rccontrol__completions_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol__completions_usage + exit + ;; + + esac + + # :command.command_filter + action="_completions" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + +} + +# :command.parse_requirements +rccontrol_backup_db_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_backup_db_usage + exit + ;; + + esac + + # :command.dependencies_filter + if ! [[ -x "$(command -v tar)" ]]; then + printf "missing dependency: tar\n" >&2 + exit 1 + fi + if ! [[ -x "$(command -v gzip)" ]]; then + printf "missing dependency: gzip\n" >&2 + exit 1 + fi + + # :command.command_filter + action="backup-db" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + +} + +# :command.parse_requirements +rccontrol_backup_data_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_backup_data_usage + exit + ;; + + esac + + # :command.dependencies_filter + if ! [[ -x "$(command -v tar)" ]]; then + printf "missing dependency: tar\n" >&2 + exit 1 + fi + + # :command.command_filter + action="backup-data" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + +} + +# :command.initialize +initialize() { + version="4.28.0.REL20221101" + long_usage='' + set -e + + # :command.environment_variables_default + export RC_CLI_VERSION_NAME="${RC_CLI_VERSION_NAME:-4.27.0}" + export RC_STACK_ROUTER_EXT="${RC_STACK_ROUTER_EXT:-.custom/docker-compose-router.override.yaml}" + export RC_STACK_METRICS_EXT="${RC_STACK_METRICS_EXT:-.custom/docker-compose-metrics.override.yaml}" + export RC_STACK_SERVICES_EXT="${RC_STACK_SERVICES_EXT:-.custom/docker-compose-services.override.yaml}" + export RC_STACK_RHODECODE_EXT="${RC_STACK_RHODECODE_EXT:-.custom/docker-compose-apps.override.yaml}" + + # src/initialize.sh + + ENV_FILE=$PWD/.env + + # bootstrap file is a config file at the same time + BOOTSTRAP_FILE='.rccontrol-bootstrap' + CONFIG_FILE='.rccontrol.ini' + + VALID_SERVICES="router metrics services rhodecode" + DOCKER_DEFS_WORK_DIR="docker_defs" + + #echo "1 ----" + #echo $RC_STACK_SERVICES_EXT + #echo $RC_STACK_METRICS_EXT + #echo "1 -----" + + #echo '2 ----' + #grep -v -e '^#' "$ENV_FILE" | xargs -I {} echo \'{}\' + #echo '2 ----' + + #expose env vars from rccontrol donfig + #if [[ -f $ENV_FILE ]]; then + # eval $(grep -v -e '^#' "$ENV_FILE" | xargs -I {} echo export \'{}\') + #fi + + #ENV_EXPAND="" + #for k in $(config_keys); do + # k_upper=${k^^} + # env_entry="-e $k_upper='$(config_get "$k")' " + # ENV_EXPAND+=$env_entry; + # #echo $env_expand + #done + + # + #echo "3 ----" + #echo $RC_STACK_SERVICES_EXT + #echo $RC_STACK_METRICS_EXT + #echo $RC_STACK_RHODECODE_EXT + #echo "3 -----" + + #env | grep RC_ + + # backup files from a docker volume into /tmp/backup.tar.gz + docker-volume-backup-compressed() { + docker run --rm -v /tmp:/backup --volumes-from "$1" debian:jessie tar -czvf /backup/backup.tar.gz "${@:2}" + } + + # restore files from /tmp/backup.tar.gz into a docker volume + docker-volume-restore-compressed() { + docker run --rm -v /tmp:/backup --volumes-from "$1" debian:jessie tar -xzvf /backup/backup.tar.gz "${@:2}" + echo "Double checking files..." + docker run --rm -v /tmp:/backup --volumes-from "$1" debian:jessie ls -lh "${@:2}" + } + + # backup files from a docker volume into /tmp/backup.tar + docker-volume-backup() { + docker run --rm -v /tmp:/backup --volumes-from "$1" busybox tar -cvf /backup/backup.tar "${@:2}" + } + + # restore files from /tmp/backup.tar into a docker volume + docker-volume-restore() { + docker run --rm -v /tmp:/backup --volumes-from "$1" busybox tar -xvf /backup/backup.tar "${@:2}" + echo "Double checking files..." + docker run --rm -v /tmp:/backup --volumes-from "$1" busybox ls -lh "${@:2}" + } + + get_started() { + echo "" + echo "~~~~ $(green Bootstrap completed) ~~~~" + echo "To start using RhodeCode run the following stacks using ./rccontrol stack [STACK_NAME]" + echo "" + echo "$(yellow_bold [Optional]): run the edge router to control domain and SSL" + echo "./rccontrol stack router up --detach" + echo "" + echo "$(green_bold [Mandatory]): run the services, like database, redis, channelstream etc..." + echo "./rccontrol stack services up --detach" + echo "" + echo "$(green_bold [Mandatory]): run the rhodecode app stack" + echo "./rccontrol stack rhodecode up --detach" + echo "" + echo "$(yellow_bold [Optional]): run the monitoring stack, this includes grafana/promethues logging/metrics system" + echo "./rccontrol stack metrics up --detach" + echo "" + echo "check services status/ports" + echo "./rccontrol status" + } +} + +# :command.run +run() { + declare -A args=() + declare -a other_args=() + declare -a input=() + normalize_input "$@" + parse_requirements "${input[@]}" + + if [[ $action == "self-update" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_self_update_usage + else + rccontrol_self_update_command + fi + + elif [[ $action == "bootstrap" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_bootstrap_usage + else + rccontrol_bootstrap_command + fi + + elif [[ $action == "get-build-artifacts" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_get_build_artifacts_usage + else + rccontrol_get_build_artifacts_command + fi + + elif [[ $action == "build" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_build_usage + else + rccontrol_build_command + fi + + elif [[ $action == "get-build-source" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_get_build_source_usage + else + rccontrol_get_build_source_command + fi + + elif [[ $action == "build-source" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_build_source_usage + else + rccontrol_build_source_command + fi + + elif [[ $action == "stack" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_stack_usage + else + rccontrol_stack_command + fi + + elif [[ $action == "stack-status" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_stack_status_usage + else + rccontrol_stack_status_command + fi + + elif [[ $action == "stack-upgrade" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_stack_upgrade_usage + else + rccontrol_stack_upgrade_command + fi + + elif [[ $action == "cli-redis" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_cli_redis_usage + else + rccontrol_cli_redis_command + fi + + elif [[ $action == "cli-db" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_cli_db_usage + else + rccontrol_cli_db_command + fi + + elif [[ $action == "cli-db-upgrade" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_cli_db_upgrade_usage + else + rccontrol_cli_db_upgrade_command + fi + + elif [[ $action == "_completions" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol__completions_usage + else + rccontrol__completions_command + fi + + elif [[ $action == "backup-db" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_backup_db_usage + else + rccontrol_backup_db_command + fi + + elif [[ $action == "backup-data" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_backup_data_usage + else + rccontrol_backup_data_command + fi + + elif [[ $action == "root" ]]; then + root_command + fi +} + +initialize +run "$@" diff --git a/service/database/rhodecode_database.dockerfile b/service/database/rhodecode_database.dockerfile deleted file mode 100644 index c173dcd..0000000 --- a/service/database/rhodecode_database.dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -ARG POSTGRES_BUILD -FROM library/postgres:$POSTGRES_BUILD - -COPY service/database/pg_customized.conf /etc/conf.d/pg_customized.conf -CMD ["postgres", "-c", "log_statement=ddl", "-c", "config_file=/etc/conf.d/pg_customized.conf"] diff --git a/service/database/rhodecode_database_mysql.dockerfile b/service/database/rhodecode_database_mysql.dockerfile deleted file mode 100644 index b2537a5..0000000 --- a/service/database/rhodecode_database_mysql.dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -ARG MYSQL_BUILD -FROM library/mysql:$MYSQL_BUILD - -RUN mkdir -p /etc/mysql/conf.d -COPY service/database/mysql_customized.conf /etc/mysql/conf.d/config-file.cnf diff --git a/service/nginx/rhodecode_nginx.dockerfile b/service/nginx/rhodecode_nginx.dockerfile deleted file mode 100644 index 8cf501b..0000000 --- a/service/nginx/rhodecode_nginx.dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -ARG NGINX_BUILD -FROM library/nginx:$NGINX_BUILD - -ENV NGINX_ENTRYPOINT_QUIET_LOGS=1 - -RUN mkdir -p /etc/nginx/sites-enabled/ -RUN mkdir -p /var/log/rhodecode/nginx -COPY service/nginx/nginx.conf /etc/nginx/nginx.conf -COPY service/nginx/http.conf /etc/nginx/sites-enabled/http.conf -COPY service/nginx/proxy.conf /etc/nginx/proxy.conf - -VOLUME /var/log/rhodecode - -#TODO enable amplify \ No newline at end of file diff --git a/service/redis/rhodecode_redis.dockerfile b/service/redis/rhodecode_redis.dockerfile deleted file mode 100644 index 0094682..0000000 --- a/service/redis/rhodecode_redis.dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -ARG REDIS_BUILD -FROM library/redis:$REDIS_BUILD -COPY service/redis/redis.conf /etc/redis/redis-rc.conf -CMD ["redis-server", "/etc/redis/redis-rc.conf"] diff --git a/service/rhodecode/rhodecode.dockerfile b/service/rhodecode/rhodecode.dockerfile index 9323a3a..9cdc637 100644 --- a/service/rhodecode/rhodecode.dockerfile +++ b/service/rhodecode/rhodecode.dockerfile @@ -6,13 +6,17 @@ ARG TZ="UTC" ARG LOCALE_TYPE=en_US.UTF-8 ARG RHODECODE_TYPE=Enterprise # binary-install -ARG RHODECODE_VERSION=4.27.1 +ARG RHODECODE_VERSION=4.28.0 ARG RHODECODE_DB=sqlite ARG RHODECODE_USER_NAME=admin ARG RHODECODE_USER_PASS=secret4 ARG RHODECODE_USER_EMAIL=support@rhodecode.com +# nix ver/channels +ARG DEV_NIX_VERSION=2.0.4 +ARG DEV_NIX_CHANNEL=nixos-18.03 + # env are runtime ENV \ TZ=${TZ} \ @@ -43,8 +47,9 @@ ENV \ # SSHD CONFIG SSHD_CONF_FILE=/etc/rhodecode/sshd_config \ \ + SHARED_CONF_DIR=/etc/rhodecode/conf \ BUILD_CONF=/etc/rhodecode/conf_build \ - BUILD_BIN_DIR=/var/opt/rhodecode_bin \ + BUILD_BIN_DIR=/usr/local/bin/rhodecode_bin \ RHODECODE_DATA_DIR=/var/opt/rhodecode_data \ RHODECODE_REPO_DIR=/var/opt/rhodecode_repo_store \ RHODECODE_HTTP_PORT=10020 \ @@ -91,8 +96,8 @@ set -eux; \ RUN \ echo "** Configure the python executable for py2/3 compat **" && \ -ISPY=$(which python3 || which python2) && \ -if [ -n $ISPY ] ; then ln -s $ISPY /usr/bin/python ; fi +IS_PY=$(which python3 || which python2) && \ +if [ -n $IS_PY ] ; then ln -s $IS_PY /usr/bin/python ; fi RUN \ echo "** Configure the locales **" && \ @@ -115,6 +120,17 @@ echo "** Create system user $RC_USER **" && \ useradd --system --gid $RC_USER --uid 999 --shell /bin/bash $RC_USER && \ usermod -G $RC_USER $RC_USER +RUN \ +echo "** Create nix-build user $NIX_BLD_USER **" && \ + groupadd --system --gid 1099 $NIX_BLD_USER && \ + useradd --system --gid $NIX_BLD_USER --uid 1099 --shell /bin/bash $NIX_BLD_USER && \ + usermod -G $NIX_BLD_USER $NIX_BLD_USER + +RUN \ +echo "** disable nix sandboxing **" && \ + mkdir /etc/nix && echo 'sandbox = false' > /etc/nix/nix.conf + + # set the defult bash shell SHELL ["/bin/bash", "-c"] @@ -129,10 +145,13 @@ RUN \ echo "** prepare rhodecode store and cache **" && \ install -d -m 0700 -o $RC_USER -g $RC_USER /nix && \ install -d -m 0755 -o $RC_USER -g $RC_USER /opt/rhodecode && \ - install -d -m 0755 -o $RC_USER -g $RC_USER /var/opt/rhodecode_bin && \ + install -d -m 0755 -o $RC_USER -g $RC_USER /usr/local/bin/rhodecode_bin && \ install -d -m 0755 -o $RC_USER -g $RC_USER $RHODECODE_REPO_DIR && \ install -d -m 0755 -o $RC_USER -g $RC_USER $RHODECODE_DATA_DIR && \ install -d -m 0755 -o $RC_USER -g $RC_USER $BUILD_CONF && \ + install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/rhodecode-vcsserver && \ + install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/rhodecode-enterprise-ce && \ + install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/rhodecode-enterprise-ee && \ install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/ && \ install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol && \ install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/cache && \ @@ -173,7 +192,7 @@ echo "**** Apache config ****" && \ # Copy artifacts COPY --chown=$RC_USER:$RC_USER .cache/* /home/$RC_USER/.rccontrol/cache/ -COPY --chown=$RC_USER:$RC_USER config/compose/rhodecode_enterprise.license /home/$RC_USER/.rccontrol/bootstrap/ +COPY --chown=$RC_USER:$RC_USER config/_shared/rhodecode_enterprise.license /home/$RC_USER/.rccontrol/bootstrap/ COPY --chown=$RC_USER:$RC_USER service/rhodecode/bootstrap/* /home/$RC_USER/.rccontrol/bootstrap/ RUN \ @@ -192,15 +211,18 @@ echo "** install rhodecode control **" && \ ${INSTALLER} --accept-license && \ ${RCCONTROL} self-init && \ cp -v /home/$RC_USER/.rccontrol-profile/etc/ca-bundle.crt $BUILD_CONF/ && \ - echo "Done" + echo "Done installing rhodecode control" RUN \ echo "** install vcsserver ${RHODECODE_VERSION} **" && \ ${RCCONTROL} install VCSServer --version ${RHODECODE_VERSION} --start-at-boot=yes --accept-license --offline \ '{"host":"'"$RHODECODE_VCS_HOST"'", "port":"'"$RHODECODE_VCS_PORT"'"}' && \ VCSSERVER_PATH=/home/$RC_USER/.rccontrol/vcsserver-1 && \ - rm -rf $BUILD_BIN_DIR/vcs_bin && ln -s ${VCSSERVER_PATH}/profile/bin $BUILD_BIN_DIR/vcs_bin && \ - cp -v ${VCSSERVER_PATH}/vcsserver.ini $BUILD_CONF/vcsserver.ini + rm -rf $BUILD_BIN_DIR/vcs_bin && \ + ln -s ${VCSSERVER_PATH}/profile/bin $BUILD_BIN_DIR/vcs_bin && \ + cp -v ${VCSSERVER_PATH}/vcsserver.ini $BUILD_CONF/vcsserver.ini && \ + cp -v ${VCSSERVER_PATH}/gunicorn_conf.py $BUILD_CONF/gunicorn_conf_vcs.py && \ + echo "Done installing vcsserver" RUN \ echo "** install rhodecode ${RHODECODE_TYPE} ${RHODECODE_VERSION} **" && \ @@ -213,31 +235,34 @@ echo "** install rhodecode ${RHODECODE_TYPE} ${RHODECODE_VERSION} **" && \ cp -v ${RHODECODE_PATH}/gunicorn_conf.py $BUILD_CONF/gunicorn_conf.py && \ cp -v ${RHODECODE_PATH}/search_mapping.ini $BUILD_CONF/search_mapping.ini && \ mkdir -p $RHODECODE_DATA_DIR/static && cp -r ${RHODECODE_PATH}/public/* $RHODECODE_DATA_DIR/static/ && \ - rm ${RHODECODE_PATH}/rhodecode.db + rm ${RHODECODE_PATH}/rhodecode.db && \ + echo "Done installing rhodecode" RUN \ echo "** configure supervisord **" && \ cp -v ${SUPERVISOR_CONF} $BUILD_CONF/ && \ sed -i "s/self_managed_supervisor = False/self_managed_supervisor = True/g" /home/$RC_USER/.rccontrol.ini && \ - echo "done" + echo "Done installing supervisord" USER root - RUN \ echo "**** cleanup ****" && \ apt-get remove -y $PYTHON_DEPS && \ apt-get autoclean -y && \ rm -f /tmp/* && \ + rm -rf /var/lib/apt/lists/* && \ + rm -rf /var/cache/apk/* && \ rm -f /home/$RC_USER/.rccontrol/cache/RhodeCode-installer-* && \ rm -f /home/$RC_USER/.rccontrol/cache/*.bz2 && \ - rm -rf /var/lib/apt/lists/* \ - rm -rf /var/cache/apk/* \ - rm ${SUPERVISOR_CONF} + rm -f ${SUPERVISOR_CONF} && \ + echo "Done cleanup" + # copy entrypoints COPY entrypoints.d/entrypoint.sh /opt/entrypoints.d/entrypoint.sh + RUN chmod +x /opt/entrypoints.d/entrypoint.sh # config volume @@ -255,4 +280,4 @@ VOLUME /var/log/rhodecode ENTRYPOINT ["/opt/entrypoints.d/entrypoint.sh"] # compose can override this -CMD ["supervisord", "--nodaemon", "-c", "/etc/rhodecode/conf/supervisord.ini"] +CMD [ "$BUILD_BIN_DIR/bin/gunicorn", "--error-logfile=-", "--paster=/etc/rhodecode/conf_build/run.ini", "--config=/etc/rhodecode/conf_build/gunicorn_conf.py" ] diff --git a/service/rhodecode/rhodecode_source.dockerfile b/service/rhodecode/rhodecode_source.dockerfile index 5ef2e4b..04e4ff2 100644 --- a/service/rhodecode/rhodecode_source.dockerfile +++ b/service/rhodecode/rhodecode_source.dockerfile @@ -6,7 +6,7 @@ ARG TZ="UTC" ARG LOCALE_TYPE=en_US.UTF-8 ARG RHODECODE_TYPE=Enterprise # source-install -ARG RHODECODE_VERSION=4.26.0 +ARG RHODECODE_VERSION=4.28.0 ARG RHODECODE_DB=sqlite ARG RHODECODE_USER_NAME=admin @@ -47,8 +47,9 @@ ENV \ # SSHD CONFIG SSHD_CONF_FILE=/etc/rhodecode/sshd_config \ \ + SHARED_CONF_DIR=/etc/rhodecode/conf \ BUILD_CONF=/etc/rhodecode/conf_build \ - BUILD_BIN_DIR=/var/opt/rhodecode_bin \ + BUILD_BIN_DIR=/usr/local/bin/rhodecode_bin \ RHODECODE_DATA_DIR=/var/opt/rhodecode_data \ RHODECODE_REPO_DIR=/var/opt/rhodecode_repo_store \ RHODECODE_HTTP_PORT=10020 \ @@ -65,7 +66,7 @@ ENV \ ENV SVN_LOCALE_DEPS apache2 apache2-utils libapache2-mod-svn ENV SSH_LOCALE_DEPS openssh-server ENV PYTHON_DEPS python2 -ENV EXTRA_DEPS vim +ENV EXTRA_DEPS "" ENV \ PATH=$PATH:/nix/var/nix/profiles/per-user/$RC_USER/profile/bin:/home/$RC_USER/rhodecode-enterprise-ee/profile/bin \ @@ -102,8 +103,8 @@ set -eux; \ RUN \ echo "** Configure the python executable for py2/3 compat **" && \ -ISPY=$(which python3 || which python2) && \ -if [ -n $ISPY ] ; then ln -s $ISPY /usr/bin/python ; fi +IS_PY=$(which python3 || which python2) && \ +if [ -n $IS_PY ] ; then ln -s $IS_PY /usr/bin/python ; fi RUN \ echo "** Configure the locales **" && \ @@ -151,7 +152,7 @@ RUN \ echo "** prepare rhodecode store and cache **" && \ install -d -m 0700 -o $RC_USER -g $RC_USER /nix && \ install -d -m 0755 -o $RC_USER -g $RC_USER /opt/rhodecode && \ - install -d -m 0755 -o $RC_USER -g $RC_USER /var/opt/rhodecode_bin && \ + install -d -m 0755 -o $RC_USER -g $RC_USER /usr/local/bin/rhodecode_bin && \ install -d -m 0755 -o $RC_USER -g $RC_USER $RHODECODE_REPO_DIR && \ install -d -m 0755 -o $RC_USER -g $RC_USER $RHODECODE_DATA_DIR && \ install -d -m 0755 -o $RC_USER -g $RC_USER $BUILD_CONF && \ @@ -162,7 +163,8 @@ echo "** prepare rhodecode store and cache **" && \ install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol && \ install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/cache && \ install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/bootstrap && \ - install -d -m 0700 -o $RC_USER -g $RC_USER /home/$RC_USER/.ssh + install -d -m 0700 -o $RC_USER -g $RC_USER /home/$RC_USER/.ssh && \ + install -d -m 0700 -o $RC_USER -g $RC_USER /home/$RC_USER/.rhoderc # expose our custom sshd config COPY service/sshd/sshd_config $SSHD_CONF_FILE @@ -195,11 +197,18 @@ echo "**** Apache config ****" && \ # Copy artifacts -COPY --chown=$RC_USER:$RC_USER .source/ /home/$RC_USER/ + COPY --chown=$RC_USER:$RC_USER .cache/* /home/$RC_USER/.rccontrol/cache/ -COPY --chown=$RC_USER:$RC_USER config/compose/rhodecode_enterprise.license /home/$RC_USER/.rccontrol/bootstrap/ +COPY --chown=$RC_USER:$RC_USER config/_shared/rhodecode_enterprise.license /home/$RC_USER/.rccontrol/bootstrap/ COPY --chown=$RC_USER:$RC_USER service/rhodecode/bootstrap/* /home/$RC_USER/.rccontrol/bootstrap/ +COPY --chown=$RC_USER:$RC_USER .source/ /home/$RC_USER/ +RUN \ +echo "** prepare rhodecode dirs **" && \ + install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/vcsserver-1 && \ + install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/community-1 && \ + install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/enterprise-1 && \ + RUN \ echo "**** locale-archive path ****" && \ mv -v /home/$RC_USER/.rccontrol/cache/locale-archive /var/opt/locale-archive @@ -235,16 +244,19 @@ echo "** install rhodecode control **" && \ # ${INSTALLER} --accept-license && \ # ${RCCONTROL} self-init && \ # cp -v /home/$RC_USER/.rccontrol-profile/etc/ca-bundle.crt $BUILD_CONF/ && \ - echo "Done" + echo "Done installing rhodecode control" RUN \ -echo "** install build vcsserver ${RHODECODE_VERSION} **" && \ +echo "** install vcsserver ${RHODECODE_VERSION} **" && \ . /home/rhodecode/.nix-profile/etc/profile.d/nix.sh && \ - nix-build --show-trace --cores 0 --max-jobs 4 --no-build-output --out-link rhodecode-vcsserver/result rhodecode-vcsserver/default.nix && \ - nix-shell --command 'echo ok' rhodecode-vcsserver/default.nix && \ + nix-build --show-trace --cores 0 --max-jobs 4 --no-build-output --out-link /home/$RC_USER/.rccontrol/vcsserver-1/profile rhodecode-vcsserver/default.nix && \ + nix-shell --command 'echo COMMAND FROM NIX-SHELL TEST' rhodecode-vcsserver/default.nix && \ VCSSERVER_PATH=/home/$RC_USER/rhodecode-vcsserver && \ - rm -rf $BUILD_BIN_DIR/vcs_bin && ln -s ${VCSSERVER_PATH}/result/bin $BUILD_BIN_DIR/vcs_bin && \ - cp -v ${VCSSERVER_PATH}/configs/production.ini $BUILD_CONF/vcsserver.ini + rm -rf $BUILD_BIN_DIR/vcs_bin && \ + cp -rv --preserve=links /home/$RC_USER/.rccontrol/vcsserver-1/profile/bin $BUILD_BIN_DIR/vcs_bin && \ + cp -v ${VCSSERVER_PATH}/configs/production.ini $BUILD_CONF/vcsserver.ini && \ + cp -v ${VCSSERVER_PATH}/configs/gunicorn_config.py $BUILD_CONF/gunicorn_conf_vcs.py && \ + echo "Done installing vcsserver" RUN \ echo "** install build Community ${RHODECODE_VERSION} **" && \ @@ -252,26 +264,27 @@ echo "** install build Community ${RHODECODE_VERSION} **" && \ echo "done" RUN \ -echo "** install build Enterprise ${RHODECODE_VERSION} **" && \ +echo "** install rhodecode ${RHODECODE_VERSION} **" && \ . /home/rhodecode/.nix-profile/etc/profile.d/nix.sh && \ - nix-build --show-trace --cores 0 --max-jobs 4 --no-build-output --out-link rhodecode-enterprise-ee/result rhodecode-enterprise-ee/default.nix && \ + nix-build --show-trace --cores 0 --max-jobs 4 --no-build-output --out-link /home/$RC_USER/.rccontrol/enterprise-1/profile rhodecode-enterprise-ee/default.nix && \ nix-shell --command 'echo ok' rhodecode-enterprise-ee/default.nix && \ RHODECODE_PATH=/home/$RC_USER/rhodecode-enterprise-ee && \ - rm -rf $BUILD_BIN_DIR/bin && ln -s ${RHODECODE_PATH}/result/bin $BUILD_BIN_DIR/ && \ + rm -rf $BUILD_BIN_DIR/bin && \ + cp -rv --preserve=links /home/$RC_USER/.rccontrol/enterprise-1/profile/bin $BUILD_BIN_DIR/ && \ cp -v ${RHODECODE_PATH}/configs/production.ini $BUILD_CONF/rhodecode.ini && \ cp -v ${RHODECODE_PATH}/configs/gunicorn_config.py $BUILD_CONF/gunicorn_conf.py && \ - mkdir -p $RHODECODE_DATA_DIR/static && cp -r ${RHODECODE_PATH}/result/etc/static/* $RHODECODE_DATA_DIR/static/ + mkdir -p $RHODECODE_DATA_DIR/static && cp -r /home/$RC_USER/.rccontrol/enterprise-1/profile/etc/static/* $RHODECODE_DATA_DIR/static/ && \ + echo "Done installing rhodecode" RUN \ echo "** configure supervisord **" && \ #cp -v ${SUPERVISOR_CONF} $BUILD_CONF/ && \ #sed -i "s/self_managed_supervisor = False/self_managed_supervisor = True/g" /home/$RC_USER/.rccontrol.ini && \ - echo "done" + echo "Done installing supervisord" USER root - RUN \ echo "**** cleanup ****" && \ apt-get remove -y $PYTHON_DEPS && \ @@ -279,12 +292,18 @@ echo "**** cleanup ****" && \ rm -f /tmp/* && \ rm -f /home/$RC_USER/.rccontrol/cache/RhodeCode-installer-* && \ rm -f /home/$RC_USER/.rccontrol/cache/*.bz2 && \ - rm -rf /var/lib/apt/lists/* \ - rm -rf /var/cache/apk/* \ - rm ${SUPERVISOR_CONF} + rm -rf /var/lib/apt/lists/* && \ + rm -rf /var/cache/apk/* && \ + rm -f ${SUPERVISOR_CONF} && \ + rm -rf /home/$RC_USER/rhodecode-vcsserver && \ + rm -rf /home/$RC_USER/rhodecode-enterprise-ce && \ + rm -rf /home/$RC_USER/rhodecode-enterprise-ee && \ + echo "Done cleanup" + # copy entrypoints COPY entrypoints.d/entrypoint.sh /opt/entrypoints.d/entrypoint.sh + RUN chmod +x /opt/entrypoints.d/entrypoint.sh # config volume @@ -302,4 +321,4 @@ VOLUME /var/log/rhodecode ENTRYPOINT ["/opt/entrypoints.d/entrypoint.sh"] # compose can override this -CMD ["supervisord", "--nodaemon", "-c", "/etc/rhodecode/conf/supervisord.ini"] +CMD [ "$BUILD_BIN_DIR/bin/gunicorn", "--error-logfile=-", "--paster=/etc/rhodecode/conf_build/run.ini", "--config=/etc/rhodecode/conf_build/gunicorn_conf.py" ] diff --git a/service/sshd/rhodecode_sshd.dockerfile b/service/sshd/rhodecode_sshd.dockerfile index 3a3d265..beef5f4 100644 --- a/service/sshd/rhodecode_sshd.dockerfile +++ b/service/sshd/rhodecode_sshd.dockerfile @@ -1,4 +1,5 @@ -FROM debian:buster +FROM ubuntu:22.04 +# Using 22.04 LTS Release MAINTAINER RhodeCode Inc. # env are runtime/build diff --git a/service/svn/rhodecode_svn.dockerfile b/service/svn/rhodecode_svn.dockerfile index d38b401..fccd915 100644 --- a/service/svn/rhodecode_svn.dockerfile +++ b/service/svn/rhodecode_svn.dockerfile @@ -5,20 +5,36 @@ MAINTAINER RhodeCode Inc. ENV \ RC_USER=rhodecode \ - MOD_DAV_SVN_CONF_FILE=/etc/rhodecode/conf/svn/mod_dav_svn.conf + # SVN CONFIG + MOD_DAV_SVN_CONF_FILE=/etc/rhodecode/conf/svn/mod_dav_svn.conf \ + MOD_DAV_SVN_PORT=8090 + +ENV SVN_LOCALE_DEPS apache2 apache2-utils libapache2-mod-svn subversion +#apache2-webdav RUN \ -echo "** install base svn packages **" && \ - apt-get update && \ - apt-get install --no-cache \ - tini \ - bash \ - curl \ - apache2 \ - apache2-utils \ - apache2-webdav \ - mod_dav_svn \ - subversion +echo "** install svn base packages **" && \ +set -eux; \ + \ + savedAptMark="$(apt-mark showmanual)"; \ + apt-get update; \ + DEBIAN_FRONTEND="noninteractive" \ + apt-get install -y --no-install-recommends \ + tini \ + bash \ + binutils \ + tzdata \ + locales \ + openssl \ + curl \ + sudo \ + gosu \ + bzip2 \ + ca-certificates \ + $SVN_LOCALE_DEPS \ + ; \ + rm -rf /var/lib/apt/lists/*; + # configure the system user # explicitly set uid/gid to guarantee that it won't change in the future @@ -26,47 +42,58 @@ echo "** install base svn packages **" && \ RUN \ echo "** Create system user $RC_USER **" && \ groupadd --system --gid 999 $RC_USER && \ - useradd --system --gid $RC_USER --uid 999 --shell /bin/bash $RC_USER + useradd --system --gid $RC_USER --uid 999 --shell /bin/bash $RC_USER && \ + usermod -G $RC_USER $RC_USER + +# Apache SVN setup +RUN \ + echo "**** Apache config cleanup ****" && \ + rm -f /etc/apache2/conf.d/info.conf \ + /etc/apache2/conf.d/mpm.conf \ + /etc/apache2/conf.d/userdir.conf && \ + rm -f /etc/apache2/sites-enabled/* && \ + rm -f /etc/apache2/sites-available/* +# custom SVN virtualhost +COPY service/svn/virtualhost.conf /etc/apache2/sites-enabled/ + +RUN \ +echo "**** Apache config ****" && \ + echo $(strings /usr/lib/apache2/modules/mod_dav_svn.so | grep 'Powered by') > /var/opt/dav.version && \ + mkdir -p /run/apache2 && \ + mkdir -p /var/opt/www && \ + echo "unset HOME" > /etc/apache2/envvars && \ + echo "export APACHE_RUN_USER=${RC_USER}" >> /etc/apache2/envvars && \ + echo "export APACHE_PID_FILE=/var/run/apache2/apache2.pid" >> /etc/apache2/envvars && \ + echo "export APACHE_RUN_DIR=/var/run/apache2" >> /etc/apache2/envvars && \ + echo "export APACHE_LOCK_DIR=/var/lock/apache2" >> /etc/apache2/envvars && \ + echo "export APACHE_RUN_USER=${RC_USER}" >> /etc/apache2/envvars && \ + echo "export APACHE_RUN_GROUP=${RC_USER}" >> /etc/apache2/envvars && \ + sed -i "s/Listen 80/Listen ${MOD_DAV_SVN_PORT}/g" /etc/apache2/ports.conf + +USER root RUN \ echo "**** cleanup ****" && \ - apt-get purge tzdata python2 && \ + apt-get autoclean -y && \ rm -f /tmp/* && \ rm -rf /var/lib/apt/lists/* \ rm -rf /var/cache/apk/* -RUN \ - echo "**** Apache config cleanup ****" && \ - rm -f /etc/apache2/conf.d/info.conf \ - /etc/apache2/conf.d/mpm.conf \ - /etc/apache2/conf.d/userdir.conf - - -COPY svn/virtualhost.conf /etc/apache2/conf.d/ # copy entrypoints COPY entrypoints.d/svn-entrypoint.sh /opt/entrypoints.d/svn-entrypoint.sh RUN chmod +x /opt/entrypoints.d/svn-entrypoint.sh -RUN \ - echo $(strings /usr/lib/apache2/mod_dav_svn.so | grep 'Powered by') > /var/opt/dav.version && \ - mkdir -p /run/apache2 && \ - mkdir -p /var/opt/www && \ - echo "export APACHE_RUN_USER=${RC_USER}" > /etc/apache2/envvars && \ - echo "export APACHE_RUN_GROUP=${RC_USER}" >> /etc/apache2/envvars && \ - sed -i "s/User apache/User ${RC_USER}/g" /etc/apache2/httpd.conf && \ - sed -i "s/Group apache/Group ${RC_USER}/g" /etc/apache2/httpd.conf +# config volume +VOLUME /etc/rhodecode/conf # repo store volume VOLUME /var/opt/rhodecode_repo_store -# config volume -VOLUME /etc/rhodecode/conf - # logs volume VOLUME /var/log/rhodecode ENTRYPOINT ["/opt/entrypoints.d/svn-entrypoint.sh"] -CMD ["apachectl", "-D", "FOREGROUND"] +CMD ["apachectl", "-D", "FOREGROUND"] \ No newline at end of file diff --git a/service/svn/virtualhost.conf b/service/svn/virtualhost.conf index bc79470..a783a47 100644 --- a/service/svn/virtualhost.conf +++ b/service/svn/virtualhost.conf @@ -2,11 +2,22 @@ LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so LoadModule authn_anon_module /usr/lib/apache2/modules/mod_authn_anon.so LoadModule dav_svn_module /usr/lib/apache2/modules/mod_dav_svn.so +ErrorLogFormat "{ \"time\":\"%{%Y-%m-%d}tT%{%T}t.%{msec_frac}tZ\", \"function\" : \"[%-m:%l]\" , \"process\" : \"[pid %P:tid %T]\" , \"message\" : \"%M\" ,\ \"referer\"\ : \"%{Referer}i\" }" +LogFormat "{ \"time\":\"%{%Y-%m-%d}tT%{%T}t.%{msec_frac}tZ\", \"process\":\"%D\", \"filename\":\"%f\", \"remoteIP\":\"%a\", \"host\":\"%V\", \"request\":\"%U\", \"query\":\"%q\", \"method\":\"%m\", \"status\":\"%>s\", \"userAgent\":\"%{User-agent}i\", \"referer\":\"%{Referer}i\" }" combined + +ErrorLog /dev/stderr +CustomLog /dev/stdout combined + + ServerAdmin admin@localhost DocumentRoot /var/opt/www - ErrorLog ${APACHE_LOG_DIR}/svn_error.log - CustomLog ${APACHE_LOG_DIR}/svn_access.log combined + + ErrorLogFormat "{ \"time\":\"%{%Y-%m-%d}tT%{%T}t.%{msec_frac}tZ\", \"function\" : \"[%-m:%l]\" , \"process\" : \"[pid %P:tid %T]\" , \"message\" : \"%M\" ,\ \"referer\"\ : \"%{Referer}i\" }" + LogFormat "{ \"time\":\"%{%Y-%m-%d}tT%{%T}t.%{msec_frac}tZ\", \"process\":\"%D\", \"filename\":\"%f\", \"remoteIP\":\"%a\", \"host\":\"%V\", \"request\":\"%U\", \"query\":\"%q\", \"method\":\"%m\", \"status\":\"%>s\", \"userAgent\":\"%{User-agent}i\", \"referer\":\"%{Referer}i\" }" combined + + ErrorLog /dev/stderr + CustomLog /dev/stdout combined LogLevel info