diff --git a/.custom/README.md b/.custom/README.md new file mode 100644 index 0000000..f8bbf46 --- /dev/null +++ b/.custom/README.md @@ -0,0 +1,12 @@ +Put any custom overrides and compose files here. + +e.g + +- docker-compose-services.override.yaml +- docker-compose-router.override.yaml +- docker-compose-metrics.override.yaml +- docker-compose-apps.override.yaml + +Passing variables to containers + +.runtime.env \ No newline at end of file diff --git a/.env b/.env index 3decc97..ed83fc2 100644 --- a/.env +++ b/.env @@ -9,8 +9,8 @@ RC_VERSION="4.28.0.REL20221120_SOURCE" # ce, or ee RC_EDITION=ee -RC_IMAGE_CE="rhodecode/rhodecode-ee:$RC_VERSION" -RC_IMAGE_EE="rhodecode/rhodecode-ce:$RC_VERSION" +RC_IMAGE_CE="rhodecode/rhodecode-ce:$RC_VERSION" +RC_IMAGE_EE="rhodecode/rhodecode-ee:$RC_VERSION" # Database bootstrap/access credentials DB_NAME=rhodecode diff --git a/.gitignore b/.gitignore index 7f1b3a7..44774d4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,9 @@ # Source code downloaded .source/* - # Custom / override wont be tracked + # Custom / override wont be tracked, except README file .custom/* +!.custom/README.md # LOGS logs/*.log @@ -19,5 +20,4 @@ logs/postgres/*.log .idea config/_shared/.rcmetadata.json -.rccontrol-bootstrap .rccontrol.ini \ No newline at end of file diff --git a/config/nginx/http.conf b/config/nginx/http.conf index 703c8da..2e65632 100644 --- a/config/nginx/http.conf +++ b/config/nginx/http.conf @@ -41,6 +41,7 @@ log_format json_log_custom escape=json '"remote_user":"$remote_user",' '"time_local":"$time_local",' '"remote_addr":"$remote_addr",' + '"host":"$host",' '"proxy_x_forwarded_for":"$proxy_add_x_forwarded_for",' '"request":"$request",' '"status": "$status",' @@ -49,6 +50,7 @@ log_format json_log_custom escape=json '"request_time":"$request_time",' '"upstream_response_time":"$upstream_response_time",' '"http_referrer":"$http_referer",' + '"http_scheme":"$scheme",' '"http_user_agent":"$http_user_agent"' '}'; @@ -56,7 +58,7 @@ log_format json_log_custom escape=json server { listen 80 default; # ensure we get the proper Docker DNS resolver for load balancing. - resolver 127.0.0.11 ipv6=off; + resolver 127.0.0.11 ipv6=off valid=10s; server_name localhost 127.0.0.1; access_log /dev/stdout json_log_custom; error_log /dev/stdout; @@ -68,6 +70,11 @@ server { # maximum number and size of buffers for large headers to read from client request large_client_header_buffers 16 256k; + location /_health { + add_header 'Content-Type' 'application/json'; + return 200 '{"status":"UP"}'; + } + ## serve static files by nginx, recommended location /_static/rhodecode { gzip on; @@ -84,6 +91,7 @@ server { ## channelstream location handler, if channelstream live chat and notifications ## are enable this will proxy the requests to channelstream websocket server + set $upstream_channelstream channelstream:8000; location /_channelstream { rewrite /_channelstream/(.*) /$1 break; gzip off; @@ -94,17 +102,18 @@ server { proxy_read_timeout 10m; proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Referer $http_referer; proxy_set_header X-Url-Scheme $scheme; - proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; - set $upstream_channelstream http://channelstream:8000; - proxy_pass $upstream_channelstream; + proxy_pass http://$upstream_channelstream; + proxy_redirect http://$upstream_channelstream/ /_channelstream; } # ## rate limit this endpoint to prevent login page brute-force attacks @@ -113,16 +122,16 @@ server { # try_files $uri @rhodecode_http; # } - location / { - include /etc/nginx/proxy.conf; - try_files $uri @rhodecode_http; - } - - location @rhodecode_http { - set $upstream http://rhodecode:10020; - include /etc/nginx/proxy.conf; - proxy_pass $upstream; - } +# location / { +# include /etc/nginx/proxy.conf; +# try_files $uri @rhodecode_http; +# } +# +# set $upstream http://rhodecode:10020; +# location @rhodecode_http { +# include /etc/nginx/proxy.conf; +# proxy_pass $upstream; +# } ## Custom 502 error page. ## Will be displayed while RhodeCode server is turned off diff --git a/config/nginx/proxy.conf b/config/nginx/proxy.conf index 7741345..f2b90d7 100644 --- a/config/nginx/proxy.conf +++ b/config/nginx/proxy.conf @@ -27,9 +27,9 @@ proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Proxy-host $proxy_host; -proxy_connect_timeout 7200; -proxy_send_timeout 7200; -proxy_read_timeout 7200; +proxy_connect_timeout 21600; +proxy_send_timeout 21600; +proxy_read_timeout 21600; proxy_buffers 8 32k; add_header X-Frame-Options SAMEORIGIN; diff --git a/config/traefik/traefik.yaml b/config/traefik/traefik.yaml index 4a223e7..95f766e 100644 --- a/config/traefik/traefik.yaml +++ b/config/traefik/traefik.yaml @@ -7,8 +7,8 @@ entryPoints: https: address: ":443" -# ssh: -# address: ":${RC_SSH_PORT:?must-specify-ssh-port}" + ssh: + address: ":${RC_SSH_PORT:?must-specify-ssh-port}" traefik: address: ":7000" diff --git a/dev.sh b/dev.sh deleted file mode 100755 index e97cc93..0000000 --- a/dev.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -IFS=$'\n\t' -CURRENT_USER_UID=$(id -u) -CURRENT_USER_GID=$(id -g) -USER_UID=${USER_UID:-$CURRENT_USER_UID} -USER_GID=${USER_GID:-$CURRENT_USER_GID} -CMD="docker-compose -f docker-compose.yml" -BUILD_CMD="docker-compose -f docker-compose.yaml -f docker-compose.source.yaml" -DEV_CMD="docker-compose -f docker-compose.yaml -f docker-compose.source.yaml -f docker-compose.dev.yaml" -DEV_CMD="docker-compose -f docker-compose.yaml -f docker-compose.source.yaml -f docker-compose.dev.yaml" -METRICS_CMD="docker-compose -f docker-compose-grafana.yaml" - -usage="$(basename "$0") ./dev.sh -- run docker command -where : - [v]cs -- run bash console for vcsserver - [r]c -- run bash console for rhodecode-ee - ce -- run bash console for rhodecode-ce - [t]est[s] -- run bash console for rhodecode-ce tests - test-mysql -- run bash console for rhodecode-ce tests with mysql - test-pgsql -- run bash console for rhodecode-ce tests with postgres - dev-build -- build rhodecode dev app for use without cache use - dev-env -- run the dev stack required for development - redis -- run the redis CLI attached to rc_cluster_redis_1 - db -- run the db bash attached to rc_cluster_database_1 - [m]etrics -- run the metrics stack - run -- run " - -case ${1:-} in - v|vcs ) - eval "(docker stop dev-vcsserver || echo skip-stop ) && ${DEV_CMD} run --service-ports --rm --use-aliases --workdir="/home/rhodecode/rhodecode-vcsserver" --name="dev-vcsserver" vcsserver bash" - exit - ;; - r|rc ) - eval "(docker stop dev-enterprise-ee || echo skip-stop ) && ${DEV_CMD} run --publish 8080:8080 --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ee" --name="dev-enterprise-ee" rhodecode bash" - exit - ;; - ce ) - eval "(docker stop dev-enterprise-ce || echo skip-stop ) && ${DEV_CMD} run --publish 8081:8080 --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ce" --name="dev-enterprise-ce" rhodecode bash" - exit - ;; - t|test|tests ) - eval "${DEV_CMD} run --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ce" --name="enterprise-dev-test" rhodecode bash" - exit - ;; - dev-build ) - eval "${BUILD_CMD} up --detach database && ${BUILD_CMD} build --no-cache --progress=plain rhodecode" - exit - ;; - dev-env ) - eval "${BUILD_CMD} up --detach database redis channelstream nginx celery beat elasticsearch" - exit - ;; - redis ) - eval "docker exec --interactive --tty rc_cluster_redis_1 redis-cli" - exit - ;; - db ) - eval "docker exec --interactive --tty rc_cluster_database_1 /bin/bash -c 'psql -U $$POSTGRES_USER' " - exit - ;; - m|metrics ) - eval "${METRICS_CMD} up --detach" - exit - ;; - run ) - command=${@:2} - eval "${DEV_CMD} run --rm rhodecode ${command}" - exit - ;; - * ) - echo "${usage}" - exit - ;; -esac diff --git a/docker-compose-apps.yaml b/docker-compose-apps.yaml index 7ec066b..f6e6020 100644 --- a/docker-compose-apps.yaml +++ b/docker-compose-apps.yaml @@ -11,13 +11,6 @@ x-logging: &custom-logging loki-timeout: "1s" loki-max-backoff: "800ms" -volumes: - - # volume for RhodeCode repo-store, it's where the repositories will be stored - rhodecode_repos: - labels: - "keep": 1 - services: @@ -35,8 +28,11 @@ services: "--name=gunicorn-rhodecode-1", "--error-logfile=-", "--paster=/etc/rhodecode/conf/rhodecode.optimized.ini", - "--config=/etc/rhodecode/conf/gunicorn_conf.py" + "--config=/etc/rhodecode/conf/gunicorn_conf_rc.py" ] + deploy: + # override this in .custom/docker-compose-apps.override.yaml to scale up + replicas: 1 # ports: # - "127.0.0.1::10020" @@ -85,7 +81,7 @@ services: volumes: - confvolume:/etc/rhodecode/conf - logvolume:/var/log/rhodecode - - rhodecode_repos:/var/opt/rhodecode_repo_store + - rc_reposvolume:/var/opt/rhodecode_repo_store - rc_datavolume:/var/opt/rhodecode_data tmpfs: @@ -95,7 +91,10 @@ services: *custom-logging labels: - - "traefik.enable=false" + - "traefik.enable=true" + - "traefik.http.routers.rhodecode.entrypoints=http" + - "traefik.http.routers.rhodecode.rule=Host(`${RHODECODE_HOSTNAME:?must-specify-rhodecode-hostname}`)" + - "traefik.http.services.rhodecode.loadbalancer.server.port=10020" vcsserver: networks: @@ -113,6 +112,9 @@ services: "--paster=/etc/rhodecode/conf/vcsserver.optimized.ini", "--config=/etc/rhodecode/conf/gunicorn_conf_vcs.py" ] + deploy: + # override this in .custom/docker-compose-apps.override.yaml to scale up + replicas: 1 # ports: # - "127.0.0.1::10010" @@ -137,7 +139,7 @@ services: volumes: - confvolume:/etc/rhodecode/conf - logvolume:/var/log/rhodecode - - rhodecode_repos:/var/opt/rhodecode_repo_store + - rc_reposvolume:/var/opt/rhodecode_repo_store - rc_datavolume:/var/opt/rhodecode_data logging: @@ -178,7 +180,7 @@ services: volumes: - confvolume:/etc/rhodecode/conf - logvolume:/var/log/rhodecode - - rhodecode_repos:/var/opt/rhodecode_repo_store + - rc_reposvolume:/var/opt/rhodecode_repo_store - rc_datavolume:/var/opt/rhodecode_data logging: @@ -222,7 +224,7 @@ services: volumes: - confvolume:/etc/rhodecode/conf - logvolume:/var/log/rhodecode - - rhodecode_repos:/var/opt/rhodecode_repo_store + - rc_reposvolume:/var/opt/rhodecode_repo_store - rc_datavolume:/var/opt/rhodecode_data logging: @@ -269,7 +271,7 @@ services: volumes: - confvolume:/etc/rhodecode/conf - logvolume:/var/log/rhodecode - - rhodecode_repos:/var/opt/rhodecode_repo_store + - rc_reposvolume:/var/opt/rhodecode_repo_store logging: *custom-logging @@ -297,7 +299,7 @@ services: volumes: - confvolume:/etc/rhodecode/conf - logvolume:/var/log/rhodecode - - rhodecode_repos:/var/opt/rhodecode_repo_store + - rc_reposvolume:/var/opt/rhodecode_repo_store - rc_datavolume:/var/opt/rhodecode_data logging: diff --git a/docker-compose-base.yaml b/docker-compose-base.yaml index a95e1c7..3e7f174 100644 --- a/docker-compose-base.yaml +++ b/docker-compose-base.yaml @@ -35,6 +35,11 @@ volumes: rc_datavolume: external: true + # volume for RhodeCode repo-store, it's where the repositories will be stored + # must be run via: docker volume create --name=rc_reposvolume + rc_reposvolume: + external: true + networks: # SHARED network for all containers diff --git a/docker-compose-metrics.yaml b/docker-compose-metrics.yaml index 7e9ec40..14e7aff 100644 --- a/docker-compose-metrics.yaml +++ b/docker-compose-metrics.yaml @@ -108,10 +108,6 @@ services: volumes: - ./config/loki:/etc/loki - labels: - - "traefik.enable=false" - - "traefik.http.services.loki.loadbalancer.server.port=3100" - - "traefik.http.services.loki.loadbalancer.server.port=9095" promtail: image: grafana/promtail:latest diff --git a/docker-compose-router.yaml b/docker-compose-router.yaml index 74e46c6..65ff197 100644 --- a/docker-compose-router.yaml +++ b/docker-compose-router.yaml @@ -45,34 +45,3 @@ services: logging: *custom-logging - - portainer: - # Run with COMPOSE_PROFILES=portainer - # to access portainer set HEADER `X-Docker-Host=portainer` - image: portainer/portainer-ce:latest - restart: always - volumes: - - portainer_data:/data - - /var/run/docker.sock:/var/run/docker.sock - deploy: - mode: replicated - replicas: 1 - placement: - constraints: - # limit swarm deploy to MANAGER only - - node.role == manager - - networks: - - rhodecode_network - labels: - - "traefik.enable=true" - - "traefik.http.services.portainer.loadbalancer.server.port=9000" - - "traefik.http.routers.portainer.entrypoints=https" - - "traefik.http.routers.portainer.rule=Headers(`X-Docker-Host`, `portainer`)" - - profiles: - ["portainer"] - -volumes: - portainer_data: - external: true \ No newline at end of file diff --git a/docker-compose-services.yaml b/docker-compose-services.yaml index 219a1fb..6e9a5ae 100644 --- a/docker-compose-services.yaml +++ b/docker-compose-services.yaml @@ -43,10 +43,10 @@ services: # ports: # - "127.0.0.1:9800:9800" - command: ["channelstream", "-i", "/etc/rhodecode/conf/channelstream.ini"] + command: ["channelstream"] - environment: - CHANNELSTREAM_ALLOW_POSTING_FROM: 0.0.0.0 + env_file: + - .custom/.runtime.env healthcheck: test: [ "CMD", "curl", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://channelstream:8000/admin/sign_in" ] @@ -81,7 +81,7 @@ services: healthcheck: # change port 80 to 443 when only using SSL - test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:80/_admin/ops/ping" ] + test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:80/_health" ] timeout: 30s interval: 60s retries: 10 @@ -102,8 +102,8 @@ services: labels: - "traefik.enable=true" - "traefik.http.routers.nginx.entrypoints=http" + - "traefik.http.routers.nginx.rule=Host(`${RHODECODE_HOSTNAME:?must-specify-rhodecode-hostname}`) && ( PathPrefix(`/_health`) || PathPrefix(`/_channelstream`) || PathPrefix(`/_static/rhodecode`) )" - "traefik.http.services.nginx.loadbalancer.server.port=80" - - "traefik.http.routers.nginx.rule=Host(`${RHODECODE_HOSTNAME:?must-specify-rhodecode-hostname}`)" elasticsearch: networks: diff --git a/scripts/rccontrol/rccontrol b/scripts/rccontrol/rccontrol index 0d28326..83964ff 100755 --- a/scripts/rccontrol/rccontrol +++ b/scripts/rccontrol/rccontrol @@ -36,9 +36,6 @@ rccontrol_usage() { printf "Commands:\n" echo " self-update update rccontrol and it's docker definitions" echo " bootstrap Bootstrap this machine, check docker version and install rhodecode-network" - echo " stack run one of available cluster stacks, use -h for more details" - echo " stack-status show stack status" - echo " stack-upgrade upgrade ALL stack status" echo printf "Build Commands:\n" echo " get-build-artifacts Fetch Artifacts to run installer based build" @@ -46,10 +43,13 @@ rccontrol_usage() { echo " get-build-source Fetch RhodeCode sources, store in .source dir to run a source-based builds" echo " build-source Build RhodeCode image from source, requires upgrade-source initially" echo + printf "Stack Commands:\n" + echo " stack run one of available cluster stacks, use -h for more details" + echo " stack-status Show stack status" + echo " stack-upgrade upgrade ALL stack status" + echo printf "CLI Commands:\n" - echo " cli-redis CLI" - echo " cli-db CLI" - echo " cli-db-upgrade CLI" + echo " cli run various CLI tools" echo printf "Backup Commands:\n" echo " backup-db CLI" @@ -172,7 +172,7 @@ rccontrol_bootstrap_usage() { fi - printf "Alias: install\n" + printf "Alias: init\n" echo printf "Usage:\n" @@ -318,7 +318,7 @@ rccontrol_get_build_source_usage() { fi printf "Usage:\n" - printf " rccontrol get-build-source REVISION [OPTIONS]\n" + printf " rccontrol get-build-source [OPTIONS]\n" printf " rccontrol get-build-source --help | -h\n" echo @@ -333,6 +333,12 @@ rccontrol_get_build_source_usage() { # :command.usage_flags # :flag.usage + echo " --revision REVISION" + printf " revision to download\n" + printf " Default: default\n" + echo + + # :flag.usage echo " --auth-token AUTH_TOKEN" printf " Specify AUTH TOKEN to obtain sources\n" echo @@ -343,14 +349,6 @@ rccontrol_get_build_source_usage() { printf " Default: https://code.rhodecode.com\n" echo - # :command.usage_args - printf "Arguments:\n" - - # :argument.usage - echo " REVISION" - printf " revision to download\n" - echo - # :command.usage_examples printf "Examples:\n" printf " rccontrol3 get-sources $RC_CLI_VERSION_NAME\n" @@ -414,8 +412,86 @@ rccontrol_stack_usage() { fi printf "Usage:\n" - printf " rccontrol stack NAME [SERVICES PARAMS...]\n" - printf " rccontrol stack --help | -h\n" + printf " rccontrol stack COMMAND\n" + printf " rccontrol stack [COMMAND] --help | -h\n" + echo + # :command.usage_commands + printf "Commands:\n" + echo " router run the router stack" + echo " metrics run the router stack" + echo " services run the router stack" + echo " rhodecode run the router stack" + echo " all run all stacks" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_examples + printf "Examples:\n" + printf " - ./rccontrol3 stack router up # run router stack with output to\n console\n - ./rccontrol3 stack router up --detach # run router stack detached\n - ./rccontrol3 stack router down # stop whole router stack\n - ./rccontrol3 stack router ps # check status of router stack\n - ./rccontrol3 stack router -f docker-overrides.yaml up -d # run router stack\n with your overrides\n" + echo + + fi +} + +# :command.usage +rccontrol_stack_router_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol stack router - run the router stack\n" + echo + + else + printf "rccontrol stack router - run the router stack\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol stack router [SERVICES PARAMS...]\n" + printf " rccontrol stack router --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_args + printf "Arguments:\n" + + echo " SERVICES PARAMS..." + printf " Additional arguments or flags for services command\n" + echo + + fi +} + +# :command.usage +rccontrol_stack_metrics_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol stack metrics - run the router stack\n" + echo + + else + printf "rccontrol stack metrics - run the router stack\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol stack metrics [SERVICES PARAMS...]\n" + printf " rccontrol stack metrics --help | -h\n" echo # :command.long_usage @@ -430,19 +506,116 @@ rccontrol_stack_usage() { # :command.usage_args printf "Arguments:\n" - # :argument.usage - echo " NAME" - printf " Stack name\n" - printf " Allowed: router, metrics, services, rhodecode\n" + echo " SERVICES PARAMS..." + printf " Additional arguments or flags for services command\n" + echo + + fi +} + +# :command.usage +rccontrol_stack_services_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol stack services - run the router stack\n" + echo + + else + printf "rccontrol stack services - run the router stack\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol stack services [SERVICES PARAMS...]\n" + printf " rccontrol stack services --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" echo + # :command.usage_args + printf "Arguments:\n" + echo " SERVICES PARAMS..." printf " Additional arguments or flags for services command\n" echo - # :command.usage_examples - printf "Examples:\n" - printf " - ./rccontrol3 stack router up # run router stack with output to\n console\n - ./rccontrol3 stack router up --detach # run router stack detached\n - ./rccontrol3 stack router down # stop whole router stack\n - ./rccontrol3 stack router ps # check status of router stack\n - ./rccontrol3 stack router -f docker-overrides.yaml up -d # run router stack\n with your overrides\n" + fi +} + +# :command.usage +rccontrol_stack_rhodecode_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol stack rhodecode - run the router stack\n" + echo + + else + printf "rccontrol stack rhodecode - run the router stack\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol stack rhodecode [SERVICES PARAMS...]\n" + printf " rccontrol stack rhodecode --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_args + printf "Arguments:\n" + + echo " SERVICES PARAMS..." + printf " Additional arguments or flags for services command\n" + echo + + fi +} + +# :command.usage +rccontrol_stack_all_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol stack all - run all stacks\n" + echo + + else + printf "rccontrol stack all - run all stacks\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol stack all [SERVICES PARAMS...]\n" + printf " rccontrol stack all --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + # :command.usage_args + printf "Arguments:\n" + + echo " SERVICES PARAMS..." + printf " Additional arguments or flags for services command\n" echo fi @@ -451,11 +624,11 @@ rccontrol_stack_usage() { # :command.usage rccontrol_stack_status_usage() { if [[ -n $long_usage ]]; then - printf "rccontrol stack-status - show stack status\n" + printf "rccontrol stack-status - Show stack status\n" echo else - printf "rccontrol stack-status - show stack status\n" + printf "rccontrol stack-status - Show stack status\n" echo fi @@ -510,20 +683,56 @@ rccontrol_stack_upgrade_usage() { } # :command.usage +rccontrol_cli_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol cli - run various CLI tools\n" + echo + + else + printf "rccontrol cli - run various CLI tools\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol cli COMMAND\n" + printf " rccontrol cli [COMMAND] --help | -h\n" + echo + # :command.usage_commands + printf "Commands:\n" + echo " redis CLI for Redis" + echo " db CLI with Database connection" + echo " db-upgrade CLI to run db upgrade" + echo " storage CLI for repository storage" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + fi +} + +# :command.usage rccontrol_cli_redis_usage() { if [[ -n $long_usage ]]; then - printf "rccontrol cli-redis - CLI\n" + printf "rccontrol cli redis - CLI for Redis\n" echo else - printf "rccontrol cli-redis - CLI\n" + printf "rccontrol cli redis - CLI for Redis\n" echo fi printf "Usage:\n" - printf " rccontrol cli-redis\n" - printf " rccontrol cli-redis --help | -h\n" + printf " rccontrol cli redis\n" + printf " rccontrol cli redis --help | -h\n" echo # :command.long_usage @@ -541,18 +750,18 @@ rccontrol_cli_redis_usage() { # :command.usage rccontrol_cli_db_usage() { if [[ -n $long_usage ]]; then - printf "rccontrol cli-db - CLI\n" + printf "rccontrol cli db - CLI with Database connection\n" echo else - printf "rccontrol cli-db - CLI\n" + printf "rccontrol cli db - CLI with Database connection\n" echo fi printf "Usage:\n" - printf " rccontrol cli-db\n" - printf " rccontrol cli-db --help | -h\n" + printf " rccontrol cli db\n" + printf " rccontrol cli db --help | -h\n" echo # :command.long_usage @@ -570,18 +779,47 @@ rccontrol_cli_db_usage() { # :command.usage rccontrol_cli_db_upgrade_usage() { if [[ -n $long_usage ]]; then - printf "rccontrol cli-db-upgrade - CLI\n" + printf "rccontrol cli db-upgrade - CLI to run db upgrade\n" + echo + + else + printf "rccontrol cli db-upgrade - CLI to run db upgrade\n" + echo + + fi + + printf "Usage:\n" + printf " rccontrol cli db-upgrade\n" + printf " rccontrol cli db-upgrade --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "Options:\n" + + # :command.usage_fixed_flags + echo " --help, -h" + printf " Show this help\n" + echo + + fi +} + +# :command.usage +rccontrol_cli_storage_usage() { + if [[ -n $long_usage ]]; then + printf "rccontrol cli storage - CLI for repository storage\n" echo else - printf "rccontrol cli-db-upgrade - CLI\n" + printf "rccontrol cli storage - CLI for repository storage\n" echo fi printf "Usage:\n" - printf " rccontrol cli-db-upgrade\n" - printf " rccontrol cli-db-upgrade --help | -h\n" + printf " rccontrol cli storage\n" + printf " rccontrol cli storage --help | -h\n" echo # :command.long_usage @@ -733,6 +971,23 @@ inspect_args() { } # :command.user_lib +# src/lib/check_bootstrap.sh + +check_bootstrap() { + # Avoid destroying bootstrapping by simple start/stop +for stage in $BOOTSTRAP_STAGES; do + + stage_name=$(echo $stage | cut -d ":" -f 1) + stage_func=$(echo $stage | cut -d ":" -f 2) + if ! config_has_key $stage_name ; then + echo "$(yellow WARNING:) bootstrap key $stage_name not found in config file $CONFIG_FILE!" + echo "$(yellow NOTICE:) Please run ./rccontrol bootstrap first" + exit + fi +done + +} + # src/lib/colors.sh print_in_color() { local color="$1" @@ -861,22 +1116,6 @@ config_has_key() { [[ $(config_get "$1") ]] } -# src/lib/sample_function.sh - -docker_ping_host() { - PING_HOST="$1" - docker run --network rhodecode_network --rm alpine ping "$PING_HOST" -} - -check_bootstrap() { - # Avoid destroying bootstrapping by simple start/stop - if [[ ! -e $BOOTSTRAP_FILE ]]; then - echo "$(yellow WARNING:) initial bootstrap file $BOOTSTRAP_FILE not found !" - echo "$(yellow NOTICE:) Please run ./rccontrol bootstrap first" - exit - fi -} - # src/lib/send_completions.sh send_completions() { echo $'# rccontrol3 completion -*- shell-script -*-' @@ -985,7 +1224,6 @@ rccontrol_bootstrap_command() { force=${args[--force]} check_bash_version() { - if [ ! "${BASH_VERSINFO:-0}" -ge 4 ]; then echo "$(red Bash version 4 or greater is required, please update your bash version!)" exit @@ -996,7 +1234,7 @@ rccontrol_bootstrap_command() { (which docker || which docker.io) &>/dev/null } - check_and_install_docker() { + bootstrap_docker_install() { failMsg="Failed to find docker on your PATH" if ! check_docker; then @@ -1012,12 +1250,17 @@ rccontrol_bootstrap_command() { fi } - docker_bootstrap() { - check_and_install_docker + bootstrap_docker_commons() { + echo 'Docker: Running bootstrap.' - echo "Docker: creating network 'rc_datavolume'" - docker volume create --name=rc_datavolume + echo "Docker: creating volume 'rc_datavolume'" + docker volume create --label keep=1 --name=rc_datavolume + echo "Docker: done" + echo "" + + echo "Docker: creating volume 'rc_reposvolume'" + docker volume create --label keep=1 --name=rc_reposvolume echo "Docker: done" echo "" @@ -1035,16 +1278,89 @@ rccontrol_bootstrap_command() { fi } - definitions_bootstrap() { + bootstrap_config() { + shared_key=$(echo $RANDOM | md5sum | head -c 32) - SOURCE_DIR=$PWD - RHODECODE_DOCKER_HASH='master' + if [[ ! -f "$CONFIG_FILE" ]]; then + echo "init config at: $CONFIG_FILE" + config_init + else + echo "re-using config at: $CONFIG_FILE" + fi - AUTH_TOKEN=${args[--auth-token]} - SERVER_URL=${args[--server-url]} + if ! config_has_key "rc_encrypted_secret" ; then + config_set "rc_encrypted_secret" $shared_key + fi + + if ! config_has_key "rc_db_url" ; then + key=$(echo $RANDOM | md5sum | head -c 32) + config_set "rc_db_url" "postgresql://$DB_USER:$key@database/$DB_NAME" + fi + + if ! config_has_key "rc_license_token" ; then + config_set "rc_license_token" abra-cada-bra1-rce4 + fi + + if ! config_has_key "rc_base_url" ; then + config_set "rc_base_url" http://docker-dev + fi + + if ! config_has_key "rc_log_formatter" ; then + # json is another option + config_set "rc_log_formatter" generic + fi + + if ! config_has_key "rc_use_celery" ; then + config_set "rc_use_celery" true + fi + + if ! config_has_key "channelstream_secret" ; then + config_set "channelstream_secret" $shared_key + fi + + if ! config_has_key "channelstream_admin_secret" ; then + config_set "channelstream_admin_secret" $shared_key + fi + + if ! config_has_key "channelstream_allow_posting_from" ; then + config_set "channelstream_allow_posting_from" 0.0.0.0 + fi + + if ! config_has_key "rc_channelstream_ws_url" ; then + config_set "rc_channelstream_ws_url" ws:/docker-dev/_channelstream + fi + + mkdir -p $PWD/.custom + BOOTSTRAP_RUNTIME_ENV=$PWD/.custom/.runtime.env + if [[ ! -f "$BOOTSTRAP_RUNTIME_ENV" ]]; then + echo "init runtime env config at: $BOOTSTRAP_RUNTIME_ENV" + touch BOOTSTRAP_RUNTIME_ENV + + #ENV_EXPAND="" + for k in $(config_keys); do + k_upper=${k^^} + echo "$k_upper='$(config_get "$k")'" >> $BOOTSTRAP_RUNTIME_ENV + done + + fi + } + + bootstrap_definitions() { + + SOURCE_DIR=$PWD + RHODECODE_DOCKER_HASH='master' + + AUTH_TOKEN=${args[--auth-token]} + SERVER_URL=${args[--server-url]} DEFINITIONS_EXIST="" - CHECK_FILES="rccontrol .env docker-compose-services.yaml docker-compose-apps.yaml" + CHECK_FILES="\ + .env \ + docker-compose-services.yaml \ + docker-compose-apps.yaml \ + docker-compose-metrics.yaml \ + docker-compose-router.yaml \ + " for check_file in $CHECK_FILES; do if [[ -f "$check_file" ]]; then DEFINITIONS_EXIST="1" @@ -1068,8 +1384,6 @@ rccontrol_bootstrap_command() { done fi - exit - # download sources echo "Files: download rhodecode docker definitions from $SERVER_URL" echo "" @@ -1094,83 +1408,56 @@ rccontrol_bootstrap_command() { echo "$(green_bold DONE: docker definitions extracted to $SOURCE_DIR)" } - config_bootstrap() { - - if [[ ! -f "$CONFIG_FILE" ]]; then - echo "init config at: $CONFIG_FILE" - config_init - else - echo "re-using config at: $CONFIG_FILE" - fi - - if ! config_has_key "rc_encrypted_secret" ; then - key=$(echo $RANDOM | md5sum | head -c 32) - config_set "rc_encrypted_secret" $key - fi - - if ! config_has_key "rc_db_url" ; then - key=$(echo $RANDOM | md5sum | head -c 32) - config_set "rc_db_url" "postgresql://$DB_USER:$key@database/$DB_NAME" - fi - - if ! config_has_key "rc_license_token" ; then - config_set "rc_license_token" abra-cada-bra1-rce4 - fi - - if ! config_has_key "rc_base_url" ; then - config_set "rc_base_url" http://docker-dev - fi - - if ! config_has_key "rc_log_formatter" ; then - # json is another option - config_set "rc_log_formatter" generic - fi - - if ! config_has_key "rc_use_celery" ; then - config_set "rc_use_celery" true - fi - - BOOTSTRAP_RUNTIME_ENV=$PWD/.custom/.runtime.env - if [[ ! -f "$BOOTSTRAP_RUNTIME_ENV" ]]; then - echo "init runtime env config at: $BOOTSTRAP_RUNTIME_ENV" - touch BOOTSTRAP_RUNTIME_ENV - - #ENV_EXPAND="" - for k in $(config_keys); do - k_upper=${k^^} - echo "$k_upper='$(config_get "$k")'" >> $BOOTSTRAP_RUNTIME_ENV - done + bootstrap_overrides() { + templates=$(find $PWD/templates/*.yaml -printf "%f\n") - fi + target_dir=$PWD/.custom/ + for o_file in $templates; do + target_file=$target_dir/$o_file + if [[ ! -f "$target_file" ]]; then + echo "copy override file $o_file" + cp -v $PWD/templates/$o_file $target_file + else + echo "file $target_file existing, skipping..." + fi + done + echo "$(green_bold DONE: overrides extracted to $target_dir)" } cur_date=$(date '+%Y-%m-%d %H:%M:%S') check_bash_version - if [[ ! -e $BOOTSTRAP_FILE ]]; then - echo "initial bootstrap file $BOOTSTRAP_FILE not found !" - - docker_bootstrap - definitions_bootstrap - config_bootstrap - - echo "$cur_date" > "$BOOTSTRAP_FILE" - + # Init the config ! + if [[ ! -f "$CONFIG_FILE" ]]; then + echo "config: init new config at: $CONFIG_FILE" + config_init else - if [ $force ]; then + echo "config: re-using present config at: $CONFIG_FILE" + fi - docker_bootstrap $force - definitions_bootstrap $force ## TODO: remove old - config_bootstrap $force + for stage in $BOOTSTRAP_STAGES; do - echo "$cur_date" > "$BOOTSTRAP_FILE" + stage_name=$(echo $stage | cut -d ":" -f 1) + stage_func=$(echo $stage | cut -d ":" -f 2) - exit - fi + if ! config_has_key $stage_name ; then + echo "$(green \* bootstrap: \'$stage_name\' stage not found\; running now... )" + $stage_func + config_set "$stage_name" $cur_date + else + if [ $force ]; then + echo "$(green \* bootstrap: \'$stage_name\' is present!\; FORCE running now... )" + $stage_func $force + config_set "$stage_name" $cur_date + else + echo "$(yellow \* bootstrap: \'$stage_name\' already present, use --force to run it again)" + fi + fi - echo "bootstrap file $BOOTSTRAP_FILE was found add --force to force bootstrap" - fi + done + + exit get_started } @@ -1263,7 +1550,7 @@ rccontrol_get_build_source_command() { AUTH_TOKEN=${args[--auth-token]} SERVER_URL=${args[--server-url]} - revision=${args[revision]} + revision=${args[--revision]} SOURCE_DIR=$PWD/.source @@ -1279,11 +1566,17 @@ rccontrol_get_build_source_command() { #TODO: fix just CE build... curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-ee/archive/$RHODECODE_EE_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR - rm -rf $SOURCE_DIR/rhodecode-vcsserver && mv $SOURCE_DIR/*rhodecode-vcsserver-plain $SOURCE_DIR/rhodecode-vcsserver - rm -rf $SOURCE_DIR/rhodecode-enterprise-ce && mv $SOURCE_DIR/*rhodecode-enterprise-ce-plain $SOURCE_DIR/rhodecode-enterprise-ce - rm -rf $SOURCE_DIR/rhodecode-enterprise-ee && cp -r $SOURCE_DIR/*rhodecode-enterprise-ee-plain $SOURCE_DIR/rhodecode-enterprise-ee + rm -rf $SOURCE_DIR/rhodecode-vcsserver && \ + mv $SOURCE_DIR/*rhodecode-vcsserver-plain $SOURCE_DIR/rhodecode-vcsserver + + rm -rf $SOURCE_DIR/rhodecode-enterprise-ce && \ + mv $SOURCE_DIR/*rhodecode-enterprise-ce-plain $SOURCE_DIR/rhodecode-enterprise-ce + + rm -rf $SOURCE_DIR/rhodecode-enterprise-ee && \ + mv $SOURCE_DIR/*rhodecode-enterprise-ee-plain $SOURCE_DIR/rhodecode-enterprise-ee echo "downloading sources done to $SOURCE_DIR" + ls -l $SOURCE_DIR } @@ -1301,12 +1594,11 @@ rccontrol_build_source_command() { } # :command.function -rccontrol_stack_command() { - # src/stack_command.sh +rccontrol_stack_router_command() { + # src/stack_router_command.sh check_bootstrap DEBUG=${args[--debug]} - service_name=${args[name]} if [[ ! -f $RC_STACK_ROUTER_EXT ]]; then RC_STACK_ROUTER_EXT="" @@ -1322,6 +1614,55 @@ rccontrol_stack_command() { -f docker-compose-base.yaml \ -f docker-compose-router.yaml $RC_STACK_ROUTER_EXT" + if [[ $DEBUG ]]; then + echo "---" + echo "stacks docker: $RC_STACK_ROUTER_EXT_LCL" + echo "running command: ${CMD_ROUTER}" + echo "ARGS: ${other_args[*]}" + echo "---" + fi + eval "${CMD_ROUTER} ${other_args[*]}" + +} + +# :command.function +rccontrol_stack_metrics_command() { + # src/stack_metrics_command.sh + check_bootstrap + + DEBUG=${args[--debug]} + + if [[ ! -f $RC_STACK_METRICS_EXT ]]; then + RC_STACK_METRICS_EXT_LCL="" + else + RC_STACK_METRICS_EXT_LCL="-f $RC_STACK_METRICS_EXT" + fi + + CMD_METRICS="\ + docker compose \ + --env-file $ENV_FILE \ + $ENV_EXPAND \ + -p rc_cluster_metrics \ + -f docker-compose-base.yaml \ + -f docker-compose-metrics.yaml $RC_STACK_METRICS_EXT_LCL" + + if [[ $DEBUG ]]; then + echo "---" + echo "stacks docker: $RC_STACK_METRICS_EXT_LCL" + echo "running command: ${CMD_METRICS}" + echo "ARGS: ${other_args[*]}" + echo "---" + fi + eval "${CMD_METRICS} ${other_args[*]}" +} + +# :command.function +rccontrol_stack_services_command() { + # src/stack_services_command.sh + check_bootstrap + + DEBUG=${args[--debug]} + if [[ ! -f $RC_STACK_SERVICES_EXT ]]; then RC_STACK_SERVICES_EXT_LCL="" else @@ -1339,19 +1680,23 @@ rccontrol_stack_command() { -f docker-compose-base.yaml \ -f docker-compose-services.yaml $RC_STACK_SERVICES_EXT_LCL" - if [[ ! -f $RC_STACK_METRICS_EXT ]]; then - RC_STACK_METRICS_EXT_LCL="" - else - RC_STACK_METRICS_EXT_LCL="-f $RC_STACK_METRICS_EXT" + if [[ $DEBUG ]]; then + echo "---" + echo "stacks docker: $RC_STACK_SERVICES_EXT_LCL" + echo "running command: ${CMD_SERVICES}" + echo "ARGS: ${other_args[*]}" + echo "---" fi - CMD_METRICS="\ - docker compose \ - --env-file $ENV_FILE \ - $ENV_EXPAND \ - -p rc_cluster_metrics \ - -f docker-compose-base.yaml \ - -f docker-compose-metrics.yaml $RC_STACK_METRICS_EXT_LCL" + eval "${CMD_SERVICES} ${other_args[*]}" +} + +# :command.function +rccontrol_stack_rhodecode_command() { + # src/stack_rhodecode_command.sh + check_bootstrap + + DEBUG=${args[--debug]} if [[ ! -f $RC_STACK_RHODECODE_EXT ]]; then RC_STACK_RHODECODE_EXT_LCL="" @@ -1376,54 +1721,29 @@ rccontrol_stack_command() { -f docker-compose-apps.yaml \ -f docker-compose-apps.source.yaml $RC_STACK_RHODECODE_EXT_LCL" - case $service_name in - - services ) - if [[ $DEBUG ]]; then - echo "---" - echo "stacks docker: $RC_STACK_SERVICES_EXT_LCL" - echo "running command: ${CMD_SERVICES}" - echo "ARGS: ${other_args[*]}" - echo "---" - fi - - eval "${CMD_SERVICES} ${other_args[*]}" - exit - ;; - router ) - if [[ $DEBUG ]]; then - echo "---" - echo "stacks docker: $RC_STACK_ROUTER_EXT_LCL" - echo "running command: ${CMD_ROUTER}" - echo "ARGS: ${other_args[*]}" - echo "---" - fi - eval "${CMD_ROUTER} ${other_args[*]}" - exit - ;; - metrics ) - if [[ $DEBUG ]]; then - echo "---" - echo "stacks docker: $RC_STACK_METRICS_EXT_LCL" - echo "running command: ${CMD_METRICS}" - echo "ARGS: ${other_args[*]}" - echo "---" - fi - eval "${CMD_METRICS} ${other_args[*]}" - exit - ;; - rhodecode ) - if [[ $DEBUG ]]; then - echo "---" - echo "stacks docker: $RC_STACK_RHODECODE_EXT_LCL" - echo "running command: ${CMD_RHODECODE}" - echo "ARGS: ${other_args[*]}" - echo "---" - fi - eval "${CMD_RHODECODE} ${other_args[*]}" - exit - ;; - esac + if [[ $DEBUG ]]; then + echo "---" + echo "stacks docker: $RC_STACK_RHODECODE_EXT_LCL" + echo "running command: ${CMD_RHODECODE}" + echo "ARGS: ${other_args[*]}" + echo "---" + fi + eval "${CMD_RHODECODE} ${other_args[*]}" + +} + +# :command.function +rccontrol_stack_all_command() { + # src/stack_all_command.sh + check_bootstrap + + DEBUG=${args[--debug]} + + stack_lst=$VALID_SERVICES + + for stack in $stack_lst; do + ./rccontrol stack $stack ${other_args[*]} + done } @@ -1552,6 +1872,52 @@ rccontrol_cli_db_upgrade_command() { } # :command.function +rccontrol_cli_storage_command() { + # src/cli_storage_command.sh + check_bootstrap + + DEBUG=${args[--debug]} + + source $ENV_FILE + target_container=$RC_IMAGE_EE + + image_id=$(docker image ls $RC_IMAGE_EE -q) + + echo "Attaching storage from $target_container" + + if [[ $DEBUG ]]; then + echo "image id: $image_id, based on $target_container filter" + docker image ls + echo "---" + fi + + if [[ $DEBUG ]]; then + echo "rc_datavolume inspection..." + docker volume inspect rc_datavolume + + echo "rhodecode_repos inspection..." + docker volume inspect rhodecode_repos + fi + + # + #if [[ $DEBUG ]]; then + # echo "image id: $image_id, based on $target_container filter" + # docker image ls + # echo "---" + #fi + + docker run --rm \ + -it \ + --volume $PWD/.custom/storage:/vol/backupvolume \ + --volume rc_datavolume:/vol/datavolume \ + --volume rc_reposvolume:/vol/repovolume \ + --workdir="/vol" \ + debian:jessie \ + /bin/bash + +} + +# :command.function rccontrol__completions_command() { # src/_completions_command.sh send_completions @@ -1568,7 +1934,7 @@ rccontrol_backup_db_command() { target_container=rc_cluster_services-database-1 docker_id=$(docker ps --filter name=$target_container -q) - backup_name=rc_db_dump-$(date +%Y-%m-%d).tar.gz + backup_name=rc_db_dump-$(date +%Y-%m-%d).sql.gz echo "creating backup $backup_name" @@ -1597,9 +1963,12 @@ rccontrol_backup_db_command() { # echo "$(red $MSG)" # exit #fi + # docker exec -i your-db-container psql -U your-db-user -d your-db-name gunzip < your_dump.sql.gz | if [[ $DEBUG ]]; then - echo "docker exec -e PGPASSWORD=$DB_PASSWORD $docker_id /bin/bash -c 'pg_dump --inserts -U $DB_USER -h 127.0.0.1 --dbname=$DB_NAME | gzip > /var/rc-data-dump/$backup_name'" + echo "docker exec -e PGPASSWORD=$DB_PASSWORD $docker_id /bin/bash -c 'mkdir -p /var/rc-data-dump && pg_dump --inserts -U $DB_USER -h 127.0.0.1 --dbname=$DB_NAME | gzip > /var/rc-data-dump/$backup_name'" + echo "dump placed in /var/rc-data-dump/$backup_name" + echo "run docker cp $docker_id:/var/rc-data-dump/$backup_name $PWD to copy the file into your host machine" fi eval "docker exec -e PGPASSWORD=$DB_PASSWORD $docker_id /bin/bash -c 'pg_dump --inserts -U $DB_USER -h 127.0.0.1 --dbname=$DB_NAME | gzip > /var/rc-data-dump/$backup_name'" @@ -1664,7 +2033,7 @@ parse_requirements() { shift $# ;; - bootstrap | install ) + bootstrap | init ) action="bootstrap" shift rccontrol_bootstrap_parse_requirements "$@" @@ -1720,24 +2089,10 @@ parse_requirements() { shift $# ;; - cli-redis ) - action="cli-redis" - shift - rccontrol_cli_redis_parse_requirements "$@" - shift $# - ;; - - cli-db ) - action="cli-db" - shift - rccontrol_cli_db_parse_requirements "$@" - shift $# - ;; - - cli-db-upgrade ) - action="cli-db-upgrade" + cli ) + action="cli" shift - rccontrol_cli_db_upgrade_parse_requirements "$@" + rccontrol_cli_parse_requirements "$@" shift $# ;; @@ -2158,6 +2513,21 @@ rccontrol_get_build_source_parse_requirements() { key="$1" case "$key" in # :flag.case + --revision ) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args[--revision]="$2" + shift + shift + else + printf "%s\n" "--revision requires an argument: --revision REVISION" >&2 + exit 1 + fi + ;; + + # :flag.case --auth-token ) # :flag.case_arg @@ -2195,27 +2565,16 @@ rccontrol_get_build_source_parse_requirements() { * ) # :command.parse_requirements_case # :command.parse_requirements_case_simple - if [[ -z ${args[revision]+x} ]]; then - - args[revision]=$1 - shift - else - printf "invalid argument: %s\n" "$key" >&2 - exit 1 - fi + printf "invalid argument: %s\n" "$key" >&2 + exit 1 ;; esac done - # :command.required_args_filter - if [[ -z ${args[revision]+x} ]]; then - printf "missing required argument: REVISION\nusage: rccontrol get-build-source REVISION [OPTIONS]\n" >&2 - exit 1 - fi - # :command.default_assignments + [[ -n ${args[--revision]:-} ]] || args[--revision]="default" [[ -n ${args[--server-url]:-} ]] || args[--server-url]="https://code.rhodecode.com" } @@ -2288,46 +2647,270 @@ rccontrol_stack_parse_requirements() { esac # :command.command_filter - action="stack" + action=${1:-} - # :command.parse_requirements_while - while [[ $# -gt 0 ]]; do - key="$1" - case "$key" in + case $action in + -* ) + ;; - -?* ) - other_args+=("$1") - shift + router ) + action="router" + shift + rccontrol_stack_router_parse_requirements "$@" + shift $# + ;; + + metrics ) + action="metrics" + shift + rccontrol_stack_metrics_parse_requirements "$@" + shift $# + ;; + + services ) + action="services" + shift + rccontrol_stack_services_parse_requirements "$@" + shift $# + ;; + + rhodecode ) + action="rhodecode" + shift + rccontrol_stack_rhodecode_parse_requirements "$@" + shift $# + ;; + + all ) + action="all" + shift + rccontrol_stack_all_parse_requirements "$@" + shift $# + ;; + + # :command.command_fallback + "" ) + rccontrol_stack_usage >&2 + exit 1 + ;; + + * ) + printf "invalid command: %s\n" "$action" >&2 + exit 1 + ;; + + esac + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + +} + +# :command.parse_requirements +rccontrol_stack_router_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_stack_router_usage + exit + ;; + + esac + + # :command.command_filter + action="stack router" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + other_args+=("$1") + shift ;; * ) # :command.parse_requirements_case # :command.parse_requirements_case_catch_all - if [[ -z ${args[name]+x} ]]; then + other_args+=("$1") + shift - args[name]=$1 - shift - else - other_args+=("$1") - shift - fi + ;; + + esac + done + +} + +# :command.parse_requirements +rccontrol_stack_metrics_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_stack_metrics_usage + exit + ;; + + esac + + # :command.command_filter + action="stack metrics" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + other_args+=("$1") + shift + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_catch_all + other_args+=("$1") + shift ;; esac done - # :command.required_args_filter - if [[ -z ${args[name]+x} ]]; then - printf "missing required argument: NAME\nusage: rccontrol stack NAME [SERVICES PARAMS...]\n" >&2 - exit 1 - fi +} - # :command.whitelist_filter - if [[ ! ${args[name]} =~ ^(router|metrics|services|rhodecode)$ ]]; then - printf "%s\n" "name must be one of: router, metrics, services, rhodecode" >&2 - exit 1 - fi +# :command.parse_requirements +rccontrol_stack_services_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_stack_services_usage + exit + ;; + + esac + + # :command.command_filter + action="stack services" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + other_args+=("$1") + shift + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_catch_all + other_args+=("$1") + shift + + ;; + + esac + done + +} + +# :command.parse_requirements +rccontrol_stack_rhodecode_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_stack_rhodecode_usage + exit + ;; + + esac + + # :command.command_filter + action="stack rhodecode" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + other_args+=("$1") + shift + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_catch_all + other_args+=("$1") + shift + + ;; + + esac + done + +} + +# :command.parse_requirements +rccontrol_stack_all_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_stack_all_usage + exit + ;; + + esac + + # :command.command_filter + action="stack all" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + other_args+=("$1") + shift + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_catch_all + other_args+=("$1") + shift + + ;; + + esac + done } @@ -2408,6 +2991,89 @@ rccontrol_stack_upgrade_parse_requirements() { } # :command.parse_requirements +rccontrol_cli_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_cli_usage + exit + ;; + + esac + + # :command.command_filter + action=${1:-} + + case $action in + -* ) + ;; + + redis ) + action="redis" + shift + rccontrol_cli_redis_parse_requirements "$@" + shift $# + ;; + + db ) + action="db" + shift + rccontrol_cli_db_parse_requirements "$@" + shift $# + ;; + + db-upgrade ) + action="db-upgrade" + shift + rccontrol_cli_db_upgrade_parse_requirements "$@" + shift $# + ;; + + storage ) + action="storage" + shift + rccontrol_cli_storage_parse_requirements "$@" + shift $# + ;; + + # :command.command_fallback + "" ) + rccontrol_cli_usage >&2 + exit 1 + ;; + + * ) + printf "invalid command: %s\n" "$action" >&2 + exit 1 + ;; + + esac + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + +} + +# :command.parse_requirements rccontrol_cli_redis_parse_requirements() { # :command.fixed_flags_filter case "${1:-}" in @@ -2420,7 +3086,7 @@ rccontrol_cli_redis_parse_requirements() { esac # :command.command_filter - action="cli-redis" + action="cli redis" # :command.parse_requirements_while while [[ $# -gt 0 ]]; do @@ -2458,7 +3124,7 @@ rccontrol_cli_db_parse_requirements() { esac # :command.command_filter - action="cli-db" + action="cli db" # :command.parse_requirements_while while [[ $# -gt 0 ]]; do @@ -2496,7 +3162,45 @@ rccontrol_cli_db_upgrade_parse_requirements() { esac # :command.command_filter - action="cli-db-upgrade" + action="cli db-upgrade" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + + -?* ) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + * ) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + +} + +# :command.parse_requirements +rccontrol_cli_storage_parse_requirements() { + # :command.fixed_flags_filter + case "${1:-}" in + --help | -h ) + long_usage=yes + rccontrol_cli_storage_usage + exit + ;; + + esac + + # :command.command_filter + action="cli storage" # :command.parse_requirements_while while [[ $# -gt 0 ]]; do @@ -2669,12 +3373,21 @@ initialize() { ENV_FILE=$PWD/.env # bootstrap file is a config file at the same time - BOOTSTRAP_FILE='.rccontrol-bootstrap' CONFIG_FILE='.rccontrol.ini' + # STACK_LIST VALID_SERVICES="router metrics services rhodecode" DOCKER_DEFS_WORK_DIR="docker_defs" + # stage key, saved in .rccontrol.ini : stage func to execute + BOOTSTRAP_STAGES="\ + bootstrap_docker_install:bootstrap_docker_install \ + bootstrap_docker_commons:bootstrap_docker_commons \ + bootstrap_config:bootstrap_config \ + bootstrap_definitions:bootstrap_definitions \ + bootstrap_overrides:bootstrap_overrides \ + " + #echo "1 ----" #echo $RC_STACK_SERVICES_EXT #echo $RC_STACK_METRICS_EXT @@ -2706,6 +3419,11 @@ initialize() { #env | grep RC_ + docker_ping_host() { + PING_HOST="$1" + docker run --network rhodecode_network --rm alpine ping "$PING_HOST" + } + # backup files from a docker volume into /tmp/backup.tar.gz docker-volume-backup-compressed() { docker run --rm -v /tmp:/backup --volumes-from "$1" debian:jessie tar -czvf /backup/backup.tar.gz "${@:2}" @@ -2816,6 +3534,46 @@ run() { rccontrol_stack_command fi + elif [[ $action == "stack router" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_stack_router_usage + else + rccontrol_stack_router_command + fi + + elif [[ $action == "stack metrics" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_stack_metrics_usage + else + rccontrol_stack_metrics_command + fi + + elif [[ $action == "stack services" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_stack_services_usage + else + rccontrol_stack_services_command + fi + + elif [[ $action == "stack rhodecode" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_stack_rhodecode_usage + else + rccontrol_stack_rhodecode_command + fi + + elif [[ $action == "stack all" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_stack_all_usage + else + rccontrol_stack_all_command + fi + elif [[ $action == "stack-status" ]]; then if [[ ${args[--help]:-} ]]; then long_usage=yes @@ -2832,7 +3590,15 @@ run() { rccontrol_stack_upgrade_command fi - elif [[ $action == "cli-redis" ]]; then + elif [[ $action == "cli" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_cli_usage + else + rccontrol_cli_command + fi + + elif [[ $action == "cli redis" ]]; then if [[ ${args[--help]:-} ]]; then long_usage=yes rccontrol_cli_redis_usage @@ -2840,7 +3606,7 @@ run() { rccontrol_cli_redis_command fi - elif [[ $action == "cli-db" ]]; then + elif [[ $action == "cli db" ]]; then if [[ ${args[--help]:-} ]]; then long_usage=yes rccontrol_cli_db_usage @@ -2848,7 +3614,7 @@ run() { rccontrol_cli_db_command fi - elif [[ $action == "cli-db-upgrade" ]]; then + elif [[ $action == "cli db-upgrade" ]]; then if [[ ${args[--help]:-} ]]; then long_usage=yes rccontrol_cli_db_upgrade_usage @@ -2856,6 +3622,14 @@ run() { rccontrol_cli_db_upgrade_command fi + elif [[ $action == "cli storage" ]]; then + if [[ ${args[--help]:-} ]]; then + long_usage=yes + rccontrol_cli_storage_usage + else + rccontrol_cli_storage_command + fi + elif [[ $action == "_completions" ]]; then if [[ ${args[--help]:-} ]]; then long_usage=yes diff --git a/service/rhodecode/rhodecode.dockerfile b/service/rhodecode/rhodecode.dockerfile index 9cdc637..4240f22 100644 --- a/service/rhodecode/rhodecode.dockerfile +++ b/service/rhodecode/rhodecode.dockerfile @@ -11,7 +11,7 @@ ARG RHODECODE_VERSION=4.28.0 ARG RHODECODE_DB=sqlite ARG RHODECODE_USER_NAME=admin ARG RHODECODE_USER_PASS=secret4 -ARG RHODECODE_USER_EMAIL=support@rhodecode.com +ARG RHODECODE_USER_EMAIL=admin@server.local # nix ver/channels ARG DEV_NIX_VERSION=2.0.4 @@ -232,7 +232,7 @@ echo "** install rhodecode ${RHODECODE_TYPE} ${RHODECODE_VERSION} **" && \ RHODECODE_PATH=/home/$RC_USER/.rccontrol/${RC_TYPE_ID} && \ rm -rf $BUILD_BIN_DIR/bin && ln -s ${RHODECODE_PATH}/profile/bin $BUILD_BIN_DIR/ && \ cp -v ${RHODECODE_PATH}/rhodecode.ini $BUILD_CONF/rhodecode.ini && \ - cp -v ${RHODECODE_PATH}/gunicorn_conf.py $BUILD_CONF/gunicorn_conf.py && \ + cp -v ${RHODECODE_PATH}/gunicorn_conf.py $BUILD_CONF/gunicorn_conf_rc.py && \ cp -v ${RHODECODE_PATH}/search_mapping.ini $BUILD_CONF/search_mapping.ini && \ mkdir -p $RHODECODE_DATA_DIR/static && cp -r ${RHODECODE_PATH}/public/* $RHODECODE_DATA_DIR/static/ && \ rm ${RHODECODE_PATH}/rhodecode.db && \ @@ -280,4 +280,4 @@ VOLUME /var/log/rhodecode ENTRYPOINT ["/opt/entrypoints.d/entrypoint.sh"] # compose can override this -CMD [ "$BUILD_BIN_DIR/bin/gunicorn", "--error-logfile=-", "--paster=/etc/rhodecode/conf_build/run.ini", "--config=/etc/rhodecode/conf_build/gunicorn_conf.py" ] +CMD [ "$BUILD_BIN_DIR/bin/gunicorn", "--error-logfile=-", "--paster=/etc/rhodecode/conf_build/run.ini", "--config=/etc/rhodecode/conf_build/gunicorn_conf_rc.py" ] diff --git a/service/rhodecode/rhodecode_source.dockerfile b/service/rhodecode/rhodecode_source.dockerfile index 04e4ff2..557ba01 100644 --- a/service/rhodecode/rhodecode_source.dockerfile +++ b/service/rhodecode/rhodecode_source.dockerfile @@ -11,7 +11,7 @@ ARG RHODECODE_VERSION=4.28.0 ARG RHODECODE_DB=sqlite ARG RHODECODE_USER_NAME=admin ARG RHODECODE_USER_PASS=secret4 -ARG RHODECODE_USER_EMAIL=support@rhodecode.com +ARG RHODECODE_USER_EMAIL=admin@server.local # nix ver/channels ARG DEV_NIX_VERSION=2.0.4 @@ -66,7 +66,7 @@ ENV \ ENV SVN_LOCALE_DEPS apache2 apache2-utils libapache2-mod-svn ENV SSH_LOCALE_DEPS openssh-server ENV PYTHON_DEPS python2 -ENV EXTRA_DEPS "" +ENV EXTRA_DEPS "" ENV \ PATH=$PATH:/nix/var/nix/profiles/per-user/$RC_USER/profile/bin:/home/$RC_USER/rhodecode-enterprise-ee/profile/bin \ @@ -202,16 +202,17 @@ COPY --chown=$RC_USER:$RC_USER .cache/* /home/$RC_USER/.rccontrol/cache/ COPY --chown=$RC_USER:$RC_USER config/_shared/rhodecode_enterprise.license /home/$RC_USER/.rccontrol/bootstrap/ COPY --chown=$RC_USER:$RC_USER service/rhodecode/bootstrap/* /home/$RC_USER/.rccontrol/bootstrap/ +RUN \ +echo "**** locale-archive path ****" && \ + mv -v /home/$RC_USER/.rccontrol/cache/locale-archive /var/opt/locale-archive + COPY --chown=$RC_USER:$RC_USER .source/ /home/$RC_USER/ RUN \ echo "** prepare rhodecode dirs **" && \ install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/vcsserver-1 && \ install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/community-1 && \ install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/enterprise-1 && \ - -RUN \ -echo "**** locale-archive path ****" && \ - mv -v /home/$RC_USER/.rccontrol/cache/locale-archive /var/opt/locale-archive + echo "Done rhodecode dirs" # change to non-root user for RUN commands USER $RC_USER @@ -220,13 +221,15 @@ WORKDIR /home/$RC_USER RUN \ echo "** download and install nix ** from $DEV_NIX_VERSION/install" && \ - curl -L https://releases.nixos.org/nix/nix-$DEV_NIX_VERSION/install | USER=$RC_USER /bin/bash + curl -L https://releases.nixos.org/nix/nix-$DEV_NIX_VERSION/install | USER=$RC_USER /bin/bash && \ + echo "Done nix install" RUN \ echo "** update nix package database and set channel to $DEV_NIX_CHANNEL **" && \ . /home/rhodecode/.nix-profile/etc/profile.d/nix.sh && \ nix-channel --add https://channels.nixos.org/$DEV_NIX_CHANNEL nixpkgs && \ - nix-channel --update + nix-channel --update && \ + echo "done nix package updates" RUN \ @@ -272,7 +275,7 @@ echo "** install rhodecode ${RHODECODE_VERSION} **" && \ rm -rf $BUILD_BIN_DIR/bin && \ cp -rv --preserve=links /home/$RC_USER/.rccontrol/enterprise-1/profile/bin $BUILD_BIN_DIR/ && \ cp -v ${RHODECODE_PATH}/configs/production.ini $BUILD_CONF/rhodecode.ini && \ - cp -v ${RHODECODE_PATH}/configs/gunicorn_config.py $BUILD_CONF/gunicorn_conf.py && \ + cp -v ${RHODECODE_PATH}/configs/gunicorn_config.py $BUILD_CONF/gunicorn_conf_rc.py && \ mkdir -p $RHODECODE_DATA_DIR/static && cp -r /home/$RC_USER/.rccontrol/enterprise-1/profile/etc/static/* $RHODECODE_DATA_DIR/static/ && \ echo "Done installing rhodecode" @@ -321,4 +324,4 @@ VOLUME /var/log/rhodecode ENTRYPOINT ["/opt/entrypoints.d/entrypoint.sh"] # compose can override this -CMD [ "$BUILD_BIN_DIR/bin/gunicorn", "--error-logfile=-", "--paster=/etc/rhodecode/conf_build/run.ini", "--config=/etc/rhodecode/conf_build/gunicorn_conf.py" ] +CMD [ "$BUILD_BIN_DIR/bin/gunicorn", "--error-logfile=-", "--paster=/etc/rhodecode/conf_build/run.ini", "--config=/etc/rhodecode/conf_build/gunicorn_conf_rc.py" ] diff --git a/templates/docker-compose-apps.override.yaml b/templates/docker-compose-apps.override.yaml new file mode 100644 index 0000000..4b6e433 --- /dev/null +++ b/templates/docker-compose-apps.override.yaml @@ -0,0 +1,31 @@ +version: '3.9' + +services: + + rhodecode: + ports: + - "10020:10020" + environment: + DB_UPGRADE: 0 # run the DB upgrade + SETUP_APP: 0 # run the application default settings setup, can be turned off after initial run + + deploy: + replicas: 1 + + vcsserver: + environment: + {} + deploy: + replicas: 1 + + sshd: + {} + + svn: + {} + + celery-beat: + {} + + celery: + {} diff --git a/templates/docker-compose-metrics.override.yaml b/templates/docker-compose-metrics.override.yaml new file mode 100644 index 0000000..1641508 --- /dev/null +++ b/templates/docker-compose-metrics.override.yaml @@ -0,0 +1,31 @@ +version: '3.9' + +services: + + promtail: + {} + + loki: + labels: + - "traefik.enable=false" +# - "traefik.http.services.loki.loadbalancer.server.port=3100" +# - "traefik.http.services.loki.loadbalancer.server.port=9095" + + prometheus: + {} + + node-exporter: + {} + + statsd-exporter: + {} + + grafana: + + labels: + # SSL + proxy prefix + - "traefik.http.routers.grafana-rt.service=grafana-ssl" + - "traefik.http.routers.grafana-rt.entrypoints=https" + - "traefik.http.routers.grafana-rt.rule=Host(`your-domain.com`) && PathPrefix(`/_grafana`)" + - "traefik.http.routers.grafana-rt.tls=true" + - "traefik.http.services.grafana-ssl.loadbalancer.server.port=3000" \ No newline at end of file diff --git a/templates/docker-compose-router.override.yaml b/templates/docker-compose-router.override.yaml new file mode 100644 index 0000000..bb6fa0f --- /dev/null +++ b/templates/docker-compose-router.override.yaml @@ -0,0 +1,43 @@ +version: '3.9' + +## docker network create -d overlay lb-net +services: + + traefik: + + volumes: + # So that Traefik can listen to the Docker events + - /var/run/docker.sock:/var/run/docker.sock + # custom traefik config, enable SSL etc.. + - $PWD/.custom/traefik_custom:/etc/traefik + + portainer: + # Run with COMPOSE_PROFILES=portainer + # to access portainer set HEADER `X-Docker-Host=portainer` + image: portainer/portainer-ce:latest + restart: always + volumes: + - portainer_data:/data + - /var/run/docker.sock:/var/run/docker.sock + deploy: + mode: replicated + replicas: 1 + placement: + constraints: + # limit swarm deploy to MANAGER only + - node.role == manager + + networks: + - rhodecode_network + labels: + - "traefik.enable=true" + - "traefik.http.routers.portainer.entrypoints=https" + - "traefik.http.routers.portainer.rule=Headers(`X-Docker-Host`, `portainer`)" + - "traefik.http.services.portainer.loadbalancer.server.port=9000" + + profiles: + ["portainer"] + +volumes: + portainer_data: + external: true \ No newline at end of file diff --git a/templates/docker-compose-services.override.yaml b/templates/docker-compose-services.override.yaml new file mode 100644 index 0000000..ef2af63 --- /dev/null +++ b/templates/docker-compose-services.override.yaml @@ -0,0 +1,24 @@ +version: '3.9' +## This is a final override file for services stack +## Add your own customizations + +## docker network create -d overlay lb-net +services: + + database: + {} + + database-mysql: + {} + + redis: + {} + + elasticsearch: + {} + + nginx: + {} + + channelstream: + {} \ No newline at end of file