multiple changes

- nginx less rhodecode
- traefik changes
- new custom override templates
- service updates
- new docker volumes naming

.custom/README.md

@@ -0,0 +1,12 b''
+Put any custom overrides and compose files here.
+
+e.g
+
+- docker-compose-services.override.yaml
+- docker-compose-router.override.yaml
+- docker-compose-metrics.override.yaml
+- docker-compose-apps.override.yaml
+
+Passing variables to containers
+
+.runtime.env
  No newline at end of file

templates/docker-compose-apps.override.yaml

@@ -0,0 +1,31 b''
+version: '3.9'
+
+services:
+
+  rhodecode:
+    ports:
+      - "10020:10020"
+    environment:
+      DB_UPGRADE: 0  # run the DB upgrade
+      SETUP_APP: 0   # run the application default settings setup, can be turned off after initial run
+
+    deploy:
+      replicas: 1
+
+  vcsserver:
+    environment:
+      {}
+    deploy:
+      replicas: 1
+
+  sshd:
+    {}
+
+  svn:
+    {}
+
+  celery-beat:
+    {}
+
+  celery:
+    {}

templates/docker-compose-metrics.override.yaml

@@ -0,0 +1,31 b''
+version: '3.9'
+
+services:
+
+  promtail:
+    {}
+
+  loki:
+    labels:
+      - "traefik.enable=false"
+#      - "traefik.http.services.loki.loadbalancer.server.port=3100"
+#      - "traefik.http.services.loki.loadbalancer.server.port=9095"
+
+  prometheus:
+    {}
+
+  node-exporter:
+    {}
+
+  statsd-exporter:
+    {}
+
+  grafana:
+
+    labels:
+      # SSL + proxy prefix
+      - "traefik.http.routers.grafana-rt.service=grafana-ssl"
+      - "traefik.http.routers.grafana-rt.entrypoints=https"
+      - "traefik.http.routers.grafana-rt.rule=Host(`your-domain.com`) && PathPrefix(`/_grafana`)"
+      - "traefik.http.routers.grafana-rt.tls=true"
+      - "traefik.http.services.grafana-ssl.loadbalancer.server.port=3000"
  No newline at end of file

templates/docker-compose-router.override.yaml

@@ -0,0 +1,43 b''
+version: '3.9'
+
+## docker network create -d overlay lb-net
+services:
+
+  traefik:
+
+    volumes:
+      # So that Traefik can listen to the Docker events
+      - /var/run/docker.sock:/var/run/docker.sock
+      # custom traefik config, enable SSL etc..
+      - $PWD/.custom/traefik_custom:/etc/traefik
+
+  portainer:
+    # Run with COMPOSE_PROFILES=portainer
+    # to access portainer set HEADER `X-Docker-Host=portainer`
+    image: portainer/portainer-ce:latest
+    restart: always
+    volumes:
+      - portainer_data:/data
+      - /var/run/docker.sock:/var/run/docker.sock
+    deploy:
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints:
+          # limit swarm deploy to MANAGER only
+          - node.role == manager
+
+    networks:
+      - rhodecode_network
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.portainer.entrypoints=https"
+      - "traefik.http.routers.portainer.rule=Headers(`X-Docker-Host`, `portainer`)"
+      - "traefik.http.services.portainer.loadbalancer.server.port=9000"
+
+    profiles:
+      ["portainer"]
+
+volumes:
+  portainer_data:
+    external: true
  No newline at end of file

templates/docker-compose-services.override.yaml

@@ -0,0 +1,24 b''
+version: '3.9'
+## This is a final override file for services stack
+## Add your own customizations
+
+## docker network create -d overlay lb-net
+services:
+
+  database:
+    {}
+
+  database-mysql:
+    {}
+
+  redis:
+    {}
+
+  elasticsearch:
+    {}
+
+  nginx:
+    {}
+
+  channelstream:
+    {}
  No newline at end of file

.env

@@ -1,56 +1,56 b''
 #COMPOSE_PROJECT_NAME=rc_cluster
 TZ="UTC"
 
 
 # Version to deploy and run
 ## VERSION_INFO
 RC_VERSION="4.28.0.REL20221120_SOURCE"
 #RC_CLI_VERSION_NAME="4.27.0"
 
 # ce, or ee
 RC_EDITION=ee
-RC_IMAGE_CE="rhodecode/rhodecode-ee:$RC_VERSION"
-RC_IMAGE_EE="rhodecode/rhodecode-ce:$RC_VERSION"
+RC_IMAGE_CE="rhodecode/rhodecode-ce:$RC_VERSION"
+RC_IMAGE_EE="rhodecode/rhodecode-ee:$RC_VERSION"
 
 # Database bootstrap/access credentials
 DB_NAME=rhodecode
 DB_USER=rhodecode
 
 # Bootstrap container with specific password
 # This password will be used to generate new account and password for the database
 DB_PASSWORD=[GENERATED_AT_BOOTSTRAP]
 
 # set limit of redis in-memory cache before keys get evicted using LRU cache
 RC_REDIS_MAXMEMORY=1024mb
 
 # set the size of ramdisk used for locking files
 RC_DATA_RAMDISK_SIZE=1G
 
 # base url for running app, this is used for traefik hasname matching, so accessing this host will redirect
 # traffic to the running instance
 # replace this with your name of host. e.g http://yourdomain.com or https://rc.company.local
 RHODECODE_HOSTNAME=docker-dev
 
 
 # user/password for first admin user created for access
 # this is a bootstrap super-admin account to access the web interface
 RHODECODE_USER_EMAIL=admin@server-com
 RHODECODE_USER_NAME=admin
 RHODECODE_USER_PASS=secret4
 
 
 
 
 # HTTP and HTTPS ports for running app
 RC_HTTP_PORT=8888
 
 # SSH Port exposed, increased security is to not used default 22
 RC_SSH_PORT=3022
 
 
 ## RHODECODE RUNTIME VARS
 
 ## New since 4.27.0, default GIT branch name can be changed
 GIT_DEFAULT_BRANCH_NAME=master
 
 

.gitignore

@@ -1,23 +1,23 b''
 # CACHE FILES
 .cache/RhodeCode-installer*
 .cache/*.tar.bz2
 .cache/locale-archive
 
 # Source code downloaded
 .source/*
 
- # Custom / override wont be tracked
+ # Custom / override wont be tracked, except README file
 .custom/*
+!.custom/README.md
 
 # LOGS
 logs/*.log
 logs/nginx/*.log
 logs/svn/*.log
 logs/apps/*.log
 logs/postgres/*.log
 
 .idea
 config/_shared/.rcmetadata.json
 
-.rccontrol-bootstrap
 .rccontrol.ini
  No newline at end of file

config/nginx/http.conf

@@ -1,135 +1,144 b''
 # read more here http://tautt.com/best-nginx-configuration-for-security/
 
 # config to don't allow the browser to render the page inside an frame or iframe
 # and avoid clickjacking http://en.wikipedia.org/wiki/Clickjacking
 # if you need to allow [i]frames, you can use SAMEORIGIN or even set an uri with ALLOW-FROM uri
 # https://developer.mozilla.org/en-US/docs/HTTP/X-Frame-Options
 #add_header X-Frame-Options SAMEORIGIN;
 
 # when serving user-supplied content, include a X-Content-Type-Options: nosniff header along with the Content-Type: header,
 # to disable content-type sniffing on some browsers.
 # https://www.owasp.org/index.php/List_of_useful_HTTP_headers
 # currently suppoorted in IE > 8 http://blogs.msdn.com/b/ie/archive/2008/09/02/ie8-security-part-vi-beta-2-update.aspx
 # http://msdn.microsoft.com/en-us/library/ie/gg622941(v=vs.85).aspx
 # 'soon' on Firefox https://bugzilla.mozilla.org/show_bug.cgi?id=471020
 #add_header X-Content-Type-Options nosniff;
 
 # This header enables the Cross-site scripting (XSS) filter built into most recent web browsers.
 # It's usually enabled by default anyway, so the role of this header is to re-enable the filter for
 # this particular website if it was disabled by the user.
 # https://www.owasp.org/index.php/List_of_useful_HTTP_headers
 #add_header X-XSS-Protection "1; mode=block";
 
 # with Content Security Policy (CSP) enabled(and a browser that supports it(http://caniuse.com/#feat=contentsecuritypolicy),
 # you can tell the browser that it can only download content from the domains you explicitly allow
 # http://www.html5rocks.com/en/tutorials/security/content-security-policy/
 # https://www.owasp.org/index.php/Content_Security_Policy
 # I need to change our application code so we can increase security by disabling 'unsafe-inline' 'unsafe-eval'
 # directives for css and js(if you have inline css or js, you will need to keep it too).
 # more: http://www.html5rocks.com/en/tutorials/security/content-security-policy/#inline-code-considered-harmful
 #add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval' https://ssl.google-analytics.com https://assets.zendesk.com https://connect.facebook.net; img-src 'self' https://ssl.google-analytics.com https://s-static.ak.facebook.com https://assets.zendesk.com; style-src 'self' 'unsafe-inline' https://fonts.googleapis.com https://assets.zendesk.com; font-src 'self' https://themes.googleusercontent.com; frame-src https://assets.zendesk.com https://www.facebook.com https://s-static.ak.facebook.com https://tautt.zendesk.com; object-src 'none'";
 
 ## rate limiter for certain pages to prevent brute force attacks
 limit_req_zone  $binary_remote_addr  zone=http_req_limit:10m   rate=1r/s;
 
 ## custom log format
 log_format http_log_custom '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time $upstream_response_time $pipe';
 
 log_format json_log_custom escape=json
   '{'
     '"source":"nginx",'
     '"remote_user":"$remote_user",'
     '"time_local":"$time_local",'
     '"remote_addr":"$remote_addr",'
+    '"host":"$host",'
     '"proxy_x_forwarded_for":"$proxy_add_x_forwarded_for",'
     '"request":"$request",'
     '"status": "$status",'
     '"request_method": "$request_method",'
     '"body_bytes_sent":"$body_bytes_sent",'
     '"request_time":"$request_time",'
     '"upstream_response_time":"$upstream_response_time",'
     '"http_referrer":"$http_referer",'
+    '"http_scheme":"$scheme",'
     '"http_user_agent":"$http_user_agent"'
   '}';
 
 
 server {
   listen 80 default;
   # ensure we get the proper Docker DNS resolver for load balancing.
-  resolver 127.0.0.11 ipv6=off;
+  resolver 127.0.0.11 ipv6=off valid=10s;
   server_name localhost 127.0.0.1;
   access_log      /dev/stdout json_log_custom;
   error_log       /dev/stdout;
 
   # uncomment to redirect http traffic to https (not applicable in Docker setup)
   #return 301 https://$host$request_uri;
 
   client_body_buffer_size     128k;
   # maximum number and size of buffers for large headers to read from client request
   large_client_header_buffers 16 256k;
 
+  location /_health {
+    add_header 'Content-Type' 'application/json';
+    return 200 '{"status":"UP"}';
+  }
+
   ## serve static files by nginx, recommended
   location /_static/rhodecode {
       gzip on;
       gzip_min_length  500;
       gzip_proxied     any;
       gzip_comp_level 4;
       gzip_types  text/css text/javascript text/xml text/plain text/x-component application/javascript application/json application/xml application/rss+xml font/truetype font/opentype application/vnd.ms-fontobject image/svg+xml;
       gzip_vary on;
       gzip_disable     "msie6";
       expires 60d;
 
       alias /var/opt/rhodecode_data/static;
   }
 
   ## channelstream location handler, if channelstream live chat and notifications
   ## are enable this will proxy the requests to channelstream websocket server
+  set $upstream_channelstream  channelstream:8000;
   location /_channelstream {
       rewrite /_channelstream/(.*) /$1 break;
       gzip                         off;
       tcp_nodelay                  off;
 
       proxy_connect_timeout        10;
       proxy_send_timeout           10m;
       proxy_read_timeout           10m;
 
       proxy_set_header             Host $host;
-      proxy_set_header             X-Real-IP $remote_addr;
+      proxy_set_header             Referer $http_referer;
       proxy_set_header             X-Url-Scheme $scheme;
-      proxy_set_header             X-Forwarded-Proto $scheme;
+      proxy_set_header             X-Real-IP $remote_addr;
       proxy_set_header             X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header             X-Forwarded-Proto $scheme;
 
       proxy_http_version           1.1;
       proxy_set_header Upgrade     $http_upgrade;
       proxy_set_header Connection  "upgrade";
 
-      set $upstream_channelstream  http://channelstream:8000;
-      proxy_pass                   $upstream_channelstream;
+      proxy_pass                  http://$upstream_channelstream;
+      proxy_redirect              http://$upstream_channelstream/ /_channelstream;
   }
 
 #   ## rate limit this endpoint to prevent login page brute-force attacks
 #   location /_admin/login {
 #       limit_req  zone=http_req_limit  burst=10  nodelay;
 #       try_files $uri @rhodecode_http;
 #   }
 
-  location / {
-      include    /etc/nginx/proxy.conf;
-      try_files  $uri @rhodecode_http;
-  }
-
-  location @rhodecode_http {
-      set $upstream  http://rhodecode:10020;
-      include        /etc/nginx/proxy.conf;
-      proxy_pass     $upstream;
-  }
+#   location / {
+#       include    /etc/nginx/proxy.conf;
+#       try_files  $uri @rhodecode_http;
+#   }
+#
+#   set $upstream  http://rhodecode:10020;
+#   location @rhodecode_http {
+#       include        /etc/nginx/proxy.conf;
+#       proxy_pass     $upstream;
+#   }
 
   ## Custom 502 error page.
   ## Will be displayed while RhodeCode server is turned off
   error_page 502 /502.html;
   location = /502.html {
       root /var/opt/rhodecode_data/static;
       internal;
   }
 
 }

config/nginx/proxy.conf

@@ -1,36 +1,36 b''
 proxy_redirect              off;
 proxy_set_header            Host $http_host;
 
 ## If you use HTTPS make sure you disable gzip compression
 ## to be safe against BREACH attack.
 gzip                        off;
 
 # Don't buffer requests in NGINX stream them using chunked-encoding
 proxy_buffering             off;
 
 ## This is also required for later GIT to use streaming.
 ## Works only for Nginx 1.7.11 and newer
 proxy_request_buffering off;
 proxy_http_version 1.1;
 
 ## Set this to a larger number if you experience timeouts
 ## or 413 Request Entity Too Large, 10GB is enough for most cases
 client_max_body_size        10240m;
 
 ## needed for container auth
 # proxy_set_header          REMOTE_USER $remote_user;
 # proxy_set_header          X-Forwarded-User $remote_user;
 
 proxy_set_header            X-Url-Scheme $scheme;
 proxy_set_header            X-Host $http_host;
 proxy_set_header            X-Real-IP $remote_addr;
 proxy_set_header            X-Forwarded-For $proxy_add_x_forwarded_for;
 proxy_set_header            Proxy-host $proxy_host;
 
-proxy_connect_timeout       7200;
-proxy_send_timeout          7200;
-proxy_read_timeout          7200;
+proxy_connect_timeout       21600;
+proxy_send_timeout          21600;
+proxy_read_timeout          21600;
 proxy_buffers               8 32k;
 
 add_header X-Frame-Options SAMEORIGIN;
 add_header Strict-Transport-Security "max-age=31536000; includeSubdomains;";

config/traefik/traefik.yaml

@@ -1,42 +1,42 b''
 ## Static configuration, needs traefik reload
 
 entryPoints:
   http:
     address: ":80"
 
   https:
     address: ":443"
 
-#  ssh:
-#    address: ":${RC_SSH_PORT:?must-specify-ssh-port}"
+  ssh:
+    address: ":${RC_SSH_PORT:?must-specify-ssh-port}"
 
   traefik:
     address: ":7000"
 
 api:
   dashboard: false
   insecure: false # change to true to expose dashboard
 
 accessLog: {}
 
 log:
   # DEBUG / INFO
   level: INFO
 
 metrics:
   prometheus: {}
 
 
 providers:
   docker:
     exposedByDefault: false
     swarmMode: false
 
   file:
     filename: "/etc/traefik/traefik_dynamic.yaml"
     watch: true
 
 
 serversTransport:
   #this disables SSL certificate verification for legacy own cert services like NextCloud etc
   insecureSkipVerify: true

docker-compose-apps.yaml

@@ -1,312 +1,314 b''
 version: '3.9'
 
 x-logging: &custom-logging
   # docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
   # NOTE: loki logging driver ONLY works for host type networks...
   driver: loki
   options:
     loki-url: "http://127.0.0.1:3100/loki/api/v1/push"
     #loki-url: "http://loki:3100/loki/api/v1/push"
     loki-retries: "5"
     loki-timeout: "1s"
     loki-max-backoff: "800ms"
 
-volumes:
-
-  # volume for RhodeCode repo-store, it's where the repositories will be stored
-  rhodecode_repos:
-    labels:
-      "keep": 1
-
 
 services:
 
   rhodecode:
     networks:
       - rhodecode_network
     image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var}
     env_file:
       - .custom/.runtime.env
     stdin_open: true
     tty: true
     restart: always
     command: [
       "/usr/local/bin/rhodecode_bin/bin/gunicorn",
       "--name=gunicorn-rhodecode-1",
       "--error-logfile=-",
       "--paster=/etc/rhodecode/conf/rhodecode.optimized.ini",
-      "--config=/etc/rhodecode/conf/gunicorn_conf.py"
+      "--config=/etc/rhodecode/conf/gunicorn_conf_rc.py"
     ]
+    deploy:
+      # override this in .custom/docker-compose-apps.override.yaml to scale up
+      replicas: 1
 
 #    ports:
 #      - "127.0.0.1::10020"
 
     build:
       context: .
       dockerfile: service/rhodecode/rhodecode.dockerfile
       args:
         TZ: ${TZ}
         RHODECODE_VERSION: ${RC_VERSION:?specify-RC_VERSION-env-var}
         RHODECODE_DB: postgresql://rhodecode:${DB_PASSWORD:?must-specify-db-password}@database/${DB_NAME:?must-specify-db-name}
         RHODECODE_USER_NAME: ${RHODECODE_USER_NAME}
         RHODECODE_USER_PASS: ${RHODECODE_USER_PASS}
         RHODECODE_USER_EMAIL: ${RHODECODE_USER_EMAIL}
 
     environment:
       RC_APP_TYPE: rhodecode_http
       RC_APP_PROC: 1
       SSL_CERT_FILE: "/etc/rhodecode/conf/ca-bundle.crt"
       REQUESTS_CA_BUNDLE: "/etc/rhodecode/conf/ca-bundle.crt"
       GIT_SSL_CAINFO: "/etc/rhodecode/conf/ca-bundle.crt"
       GEVENT_RESOLVER: "ares"
 
       DB_UPGRADE: 1  # run the DB upgrade
       SETUP_APP: 1 # run the application default settings setup, can be turned off after initial run
       #FORCE_DB_INIT_FILE: 1 # force the database init, warning: destroys old DB
       #FORCE_RC_SETUP_APP: 1 # force running setup scripts for configuration/license application
       MAIN_INI_PATH: /etc/rhodecode/conf/rhodecode.optimized.ini
 
       # SVN Specific
       MOD_DAV_SVN_PORT: 8090
       APACHE_LOG_DIR: /var/log/rhodecode/svn
       MOD_DAV_SVN_CONF_FILE: /etc/rhodecode/conf/svn/mod_dav_svn.conf
 
     healthcheck:
       test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:10020/_admin/ops/ping" ]
       timeout: 30s
       interval: 60s
       retries: 10
 
 #    depends_on:
 #      - database
 #      - redis
 #      - channelstream
 
     volumes:
       - confvolume:/etc/rhodecode/conf
       - logvolume:/var/log/rhodecode
-      - rhodecode_repos:/var/opt/rhodecode_repo_store
+      - rc_reposvolume:/var/opt/rhodecode_repo_store
       - rc_datavolume:/var/opt/rhodecode_data
 
     tmpfs:
       - /data_ramdisk:size=${RC_DATA_RAMDISK_SIZE:?specify-RC_DATA_RAMDISK_SIZE-env-var}
 
     logging:
       *custom-logging
 
     labels:
-      - "traefik.enable=false"
+      - "traefik.enable=true"
+      - "traefik.http.routers.rhodecode.entrypoints=http"
+      - "traefik.http.routers.rhodecode.rule=Host(`${RHODECODE_HOSTNAME:?must-specify-rhodecode-hostname}`)"
+      - "traefik.http.services.rhodecode.loadbalancer.server.port=10020"
 
   vcsserver:
     networks:
       - rhodecode_network
     image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var}
     env_file:
       - .custom/.runtime.env
     stdin_open: true
     tty: true
     restart: always
     command: [
        "/home/rhodecode/.rccontrol/vcsserver-1/profile/bin/gunicorn",
        "--name=gunicorn-vcsserver-1",
        "--error-logfile=-",
        "--paster=/etc/rhodecode/conf/vcsserver.optimized.ini",
        "--config=/etc/rhodecode/conf/gunicorn_conf_vcs.py"
     ]
+    deploy:
+      # override this in .custom/docker-compose-apps.override.yaml to scale up
+      replicas: 1
 
 #    ports:
 #      - "127.0.0.1::10010"
 
     healthcheck:
       test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:10010/status" ]
       timeout: 30s
       interval: 60s
       retries: 10
 
     environment:
       RC_APP_TYPE: rhodecode_vcsserver
       RC_APP_PROC: 1
       MAIN_INI_PATH: /etc/rhodecode/conf/vcsserver.optimized.ini
       SSL_CERT_FILE: "/etc/rhodecode/conf/ca-bundle.crt"
       REQUESTS_CA_BUNDLE: "/etc/rhodecode/conf/ca-bundle.crt"
       GIT_SSL_CAINFO: "/etc/rhodecode/conf/ca-bundle.crt"
 
 #    depends_on:
 #      - redis
 
     volumes:
       - confvolume:/etc/rhodecode/conf
       - logvolume:/var/log/rhodecode
-      - rhodecode_repos:/var/opt/rhodecode_repo_store
+      - rc_reposvolume:/var/opt/rhodecode_repo_store
       - rc_datavolume:/var/opt/rhodecode_data
 
     logging:
       *custom-logging
 
   celery:
     networks:
       - rhodecode_network
     image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var}
     env_file:
       - .custom/.runtime.env
     stdin_open: true
     tty: true
     restart: always
     command: [
         "/usr/local/bin/rhodecode_bin/bin/celery",
         "worker",
         "--task-events",
         "--autoscale=10,2",
         "--no-color",
         "--app=rhodecode.lib.celerylib.loader",
         "--loglevel=DEBUG",
         "--ini=/etc/rhodecode/conf/rhodecode.optimized.ini"
     ]
 
     environment:
       RC_APP_TYPE: rhodecode_celery
       RC_APP_PROC: 1
       MAIN_INI_PATH: /etc/rhodecode/conf/rhodecode.optimized.ini
       SSL_CERT_FILE: "/etc/rhodecode/conf/ca-bundle.crt"
       REQUESTS_CA_BUNDLE: "/etc/rhodecode/conf/ca-bundle.crt"
       GIT_SSL_CAINFO: "/etc/rhodecode/conf/ca-bundle.crt"
 
 #    depends_on:
 #      - database
 #      - redis
 
     volumes:
       - confvolume:/etc/rhodecode/conf
       - logvolume:/var/log/rhodecode
-      - rhodecode_repos:/var/opt/rhodecode_repo_store
+      - rc_reposvolume:/var/opt/rhodecode_repo_store
       - rc_datavolume:/var/opt/rhodecode_data
 
     logging:
       *custom-logging
 
     labels:
       - "traefik.enable=false"
 
   celery-beat:
     # This service is not scalable
     networks:
       - rhodecode_network
     image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var}
     env_file:
       - .custom/.runtime.env
     stdin_open: true
     tty: true
     restart: always
     command: [
       "/usr/local/bin/rhodecode_bin/bin/celery",
       "beat",
       "--no-color",
       "--app=rhodecode.lib.celerylib.loader",
       "--scheduler=rhodecode.lib.celerylib.scheduler.RcScheduler",
       "--loglevel=DEBUG",
       "--ini=/etc/rhodecode/conf/rhodecode.optimized.ini"
     ]
 
     environment:
       RC_APP_TYPE: rhodecode_beat
       RC_APP_PROC: 1
       MAIN_INI_PATH: /etc/rhodecode/conf/rhodecode.optimized.ini
       SSL_CERT_FILE: "/etc/rhodecode/conf/ca-bundle.crt"
       REQUESTS_CA_BUNDLE: "/etc/rhodecode/conf/ca-bundle.crt"
       GIT_SSL_CAINFO: "/etc/rhodecode/conf/ca-bundle.crt"
 
 #    depends_on:
 #      - database
 #      - redis
 
     volumes:
       - confvolume:/etc/rhodecode/conf
       - logvolume:/var/log/rhodecode
-      - rhodecode_repos:/var/opt/rhodecode_repo_store
+      - rc_reposvolume:/var/opt/rhodecode_repo_store
       - rc_datavolume:/var/opt/rhodecode_data
 
     logging:
       *custom-logging
 
     labels:
       - "traefik.enable=false"
 
   svn:
     networks:
       - rhodecode_network
     image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var}
     env_file:
       - .custom/.runtime.env
 
 #    build:
 #      context: .
 #      dockerfile: service/svn/rhodecode_svn.dockerfile
 #      args:
 #        APACHE_VER: 1.3
 
     stdin_open: true
     tty: true
     restart: always
     command: ["apachectl", "-D", "FOREGROUND"]
 
     environment:
       RC_APP_TYPE: rhodecode_svn
 
       # SVN Specific
       MOD_DAV_SVN_PORT: 8090
       APACHE_LOG_DIR: /var/log/rhodecode/svn
       MOD_DAV_SVN_CONF_FILE: /etc/rhodecode/conf/svn/mod_dav_svn.conf
 
 #    ports:
 #      - "127.0.0.1::8090"
 
     healthcheck:
       test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:8090/_server_status" ]
       timeout: 30s
       interval: 60s
       retries: 10
 
     volumes:
       - confvolume:/etc/rhodecode/conf
       - logvolume:/var/log/rhodecode
-      - rhodecode_repos:/var/opt/rhodecode_repo_store
+      - rc_reposvolume:/var/opt/rhodecode_repo_store
 
     logging:
       *custom-logging
 
   sshd:
     networks:
       - rhodecode_network
     image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:?specify-RC_VERSION-env-var}
     env_file:
       - .custom/.runtime.env
 
     stdin_open: true
     tty: true
     restart: always
     command: ["/usr/sbin/sshd", "-f", "/etc/rhodecode/sshd_config", "-D", "-e"]
 
     environment:
       RC_APP_TYPE: rhodecode_sshd
       SSH_BOOTSTRAP: 1
 
 #    ports:
 #      # set from .env file
 #      - "${RC_SSH_PORT:?must-specify-ssh-port}:22"
 
     volumes:
       - confvolume:/etc/rhodecode/conf
       - logvolume:/var/log/rhodecode
-      - rhodecode_repos:/var/opt/rhodecode_repo_store
+      - rc_reposvolume:/var/opt/rhodecode_repo_store
       - rc_datavolume:/var/opt/rhodecode_data
 
     logging:
       *custom-logging
 
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.sshd.entrypoints=ssh"
       - "traefik.http.routers.sshd.rule=Host(`*`)"
       - "traefik.http.services.sshd.loadbalancer.server.port=${RC_SSH_PORT:?must-specify-ssh-port}"
 
 

docker-compose-base.yaml

@@ -1,46 +1,51 b''
 version: '3.9'
 
 ## Shared base stuff for all compose files in stack
 
 x-logging: &custom-logging
   # docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
   # NOTE: loki logging driver ONLY works for host type networks...
   driver: loki
   options:
     loki-url: "http://127.0.0.1:3100/loki/api/v1/push"
     #loki-url: "http://loki:3100/loki/api/v1/push"
     loki-retries: "5"
     loki-timeout: "1s"
     loki-max-backoff: "800ms"
 
 volumes:
   # shared logvolume
   logvolume:
     driver: local
     driver_opts:
       type: none
       o: bind
       device: $PWD/logs
 
   # bind-mount with configs
   confvolume:
     driver: local
     driver_opts:
      type: none
      o: bind
      device: $PWD/config/_shared
 
   # SHARED volume for rhodecode caches, archive caches, nginx static,
   # must be run via: docker volume create --name=rc_datavolume
   rc_datavolume:
     external: true
 
+  # volume for RhodeCode repo-store, it's where the repositories will be stored
+  # must be run via: docker volume create --name=rc_reposvolume
+  rc_reposvolume:
+    external: true
+
 networks:
 
   # SHARED network for all containers
   # must be run via: docker network create --name=rhodecode_network
   rhodecode_network:
     name: rhodecode_network
     driver: bridge
     external: true
 

docker-compose-metrics.yaml

@@ -1,128 +1,124 b''
 version: '3.9'
 
 volumes:
 
   prometheus_data:
     labels:
       "keep": 1
 
   grafana_data:
     labels:
       "keep": 1
 
 services:
 
   statsd-exporter:
     image: prom/statsd-exporter:v0.22.8
     restart: always
     command: [
       '--statsd.mapping-config=/etc/statsd/statsd.yaml',
       '--statsd.listen-udp=:9125',
       '--web.listen-address=:9102'
     ]
     networks:
       - rhodecode_network
 #    ports:
 #      - "9125:9125/udp"
 #      - "9102:9102"
 
     volumes:
       - ./config/statsd-exporter:/etc/statsd
     labels:
       - "traefik.enable=false"
       - "traefik.http.services.statsd-exporter.loadbalancer.server.port=9125"
       - "traefik.http.services.statsd-exporter.loadbalancer.server.port=9102"
 
   node-exporter:
     image: prom/node-exporter:v1.4.0
     command:
       - "--path.procfs=/host/proc"
       - "--path.rootfs=/rootfs"
       - "--path.sysfs=/host/sys"
       - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
       - "--collector.systemd"
       - "--collector.processes"
     networks:
       - rhodecode_network
     pid: host
     restart: always
 #    ports:
 #      - "9100:9100"
     volumes:
       - "/proc:/host/proc:ro"
       - "/sys:/host/sys:ro"
       - "/:/rootfs:ro"
     labels:
       - "traefik.enable=false"
       - "traefik.http.services.node-exporter.loadbalancer.server.port=9100"
 
   prometheus:
     image: prom/prometheus:v2.40.2
     restart: always
     command:
       - "--config.file=/etc/prometheus/prometheus.yml"
       - "--storage.tsdb.path=/prometheus"
       - "--web.console.libraries=/etc/prometheus/console_libraries"
       - "--web.console.templates=/etc/prometheus/consoles"
       - "--storage.tsdb.retention.time=24h"
       - "--web.enable-lifecycle"
 #    ports:
 #      - "9090:9090"
     networks:
       - rhodecode_network
     volumes:
       - ./config/prometheus:/etc/prometheus
       - prometheus_data:/prometheus
     labels:
       - "traefik.enable=false"
       - "traefik.http.services.prometheus.loadbalancer.server.port=9090"
 
   grafana:
     image: grafana/grafana:9.2.5
     restart: always
 #    ports:
 #      - "3000:3000"
     volumes:
       - grafana_data:/var/lib/grafana
       - ./config/grafana:/etc/grafana
     networks:
       - rhodecode_network
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.grafana.entrypoints=http"
       - "traefik.http.routers.grafana.rule=PathPrefix(`/_grafana`)"
       - "traefik.http.routers.grafana.service=grafana-http"
       - "traefik.http.services.grafana-http.loadbalancer.server.port=3000"
 
   loki:
     image: grafana/loki:2.7.0
     restart: always
 #    ports:
 #      - "3100:3100"
 #      - "9095:9095"
     command: [
       "-config.file=/etc/loki/loki-config.yaml"
     ]
     networks:
       - rhodecode_network
     volumes:
       - ./config/loki:/etc/loki
 
-    labels:
-      - "traefik.enable=false"
-      - "traefik.http.services.loki.loadbalancer.server.port=3100"
-      - "traefik.http.services.loki.loadbalancer.server.port=9095"
 
   promtail:
     image: grafana/promtail:latest
     command: [
       "-config.file=/etc/promtail/promtail-config.yaml"
     ]
     networks:
       - rhodecode_network
     volumes:
       - /var/log:/var/log
       - logvolume:/var/log_volume
       - ./config/promtail:/etc/promtail
     labels:
       - "traefik.enable=false"

docker-compose-router.yaml

@@ -1,78 +1,47 b''
 version: '3.9'
 
 x-logging: &custom-logging
   # docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
   # NOTE: loki logging driver ONLY works for host type networks...
   driver: loki
   options:
     loki-url: "http://127.0.0.1:3100/loki/api/v1/push"
     #loki-url: "http://loki:3100/loki/api/v1/push"
     loki-retries: "5"
     loki-timeout: "1s"
     loki-max-backoff: "800ms"
 
 ## docker network create -d overlay lb-net
 services:
 
   traefik:
 
     image: traefik:v2.9.5
 
     ports:
       # The HTTP port, exposed as http entrypoint
       - "80:80"
       # The HTTPS port, exposed as https entrypoint
       - "443:443"
       # The SSH port
       - "${RC_SSH_PORT}:${RC_SSH_PORT}"
       # The Web UI (enabled by --api.insecure=true)
       # uncomment to expose dashboard at port :7000
       #- "7000:7000"
     volumes:
       # So that Traefik can listen to the Docker events
       - /var/run/docker.sock:/var/run/docker.sock
       - ./config/traefik:/etc/traefik
     deploy:
       placement:
         constraints:
           # limit swarm deploy to MANAGER only
           - node.role == manager
     networks:
       - rhodecode_network
 
     labels:
       - "traefik.enable=true"
 
     logging:
       *custom-logging
-
-  portainer:
-    # Run with COMPOSE_PROFILES=portainer
-    # to access portainer set HEADER `X-Docker-Host=portainer`
-    image: portainer/portainer-ce:latest
-    restart: always
-    volumes:
-      - portainer_data:/data
-      - /var/run/docker.sock:/var/run/docker.sock
-    deploy:
-      mode: replicated
-      replicas: 1
-      placement:
-        constraints:
-          # limit swarm deploy to MANAGER only
-          - node.role == manager
-
-    networks:
-      - rhodecode_network
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.services.portainer.loadbalancer.server.port=9000"
-      - "traefik.http.routers.portainer.entrypoints=https"
-      - "traefik.http.routers.portainer.rule=Headers(`X-Docker-Host`, `portainer`)"
-
-    profiles:
-      ["portainer"]
-
-volumes:
-  portainer_data:
-    external: true
  No newline at end of file

docker-compose-services.yaml

@@ -1,240 +1,240 b''
 version: '3.9'
 
 x-logging: &custom-logging
   # docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
   # NOTE: loki logging driver ONLY works for host type networks...
   driver: loki
   options:
     loki-url: "http://127.0.0.1:3100/loki/api/v1/push"
     #loki-url: "http://loki:3100/loki/api/v1/push"
     loki-retries: "5"
     loki-timeout: "1s"
     loki-max-backoff: "800ms"
 
 volumes:
 
   # volume for redis data store
   redis_data:
     labels:
       "keep": 1
 
    # volume for Postgres db store
 
   # volume for Postgres Data
   pg_data:
     labels:
       "keep": 1
 
   # volume for rhodecode elasticsearch
   es_data:
     labels:
       "keep": 1
 
 
 services:
 
   channelstream:
     networks:
       - rhodecode_network
     image: channelstream/channelstream:0.7.1
 
     restart: always
 
 #    ports:
 #      - "127.0.0.1:9800:9800"
 
-    command: ["channelstream", "-i", "/etc/rhodecode/conf/channelstream.ini"]
+    command: ["channelstream"]
 
-    environment:
-      CHANNELSTREAM_ALLOW_POSTING_FROM: 0.0.0.0
+    env_file:
+      - .custom/.runtime.env
 
     healthcheck:
         test: [ "CMD", "curl", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://channelstream:8000/admin/sign_in" ]
         timeout: 5s
         interval: 60s
         retries: 10
 
     volumes:
       - confvolume:/etc/rhodecode/conf
       - logvolume:/var/log/rhodecode
 
     logging:
       *custom-logging
 
     labels:
       - "traefik.enable=false"
       #- "traefik.http.routers.channelstream.entrypoints=http"
       #- "traefik.http.services.channelstream.loadbalancer.server.port=9800"
 
     profiles:
       ["channelstream"]
 
   nginx:
     networks:
       - rhodecode_network
     image: library/nginx:1.23.2
 
     restart: always
 
     environment:
       NGINX_ENTRYPOINT_QUIET_LOGS: 1
 
     healthcheck:
       # change port 80 to 443 when only using SSL
-      test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:80/_admin/ops/ping" ]
+      test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "http://127.0.0.1:80/_health" ]
       timeout: 30s
       interval: 60s
       retries: 10
 
 #    depends_on:
 #      - channelstream
 
     volumes:
       - ./config/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
       - ./config/nginx/http.conf:/etc/nginx/http.conf:ro
       - ./config/nginx/proxy.conf:/etc/nginx/proxy.conf:ro
       - logvolume:/var/log/rhodecode
       - rc_datavolume:/var/opt/rhodecode_data
 
     logging:
       *custom-logging
 
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.nginx.entrypoints=http"
+      - "traefik.http.routers.nginx.rule=Host(`${RHODECODE_HOSTNAME:?must-specify-rhodecode-hostname}`) && ( PathPrefix(`/_health`) || PathPrefix(`/_channelstream`) || PathPrefix(`/_static/rhodecode`) )"
       - "traefik.http.services.nginx.loadbalancer.server.port=80"
-      - "traefik.http.routers.nginx.rule=Host(`${RHODECODE_HOSTNAME:?must-specify-rhodecode-hostname}`)"
 
   elasticsearch:
     networks:
       - rhodecode_network
     image: elasticsearch:6.8.23
 
     environment:
       - cluster.name=elasticsearch-cluster
       - network.host=0.0.0.0
       - bootstrap.memory_lock=true
       - discovery.type=single-node
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
     healthcheck:
       # change port 80 to 443 when only using SSL
       test: [ "CMD", "curl", "-A", "RhodeCode-Healthcheck", "-s", "-o", "/dev/null", "-w", "'%{http_code}'", "elasticsearch:9200/_cat/health" ]
       timeout: 30s
       interval: 60s
       retries: 10
 
     ulimits:
       memlock:
         soft: -1
         hard: -1
 
     volumes:
       - es_data:/usr/share/elasticsearch/data
 
     logging:
       *custom-logging
 
     profiles:
       ["elasticsearch"]
 
   redis:
     networks:
       - rhodecode_network
     image: library/redis:7.0.5
 
     restart: always
     command:
       - "redis-server"
       - "/etc/redis/redis.conf"
       - "--maxmemory-policy allkeys-lru"
       - "--maxmemory ${RC_REDIS_MAXMEMORY}"
 
 #    ports:
 #      - "127.0.0.1::6379"
 
     healthcheck:
       test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
       interval: 60s
 
     volumes:
       - ./config/redis/redis.conf:/etc/redis/redis.conf:ro
       - logvolume:/var/log/rhodecode
       - redis_data:/data
 
     profiles:
       ["redis"]
 
     logging:
       *custom-logging
 
   database:
     networks:
       - rhodecode_network
     image: library/postgres:14.6
 
     environment:
       POSTGRES_DB: ${DB_NAME:?must-specify-db-name}
       POSTGRES_USER: ${DB_USER:?must-specify-db-user}
       PGUSER: ${DB_USER:?must-specify-db-user}
       POSTGRES_PASSWORD: ${DB_PASSWORD:?must-specify-db-password}
       POSTGRES_HOST_AUTH_METHOD: md5
       POSTGRES_INITDB_ARGS: "--auth-host=md5 --auth-local=md5"
 
     restart: unless-stopped
     command:
       - "postgres"
       - "-c"
       - "log_statement=ddl"
       - "-c"
       - "config_file=/etc/conf.d/pg_customized.conf"
 
 #    ports:
 #      - "127.0.0.1::5432"
 
     healthcheck:
       test: ["CMD-SHELL", "pg_isready", '-U', "$DB_USER"]
       interval: 10s
       timeout: 5s
       retries: 5
 
     volumes:
       - $PWD/config/database/pg_customized.conf:/etc/conf.d/pg_customized.conf:ro
       # db dumps reverse mount
       - $PWD/.custom/db_dump:/var/rc-data-dump
       # save the pg_data volume
       - pg_data:/var/lib/postgresql/data
       - logvolume:/var/log/rhodecode
 
     profiles:
       ["postgres"]
 
     logging:
       *custom-logging
 
   database-mysql:
       networks:
         - rhodecode_network
       image: library/mysql:8.0.31
 
       environment:
         MYSQL_DATABASE: ${DB_NAME:?must-specify-db-name}
         MYSQL_USER: ${DB_USER:?must-specify-db-user}
         MYSQL_PASSWORD: ${DB_PASSWORD:?must-specify-db-password}
         MYSQL_ROOT_PASSWORD: ${DB_PASSWORD:?must-specify-db-password}
 
       restart: unless-stopped
 
 #      ports:
 #        - "127.0.0.1::3306"
 
       volumes:
         - ./config/database/mysql_customized.conf:/etc/mysql/conf.d/config-file.cnf:ro
         # save the mysql_data volume
         - $PWD/mysql_dir:/var/lib/mysql
         - logvolume:/var/log/rhodecode
 
       profiles:
         ["mysql"]
 
       logging:
         *custom-logging

scripts/rccontrol/rccontrol

@@ -1,2889 +1,3663 b''
 #!/usr/bin/env bash
 # This script was generated by bashly 0.8.10 (https://bashly.dannyb.co)
 # Modifying it manually is not recommended
 
 # :wrapper.bash3_bouncer
 if [[ "${BASH_VERSINFO:-0}" -lt 4 ]]; then
   printf "bash version 4 or higher is required\n" >&2
   exit 1
 fi
 
 # :command.master_script
 
 # :command.version_command
 version_command() {
   echo "$version"
 }
 
 # :command.usage
 rccontrol_usage() {
   if [[ -n $long_usage ]]; then
     printf "rccontrol - RhodeCode Control - CLI for manaing RhodeCode Cluster Stack\n"
     echo
 
   else
     printf "rccontrol - RhodeCode Control - CLI for manaing RhodeCode Cluster Stack\n"
     echo
 
   fi
 
   printf "Usage:\n"
   printf "  rccontrol [OPTIONS] COMMAND\n"
   printf "  rccontrol [COMMAND] --help | -h\n"
   printf "  rccontrol --version | -v\n"
   echo
   # :command.usage_commands
   printf "Commands:\n"
   echo "  self-update           update rccontrol and it's docker definitions"
   echo "  bootstrap             Bootstrap this machine, check docker version and install rhodecode-network"
-  echo "  stack                 run one of available cluster stacks, use -h for more details"
-  echo "  stack-status          show stack status"
-  echo "  stack-upgrade         upgrade ALL stack status"
   echo
   printf "Build Commands:\n"
   echo "  get-build-artifacts   Fetch Artifacts to run installer based build"
   echo "  build                 Build RhodeCode image from installer"
   echo "  get-build-source      Fetch RhodeCode sources, store in .source dir to run a source-based builds"
   echo "  build-source          Build RhodeCode image from source, requires upgrade-source initially"
   echo
+  printf "Stack Commands:\n"
+  echo "  stack                 run one of available cluster stacks, use -h for more details"
+  echo "  stack-status          Show stack status"
+  echo "  stack-upgrade         upgrade ALL stack status"
+  echo
   printf "CLI Commands:\n"
-  echo "  cli-redis             CLI"
-  echo "  cli-db                CLI"
-  echo "  cli-db-upgrade        CLI"
+  echo "  cli                   run various CLI tools"
   echo
   printf "Backup Commands:\n"
   echo "  backup-db             CLI"
   echo "  backup-data           CLI"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
     echo "  --version, -v"
     printf "    Show version number\n"
     echo
 
     # :command.usage_flags
     # :flag.usage
     echo "  --debug"
     printf "    Enable debug and detailed output\n"
     echo
 
     # :command.usage_environment_variables
     printf "Environment Variables:\n"
 
     # :environment_variable.usage
     echo "  AUTH_TOKEN"
     printf "    Set your RhodeCode AUTH Token\n"
     echo
 
     # :environment_variable.usage
     echo "  RC_CLI_VERSION_NAME"
     printf "    default version to build and install\n"
     printf "    Default: 4.27.0\n"
     echo
 
     # :environment_variable.usage
     echo "  RC_STACK_ROUTER_EXT"
     printf "\n"
     printf "    Default: .custom/docker-compose-router.override.yaml\n"
     echo
 
     # :environment_variable.usage
     echo "  RC_STACK_METRICS_EXT"
     printf "\n"
     printf "    Default: .custom/docker-compose-metrics.override.yaml\n"
     echo
 
     # :environment_variable.usage
     echo "  RC_STACK_SERVICES_EXT"
     printf "\n"
     printf "    Default: .custom/docker-compose-services.override.yaml\n"
     echo
 
     # :environment_variable.usage
     echo "  RC_STACK_RHODECODE_EXT"
     printf "\n"
     printf "    Default: .custom/docker-compose-apps.override.yaml\n"
     echo
 
     # :command.footer
     printf "RhodeCode Inc 2022\n\n"
     echo
 
   fi
 }
 
 # :command.usage
 rccontrol_self_update_usage() {
   if [[ -n $long_usage ]]; then
     printf "rccontrol self-update - update rccontrol and it's docker definitions\n"
     echo
 
   else
     printf "rccontrol self-update - update rccontrol and it's docker definitions\n"
     echo
 
   fi
 
   printf "Usage:\n"
   printf "  rccontrol self-update [OPTIONS]\n"
   printf "  rccontrol self-update --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
     # :command.usage_flags
     # :flag.usage
     echo "  --auth-token AUTH_TOKEN"
     printf "    Optionally specify AUTH TOKEN to obtain sources\n"
     echo
 
     # :flag.usage
     echo "  --server-url SERVER_URL"
     printf "    Specify RhodeCode server location where projects should be downloaded\n"
     printf "    Default: https://code.rhodecode.com\n"
     echo
 
   fi
 }
 
 # :command.usage
 rccontrol_bootstrap_usage() {
   if [[ -n $long_usage ]]; then
     printf "rccontrol bootstrap - Bootstrap this machine, check docker version and install rhodecode-network\n"
     echo
 
   else
     printf "rccontrol bootstrap - Bootstrap this machine, check docker version and install rhodecode-network\n"
     echo
 
   fi
 
-  printf "Alias: install\n"
+  printf "Alias: init\n"
   echo
 
   printf "Usage:\n"
   printf "  rccontrol bootstrap [OPTIONS]\n"
   printf "  rccontrol bootstrap --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
     # :command.usage_flags
     # :flag.usage
     echo "  --force, -f"
     printf "    Overwrite existing files\n"
     echo
 
     # :flag.usage
     echo "  --auth-token AUTH_TOKEN"
     printf "    Optionally specify AUTH TOKEN to obtain sources\n"
     echo
 
     # :flag.usage
     echo "  --server-url SERVER_URL"
     printf "    Specify RhodeCode server location where projects should be downloaded\n"
     printf "    Default: https://code.rhodecode.com\n"
     echo
 
     # :command.usage_examples
     printf "Examples:\n"
     printf "  rccontrol3 bootstrap\n"
     printf "  rccontrol3 bootstrap --force\n"
     echo
 
   fi
 }
 
 # :command.usage
 rccontrol_get_build_artifacts_usage() {
   if [[ -n $long_usage ]]; then
     printf "rccontrol get-build-artifacts - Fetch Artifacts to run installer based build\n"
     echo
 
   else
     printf "rccontrol get-build-artifacts - Fetch Artifacts to run installer based build\n"
     echo
 
   fi
 
   printf "Usage:\n"
   printf "  rccontrol get-build-artifacts [OPTIONS]\n"
   printf "  rccontrol get-build-artifacts --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
     # :command.usage_flags
     # :flag.usage
     echo "  --auth AUTH"
     printf "    Specify custom auth for curl e.g -u admin:secret\n"
     printf "    Default: \n"
     echo
 
     # :flag.usage
     echo "  --installer-url INSTALLER_URL"
     printf "    Installer Download URL\n"
     printf "    Default: https://dls.rhodecode.com/dls/N2E2ZTY1NzA3NjYxNDA2NTc1NjI3MTcyNzA2MjcxNzIyZTcwNjI3YQ==/rhodecode-control/latest-linux-ee\n"
     echo
 
     # :flag.usage
     echo "  --manifest-url MANIFEST_URL"
     printf "    Manifest file url\n"
     printf "    Default: https://dls.rhodecode.com/linux/MANIFEST\n"
     echo
 
     # :flag.usage
     echo "  --version-name VERSION_NAME"
     printf "    Specify custom build ver e.g $RC_CLI_VERSION_NAME\n"
     printf "    Default: $RC_CLI_VERSION_NAME\n"
     echo
 
   fi
 }
 
 # :command.usage
 rccontrol_build_usage() {
   if [[ -n $long_usage ]]; then
     printf "rccontrol build - Build RhodeCode image from installer\n"
     echo
 
   else
     printf "rccontrol build - Build RhodeCode image from installer\n"
     echo
 
   fi
 
   printf "Usage:\n"
   printf "  rccontrol build [OPTIONS]\n"
   printf "  rccontrol build --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
     # :command.usage_flags
     # :flag.usage
     echo "  --version-name VERSION_NAME"
     printf "    Specify custom build ver e.g $RC_CLI_VERSION_NAME\n"
     printf "    Default: $RC_CLI_VERSION_NAME\n"
     echo
 
   fi
 }
 
 # :command.usage
 rccontrol_get_build_source_usage() {
   if [[ -n $long_usage ]]; then
     printf "rccontrol get-build-source - Fetch RhodeCode sources, store in .source dir to run a source-based builds\n"
     echo
 
   else
     printf "rccontrol get-build-source - Fetch RhodeCode sources, store in .source dir to run a source-based builds\n"
     echo
 
   fi
 
   printf "Usage:\n"
-  printf "  rccontrol get-build-source REVISION [OPTIONS]\n"
+  printf "  rccontrol get-build-source [OPTIONS]\n"
   printf "  rccontrol get-build-source --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
     # :command.usage_flags
     # :flag.usage
+    echo "  --revision REVISION"
+    printf "    revision to download\n"
+    printf "    Default: default\n"
+    echo
+
+    # :flag.usage
     echo "  --auth-token AUTH_TOKEN"
     printf "    Specify AUTH TOKEN to obtain sources\n"
     echo
 
     # :flag.usage
     echo "  --server-url SERVER_URL"
     printf "    Specify RhodeCode server location where projects should be downloaded\n"
     printf "    Default: https://code.rhodecode.com\n"
     echo
 
-    # :command.usage_args
-    printf "Arguments:\n"
-
-    # :argument.usage
-    echo "  REVISION"
-    printf "    revision to download\n"
-    echo
-
     # :command.usage_examples
     printf "Examples:\n"
     printf "  rccontrol3 get-sources $RC_CLI_VERSION_NAME\n"
     printf "  rccontrol3 get-sources default --auth-token xyxyxyx --server-url\n  https://secret.repo/\n"
     echo
 
   fi
 }
 
 # :command.usage
 rccontrol_build_source_usage() {
   if [[ -n $long_usage ]]; then
     printf "rccontrol build-source - Build RhodeCode image from source, requires upgrade-source initially\n"
     echo
 
   else
     printf "rccontrol build-source - Build RhodeCode image from source, requires upgrade-source initially\n"
     echo
 
   fi
 
   printf "Usage:\n"
   printf "  rccontrol build-source [OPTIONS]\n"
   printf "  rccontrol build-source --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
     # :command.usage_flags
     # :flag.usage
     echo "  --version-name VERSION_NAME"
     printf "    Specify custom build ver e.g $RC_CLI_VERSION_NAME\n"
     printf "    Default: $RC_CLI_VERSION_NAME\n"
     echo
 
     # :command.usage_examples
     printf "Examples:\n"
     printf "  build foo\n"
     echo
 
   fi
 }
 
 # :command.usage
 rccontrol_stack_usage() {
   if [[ -n $long_usage ]]; then
     printf "rccontrol stack - run one of available cluster stacks, use -h for more details\n"
     echo
 
   else
     printf "rccontrol stack - run one of available cluster stacks, use -h for more details\n"
     echo
 
   fi
 
   printf "Usage:\n"
-  printf "  rccontrol stack NAME [SERVICES PARAMS...]\n"
-  printf "  rccontrol stack --help | -h\n"
+  printf "  rccontrol stack COMMAND\n"
+  printf "  rccontrol stack [COMMAND] --help | -h\n"
+  echo
+  # :command.usage_commands
+  printf "Commands:\n"
+  echo "  router      run the router stack"
+  echo "  metrics     run the router stack"
+  echo "  services    run the router stack"
+  echo "  rhodecode   run the router stack"
+  echo "  all         run all stacks"
+  echo
+
+  # :command.long_usage
+  if [[ -n $long_usage ]]; then
+    printf "Options:\n"
+
+    # :command.usage_fixed_flags
+    echo "  --help, -h"
+    printf "    Show this help\n"
+    echo
+
+    # :command.usage_examples
+    printf "Examples:\n"
+    printf "  - ./rccontrol3 stack router up          # run router stack with output to\n  console\n  - ./rccontrol3 stack router up --detach # run router stack detached\n  - ./rccontrol3 stack router down        # stop whole router stack\n  - ./rccontrol3 stack router ps          # check status of router stack\n  - ./rccontrol3 stack router -f docker-overrides.yaml up -d # run router stack\n  with your overrides\n"
+    echo
+
+  fi
+}
+
+# :command.usage
+rccontrol_stack_router_usage() {
+  if [[ -n $long_usage ]]; then
+    printf "rccontrol stack router - run the router stack\n"
+    echo
+
+  else
+    printf "rccontrol stack router - run the router stack\n"
+    echo
+
+  fi
+
+  printf "Usage:\n"
+  printf "  rccontrol stack router [SERVICES PARAMS...]\n"
+  printf "  rccontrol stack router --help | -h\n"
+  echo
+
+  # :command.long_usage
+  if [[ -n $long_usage ]]; then
+    printf "Options:\n"
+
+    # :command.usage_fixed_flags
+    echo "  --help, -h"
+    printf "    Show this help\n"
+    echo
+
+    # :command.usage_args
+    printf "Arguments:\n"
+
+    echo "  SERVICES PARAMS..."
+    printf "    Additional arguments or flags for services command\n"
+    echo
+
+  fi
+}
+
+# :command.usage
+rccontrol_stack_metrics_usage() {
+  if [[ -n $long_usage ]]; then
+    printf "rccontrol stack metrics - run the router stack\n"
+    echo
+
+  else
+    printf "rccontrol stack metrics - run the router stack\n"
+    echo
+
+  fi
+
+  printf "Usage:\n"
+  printf "  rccontrol stack metrics [SERVICES PARAMS...]\n"
+  printf "  rccontrol stack metrics --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
     # :command.usage_args
     printf "Arguments:\n"
 
-    # :argument.usage
-    echo "  NAME"
-    printf "    Stack name\n"
-    printf "    Allowed: router, metrics, services, rhodecode\n"
+    echo "  SERVICES PARAMS..."
+    printf "    Additional arguments or flags for services command\n"
+    echo
+
+  fi
+}
+
+# :command.usage
+rccontrol_stack_services_usage() {
+  if [[ -n $long_usage ]]; then
+    printf "rccontrol stack services - run the router stack\n"
+    echo
+
+  else
+    printf "rccontrol stack services - run the router stack\n"
+    echo
+
+  fi
+
+  printf "Usage:\n"
+  printf "  rccontrol stack services [SERVICES PARAMS...]\n"
+  printf "  rccontrol stack services --help | -h\n"
+  echo
+
+  # :command.long_usage
+  if [[ -n $long_usage ]]; then
+    printf "Options:\n"
+
+    # :command.usage_fixed_flags
+    echo "  --help, -h"
+    printf "    Show this help\n"
     echo
 
+    # :command.usage_args
+    printf "Arguments:\n"
+
     echo "  SERVICES PARAMS..."
     printf "    Additional arguments or flags for services command\n"
     echo
 
-    # :command.usage_examples
-    printf "Examples:\n"
-    printf "  - ./rccontrol3 stack router up          # run router stack with output to\n  console\n  - ./rccontrol3 stack router up --detach # run router stack detached\n  - ./rccontrol3 stack router down        # stop whole router stack\n  - ./rccontrol3 stack router ps          # check status of router stack\n  - ./rccontrol3 stack router -f docker-overrides.yaml up -d # run router stack\n  with your overrides\n"
+  fi
+}
+
+# :command.usage
+rccontrol_stack_rhodecode_usage() {
+  if [[ -n $long_usage ]]; then
+    printf "rccontrol stack rhodecode - run the router stack\n"
+    echo
+
+  else
+    printf "rccontrol stack rhodecode - run the router stack\n"
+    echo
+
+  fi
+
+  printf "Usage:\n"
+  printf "  rccontrol stack rhodecode [SERVICES PARAMS...]\n"
+  printf "  rccontrol stack rhodecode --help | -h\n"
+  echo
+
+  # :command.long_usage
+  if [[ -n $long_usage ]]; then
+    printf "Options:\n"
+
+    # :command.usage_fixed_flags
+    echo "  --help, -h"
+    printf "    Show this help\n"
+    echo
+
+    # :command.usage_args
+    printf "Arguments:\n"
+
+    echo "  SERVICES PARAMS..."
+    printf "    Additional arguments or flags for services command\n"
+    echo
+
+  fi
+}
+
+# :command.usage
+rccontrol_stack_all_usage() {
+  if [[ -n $long_usage ]]; then
+    printf "rccontrol stack all - run all stacks\n"
+    echo
+
+  else
+    printf "rccontrol stack all - run all stacks\n"
+    echo
+
+  fi
+
+  printf "Usage:\n"
+  printf "  rccontrol stack all [SERVICES PARAMS...]\n"
+  printf "  rccontrol stack all --help | -h\n"
+  echo
+
+  # :command.long_usage
+  if [[ -n $long_usage ]]; then
+    printf "Options:\n"
+
+    # :command.usage_fixed_flags
+    echo "  --help, -h"
+    printf "    Show this help\n"
+    echo
+
+    # :command.usage_args
+    printf "Arguments:\n"
+
+    echo "  SERVICES PARAMS..."
+    printf "    Additional arguments or flags for services command\n"
     echo
 
   fi
 }
 
 # :command.usage
 rccontrol_stack_status_usage() {
   if [[ -n $long_usage ]]; then
-    printf "rccontrol stack-status - show stack status\n"
+    printf "rccontrol stack-status - Show stack status\n"
     echo
 
   else
-    printf "rccontrol stack-status - show stack status\n"
+    printf "rccontrol stack-status - Show stack status\n"
     echo
 
   fi
 
   printf "Alias: status\n"
   echo
 
   printf "Usage:\n"
   printf "  rccontrol stack-status\n"
   printf "  rccontrol stack-status --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
   fi
 }
 
 # :command.usage
 rccontrol_stack_upgrade_usage() {
   if [[ -n $long_usage ]]; then
     printf "rccontrol stack-upgrade - upgrade ALL stack status\n"
     echo
 
   else
     printf "rccontrol stack-upgrade - upgrade ALL stack status\n"
     echo
 
   fi
 
   printf "Usage:\n"
   printf "  rccontrol stack-upgrade\n"
   printf "  rccontrol stack-upgrade --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
   fi
 }
 
 # :command.usage
+rccontrol_cli_usage() {
+  if [[ -n $long_usage ]]; then
+    printf "rccontrol cli - run various CLI tools\n"
+    echo
+
+  else
+    printf "rccontrol cli - run various CLI tools\n"
+    echo
+
+  fi
+
+  printf "Usage:\n"
+  printf "  rccontrol cli COMMAND\n"
+  printf "  rccontrol cli [COMMAND] --help | -h\n"
+  echo
+  # :command.usage_commands
+  printf "Commands:\n"
+  echo "  redis        CLI for Redis"
+  echo "  db           CLI with Database connection"
+  echo "  db-upgrade   CLI to run db upgrade"
+  echo "  storage      CLI for repository storage"
+  echo
+
+  # :command.long_usage
+  if [[ -n $long_usage ]]; then
+    printf "Options:\n"
+
+    # :command.usage_fixed_flags
+    echo "  --help, -h"
+    printf "    Show this help\n"
+    echo
+
+  fi
+}
+
+# :command.usage
 rccontrol_cli_redis_usage() {
   if [[ -n $long_usage ]]; then
-    printf "rccontrol cli-redis - CLI\n"
+    printf "rccontrol cli redis - CLI for Redis\n"
     echo
 
   else
-    printf "rccontrol cli-redis - CLI\n"
+    printf "rccontrol cli redis - CLI for Redis\n"
     echo
 
   fi
 
   printf "Usage:\n"
-  printf "  rccontrol cli-redis\n"
-  printf "  rccontrol cli-redis --help | -h\n"
+  printf "  rccontrol cli redis\n"
+  printf "  rccontrol cli redis --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
   fi
 }
 
 # :command.usage
 rccontrol_cli_db_usage() {
   if [[ -n $long_usage ]]; then
-    printf "rccontrol cli-db - CLI\n"
+    printf "rccontrol cli db - CLI with Database connection\n"
     echo
 
   else
-    printf "rccontrol cli-db - CLI\n"
+    printf "rccontrol cli db - CLI with Database connection\n"
     echo
 
   fi
 
   printf "Usage:\n"
-  printf "  rccontrol cli-db\n"
-  printf "  rccontrol cli-db --help | -h\n"
+  printf "  rccontrol cli db\n"
+  printf "  rccontrol cli db --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
   fi
 }
 
 # :command.usage
 rccontrol_cli_db_upgrade_usage() {
   if [[ -n $long_usage ]]; then
-    printf "rccontrol cli-db-upgrade - CLI\n"
+    printf "rccontrol cli db-upgrade - CLI to run db upgrade\n"
+    echo
+
+  else
+    printf "rccontrol cli db-upgrade - CLI to run db upgrade\n"
+    echo
+
+  fi
+
+  printf "Usage:\n"
+  printf "  rccontrol cli db-upgrade\n"
+  printf "  rccontrol cli db-upgrade --help | -h\n"
+  echo
+
+  # :command.long_usage
+  if [[ -n $long_usage ]]; then
+    printf "Options:\n"
+
+    # :command.usage_fixed_flags
+    echo "  --help, -h"
+    printf "    Show this help\n"
+    echo
+
+  fi
+}
+
+# :command.usage
+rccontrol_cli_storage_usage() {
+  if [[ -n $long_usage ]]; then
+    printf "rccontrol cli storage - CLI for repository storage\n"
     echo
 
   else
-    printf "rccontrol cli-db-upgrade - CLI\n"
+    printf "rccontrol cli storage - CLI for repository storage\n"
     echo
 
   fi
 
   printf "Usage:\n"
-  printf "  rccontrol cli-db-upgrade\n"
-  printf "  rccontrol cli-db-upgrade --help | -h\n"
+  printf "  rccontrol cli storage\n"
+  printf "  rccontrol cli storage --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
   fi
 }
 
 # :command.usage
 rccontrol__completions_usage() {
   if [[ -n $long_usage ]]; then
     printf "rccontrol _completions - Generate completions, eval "$(./rccontrol _completions)"\n"
     echo
 
   else
     printf "rccontrol _completions - Generate completions, eval "$(./rccontrol _completions)"\n"
     echo
 
   fi
 
   printf "Usage:\n"
   printf "  rccontrol _completions\n"
   printf "  rccontrol _completions --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
   fi
 }
 
 # :command.usage
 rccontrol_backup_db_usage() {
   if [[ -n $long_usage ]]; then
     printf "rccontrol backup-db - CLI\n"
     echo
 
   else
     printf "rccontrol backup-db - CLI\n"
     echo
 
   fi
 
   printf "Usage:\n"
   printf "  rccontrol backup-db\n"
   printf "  rccontrol backup-db --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
     # :command.usage_examples
     printf "Examples:\n"
     printf "  ./rccontrol backup-db\n"
     echo
 
   fi
 }
 
 # :command.usage
 rccontrol_backup_data_usage() {
   if [[ -n $long_usage ]]; then
     printf "rccontrol backup-data - CLI\n"
     echo
 
   else
     printf "rccontrol backup-data - CLI\n"
     echo
 
   fi
 
   printf "Usage:\n"
   printf "  rccontrol backup-data\n"
   printf "  rccontrol backup-data --help | -h\n"
   echo
 
   # :command.long_usage
   if [[ -n $long_usage ]]; then
     printf "Options:\n"
 
     # :command.usage_fixed_flags
     echo "  --help, -h"
     printf "    Show this help\n"
     echo
 
   fi
 }
 
 # :command.normalize_input
 normalize_input() {
   local arg flags
 
   while [[ $# -gt 0 ]]; do
     arg="$1"
     if [[ $arg =~ ^(--[a-zA-Z0-9_\-]+)=(.+)$ ]]; then
       input+=("${BASH_REMATCH[1]}")
       input+=("${BASH_REMATCH[2]}")
     elif [[ $arg =~ ^(-[a-zA-Z0-9])=(.+)$ ]]; then
       input+=("${BASH_REMATCH[1]}")
       input+=("${BASH_REMATCH[2]}")
     elif [[ $arg =~ ^-([a-zA-Z0-9][a-zA-Z0-9]+)$ ]]; then
       flags="${BASH_REMATCH[1]}"
       for (( i=0 ; i < ${#flags} ; i++ )); do
         input+=("-${flags:i:1}")
       done
     else
       input+=("$arg")
     fi
 
     shift
   done
 }
 # :command.inspect_args
 inspect_args() {
   readarray -t sorted_keys < <(printf '%s\n' "${!args[@]}" | sort)
   if (( ${#args[@]} )); then
     echo args:
     for k in "${sorted_keys[@]}"; do echo "- \${args[$k]} = ${args[$k]}"; done
   else
     echo args: none
   fi
 
   if (( ${#other_args[@]} )); then
     echo
     echo other_args:
     echo "- \${other_args[*]} = ${other_args[*]}"
     for i in "${!other_args[@]}"; do
       echo "- \${other_args[$i]} = ${other_args[$i]}"
     done
   fi
 }
 
 # :command.user_lib
+# src/lib/check_bootstrap.sh
+
+check_bootstrap() {
+  # Avoid destroying bootstrapping by simple start/stop
+for stage in $BOOTSTRAP_STAGES; do
+
+    stage_name=$(echo $stage | cut -d ":" -f 1)
+    stage_func=$(echo $stage | cut -d ":" -f 2)
+    if ! config_has_key $stage_name ; then
+      echo "$(yellow WARNING:) bootstrap key $stage_name not found in config file $CONFIG_FILE!"
+      echo "$(yellow NOTICE:) Please run ./rccontrol bootstrap first"
+      exit
+    fi
+done
+
+}
+
 # src/lib/colors.sh
 print_in_color() {
   local color="$1"
   shift
   if [[ -z ${NO_COLOR+x} ]]; then
     printf "$color%b\e[0m\n" "$*";
   else
     printf "%b\n" "$*";
   fi
 }
 
 red() { print_in_color "\e[31m" "$*"; }
 green() { print_in_color "\e[32m" "$*"; }
 yellow() { print_in_color "\e[33m" "$*"; }
 blue() { print_in_color "\e[34m" "$*"; }
 magenta() { print_in_color "\e[35m" "$*"; }
 cyan() { print_in_color "\e[36m" "$*"; }
 bold() { print_in_color "\e[1m" "$*"; }
 underlined() { print_in_color "\e[4m" "$*"; }
 red_bold() { print_in_color "\e[1;31m" "$*"; }
 green_bold() { print_in_color "\e[1;32m" "$*"; }
 yellow_bold() { print_in_color "\e[1;33m" "$*"; }
 blue_bold() { print_in_color "\e[1;34m" "$*"; }
 magenta_bold() { print_in_color "\e[1;35m" "$*"; }
 cyan_bold() { print_in_color "\e[1;36m" "$*"; }
 red_underlined() { print_in_color "\e[4;31m" "$*"; }
 green_underlined() { print_in_color "\e[4;32m" "$*"; }
 yellow_underlined() { print_in_color "\e[4;33m" "$*"; }
 blue_underlined() { print_in_color "\e[4;34m" "$*"; }
 magenta_underlined() { print_in_color "\e[4;35m" "$*"; }
 cyan_underlined() { print_in_color "\e[4;36m" "$*"; }
 
 # src/lib/config.sh
 config_init() {
   CONFIG_FILE=${CONFIG_FILE:=config.ini}
   [[ -f "$CONFIG_FILE" ]] || touch "$CONFIG_FILE"
 }
 
 config_get() {
   local key=$1
   local regex="^$key *= *(.+)$"
   local value=""
 
   config_init
 
   while IFS= read -r line || [ -n "$line" ]; do
     if [[ $line =~ $regex ]]; then
       value="${BASH_REMATCH[1]}"
       break
     fi
   done < "$CONFIG_FILE"
 
   echo "$value"
 }
 
 config_set() {
   local key=$1
   shift
   local value="$*"
 
   config_init
 
   local regex="^($key) *= *.+$"
   local output=""
   local found_key=""
   local newline
 
   while IFS= read -r line || [ -n "$line" ]; do
     newline=$line
     if [[ $line =~ $regex ]]; then
       found_key="${BASH_REMATCH[1]}"
       newline="$key = $value"
       output="$output$newline\n"
     elif [[ $line ]]; then
       output="$output$line\n"
     fi
   done < "$CONFIG_FILE"
 
   if [[ -z $found_key ]]; then
     output="$output$key = $value\n"
   fi
 
   printf "%b\n" "$output" > "$CONFIG_FILE"
 }
 
 config_del() {
   local key=$1
 
   local regex="^($key) *="
   local output=""
 
   config_init
 
   while IFS= read -r line || [ -n "$line" ]; do
     if [[ $line ]] && [[ ! $line =~ $regex ]]; then
       output="$output$line\n"
     fi
   done < "$CONFIG_FILE"
 
   printf "%b\n" "$output" > "$CONFIG_FILE"
 }
 
 config_show() {
   config_init
   cat "$CONFIG_FILE"
 }
 
 config_keys() {
   local regex="^([a-zA-Z0-9_\-\/\.]+) *="
 
   config_init
 
   local keys=()
   local key
 
   while IFS= read -r line || [ -n "$line" ]; do
     if [[ $line =~ $regex ]]; then
       key="${BASH_REMATCH[1]}"
       keys+=("$key")
     fi
   done < "$CONFIG_FILE"
   echo "${keys[@]}"
 }
 
 config_has_key() {
   [[ $(config_get "$1") ]]
 }
 
-# src/lib/sample_function.sh
-
-docker_ping_host() {
-  PING_HOST="$1"
-  docker run --network rhodecode_network --rm alpine ping "$PING_HOST"
-}
-
-check_bootstrap() {
-  # Avoid destroying bootstrapping by simple start/stop
-  if [[ ! -e $BOOTSTRAP_FILE ]]; then
-    echo "$(yellow WARNING:) initial bootstrap file $BOOTSTRAP_FILE not found !"
-    echo "$(yellow NOTICE:) Please run ./rccontrol bootstrap first"
-    exit
-  fi
-}
-
 # src/lib/send_completions.sh
 send_completions() {
   echo $'# rccontrol3 completion                                    -*- shell-script -*-'
   echo $''
   echo $'# This bash completions script was generated by'
   echo $'# completely (https://github.com/dannyben/completely)'
   echo $'# Modifying it manually is not recommended'
   echo $''
   echo $'_rccontrol3_completions_filter() {'
   echo $'  local words="$1"'
   echo $'  local cur=${COMP_WORDS[COMP_CWORD]}'
   echo $'  local result=()'
   echo $''
   echo $'  if [[ "${cur:0:1}" == "-" ]]; then'
   echo $'    echo "$words"'
   echo $'  '
   echo $'  else'
   echo $'    for word in $words; do'
   echo $'      [[ "${word:0:1}" != "-" ]] && result+=("$word")'
   echo $'    done'
   echo $''
   echo $'    echo "${result[*]}"'
   echo $''
   echo $'  fi'
   echo $'}'
   echo $''
   echo $'_rccontrol3_completions() {'
   echo $'  local cur=${COMP_WORDS[COMP_CWORD]}'
   echo $'  local compwords=("${COMP_WORDS[@]:1:$COMP_CWORD-1}")'
   echo $'  local compline="${compwords[*]}"'
   echo $''
   echo $'  case "$compline" in'
   echo $'    \'upgrade-source\'*)'
   echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_rccontrol3_completions_filter "--auth-token --help --server-url -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    \'build-source\'*)'
   echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_rccontrol3_completions_filter "--help --version-name -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    \'self-update\'*)'
   echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_rccontrol3_completions_filter "--auth-token --help --server-url -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    \'bootstrap\'*)'
   echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_rccontrol3_completions_filter "--auth-token --force --help --server-url -f -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    \'build\'*)'
   echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_rccontrol3_completions_filter "--help --version-name -h")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    \'stack\'*)'
   echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_rccontrol3_completions_filter "--help -h metrics rhodecode router services")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'    *)'
   echo $'      while read -r; do COMPREPLY+=( "$REPLY" ); done < <( compgen -W "$(_rccontrol3_completions_filter "--help --version -h -v bootstrap build build-source self-update stack upgrade-source")" -- "$cur" )'
   echo $'      ;;'
   echo $''
   echo $'  esac'
   echo $'} &&'
   echo $'complete -F _rccontrol3_completions rccontrol3'
   echo $''
   echo $'# ex: filetype=sh'
 }
 
 # src/lib/validate_stack_exists.sh
 
 validate_stack_exists() {
   err=""
   invalid="1"
 
   for item in $VALID_SERVICES
   do
       if [ "$1" == "$item" ]; then
           invalid=""
           break
       fi
   done
 
   if [[ -n $invalid ]]; then
     err="command '$1' not in list of $VALID_SERVICES"
   fi
 
   echo $err
 }
 
 # :command.command_functions
 # :command.function
 rccontrol_self_update_command() {
   # src/self_update_command.sh
   check_bootstrap
 
   echo "# this file is located in 'src/self_update_command.sh'"
   echo "# code for 'rccontrol3 self-update' goes here"
   echo "# you can edit it freely and regenerate (it will not be overwritten)"
   inspect_args
 
 }
 
 # :command.function
 rccontrol_bootstrap_command() {
   # src/bootstrap_command.sh
   DEBUG=${args[--debug]}
   force=${args[--force]}
 
   check_bash_version() {
-
      if [ ! "${BASH_VERSINFO:-0}" -ge 4 ]; then
         echo "$(red Bash version 4 or greater is required, please update your bash version!)"
         exit
      fi
   }
 
   check_docker() {
     (which docker || which docker.io) &>/dev/null
   }
 
-  check_and_install_docker() {
+  bootstrap_docker_install() {
     failMsg="Failed to find docker on your PATH"
 
     if ! check_docker; then
       echo "$failMsg"
       read -p "Enter to install Docker directly from https://get.docker.com/ or Ctrl+C to exit"
       curl https://get.docker.com/ | sh
 
       if ! check_docker; then
         echo "$failMsg"
         echo "Docker install failed. Quitting."
         exit
       fi
     fi
   }
 
-  docker_bootstrap() {
-    check_and_install_docker
+  bootstrap_docker_commons() {
+
     echo 'Docker: Running bootstrap.'
 
-    echo "Docker: creating network 'rc_datavolume'"
-    docker volume create --name=rc_datavolume
+    echo "Docker: creating volume 'rc_datavolume'"
+    docker volume create --label keep=1 --name=rc_datavolume
+    echo "Docker: done"
+    echo ""
+
+    echo "Docker: creating volume 'rc_reposvolume'"
+    docker volume create --label keep=1 --name=rc_reposvolume
     echo "Docker: done"
     echo ""
 
     echo "Docker: creating network 'rhodecode_network'"
     docker network inspect rhodecode_network >/dev/null 2>&1 || docker network create rhodecode_network
     echo "Docker: done"
     echo ""
 
     echo "Docker: creating loki logging"
     loki_driver=$(docker plugin ls --format {{.Name}} --filter enabled=true | grep loki || echo "")
     if [[ $loki_driver == "" ]]; then
       docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
     else
       echo "Docker: loki driver already exists"
     fi
   }
 
-  definitions_bootstrap() {
-
-    SOURCE_DIR=$PWD
-    RHODECODE_DOCKER_HASH='master'
-
-    AUTH_TOKEN=${args[--auth-token]}
-    SERVER_URL=${args[--server-url]}
+  bootstrap_config() {
+    shared_key=$(echo $RANDOM | md5sum | head -c 32)
+
+    if [[ ! -f "$CONFIG_FILE" ]]; then
+      echo "init config at: $CONFIG_FILE"
+      config_init
+    else
+      echo "re-using config at: $CONFIG_FILE"
+    fi
+
+    if ! config_has_key "rc_encrypted_secret" ; then
+     config_set "rc_encrypted_secret" $shared_key
+    fi
+
+    if ! config_has_key "rc_db_url" ; then
+     key=$(echo $RANDOM | md5sum | head -c 32)
+     config_set "rc_db_url" "postgresql://$DB_USER:$key@database/$DB_NAME"
+    fi
+
+    if ! config_has_key "rc_license_token" ; then
+     config_set "rc_license_token" abra-cada-bra1-rce4
+    fi
+
+    if ! config_has_key "rc_base_url" ; then
+     config_set "rc_base_url" http://docker-dev
+    fi
+
+    if ! config_has_key "rc_log_formatter" ; then
+     # json is another option
+     config_set "rc_log_formatter" generic
+    fi
+
+    if ! config_has_key "rc_use_celery" ; then
+     config_set "rc_use_celery" true
+    fi
+
+    if ! config_has_key "channelstream_secret" ; then
+     config_set "channelstream_secret" $shared_key
+    fi
+
+    if ! config_has_key "channelstream_admin_secret" ; then
+     config_set "channelstream_admin_secret" $shared_key
+    fi
+
+    if ! config_has_key "channelstream_allow_posting_from" ; then
+     config_set "channelstream_allow_posting_from" 0.0.0.0
+    fi
+
+    if ! config_has_key "rc_channelstream_ws_url" ; then
+     config_set "rc_channelstream_ws_url" ws:/docker-dev/_channelstream
+    fi
+
+    mkdir -p $PWD/.custom
+    BOOTSTRAP_RUNTIME_ENV=$PWD/.custom/.runtime.env
+    if [[ ! -f "$BOOTSTRAP_RUNTIME_ENV" ]]; then
+      echo "init runtime env config at: $BOOTSTRAP_RUNTIME_ENV"
+      touch BOOTSTRAP_RUNTIME_ENV
+
+      #ENV_EXPAND=""
+      for k in $(config_keys); do
+        k_upper=${k^^}
+        echo "$k_upper='$(config_get "$k")'" >> $BOOTSTRAP_RUNTIME_ENV
+      done
+
+    fi
+  }
+
+  bootstrap_definitions() {
+
+    SOURCE_DIR=$PWD
+    RHODECODE_DOCKER_HASH='master'
+
+    AUTH_TOKEN=${args[--auth-token]}
+    SERVER_URL=${args[--server-url]}
 
     DEFINITIONS_EXIST=""
-    CHECK_FILES="rccontrol .env docker-compose-services.yaml docker-compose-apps.yaml"
+    CHECK_FILES="\
+    .env \
+    docker-compose-services.yaml \
+    docker-compose-apps.yaml \
+    docker-compose-metrics.yaml \
+    docker-compose-router.yaml \
+    "
     for check_file in $CHECK_FILES; do
       if [[ -f "$check_file" ]]; then
         DEFINITIONS_EXIST="1"
       fi
     done
 
     if [[ -n $DEFINITIONS_EXIST && ! $force ]]; then
         echo "$(yellow skipping docker defs creation, existing files found. Use --force to create them anyway)"
         return
     fi
 
     if [[ -n $DEFINITIONS_EXIST ]]; then
         echo "$(yellow docker definitions exists, are you sure to force re-create them?)"
         while true; do
           read -p "Would you like to continue with overriding file? [yn] " yn
           case $yn in
             [Yy]*) return 2 ;;
             [Nn]*) exit ;;
             *) echo "Please answer y or n." ;;
           esac
         done
     fi
 
-    exit
-
     # download sources
     echo "Files: download rhodecode docker definitions from $SERVER_URL"
     echo ""
 
     if [ $DEBUG ]; then
       echo "downloading: $SERVER_URL/rhodecode-enterprise-docker/archive/$RHODECODE_DOCKER_HASH.tgz?with_hash=0"
       curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-docker/archive/$RHODECODE_DOCKER_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR
 
       echo "running CP $SOURCE_DIR/*rhodecode-enterprise-docker-plain/* $SOURCE_DIR"
       cp -v -r -f --update --backup=numbered $SOURCE_DIR/*rhodecode-enterprise-docker-plain/* $SOURCE_DIR
 
       echo "removing $SOURCE_DIR/*rhodecode-enterprise-docker-plain"
       rm -r $SOURCE_DIR/*rhodecode-enterprise-docker-plain
     else
       curl -s --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-docker/archive/$RHODECODE_DOCKER_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR
 
       cp -r -f --update --backup=numbered $SOURCE_DIR/*rhodecode-enterprise-docker-plain/* $SOURCE_DIR
 
       rm -r $SOURCE_DIR/*rhodecode-enterprise-docker-plain
     fi
 
     echo "$(green_bold DONE: docker definitions extracted to $SOURCE_DIR)"
   }
 
-  config_bootstrap() {
+  bootstrap_overrides() {
+    templates=$(find $PWD/templates/*.yaml  -printf "%f\n")
 
-    if [[ ! -f "$CONFIG_FILE" ]]; then
-      echo "init config at: $CONFIG_FILE"
-      config_init
+    target_dir=$PWD/.custom/
+    for o_file in $templates; do
+      target_file=$target_dir/$o_file
+      if [[ ! -f "$target_file" ]]; then
+        echo "copy override file $o_file"
+        cp -v $PWD/templates/$o_file $target_file
     else
-      echo "re-using config at: $CONFIG_FILE"
-    fi
-
-    if ! config_has_key "rc_encrypted_secret" ; then
-     key=$(echo $RANDOM | md5sum | head -c 32)
-     config_set "rc_encrypted_secret" $key
-    fi
-
-    if ! config_has_key "rc_db_url" ; then
-     key=$(echo $RANDOM | md5sum | head -c 32)
-     config_set "rc_db_url" "postgresql://$DB_USER:$key@database/$DB_NAME"
-    fi
-
-    if ! config_has_key "rc_license_token" ; then
-     config_set "rc_license_token" abra-cada-bra1-rce4
+        echo "file $target_file existing, skipping..."
     fi
-
-    if ! config_has_key "rc_base_url" ; then
-     config_set "rc_base_url" http://docker-dev
-    fi
-
-    if ! config_has_key "rc_log_formatter" ; then
-     # json is another option
-     config_set "rc_log_formatter" generic
-    fi
-
-    if ! config_has_key "rc_use_celery" ; then
-     config_set "rc_use_celery" true
-    fi
-
-    BOOTSTRAP_RUNTIME_ENV=$PWD/.custom/.runtime.env
-    if [[ ! -f "$BOOTSTRAP_RUNTIME_ENV" ]]; then
-      echo "init runtime env config at: $BOOTSTRAP_RUNTIME_ENV"
-      touch BOOTSTRAP_RUNTIME_ENV
-
-      #ENV_EXPAND=""
-      for k in $(config_keys); do
-        k_upper=${k^^}
-        echo "$k_upper='$(config_get "$k")'" >> $BOOTSTRAP_RUNTIME_ENV
       done
-
-    fi
+    echo "$(green_bold DONE: overrides extracted to $target_dir)"
   }
 
   cur_date=$(date '+%Y-%m-%d %H:%M:%S')
 
   check_bash_version
 
-  if [[ ! -e $BOOTSTRAP_FILE ]]; then
-    echo "initial bootstrap file $BOOTSTRAP_FILE not found !"
+  # Init the config !
+  if [[ ! -f "$CONFIG_FILE" ]]; then
+    echo "config: init new config at: $CONFIG_FILE"
+    config_init
+  else
+    echo "config: re-using present config at: $CONFIG_FILE"
+  fi
 
-    docker_bootstrap
-    definitions_bootstrap
-    config_bootstrap
+  for stage in $BOOTSTRAP_STAGES; do
 
-    echo "$cur_date" > "$BOOTSTRAP_FILE"
+      stage_name=$(echo $stage | cut -d ":" -f 1)
+      stage_func=$(echo $stage | cut -d ":" -f 2)
 
+      if ! config_has_key $stage_name ; then
+        echo "$(green \* bootstrap: \'$stage_name\' stage not found\; running now... )"
+        $stage_func
+        config_set "$stage_name" $cur_date
   else
     if [ $force ]; then
+          echo "$(green \* bootstrap: \'$stage_name\' is present!\; FORCE running now... )"
+          $stage_func $force
+          config_set "$stage_name" $cur_date
+        else
+          echo "$(yellow \* bootstrap: \'$stage_name\' already present, use --force to run it again)"
+        fi
+      fi
 
-      docker_bootstrap $force
-      definitions_bootstrap $force ## TODO: remove old
-      config_bootstrap $force
-
-      echo "$cur_date" > "$BOOTSTRAP_FILE"
+  done
 
       exit
-    fi
-
-    echo "bootstrap file $BOOTSTRAP_FILE was found add --force to force bootstrap"
-  fi
 
   get_started
 }
 
 # :command.function
 rccontrol_get_build_artifacts_command() {
   # src/get_build_artifacts_command.sh
   check_bootstrap
 
   DEBUG=${args[--debug]}
   AUTH=${args[--auth]}
   INSTALLER_URL=${args[--installer-url]}
   MANIFEST_URL=${args[--manifest-url]}
   RC_VERSION=${args[--version-name]}
   VER=$RC_VERSION
 
   CACHE_DIR=$PWD/.cache
   VER_REGEX="$VER+x86_64"
 
   echo "Downloading Artifacts for version: $VER"
 
   echo "1/4 Checking available downloads from MANIFEST file"
 
   ARTS=$(curl -s $AUTH $MANIFEST_URL | grep --ignore-case "$VER_REGEX" | cut -d ' ' -f 2)
 
   if [[ $DEBUG ]]; then
     echo "DEBUG START"
     curl -s $AUTH $MANIFEST_URL | grep --ignore-case "$VER_REGEX" || echo "no regex match"
     curl -s $AUTH $MANIFEST_URL | grep --ignore-case "$VER_REGEX" | cut -d ' ' -f 2
     echo "Found following artifacts:"
     echo $ARTS
     echo "DEBUG END"
   fi
 
   if [[ $ARTS == "" ]]; then
     MSG="Failed to found any MANIFEST entries for version $VER make sure there exists release with that version or use --version to specify different version"
     echo "$(red $MSG)"
     exit
   fi
 
   echo "2/4 Downloading locale-archive"
   curl -L https://dls.rhodecode.com/assets/locale-archive -J -O
   mv -v locale-archive "$CACHE_DIR"
 
   # vcsserver/ce/ee
   echo "3/4 Downloading installer artifacts"
   for url in $ARTS; do
       echo "Downloading $url with $AUTH"
       curl $AUTH --fail-early -L ${url} -J -O
   done
 
   #for url in $(curl -s $MANIFEST_URL | grep --ignore-case -E 'control.+\+x86_64' | cut -d ' ' -f 2); do
   #    echo "Downloading $url"
   #    curl -L ${url} -J -O
   #done
 
   echo "4/4 Downloading installer from $INSTALLER_URL"
   curl $AUTH -L $INSTALLER_URL -J -O
 
   INSTALLER=$(ls -Art RhodeCode-installer-* | tail -n 1)
   if [[ -n $INSTALLER ]]; then
     chmod +x "${INSTALLER}"
   fi
 
   echo "Copying artifacts into $CACHE_DIR"
 
   mv -v "${INSTALLER}" $CACHE_DIR
   mv -v *.bz2 $CACHE_DIR
   ls -lh $CACHE_DIR
 
 }
 
 # :command.function
 rccontrol_build_command() {
   # src/build_command.sh
   check_bootstrap
 
   rc_version=${args[--version-name]}
   export RC_VERSION=$rc_version
   eval "echo INSTALLER BASED BUILDING${RC_VERSION}"
 
   RC_VERSION=$rc_version ./rccontrol stack rhodecode build --progress plain rhodecode
 
 }
 
 # :command.function
 rccontrol_get_build_source_command() {
   # src/get_build_source_command.sh
   check_bootstrap
 
   AUTH_TOKEN=${args[--auth-token]}
   SERVER_URL=${args[--server-url]}
-  revision=${args[revision]}
+  revision=${args[--revision]}
 
   SOURCE_DIR=$PWD/.source
 
   RHODECODE_VCS_HASH=$revision
   RHODECODE_CE_HASH=$revision
   RHODECODE_EE_HASH=$revision
 
   # download sources
   echo "** download rhodecode source for build from $SERVER_URL **"
 
   curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-vcsserver/archive/$RHODECODE_VCS_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR
   curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-ce/archive/$RHODECODE_CE_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR
   #TODO: fix just CE build...
   curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-ee/archive/$RHODECODE_EE_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR
 
-  rm -rf $SOURCE_DIR/rhodecode-vcsserver     && mv    $SOURCE_DIR/*rhodecode-vcsserver-plain     $SOURCE_DIR/rhodecode-vcsserver
-  rm -rf $SOURCE_DIR/rhodecode-enterprise-ce && mv    $SOURCE_DIR/*rhodecode-enterprise-ce-plain $SOURCE_DIR/rhodecode-enterprise-ce
-  rm -rf $SOURCE_DIR/rhodecode-enterprise-ee && cp -r $SOURCE_DIR/*rhodecode-enterprise-ee-plain $SOURCE_DIR/rhodecode-enterprise-ee
+  rm -rf $SOURCE_DIR/rhodecode-vcsserver && \
+  mv  $SOURCE_DIR/*rhodecode-vcsserver-plain     $SOURCE_DIR/rhodecode-vcsserver
+
+  rm -rf $SOURCE_DIR/rhodecode-enterprise-ce && \
+  mv  $SOURCE_DIR/*rhodecode-enterprise-ce-plain $SOURCE_DIR/rhodecode-enterprise-ce
+
+  rm -rf $SOURCE_DIR/rhodecode-enterprise-ee && \
+  mv  $SOURCE_DIR/*rhodecode-enterprise-ee-plain $SOURCE_DIR/rhodecode-enterprise-ee
 
   echo "downloading sources done to $SOURCE_DIR"
+  ls -l $SOURCE_DIR
 
 }
 
 # :command.function
 rccontrol_build_source_command() {
   # src/build_source_command.sh
   check_bootstrap
 
   rc_version=${args[--version-name]}
   export RC_VERSION=$rc_version
   eval "echo SOURCE BASED BUILDING${RC_VERSION}"
 
   RC_VERSION=$rc_version ./rccontrol stack rhodecode -f docker-compose-apps.source.yaml build --progress plain rhodecode
 
 }
 
 # :command.function
-rccontrol_stack_command() {
-  # src/stack_command.sh
+rccontrol_stack_router_command() {
+  # src/stack_router_command.sh
   check_bootstrap
 
   DEBUG=${args[--debug]}
-  service_name=${args[name]}
 
   if [[ ! -f $RC_STACK_ROUTER_EXT ]]; then
     RC_STACK_ROUTER_EXT=""
   else
     RC_STACK_ROUTER_EXT="-f $RC_STACK_ROUTER_EXT"
   fi
 
   CMD_ROUTER="\
   docker compose \
   --env-file $ENV_FILE \
   $ENV_EXPAND \
   -p rc_cluster_router \
   -f docker-compose-base.yaml \
   -f docker-compose-router.yaml $RC_STACK_ROUTER_EXT"
 
+  if [[ $DEBUG ]]; then
+    echo "---"
+    echo "stacks docker: $RC_STACK_ROUTER_EXT_LCL"
+    echo "running command: ${CMD_ROUTER}"
+    echo "ARGS: ${other_args[*]}"
+    echo "---"
+  fi
+  eval "${CMD_ROUTER} ${other_args[*]}"
+
+}
+
+# :command.function
+rccontrol_stack_metrics_command() {
+  # src/stack_metrics_command.sh
+  check_bootstrap
+
+  DEBUG=${args[--debug]}
+
+  if [[ ! -f $RC_STACK_METRICS_EXT ]]; then
+    RC_STACK_METRICS_EXT_LCL=""
+  else
+    RC_STACK_METRICS_EXT_LCL="-f $RC_STACK_METRICS_EXT"
+  fi
+
+  CMD_METRICS="\
+  docker compose \
+  --env-file $ENV_FILE \
+  $ENV_EXPAND \
+  -p rc_cluster_metrics \
+  -f docker-compose-base.yaml \
+  -f docker-compose-metrics.yaml $RC_STACK_METRICS_EXT_LCL"
+
+  if [[ $DEBUG ]]; then
+    echo "---"
+    echo "stacks docker: $RC_STACK_METRICS_EXT_LCL"
+    echo "running command: ${CMD_METRICS}"
+    echo "ARGS: ${other_args[*]}"
+    echo "---"
+  fi
+  eval "${CMD_METRICS} ${other_args[*]}"
+}
+
+# :command.function
+rccontrol_stack_services_command() {
+  # src/stack_services_command.sh
+  check_bootstrap
+
+  DEBUG=${args[--debug]}
+
   if [[ ! -f $RC_STACK_SERVICES_EXT ]]; then
     RC_STACK_SERVICES_EXT_LCL=""
   else
     RC_STACK_SERVICES_EXT_LCL="-f $RC_STACK_SERVICES_EXT"
   fi
 
   RC_STACK_PROFILES="--profile postgres --profile redis --profile elasticsearch --profile channelstream"
 
   CMD_SERVICES="\
   docker compose \
   --env-file $ENV_FILE \
   $ENV_EXPAND \
   $RC_STACK_PROFILES \
   -p rc_cluster_services \
   -f docker-compose-base.yaml \
   -f docker-compose-services.yaml $RC_STACK_SERVICES_EXT_LCL"
 
-  if [[ ! -f $RC_STACK_METRICS_EXT ]]; then
-    RC_STACK_METRICS_EXT_LCL=""
-  else
-    RC_STACK_METRICS_EXT_LCL="-f $RC_STACK_METRICS_EXT"
+  if [[ $DEBUG ]]; then
+    echo "---"
+    echo "stacks docker: $RC_STACK_SERVICES_EXT_LCL"
+    echo "running command: ${CMD_SERVICES}"
+    echo "ARGS: ${other_args[*]}"
+    echo "---"
   fi
 
-  CMD_METRICS="\
-  docker compose \
-  --env-file $ENV_FILE \
-  $ENV_EXPAND \
-  -p rc_cluster_metrics \
-  -f docker-compose-base.yaml \
-  -f docker-compose-metrics.yaml $RC_STACK_METRICS_EXT_LCL"
+  eval "${CMD_SERVICES} ${other_args[*]}"
+}
+
+# :command.function
+rccontrol_stack_rhodecode_command() {
+  # src/stack_rhodecode_command.sh
+  check_bootstrap
+
+  DEBUG=${args[--debug]}
 
   if [[ ! -f $RC_STACK_RHODECODE_EXT ]]; then
     RC_STACK_RHODECODE_EXT_LCL=""
   else
     RC_STACK_RHODECODE_EXT_LCL="-f $RC_STACK_RHODECODE_EXT"
   fi
 
   CMD_RHODECODE="\
   docker compose \
   --env-file $ENV_FILE \
   $ENV_EXPAND \
   -p rc_cluster_apps \
   -f docker-compose-base.yaml \
   -f docker-compose-apps.yaml $RC_STACK_RHODECODE_EXT_LCL"
 
   CMD_RHODECODE_SOURCE="\
   docker compose \
   --env-file $ENV_FILE \
   $ENV_EXPAND \
   -p rc_cluster_apps \
   -f docker-compose-base.yaml \
   -f docker-compose-apps.yaml \
   -f docker-compose-apps.source.yaml $RC_STACK_RHODECODE_EXT_LCL"
 
-  case $service_name in
-
-      services )
-          if [[ $DEBUG ]]; then
-            echo "---"
-            echo "stacks docker: $RC_STACK_SERVICES_EXT_LCL"
-            echo "running command: ${CMD_SERVICES}"
-            echo "ARGS: ${other_args[*]}"
-            echo "---"
-          fi
-
-          eval "${CMD_SERVICES} ${other_args[*]}"
-          exit
-          ;;
-      router )
-          if [[ $DEBUG ]]; then
-            echo "---"
-            echo "stacks docker: $RC_STACK_ROUTER_EXT_LCL"
-            echo "running command: ${CMD_ROUTER}"
-            echo "ARGS: ${other_args[*]}"
-            echo "---"
-          fi
-          eval "${CMD_ROUTER} ${other_args[*]}"
-          exit
-          ;;
-      metrics )
-          if [[ $DEBUG ]]; then
-            echo "---"
-            echo "stacks docker: $RC_STACK_METRICS_EXT_LCL"
-            echo "running command: ${CMD_METRICS}"
-            echo "ARGS: ${other_args[*]}"
-            echo "---"
-          fi
-          eval "${CMD_METRICS} ${other_args[*]}"
-          exit
-          ;;
-      rhodecode )
           if [[ $DEBUG ]]; then
             echo "---"
             echo "stacks docker: $RC_STACK_RHODECODE_EXT_LCL"
             echo "running command: ${CMD_RHODECODE}"
             echo "ARGS: ${other_args[*]}"
             echo "---"
           fi
           eval "${CMD_RHODECODE} ${other_args[*]}"
-          exit
-          ;;
-  esac
+
+}
+
+# :command.function
+rccontrol_stack_all_command() {
+  # src/stack_all_command.sh
+  check_bootstrap
+
+  DEBUG=${args[--debug]}
+
+  stack_lst=$VALID_SERVICES
+
+  for stack in $stack_lst; do
+    ./rccontrol stack $stack ${other_args[*]}
+  done
 
 }
 
 # :command.function
 rccontrol_stack_status_command() {
   # src/stack_status_command.sh
   check_bootstrap
   DEBUG=${args[--debug]}
   if [[ $DEBUG ]]; then
     echo "---"
 
       ps_cmd=$(docker ps --filter=name=rc_cluster --format="{{.ID}}")
 
       for service in $ps_cmd; do
 
         servicename=`docker inspect --format '{{ .Name }}' $service`
         servicename=${servicename:1}
         echo $servicename
         docker inspect $service --format='{{.State.Status}}: {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'
         echo ""
 
       done
     fi
     echo "---"
   docker ps --filter=name=rc_cluster --format="table {{.ID}}\\t{{.Image}}\\t{{.Status}}\\t{{.Names}}\\t{{.Ports}}"
 }
 
 # :command.function
 rccontrol_stack_upgrade_command() {
   # src/stack_upgrade_command.sh
   check_bootstrap
 
   echo "# this file is located in 'src/stack_upgrade_command.sh'"
   echo "# code for 'rccontrol stack-upgrade' goes here"
   echo "# you can edit it freely and regenerate (it will not be overwritten)"
   inspect_args
 
 }
 
 # :command.function
 rccontrol_cli_redis_command() {
   # src/cli_redis_command.sh
   check_bootstrap
 
   DEBUG=${args[--debug]}
 
   target_container=rc_cluster_services-redis-1
   docker_id=$(docker ps --filter name=$target_container -q)
 
   if [[ $DEBUG ]]; then
     echo "container id: $docker_id, based on $target_container filter"
     docker ps
   fi
 
   if [ "$docker_id" == "" ]; then
     MSG="Cannot find container ID with name $target_container"
     echo "$(red $MSG)"
     exit
   fi
 
   if [[ $DEBUG ]]; then
     echo "Running |docker exec --interactive --tty $docker_id $CMD|"
     echo "docker exec --interactive --tty $docker_id /bin/bash -c 'redis-cli'"
   fi
 
   eval "docker exec --interactive --tty $docker_id /bin/bash -c 'redis-cli'"
 
 }
 
 # :command.function
 rccontrol_cli_db_command() {
   # src/cli_db_command.sh
   check_bootstrap
 
   DEBUG=${args[--debug]}
 
   target_container=rc_cluster_services-database-1
   docker_id=$(docker ps --filter name=$target_container -q)
 
   if [[ $DEBUG ]]; then
     echo "container id: $docker_id, based on $target_container filter"
     docker ps
   fi
 
   if [ "$docker_id" == "" ]; then
     MSG="Cannot find container ID with name $target_container"
     echo "$(red $MSG)"
     exit
   fi
 
   if [[ $DEBUG ]]; then
     echo "Running |docker exec --interactive --tty $docker_id $CMD|"
     echo "docker exec --interactive --tty $docker_id  /bin/bash -c 'PGPASSWORD=$DB_PASSWORD psql --username=$DB_USER --dbname=$DB_NAME'"
   fi
 
   eval "docker exec --interactive --tty $docker_id  /bin/bash -c 'PGPASSWORD=$DB_PASSWORD psql --username=$DB_USER --dbname=$DB_NAME'"
 
 }
 
 # :command.function
 rccontrol_cli_db_upgrade_command() {
   # src/cli_db_upgrade_command.sh
   check_bootstrap
 
   DEBUG=${args[--debug]}
 
   target_container=rc_cluster_apps-rhodecode-1
   docker_id=$(docker ps --filter name=$target_container -q)
 
   if [[ $DEBUG ]]; then
     echo "container id: $docker_id, based on $target_container filter"
     docker ps
   fi
 
   if [ "$docker_id" == "" ]; then
     MSG="Cannot find container ID with name $target_container"
     echo "$(red $MSG)"
     exit
   fi
 
   if [[ $DEBUG ]]; then
     echo "./rccontrol stack rhodecode exec rhodecode /usr/local/bin/rhodecode_bin/bin/rc-upgrade-db /etc/rhodecode/conf/rhodecode.optimized.ini --force-yes"
   fi
 
   ./rccontrol stack rhodecode exec rhodecode /usr/local/bin/rhodecode_bin/bin/rc-upgrade-db /etc/rhodecode/conf/rhodecode.optimized.ini --force-yes
 }
 
 # :command.function
-rccontrol__completions_command() {
-  # src/_completions_command.sh
-  send_completions
-}
-
-# :command.function
-rccontrol_backup_db_command() {
-  # src/backup_db_command.sh
+rccontrol_cli_storage_command() {
+  # src/cli_storage_command.sh
   check_bootstrap
 
   DEBUG=${args[--debug]}
-  DESTINATION=${args[destination]}
 
-  target_container=rc_cluster_services-database-1
+  source $ENV_FILE
+  target_container=$RC_IMAGE_EE
 
-  docker_id=$(docker ps --filter name=$target_container -q)
-  backup_name=rc_db_dump-$(date +%Y-%m-%d).tar.gz
+  image_id=$(docker image ls $RC_IMAGE_EE -q)
 
-  echo "creating backup $backup_name"
+  echo "Attaching storage from $target_container"
 
   if [[ $DEBUG ]]; then
-    echo "container id: $docker_id, based on $target_container filter"
-    docker ps
+    echo "image id: $image_id, based on $target_container filter"
+    docker image ls
     echo "---"
   fi
 
-  #image_id=$(docker inspect "$docker_id" --format {{.Image}} | cut -c 8-)
+  if [[ $DEBUG ]]; then
+    echo "rc_datavolume inspection..."
+    docker volume inspect rc_datavolume
+
+    echo "rhodecode_repos inspection..."
+    docker volume inspect rhodecode_repos
+  fi
+
+  #
+  #if [[ $DEBUG ]]; then
+  #  echo "image id: $image_id, based on $target_container filter"
+  #  docker image ls
+  #  echo "---"
+  #fi
+
+  docker run --rm \
+    -it \
+    --volume $PWD/.custom/storage:/vol/backupvolume \
+    --volume rc_datavolume:/vol/datavolume \
+    --volume rc_reposvolume:/vol/repovolume \
+    --workdir="/vol" \
+    debian:jessie \
+    /bin/bash
+
+}
+
+# :command.function
+rccontrol__completions_command() {
+  # src/_completions_command.sh
+  send_completions
+}
+
+# :command.function
+rccontrol_backup_db_command() {
+  # src/backup_db_command.sh
+  check_bootstrap
+
+  DEBUG=${args[--debug]}
+  DESTINATION=${args[destination]}
+
+  target_container=rc_cluster_services-database-1
+
+  docker_id=$(docker ps --filter name=$target_container -q)
+  backup_name=rc_db_dump-$(date +%Y-%m-%d).sql.gz
+
+  echo "creating backup $backup_name"
+
+  if [[ $DEBUG ]]; then
+    echo "container id: $docker_id, based on $target_container filter"
+    docker ps
+    echo "---"
+  fi
+
+  #image_id=$(docker inspect "$docker_id" --format {{.Image}} | cut -c 8-)
   #
   #if [[ $DEBUG ]]; then
   #  echo "image id: $image_id, based on $target_container filter"
   #  docker image ls
   #  echo "---"
   #fi
 
   if [ "$docker_id" == "" ]; then
     MSG="Cannot find container ID with name $target_container"
     echo "$(red $MSG)"
     exit
   fi
 
   #if [ "image_id" == "" ]; then
   #  MSG="Cannot find image ID with name $target_container"
   #  echo "$(red $MSG)"
   #  exit
   #fi
+  # docker exec -i your-db-container psql -U your-db-user -d your-db-name gunzip < your_dump.sql.gz |
 
   if [[ $DEBUG ]]; then
-    echo "docker exec -e PGPASSWORD=$DB_PASSWORD $docker_id /bin/bash -c 'pg_dump --inserts -U $DB_USER -h 127.0.0.1 --dbname=$DB_NAME | gzip > /var/rc-data-dump/$backup_name'"
+    echo "docker exec -e PGPASSWORD=$DB_PASSWORD $docker_id /bin/bash -c 'mkdir -p /var/rc-data-dump && pg_dump --inserts -U $DB_USER -h 127.0.0.1 --dbname=$DB_NAME | gzip > /var/rc-data-dump/$backup_name'"
+    echo "dump placed in /var/rc-data-dump/$backup_name"
+    echo "run docker cp $docker_id:/var/rc-data-dump/$backup_name $PWD to copy the file into your host machine"
   fi
 
   eval "docker exec -e PGPASSWORD=$DB_PASSWORD $docker_id /bin/bash -c 'pg_dump --inserts -U $DB_USER -h 127.0.0.1 --dbname=$DB_NAME | gzip > /var/rc-data-dump/$backup_name'"
   echo "$(green Backup created in /var/rc-data-dump/ mount !)"
 
 }
 
 # :command.function
 rccontrol_backup_data_command() {
   # src/backup_data_command.sh
   echo "# this file is located in 'src/backup_data_command.sh'"
   echo "# code for 'rccontrol backup-data' goes here"
   echo "# you can edit it freely and regenerate (it will not be overwritten)"
   inspect_args
 
 }
 
 # :command.parse_requirements
 parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --version | -v )
     version_command
     exit
     ;;
 
   --help | -h )
     long_usage=yes
     rccontrol_usage
     exit
     ;;
 
   # :flag.case
   --debug )
 
     # :flag.case_no_arg
     args[--debug]=1
     shift
     ;;
 
   esac
 
   # :command.environment_variables_filter
   # :command.environment_variables_default
   export RC_CLI_VERSION_NAME="${RC_CLI_VERSION_NAME:-4.27.0}"
   export RC_STACK_ROUTER_EXT="${RC_STACK_ROUTER_EXT:-.custom/docker-compose-router.override.yaml}"
   export RC_STACK_METRICS_EXT="${RC_STACK_METRICS_EXT:-.custom/docker-compose-metrics.override.yaml}"
   export RC_STACK_SERVICES_EXT="${RC_STACK_SERVICES_EXT:-.custom/docker-compose-services.override.yaml}"
   export RC_STACK_RHODECODE_EXT="${RC_STACK_RHODECODE_EXT:-.custom/docker-compose-apps.override.yaml}"
 
   # :command.command_filter
   action=${1:-}
 
   case $action in
   -* )
     ;;
 
   self-update )
     action="self-update"
     shift
     rccontrol_self_update_parse_requirements "$@"
     shift $#
     ;;
 
-  bootstrap | install )
+  bootstrap | init )
     action="bootstrap"
     shift
     rccontrol_bootstrap_parse_requirements "$@"
     shift $#
     ;;
 
   get-build-artifacts )
     action="get-build-artifacts"
     shift
     rccontrol_get_build_artifacts_parse_requirements "$@"
     shift $#
     ;;
 
   build )
     action="build"
     shift
     rccontrol_build_parse_requirements "$@"
     shift $#
     ;;
 
   get-build-source )
     action="get-build-source"
     shift
     rccontrol_get_build_source_parse_requirements "$@"
     shift $#
     ;;
 
   build-source )
     action="build-source"
     shift
     rccontrol_build_source_parse_requirements "$@"
     shift $#
     ;;
 
   stack )
     action="stack"
     shift
     rccontrol_stack_parse_requirements "$@"
     shift $#
     ;;
 
   stack-status | status )
     action="stack-status"
     shift
     rccontrol_stack_status_parse_requirements "$@"
     shift $#
     ;;
 
   stack-upgrade )
     action="stack-upgrade"
     shift
     rccontrol_stack_upgrade_parse_requirements "$@"
     shift $#
     ;;
 
-  cli-redis )
-    action="cli-redis"
+  cli )
+    action="cli"
     shift
-    rccontrol_cli_redis_parse_requirements "$@"
-    shift $#
-    ;;
-
-  cli-db )
-    action="cli-db"
-    shift
-    rccontrol_cli_db_parse_requirements "$@"
-    shift $#
-    ;;
-
-  cli-db-upgrade )
-    action="cli-db-upgrade"
-    shift
-    rccontrol_cli_db_upgrade_parse_requirements "$@"
+    rccontrol_cli_parse_requirements "$@"
     shift $#
     ;;
 
   _completions )
     action="_completions"
     shift
     rccontrol__completions_parse_requirements "$@"
     shift $#
     ;;
 
   backup-db )
     action="backup-db"
     shift
     rccontrol_backup_db_parse_requirements "$@"
     shift $#
     ;;
 
   backup-data )
     action="backup-data"
     shift
     rccontrol_backup_data_parse_requirements "$@"
     shift $#
     ;;
 
   # :command.command_fallback
   "" )
     rccontrol_usage >&2
     exit 1
     ;;
 
   * )
     printf "invalid command: %s\n" "$action" >&2
     exit 1
     ;;
 
   esac
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
 
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1
       ;;
 
     * )
       # :command.parse_requirements_case
       # :command.parse_requirements_case_simple
       printf "invalid argument: %s\n" "$key" >&2
       exit 1
 
       ;;
 
     esac
   done
 
 }
 
 # :command.parse_requirements
 rccontrol_self_update_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
     rccontrol_self_update_usage
     exit
     ;;
 
   esac
 
   # :command.dependencies_filter
   if ! [[ -x "$(command -v curl)" ]]; then
     printf "missing dependency: curl\n" >&2
     exit 1
   fi
   if ! [[ -x "$(command -v tar)" ]]; then
     printf "missing dependency: tar\n" >&2
     exit 1
   fi
 
   # :command.command_filter
   action="self-update"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
     # :flag.case
     --auth-token )
 
       # :flag.case_arg
       if [[ -n ${2+x} ]]; then
 
         args[--auth-token]="$2"
         shift
         shift
       else
         printf "%s\n" "--auth-token requires an argument: --auth-token AUTH_TOKEN" >&2
         exit 1
       fi
       ;;
 
     # :flag.case
     --server-url )
 
       # :flag.case_arg
       if [[ -n ${2+x} ]]; then
 
         args[--server-url]="$2"
         shift
         shift
       else
         printf "%s\n" "--server-url requires an argument: --server-url SERVER_URL" >&2
         exit 1
       fi
       ;;
 
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1
       ;;
 
     * )
       # :command.parse_requirements_case
       # :command.parse_requirements_case_simple
       printf "invalid argument: %s\n" "$key" >&2
       exit 1
 
       ;;
 
     esac
   done
 
   # :command.default_assignments
   [[ -n ${args[--server-url]:-} ]] || args[--server-url]="https://code.rhodecode.com"
 
 }
 
 # :command.parse_requirements
 rccontrol_bootstrap_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
     rccontrol_bootstrap_usage
     exit
     ;;
 
   esac
 
   # :command.dependencies_filter
   if ! [[ -x "$(command -v curl)" ]]; then
     printf "missing dependency: curl\n" >&2
     exit 1
   fi
   if ! [[ -x "$(command -v tar)" ]]; then
     printf "missing dependency: tar\n" >&2
     exit 1
   fi
   if ! [[ -x "$(command -v md5sum)" ]]; then
     printf "missing dependency: md5sum\n" >&2
     exit 1
   fi
 
   # :command.command_filter
   action="bootstrap"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
     # :flag.case
     --force | -f )
 
       # :flag.case_no_arg
       args[--force]=1
       shift
       ;;
 
     # :flag.case
     --auth-token )
 
       # :flag.case_arg
       if [[ -n ${2+x} ]]; then
 
         args[--auth-token]="$2"
         shift
         shift
       else
         printf "%s\n" "--auth-token requires an argument: --auth-token AUTH_TOKEN" >&2
         exit 1
       fi
       ;;
 
     # :flag.case
     --server-url )
 
       # :flag.case_arg
       if [[ -n ${2+x} ]]; then
 
         args[--server-url]="$2"
         shift
         shift
       else
         printf "%s\n" "--server-url requires an argument: --server-url SERVER_URL" >&2
         exit 1
       fi
       ;;
 
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1
       ;;
 
     * )
       # :command.parse_requirements_case
       # :command.parse_requirements_case_simple
       printf "invalid argument: %s\n" "$key" >&2
       exit 1
 
       ;;
 
     esac
   done
 
   # :command.default_assignments
   [[ -n ${args[--server-url]:-} ]] || args[--server-url]="https://code.rhodecode.com"
 
 }
 
 # :command.parse_requirements
 rccontrol_get_build_artifacts_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
     rccontrol_get_build_artifacts_usage
     exit
     ;;
 
   esac
 
   # :command.command_filter
   action="get-build-artifacts"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
     # :flag.case
     --auth )
 
       # :flag.case_arg
       if [[ -n ${2+x} ]]; then
 
         args[--auth]="$2"
         shift
         shift
       else
         printf "%s\n" "--auth requires an argument: --auth AUTH" >&2
         exit 1
       fi
       ;;
 
     # :flag.case
     --installer-url )
 
       # :flag.case_arg
       if [[ -n ${2+x} ]]; then
 
         args[--installer-url]="$2"
         shift
         shift
       else
         printf "%s\n" "--installer-url requires an argument: --installer-url INSTALLER_URL" >&2
         exit 1
       fi
       ;;
 
     # :flag.case
     --manifest-url )
 
       # :flag.case_arg
       if [[ -n ${2+x} ]]; then
 
         args[--manifest-url]="$2"
         shift
         shift
       else
         printf "%s\n" "--manifest-url requires an argument: --manifest-url MANIFEST_URL" >&2
         exit 1
       fi
       ;;
 
     # :flag.case
     --version-name )
 
       # :flag.case_arg
       if [[ -n ${2+x} ]]; then
 
         args[--version-name]="$2"
         shift
         shift
       else
         printf "%s\n" "--version-name requires an argument: --version-name VERSION_NAME" >&2
         exit 1
       fi
       ;;
 
     -?* )
-      printf "invalid option: %s\n" "$key" >&2
-      exit 1
+      printf "invalid option: %s\n" "$key" >&2
+      exit 1
+      ;;
+
+    * )
+      # :command.parse_requirements_case
+      # :command.parse_requirements_case_simple
+      printf "invalid argument: %s\n" "$key" >&2
+      exit 1
+
+      ;;
+
+    esac
+  done
+
+  # :command.default_assignments
+  [[ -n ${args[--auth]:-} ]] || args[--auth]=""
+  [[ -n ${args[--installer-url]:-} ]] || args[--installer-url]="https://dls.rhodecode.com/dls/N2E2ZTY1NzA3NjYxNDA2NTc1NjI3MTcyNzA2MjcxNzIyZTcwNjI3YQ==/rhodecode-control/latest-linux-ee"
+  [[ -n ${args[--manifest-url]:-} ]] || args[--manifest-url]="https://dls.rhodecode.com/linux/MANIFEST"
+  [[ -n ${args[--version-name]:-} ]] || args[--version-name]="$RC_CLI_VERSION_NAME"
+
+}
+
+# :command.parse_requirements
+rccontrol_build_parse_requirements() {
+  # :command.fixed_flags_filter
+  case "${1:-}" in
+  --help | -h )
+    long_usage=yes
+    rccontrol_build_usage
+    exit
+    ;;
+
+  esac
+
+  # :command.command_filter
+  action="build"
+
+  # :command.parse_requirements_while
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case "$key" in
+    # :flag.case
+    --version-name )
+
+      # :flag.case_arg
+      if [[ -n ${2+x} ]]; then
+
+        args[--version-name]="$2"
+        shift
+        shift
+      else
+        printf "%s\n" "--version-name requires an argument: --version-name VERSION_NAME" >&2
+        exit 1
+      fi
+      ;;
+
+    -?* )
+      printf "invalid option: %s\n" "$key" >&2
+      exit 1
+      ;;
+
+    * )
+      # :command.parse_requirements_case
+      # :command.parse_requirements_case_simple
+      printf "invalid argument: %s\n" "$key" >&2
+      exit 1
+
+      ;;
+
+    esac
+  done
+
+  # :command.default_assignments
+  [[ -n ${args[--version-name]:-} ]] || args[--version-name]="$RC_CLI_VERSION_NAME"
+
+}
+
+# :command.parse_requirements
+rccontrol_get_build_source_parse_requirements() {
+  # :command.fixed_flags_filter
+  case "${1:-}" in
+  --help | -h )
+    long_usage=yes
+    rccontrol_get_build_source_usage
+    exit
+    ;;
+
+  esac
+
+  # :command.dependencies_filter
+  if ! [[ -x "$(command -v curl)" ]]; then
+    printf "missing dependency: curl\n" >&2
+    exit 1
+  fi
+  if ! [[ -x "$(command -v tar)" ]]; then
+    printf "missing dependency: tar\n" >&2
+    exit 1
+  fi
+
+  # :command.command_filter
+  action="get-build-source"
+
+  # :command.parse_requirements_while
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case "$key" in
+    # :flag.case
+    --revision )
+
+      # :flag.case_arg
+      if [[ -n ${2+x} ]]; then
+
+        args[--revision]="$2"
+        shift
+        shift
+      else
+        printf "%s\n" "--revision requires an argument: --revision REVISION" >&2
+        exit 1
+      fi
+      ;;
+
+    # :flag.case
+    --auth-token )
+
+      # :flag.case_arg
+      if [[ -n ${2+x} ]]; then
+
+        args[--auth-token]="$2"
+        shift
+        shift
+      else
+        printf "%s\n" "--auth-token requires an argument: --auth-token AUTH_TOKEN" >&2
+        exit 1
+      fi
+      ;;
+
+    # :flag.case
+    --server-url )
+
+      # :flag.case_arg
+      if [[ -n ${2+x} ]]; then
+
+        args[--server-url]="$2"
+        shift
+        shift
+      else
+        printf "%s\n" "--server-url requires an argument: --server-url SERVER_URL" >&2
+        exit 1
+      fi
+      ;;
+
+    -?* )
+      printf "invalid option: %s\n" "$key" >&2
+      exit 1
+      ;;
+
+    * )
+      # :command.parse_requirements_case
+      # :command.parse_requirements_case_simple
+      printf "invalid argument: %s\n" "$key" >&2
+      exit 1
+
+      ;;
+
+    esac
+  done
+
+  # :command.default_assignments
+  [[ -n ${args[--revision]:-} ]] || args[--revision]="default"
+  [[ -n ${args[--server-url]:-} ]] || args[--server-url]="https://code.rhodecode.com"
+
+}
+
+# :command.parse_requirements
+rccontrol_build_source_parse_requirements() {
+  # :command.fixed_flags_filter
+  case "${1:-}" in
+  --help | -h )
+    long_usage=yes
+    rccontrol_build_source_usage
+    exit
+    ;;
+
+  esac
+
+  # :command.command_filter
+  action="build-source"
+
+  # :command.parse_requirements_while
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case "$key" in
+    # :flag.case
+    --version-name )
+
+      # :flag.case_arg
+      if [[ -n ${2+x} ]]; then
+
+        args[--version-name]="$2"
+        shift
+        shift
+      else
+        printf "%s\n" "--version-name requires an argument: --version-name VERSION_NAME" >&2
+        exit 1
+      fi
+      ;;
+
+    -?* )
+      printf "invalid option: %s\n" "$key" >&2
+      exit 1
+      ;;
+
+    * )
+      # :command.parse_requirements_case
+      # :command.parse_requirements_case_simple
+      printf "invalid argument: %s\n" "$key" >&2
+      exit 1
+
+      ;;
+
+    esac
+  done
+
+  # :command.default_assignments
+  [[ -n ${args[--version-name]:-} ]] || args[--version-name]="$RC_CLI_VERSION_NAME"
+
+}
+
+# :command.parse_requirements
+rccontrol_stack_parse_requirements() {
+  # :command.fixed_flags_filter
+  case "${1:-}" in
+  --help | -h )
+    long_usage=yes
+    rccontrol_stack_usage
+    exit
+    ;;
+
+  esac
+
+  # :command.command_filter
+  action=${1:-}
+
+  case $action in
+  -* )
+    ;;
+
+  router )
+    action="router"
+    shift
+    rccontrol_stack_router_parse_requirements "$@"
+    shift $#
+    ;;
+
+  metrics )
+    action="metrics"
+    shift
+    rccontrol_stack_metrics_parse_requirements "$@"
+    shift $#
+    ;;
+
+  services )
+    action="services"
+    shift
+    rccontrol_stack_services_parse_requirements "$@"
+    shift $#
+    ;;
+
+  rhodecode )
+    action="rhodecode"
+    shift
+    rccontrol_stack_rhodecode_parse_requirements "$@"
+    shift $#
+    ;;
+
+  all )
+    action="all"
+    shift
+    rccontrol_stack_all_parse_requirements "$@"
+    shift $#
+    ;;
+
+  # :command.command_fallback
+  "" )
+    rccontrol_stack_usage >&2
+    exit 1
+    ;;
+
+  * )
+    printf "invalid command: %s\n" "$action" >&2
+    exit 1
+    ;;
+
+  esac
+
+  # :command.parse_requirements_while
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case "$key" in
+
+    -?* )
+      printf "invalid option: %s\n" "$key" >&2
+      exit 1
+      ;;
+
+    * )
+      # :command.parse_requirements_case
+      # :command.parse_requirements_case_simple
+      printf "invalid argument: %s\n" "$key" >&2
+      exit 1
+
+      ;;
+
+    esac
+  done
+
+}
+
+# :command.parse_requirements
+rccontrol_stack_router_parse_requirements() {
+  # :command.fixed_flags_filter
+  case "${1:-}" in
+  --help | -h )
+    long_usage=yes
+    rccontrol_stack_router_usage
+    exit
+    ;;
+
+  esac
+
+  # :command.command_filter
+  action="stack router"
+
+  # :command.parse_requirements_while
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case "$key" in
+
+    -?* )
+      other_args+=("$1")
+      shift
+      ;;
+
+    * )
+      # :command.parse_requirements_case
+      # :command.parse_requirements_case_catch_all
+      other_args+=("$1")
+      shift
+
+      ;;
+
+    esac
+  done
+
+}
+
+# :command.parse_requirements
+rccontrol_stack_metrics_parse_requirements() {
+  # :command.fixed_flags_filter
+  case "${1:-}" in
+  --help | -h )
+    long_usage=yes
+    rccontrol_stack_metrics_usage
+    exit
+    ;;
+
+  esac
+
+  # :command.command_filter
+  action="stack metrics"
+
+  # :command.parse_requirements_while
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case "$key" in
+
+    -?* )
+      other_args+=("$1")
+      shift
+      ;;
+
+    * )
+      # :command.parse_requirements_case
+      # :command.parse_requirements_case_catch_all
+      other_args+=("$1")
+      shift
+
+      ;;
+
+    esac
+  done
+
+}
+
+# :command.parse_requirements
+rccontrol_stack_services_parse_requirements() {
+  # :command.fixed_flags_filter
+  case "${1:-}" in
+  --help | -h )
+    long_usage=yes
+    rccontrol_stack_services_usage
+    exit
+    ;;
+
+  esac
+
+  # :command.command_filter
+  action="stack services"
+
+  # :command.parse_requirements_while
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case "$key" in
+
+    -?* )
+      other_args+=("$1")
+      shift
       ;;
 
     * )
       # :command.parse_requirements_case
-      # :command.parse_requirements_case_simple
-      printf "invalid argument: %s\n" "$key" >&2
-      exit 1
+      # :command.parse_requirements_case_catch_all
+      other_args+=("$1")
+      shift
 
       ;;
 
     esac
   done
 
-  # :command.default_assignments
-  [[ -n ${args[--auth]:-} ]] || args[--auth]=""
-  [[ -n ${args[--installer-url]:-} ]] || args[--installer-url]="https://dls.rhodecode.com/dls/N2E2ZTY1NzA3NjYxNDA2NTc1NjI3MTcyNzA2MjcxNzIyZTcwNjI3YQ==/rhodecode-control/latest-linux-ee"
-  [[ -n ${args[--manifest-url]:-} ]] || args[--manifest-url]="https://dls.rhodecode.com/linux/MANIFEST"
-  [[ -n ${args[--version-name]:-} ]] || args[--version-name]="$RC_CLI_VERSION_NAME"
-
 }
 
 # :command.parse_requirements
-rccontrol_build_parse_requirements() {
+rccontrol_stack_rhodecode_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
-    rccontrol_build_usage
+    rccontrol_stack_rhodecode_usage
     exit
     ;;
 
   esac
 
   # :command.command_filter
-  action="build"
+  action="stack rhodecode"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
-    # :flag.case
-    --version-name )
-
-      # :flag.case_arg
-      if [[ -n ${2+x} ]]; then
-
-        args[--version-name]="$2"
-        shift
-        shift
-      else
-        printf "%s\n" "--version-name requires an argument: --version-name VERSION_NAME" >&2
-        exit 1
-      fi
-      ;;
 
     -?* )
-      printf "invalid option: %s\n" "$key" >&2
-      exit 1
+      other_args+=("$1")
+      shift
       ;;
 
     * )
       # :command.parse_requirements_case
-      # :command.parse_requirements_case_simple
-      printf "invalid argument: %s\n" "$key" >&2
-      exit 1
+      # :command.parse_requirements_case_catch_all
+      other_args+=("$1")
+      shift
 
       ;;
 
     esac
   done
 
-  # :command.default_assignments
-  [[ -n ${args[--version-name]:-} ]] || args[--version-name]="$RC_CLI_VERSION_NAME"
-
 }
 
 # :command.parse_requirements
-rccontrol_get_build_source_parse_requirements() {
+rccontrol_stack_all_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
-    rccontrol_get_build_source_usage
+    rccontrol_stack_all_usage
     exit
     ;;
 
   esac
 
-  # :command.dependencies_filter
-  if ! [[ -x "$(command -v curl)" ]]; then
-    printf "missing dependency: curl\n" >&2
-    exit 1
-  fi
-  if ! [[ -x "$(command -v tar)" ]]; then
-    printf "missing dependency: tar\n" >&2
-    exit 1
-  fi
-
   # :command.command_filter
-  action="get-build-source"
+  action="stack all"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
-    # :flag.case
-    --auth-token )
-
-      # :flag.case_arg
-      if [[ -n ${2+x} ]]; then
-
-        args[--auth-token]="$2"
-        shift
-        shift
-      else
-        printf "%s\n" "--auth-token requires an argument: --auth-token AUTH_TOKEN" >&2
-        exit 1
-      fi
-      ;;
-
-    # :flag.case
-    --server-url )
-
-      # :flag.case_arg
-      if [[ -n ${2+x} ]]; then
-
-        args[--server-url]="$2"
-        shift
-        shift
-      else
-        printf "%s\n" "--server-url requires an argument: --server-url SERVER_URL" >&2
-        exit 1
-      fi
-      ;;
 
     -?* )
-      printf "invalid option: %s\n" "$key" >&2
-      exit 1
+      other_args+=("$1")
+      shift
       ;;
 
     * )
       # :command.parse_requirements_case
-      # :command.parse_requirements_case_simple
-      if [[ -z ${args[revision]+x} ]]; then
-
-        args[revision]=$1
+      # :command.parse_requirements_case_catch_all
+      other_args+=("$1")
         shift
-      else
-        printf "invalid argument: %s\n" "$key" >&2
-        exit 1
-      fi
 
       ;;
 
     esac
   done
 
-  # :command.required_args_filter
-  if [[ -z ${args[revision]+x} ]]; then
-    printf "missing required argument: REVISION\nusage: rccontrol get-build-source REVISION [OPTIONS]\n" >&2
-    exit 1
-  fi
-
-  # :command.default_assignments
-  [[ -n ${args[--server-url]:-} ]] || args[--server-url]="https://code.rhodecode.com"
-
 }
 
 # :command.parse_requirements
-rccontrol_build_source_parse_requirements() {
+rccontrol_stack_status_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
-    rccontrol_build_source_usage
+    rccontrol_stack_status_usage
     exit
     ;;
 
   esac
 
   # :command.command_filter
-  action="build-source"
+  action="stack-status"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
-    # :flag.case
-    --version-name )
-
-      # :flag.case_arg
-      if [[ -n ${2+x} ]]; then
-
-        args[--version-name]="$2"
-        shift
-        shift
-      else
-        printf "%s\n" "--version-name requires an argument: --version-name VERSION_NAME" >&2
-        exit 1
-      fi
-      ;;
 
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1
       ;;
 
     * )
       # :command.parse_requirements_case
       # :command.parse_requirements_case_simple
       printf "invalid argument: %s\n" "$key" >&2
       exit 1
 
       ;;
 
     esac
   done
 
-  # :command.default_assignments
-  [[ -n ${args[--version-name]:-} ]] || args[--version-name]="$RC_CLI_VERSION_NAME"
-
 }
 
 # :command.parse_requirements
-rccontrol_stack_parse_requirements() {
+rccontrol_stack_upgrade_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
-    rccontrol_stack_usage
+    rccontrol_stack_upgrade_usage
     exit
     ;;
 
   esac
 
   # :command.command_filter
-  action="stack"
+  action="stack-upgrade"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
 
     -?* )
-      other_args+=("$1")
-      shift
+      printf "invalid option: %s\n" "$key" >&2
+      exit 1
       ;;
 
     * )
       # :command.parse_requirements_case
-      # :command.parse_requirements_case_catch_all
-      if [[ -z ${args[name]+x} ]]; then
-
-        args[name]=$1
-        shift
-      else
-        other_args+=("$1")
-        shift
-      fi
+      # :command.parse_requirements_case_simple
+      printf "invalid argument: %s\n" "$key" >&2
+      exit 1
 
       ;;
 
     esac
   done
 
-  # :command.required_args_filter
-  if [[ -z ${args[name]+x} ]]; then
-    printf "missing required argument: NAME\nusage: rccontrol stack NAME [SERVICES PARAMS...]\n" >&2
-    exit 1
-  fi
-
-  # :command.whitelist_filter
-  if [[ ! ${args[name]} =~ ^(router|metrics|services|rhodecode)$ ]]; then
-    printf "%s\n" "name must be one of: router, metrics, services, rhodecode" >&2
-    exit 1
-  fi
-
 }
 
 # :command.parse_requirements
-rccontrol_stack_status_parse_requirements() {
+rccontrol_cli_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
-    rccontrol_stack_status_usage
+    rccontrol_cli_usage
     exit
     ;;
 
   esac
 
   # :command.command_filter
-  action="stack-status"
+  action=${1:-}
+
+  case $action in
+  -* )
+    ;;
+
+  redis )
+    action="redis"
+    shift
+    rccontrol_cli_redis_parse_requirements "$@"
+    shift $#
+    ;;
+
+  db )
+    action="db"
+    shift
+    rccontrol_cli_db_parse_requirements "$@"
+    shift $#
+    ;;
+
+  db-upgrade )
+    action="db-upgrade"
+    shift
+    rccontrol_cli_db_upgrade_parse_requirements "$@"
+    shift $#
+    ;;
+
+  storage )
+    action="storage"
+    shift
+    rccontrol_cli_storage_parse_requirements "$@"
+    shift $#
+    ;;
+
+  # :command.command_fallback
+  "" )
+    rccontrol_cli_usage >&2
+    exit 1
+    ;;
+
+  * )
+    printf "invalid command: %s\n" "$action" >&2
+    exit 1
+    ;;
+
+  esac
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
 
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1
       ;;
 
     * )
       # :command.parse_requirements_case
       # :command.parse_requirements_case_simple
       printf "invalid argument: %s\n" "$key" >&2
       exit 1
 
       ;;
 
     esac
   done
 
 }
 
 # :command.parse_requirements
-rccontrol_stack_upgrade_parse_requirements() {
+rccontrol_cli_redis_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
-    rccontrol_stack_upgrade_usage
+    rccontrol_cli_redis_usage
     exit
     ;;
 
   esac
 
   # :command.command_filter
-  action="stack-upgrade"
+  action="cli redis"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
 
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1
       ;;
 
     * )
       # :command.parse_requirements_case
       # :command.parse_requirements_case_simple
       printf "invalid argument: %s\n" "$key" >&2
       exit 1
 
       ;;
 
     esac
   done
 
 }
 
 # :command.parse_requirements
-rccontrol_cli_redis_parse_requirements() {
+rccontrol_cli_db_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
-    rccontrol_cli_redis_usage
+    rccontrol_cli_db_usage
     exit
     ;;
 
   esac
 
   # :command.command_filter
-  action="cli-redis"
+  action="cli db"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
 
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1
       ;;
 
     * )
       # :command.parse_requirements_case
       # :command.parse_requirements_case_simple
       printf "invalid argument: %s\n" "$key" >&2
       exit 1
 
       ;;
 
     esac
   done
 
 }
 
 # :command.parse_requirements
-rccontrol_cli_db_parse_requirements() {
+rccontrol_cli_db_upgrade_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
-    rccontrol_cli_db_usage
+    rccontrol_cli_db_upgrade_usage
     exit
     ;;
 
   esac
 
   # :command.command_filter
-  action="cli-db"
+  action="cli db-upgrade"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
 
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1
       ;;
 
     * )
       # :command.parse_requirements_case
       # :command.parse_requirements_case_simple
       printf "invalid argument: %s\n" "$key" >&2
       exit 1
 
       ;;
 
     esac
   done
 
 }
 
 # :command.parse_requirements
-rccontrol_cli_db_upgrade_parse_requirements() {
+rccontrol_cli_storage_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
-    rccontrol_cli_db_upgrade_usage
+    rccontrol_cli_storage_usage
     exit
     ;;
 
   esac
 
   # :command.command_filter
-  action="cli-db-upgrade"
+  action="cli storage"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
 
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1
       ;;
 
     * )
       # :command.parse_requirements_case
       # :command.parse_requirements_case_simple
       printf "invalid argument: %s\n" "$key" >&2
       exit 1
 
       ;;
 
     esac
   done
 
 }
 
 # :command.parse_requirements
 rccontrol__completions_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
     rccontrol__completions_usage
     exit
     ;;
 
   esac
 
   # :command.command_filter
   action="_completions"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
 
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1
       ;;
 
     * )
       # :command.parse_requirements_case
       # :command.parse_requirements_case_simple
       printf "invalid argument: %s\n" "$key" >&2
       exit 1
 
       ;;
 
     esac
   done
 
 }
 
 # :command.parse_requirements
 rccontrol_backup_db_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
     rccontrol_backup_db_usage
     exit
     ;;
 
   esac
 
   # :command.dependencies_filter
   if ! [[ -x "$(command -v tar)" ]]; then
     printf "missing dependency: tar\n" >&2
     exit 1
   fi
   if ! [[ -x "$(command -v gzip)" ]]; then
     printf "missing dependency: gzip\n" >&2
     exit 1
   fi
 
   # :command.command_filter
   action="backup-db"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
 
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1
       ;;
 
     * )
       # :command.parse_requirements_case
       # :command.parse_requirements_case_simple
       printf "invalid argument: %s\n" "$key" >&2
       exit 1
 
       ;;
 
     esac
   done
 
 }
 
 # :command.parse_requirements
 rccontrol_backup_data_parse_requirements() {
   # :command.fixed_flags_filter
   case "${1:-}" in
   --help | -h )
     long_usage=yes
     rccontrol_backup_data_usage
     exit
     ;;
 
   esac
 
   # :command.dependencies_filter
   if ! [[ -x "$(command -v tar)" ]]; then
     printf "missing dependency: tar\n" >&2
     exit 1
   fi
 
   # :command.command_filter
   action="backup-data"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
 
     -?* )
       printf "invalid option: %s\n" "$key" >&2
       exit 1
       ;;
 
     * )
       # :command.parse_requirements_case
       # :command.parse_requirements_case_simple
       printf "invalid argument: %s\n" "$key" >&2
       exit 1
 
       ;;
 
     esac
   done
 
 }
 
 # :command.initialize
 initialize() {
   version="4.28.0.REL20221101"
   long_usage=''
   set -e
 
   # :command.environment_variables_default
   export RC_CLI_VERSION_NAME="${RC_CLI_VERSION_NAME:-4.27.0}"
   export RC_STACK_ROUTER_EXT="${RC_STACK_ROUTER_EXT:-.custom/docker-compose-router.override.yaml}"
   export RC_STACK_METRICS_EXT="${RC_STACK_METRICS_EXT:-.custom/docker-compose-metrics.override.yaml}"
   export RC_STACK_SERVICES_EXT="${RC_STACK_SERVICES_EXT:-.custom/docker-compose-services.override.yaml}"
   export RC_STACK_RHODECODE_EXT="${RC_STACK_RHODECODE_EXT:-.custom/docker-compose-apps.override.yaml}"
 
   # src/initialize.sh
 
   ENV_FILE=$PWD/.env
 
   # bootstrap file is a config file at the same time
-  BOOTSTRAP_FILE='.rccontrol-bootstrap'
   CONFIG_FILE='.rccontrol.ini'
 
+  # STACK_LIST
   VALID_SERVICES="router metrics services rhodecode"
   DOCKER_DEFS_WORK_DIR="docker_defs"
 
+  # stage key, saved in .rccontrol.ini : stage func to execute
+  BOOTSTRAP_STAGES="\
+  bootstrap_docker_install:bootstrap_docker_install \
+  bootstrap_docker_commons:bootstrap_docker_commons \
+  bootstrap_config:bootstrap_config \
+  bootstrap_definitions:bootstrap_definitions \
+  bootstrap_overrides:bootstrap_overrides \
+  "
+
   #echo "1 ----"
   #echo $RC_STACK_SERVICES_EXT
   #echo $RC_STACK_METRICS_EXT
   #echo "1 -----"
 
   #echo '2 ----'
   #grep -v -e '^#' "$ENV_FILE" | xargs -I {} echo \'{}\'
   #echo '2 ----'
 
   #expose env vars from rccontrol donfig
   #if [[ -f $ENV_FILE ]]; then
   #  eval $(grep -v -e '^#' "$ENV_FILE" | xargs -I {} echo export \'{}\')
   #fi
 
   #ENV_EXPAND=""
   #for k in $(config_keys); do
   #  k_upper=${k^^}
   #  env_entry="-e $k_upper='$(config_get "$k")' "
   #  ENV_EXPAND+=$env_entry;
   #  #echo $env_expand
   #done
 
   #
   #echo "3 ----"
   #echo $RC_STACK_SERVICES_EXT
   #echo $RC_STACK_METRICS_EXT
   #echo $RC_STACK_RHODECODE_EXT
   #echo "3 -----"
 
   #env | grep RC_
 
+  docker_ping_host() {
+    PING_HOST="$1"
+    docker run --network rhodecode_network --rm alpine ping "$PING_HOST"
+  }
+
   # backup files from a docker volume into /tmp/backup.tar.gz
   docker-volume-backup-compressed() {
     docker run --rm -v /tmp:/backup --volumes-from "$1" debian:jessie tar -czvf /backup/backup.tar.gz "${@:2}"
   }
 
   # restore files from /tmp/backup.tar.gz into a docker volume
   docker-volume-restore-compressed() {
     docker run --rm -v /tmp:/backup --volumes-from "$1" debian:jessie tar -xzvf /backup/backup.tar.gz "${@:2}"
     echo "Double checking files..."
     docker run --rm -v /tmp:/backup --volumes-from "$1" debian:jessie ls -lh "${@:2}"
   }
 
   # backup files from a docker volume into /tmp/backup.tar
   docker-volume-backup() {
     docker run --rm -v /tmp:/backup --volumes-from "$1" busybox tar -cvf /backup/backup.tar "${@:2}"
   }
 
   # restore files from /tmp/backup.tar into a docker volume
   docker-volume-restore() {
     docker run --rm -v /tmp:/backup --volumes-from "$1" busybox tar -xvf /backup/backup.tar "${@:2}"
     echo "Double checking files..."
     docker run --rm -v /tmp:/backup --volumes-from "$1" busybox ls -lh "${@:2}"
   }
 
   get_started() {
     echo ""
     echo "~~~~ $(green Bootstrap completed) ~~~~"
     echo "To start using RhodeCode run the following stacks using ./rccontrol stack [STACK_NAME]"
     echo ""
     echo "$(yellow_bold [Optional]): run the edge router to control domain and SSL"
     echo "./rccontrol stack router up --detach"
     echo ""
     echo "$(green_bold [Mandatory]): run the services, like database, redis, channelstream etc..."
     echo "./rccontrol stack services up --detach"
     echo ""
     echo "$(green_bold [Mandatory]): run the rhodecode app stack"
     echo "./rccontrol stack rhodecode up --detach"
     echo ""
     echo "$(yellow_bold [Optional]): run the monitoring stack, this includes grafana/promethues logging/metrics system"
     echo "./rccontrol stack metrics up --detach"
     echo ""
     echo "check services status/ports"
     echo "./rccontrol status"
   }
 }
 
 # :command.run
 run() {
   declare -A args=()
   declare -a other_args=()
   declare -a input=()
   normalize_input "$@"
   parse_requirements "${input[@]}"
 
   if [[ $action == "self-update" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol_self_update_usage
     else
       rccontrol_self_update_command
     fi
 
   elif [[ $action == "bootstrap" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol_bootstrap_usage
     else
       rccontrol_bootstrap_command
     fi
 
   elif [[ $action == "get-build-artifacts" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol_get_build_artifacts_usage
     else
       rccontrol_get_build_artifacts_command
     fi
 
   elif [[ $action == "build" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol_build_usage
     else
       rccontrol_build_command
     fi
 
   elif [[ $action == "get-build-source" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol_get_build_source_usage
     else
       rccontrol_get_build_source_command
     fi
 
   elif [[ $action == "build-source" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol_build_source_usage
     else
       rccontrol_build_source_command
     fi
 
   elif [[ $action == "stack" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol_stack_usage
     else
       rccontrol_stack_command
     fi
 
+  elif [[ $action == "stack router" ]]; then
+    if [[ ${args[--help]:-} ]]; then
+      long_usage=yes
+      rccontrol_stack_router_usage
+    else
+      rccontrol_stack_router_command
+    fi
+
+  elif [[ $action == "stack metrics" ]]; then
+    if [[ ${args[--help]:-} ]]; then
+      long_usage=yes
+      rccontrol_stack_metrics_usage
+    else
+      rccontrol_stack_metrics_command
+    fi
+
+  elif [[ $action == "stack services" ]]; then
+    if [[ ${args[--help]:-} ]]; then
+      long_usage=yes
+      rccontrol_stack_services_usage
+    else
+      rccontrol_stack_services_command
+    fi
+
+  elif [[ $action == "stack rhodecode" ]]; then
+    if [[ ${args[--help]:-} ]]; then
+      long_usage=yes
+      rccontrol_stack_rhodecode_usage
+    else
+      rccontrol_stack_rhodecode_command
+    fi
+
+  elif [[ $action == "stack all" ]]; then
+    if [[ ${args[--help]:-} ]]; then
+      long_usage=yes
+      rccontrol_stack_all_usage
+    else
+      rccontrol_stack_all_command
+    fi
+
   elif [[ $action == "stack-status" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol_stack_status_usage
     else
       rccontrol_stack_status_command
     fi
 
   elif [[ $action == "stack-upgrade" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol_stack_upgrade_usage
     else
       rccontrol_stack_upgrade_command
     fi
 
-  elif [[ $action == "cli-redis" ]]; then
+  elif [[ $action == "cli" ]]; then
+    if [[ ${args[--help]:-} ]]; then
+      long_usage=yes
+      rccontrol_cli_usage
+    else
+      rccontrol_cli_command
+    fi
+
+  elif [[ $action == "cli redis" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol_cli_redis_usage
     else
       rccontrol_cli_redis_command
     fi
 
-  elif [[ $action == "cli-db" ]]; then
+  elif [[ $action == "cli db" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol_cli_db_usage
     else
       rccontrol_cli_db_command
     fi
 
-  elif [[ $action == "cli-db-upgrade" ]]; then
+  elif [[ $action == "cli db-upgrade" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol_cli_db_upgrade_usage
     else
       rccontrol_cli_db_upgrade_command
     fi
 
+  elif [[ $action == "cli storage" ]]; then
+    if [[ ${args[--help]:-} ]]; then
+      long_usage=yes
+      rccontrol_cli_storage_usage
+    else
+      rccontrol_cli_storage_command
+    fi
+
   elif [[ $action == "_completions" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol__completions_usage
     else
       rccontrol__completions_command
     fi
 
   elif [[ $action == "backup-db" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol_backup_db_usage
     else
       rccontrol_backup_db_command
     fi
 
   elif [[ $action == "backup-data" ]]; then
     if [[ ${args[--help]:-} ]]; then
       long_usage=yes
       rccontrol_backup_data_usage
     else
       rccontrol_backup_data_command
     fi
 
   elif [[ $action == "root" ]]; then
     root_command
   fi
 }
 
 initialize
 run "$@"

service/rhodecode/rhodecode.dockerfile

@@ -1,283 +1,283 b''
 FROM ubuntu:22.04
 # Using 22.04 LTS Release
 MAINTAINER RhodeCode Inc. <[e-mail]>
 
 ARG TZ="UTC"
 ARG LOCALE_TYPE=en_US.UTF-8
 ARG RHODECODE_TYPE=Enterprise
 # binary-install
 ARG RHODECODE_VERSION=4.28.0
 
 ARG RHODECODE_DB=sqlite
 ARG RHODECODE_USER_NAME=admin
 ARG RHODECODE_USER_PASS=secret4
-ARG RHODECODE_USER_EMAIL=[e-mail]
+ARG RHODECODE_USER_EMAIL=[e-mail]
 
 # nix ver/channels
 ARG DEV_NIX_VERSION=2.0.4
 ARG DEV_NIX_CHANNEL=nixos-18.03
 
 # env are runtime
 ENV \
     TZ=${TZ} \
     LOCALE_TYPE=${LOCALE_TYPE} \
     \
     ## Define type we build, and the instance we'll create
     RHODECODE_TYPE=${RHODECODE_TYPE} \
     RC_TYPE_ID=enterprise-1 \
     \
     ## SETUP ARGS FOR INSTALLATION ##
     ## set version we build on, get from .env or set default ver
     RHODECODE_VERSION=${RHODECODE_VERSION} \
     \
     ## set DB, default sqlite
     RHODECODE_DB=${RHODECODE_DB} \
     \
     ## set app bootstrap required data
     RHODECODE_USER_NAME=${RHODECODE_USER_NAME} \
     RHODECODE_USER_PASS=${RHODECODE_USER_PASS} \
     RHODECODE_USER_EMAIL=${RHODECODE_USER_EMAIL} \
     \
     RC_USER=rhodecode \
     \
     # SVN CONFIG
     MOD_DAV_SVN_CONF_FILE=/etc/rhodecode/conf/svn/mod_dav_svn.conf \
     MOD_DAV_SVN_PORT=8090 \
     \
     # SSHD CONFIG
     SSHD_CONF_FILE=/etc/rhodecode/sshd_config \
     \
     SHARED_CONF_DIR=/etc/rhodecode/conf \
     BUILD_CONF=/etc/rhodecode/conf_build \
     BUILD_BIN_DIR=/usr/local/bin/rhodecode_bin \
     RHODECODE_DATA_DIR=/var/opt/rhodecode_data \
     RHODECODE_REPO_DIR=/var/opt/rhodecode_repo_store \
     RHODECODE_HTTP_PORT=10020 \
     RHODECODE_VCS_PORT=10010 \
     RHODECODE_HOST=0.0.0.0 \
     RHODECODE_VCS_HOST=127.0.0.1
 
 ENV \
     RCCONTROL=/home/$RC_USER/.rccontrol-profile/bin/rccontrol \
     SUPERVISOR_CONF=/home/$RC_USER/.rccontrol/supervisor/supervisord.ini \
     # make application scripts visible
     PATH=$PATH:/home/$RC_USER/.rccontrol-profile/bin
 
 ENV SVN_LOCALE_DEPS apache2 apache2-utils libapache2-mod-svn
 ENV SSH_LOCALE_DEPS openssh-server
 ENV PYTHON_DEPS python2
 ENV EXTRA_DEPS ""
 
 RUN \
 echo "** install base packages **" && \
 set -eux; \
 	\
 	savedAptMark="$(apt-mark showmanual)"; \
 	apt-get update; \
 	DEBIAN_FRONTEND="noninteractive" \
 	apt-get install -y --no-install-recommends \
         tini \
         bash \
         binutils \
         tzdata \
         locales \
         openssl \
         curl \
         sudo \
         gosu \
         bzip2 \
         ca-certificates \
         $PYTHON_DEPS \
         $SSH_LOCALE_DEPS \
         $SVN_LOCALE_DEPS \
         $EXTRA_DEPS \
 	; \
 	rm -rf /var/lib/apt/lists/*;
 
 RUN \
 echo "** Configure the python executable for py2/3 compat **" && \
 IS_PY=$(which python3 || which python2) && \
 if [ -n $IS_PY ] ; then ln -s $IS_PY /usr/bin/python ; fi
 
 RUN \
 echo "** Configure the locales **" && \
     sed -i "s/^# ${LOCALE_TYPE}/${LOCALE_TYPE}/g" /etc/locale.gen && \
     locale-gen
 
 # locale-archive is a fix for old nix glibc2.26 locales available
 ENV \
     LOCALE_ARCHIVE=/var/opt/locale-archive \
     LANG=${LOCALE_TYPE} \
     LANGUAGE=${LOCALE_TYPE} \
     LC_ALL=${LOCALE_TYPE}
 
 # configure the system user
 # explicitly set uid/gid to guarantee that it won't change in the future
 # the values 999:999 are identical to the current user/group id assigned
 RUN \
 echo "** Create system user $RC_USER **" && \
   groupadd --system --gid 999 $RC_USER && \
   useradd --system --gid $RC_USER --uid 999 --shell /bin/bash $RC_USER && \
   usermod -G $RC_USER $RC_USER
 
 RUN \
 echo "** Create nix-build user $NIX_BLD_USER **" && \
     groupadd --system --gid 1099 $NIX_BLD_USER && \
     useradd --system --gid $NIX_BLD_USER --uid 1099 --shell /bin/bash $NIX_BLD_USER && \
     usermod -G $NIX_BLD_USER $NIX_BLD_USER
 
 RUN \
 echo "** disable nix sandboxing **" && \
     mkdir /etc/nix && echo 'sandbox = false' > /etc/nix/nix.conf
 
 
 # set the defult bash shell
 SHELL ["/bin/bash", "-c"]
 
 # Fix and set a timezone
 RUN \
 echo "** configure the timezone **" && \
 rm /etc/localtime && cp /usr/share/zoneinfo/$TZ /etc/localtime && \
 echo $TZ > /etc/timezone
 
 
 RUN \
 echo "** prepare rhodecode store and cache **" && \
   install -d -m 0700 -o $RC_USER -g $RC_USER /nix && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /opt/rhodecode && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /usr/local/bin/rhodecode_bin && \
   install -d -m 0755 -o $RC_USER -g $RC_USER $RHODECODE_REPO_DIR && \
   install -d -m 0755 -o $RC_USER -g $RC_USER $RHODECODE_DATA_DIR && \
   install -d -m 0755 -o $RC_USER -g $RC_USER $BUILD_CONF && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/rhodecode-vcsserver && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/rhodecode-enterprise-ce && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/rhodecode-enterprise-ee && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/ && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/cache && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/bootstrap && \
   install -d -m 0700 -o $RC_USER -g $RC_USER /home/$RC_USER/.ssh && \
   install -d -m 0700 -o $RC_USER -g $RC_USER /home/$RC_USER/.rhoderc
 
 # expose our custom sshd config
 COPY service/sshd/sshd_config $SSHD_CONF_FILE
 
 # Apache SVN setup
 RUN \
     echo "**** Apache config cleanup ****" && \
     rm -f /etc/apache2/conf.d/info.conf \
           /etc/apache2/conf.d/mpm.conf \
           /etc/apache2/conf.d/userdir.conf && \
     rm -f /etc/apache2/sites-enabled/* && \
     rm -f /etc/apache2/sites-available/*
 
 # custom SVN virtualhost
 COPY service/svn/virtualhost.conf /etc/apache2/sites-enabled/
 
 RUN \
 echo "**** Apache config ****" && \
     echo $(strings /usr/lib/apache2/modules/mod_dav_svn.so | grep 'Powered by') > /var/opt/dav.version && \
     mkdir -p /run/apache2 && \
     mkdir -p /var/opt/www && \
     echo "unset HOME" > /etc/apache2/envvars && \
     echo "export APACHE_RUN_USER=${RC_USER}" >> /etc/apache2/envvars && \
     echo "export APACHE_PID_FILE=/var/run/apache2/apache2.pid" >> /etc/apache2/envvars && \
     echo "export APACHE_RUN_DIR=/var/run/apache2" >> /etc/apache2/envvars && \
     echo "export APACHE_LOCK_DIR=/var/lock/apache2" >> /etc/apache2/envvars && \
     echo "export APACHE_RUN_USER=${RC_USER}" >> /etc/apache2/envvars && \
     echo "export APACHE_RUN_GROUP=${RC_USER}" >> /etc/apache2/envvars && \
     sed -i "s/Listen 80/Listen ${MOD_DAV_SVN_PORT}/g" /etc/apache2/ports.conf
 
 
 # Copy artifacts
 
 COPY --chown=$RC_USER:$RC_USER .cache/* /home/$RC_USER/.rccontrol/cache/
 COPY --chown=$RC_USER:$RC_USER config/_shared/rhodecode_enterprise.license /home/$RC_USER/.rccontrol/bootstrap/
 COPY --chown=$RC_USER:$RC_USER service/rhodecode/bootstrap/* /home/$RC_USER/.rccontrol/bootstrap/
 
 RUN \
 echo "**** locale-archive path ****" && \
     mv -v /home/$RC_USER/.rccontrol/cache/locale-archive /var/opt/locale-archive
 
 # change to non-root user for RUN commands
 USER $RC_USER
 WORKDIR /home/$RC_USER
 
 RUN \
 echo "** install rhodecode control **" && \
   cd /home/$RC_USER/.rccontrol/cache && \
   INSTALLER=$(ls -Art /home/$RC_USER/.rccontrol/cache/RhodeCode-installer-* | tail -n 1) && \
   chmod +x ${INSTALLER} && \
   ${INSTALLER} --accept-license && \
   ${RCCONTROL} self-init && \
   cp -v /home/$RC_USER/.rccontrol-profile/etc/ca-bundle.crt $BUILD_CONF/ && \
   echo "Done installing rhodecode control"
 
 RUN \
 echo "** install vcsserver ${RHODECODE_VERSION} **" && \
   ${RCCONTROL} install VCSServer --version ${RHODECODE_VERSION} --start-at-boot=yes --accept-license --offline \
   '{"host":"'"$RHODECODE_VCS_HOST"'", "port":"'"$RHODECODE_VCS_PORT"'"}' && \
   VCSSERVER_PATH=/home/$RC_USER/.rccontrol/vcsserver-1 && \
   rm -rf $BUILD_BIN_DIR/vcs_bin && \
   ln -s ${VCSSERVER_PATH}/profile/bin $BUILD_BIN_DIR/vcs_bin && \
   cp -v ${VCSSERVER_PATH}/vcsserver.ini $BUILD_CONF/vcsserver.ini && \
   cp -v ${VCSSERVER_PATH}/gunicorn_conf.py $BUILD_CONF/gunicorn_conf_vcs.py && \
   echo "Done installing vcsserver"
 
 RUN \
 echo "** install rhodecode ${RHODECODE_TYPE} ${RHODECODE_VERSION} **" && \
   RHODECODE_DB_INIT=sqlite && \
   ${RCCONTROL} install ${RHODECODE_TYPE} --version ${RHODECODE_VERSION} --start-at-boot=yes --accept-license --offline \
   '{"host":"'"$RHODECODE_HOST"'", "port":"'"$RHODECODE_HTTP_PORT"'", "username":"'"$RHODECODE_USER_NAME"'", "password":"'"$RHODECODE_USER_PASS"'", "email":"'"$RHODECODE_USER_EMAIL"'", "repo_dir":"'"$RHODECODE_REPO_DIR"'", "database": "'"$RHODECODE_DB_INIT"'", "skip_existing_db": "1"}' && \
   RHODECODE_PATH=/home/$RC_USER/.rccontrol/${RC_TYPE_ID} && \
   rm -rf $BUILD_BIN_DIR/bin && ln -s ${RHODECODE_PATH}/profile/bin $BUILD_BIN_DIR/ && \
   cp -v ${RHODECODE_PATH}/rhodecode.ini $BUILD_CONF/rhodecode.ini && \
-  cp -v ${RHODECODE_PATH}/gunicorn_conf.py $BUILD_CONF/gunicorn_conf.py && \
+  cp -v ${RHODECODE_PATH}/gunicorn_conf.py $BUILD_CONF/gunicorn_conf_rc.py && \
   cp -v ${RHODECODE_PATH}/search_mapping.ini $BUILD_CONF/search_mapping.ini && \
   mkdir -p $RHODECODE_DATA_DIR/static && cp -r ${RHODECODE_PATH}/public/* $RHODECODE_DATA_DIR/static/ && \
   rm ${RHODECODE_PATH}/rhodecode.db && \
   echo "Done installing rhodecode"
 
 
 RUN \
 echo "** configure supervisord **" && \
   cp -v ${SUPERVISOR_CONF} $BUILD_CONF/ && \
   sed -i "s/self_managed_supervisor = False/self_managed_supervisor = True/g" /home/$RC_USER/.rccontrol.ini && \
   echo "Done installing supervisord"
 
 USER root
 
 RUN \
 echo "**** cleanup ****" && \
     apt-get remove -y $PYTHON_DEPS && \
     apt-get autoclean -y && \
     rm -f /tmp/* && \
     rm -rf /var/lib/apt/lists/* && \
     rm -rf /var/cache/apk/* && \
     rm -f /home/$RC_USER/.rccontrol/cache/RhodeCode-installer-* && \
     rm -f /home/$RC_USER/.rccontrol/cache/*.bz2 && \
     rm -f ${SUPERVISOR_CONF} && \
     echo "Done cleanup"
 
 
 # copy entrypoints
 COPY entrypoints.d/entrypoint.sh /opt/entrypoints.d/entrypoint.sh
 
 RUN chmod +x /opt/entrypoints.d/entrypoint.sh
 
 # config volume
 VOLUME /etc/rhodecode/conf
 
 # repo store volume
 VOLUME /var/opt/rhodecode_repo_store
 
 # data volume
 VOLUME /var/opt/rhodecode_data
 
 # logs volume
 VOLUME /var/log/rhodecode
 
 ENTRYPOINT ["/opt/entrypoints.d/entrypoint.sh"]
 
 # compose can override this
-CMD [ "$BUILD_BIN_DIR/bin/gunicorn", "--error-logfile=-", "--paster=/etc/rhodecode/conf_build/run.ini", "--config=/etc/rhodecode/conf_build/gunicorn_conf.py" ]
+CMD [ "$BUILD_BIN_DIR/bin/gunicorn", "--error-logfile=-", "--paster=/etc/rhodecode/conf_build/run.ini", "--config=/etc/rhodecode/conf_build/gunicorn_conf_rc.py" ]

service/rhodecode/rhodecode_source.dockerfile

@@ -1,324 +1,327 b''
 FROM ubuntu:22.04
 # Using 22.04 LTS Release
 MAINTAINER RhodeCode Inc. <[e-mail]>
 
 ARG TZ="UTC"
 ARG LOCALE_TYPE=en_US.UTF-8
 ARG RHODECODE_TYPE=Enterprise
 # source-install
 ARG RHODECODE_VERSION=4.28.0
 
 ARG RHODECODE_DB=sqlite
 ARG RHODECODE_USER_NAME=admin
 ARG RHODECODE_USER_PASS=secret4
-ARG RHODECODE_USER_EMAIL=[e-mail]
+ARG RHODECODE_USER_EMAIL=[e-mail]
 
 # nix ver/channels
 ARG DEV_NIX_VERSION=2.0.4
 ARG DEV_NIX_CHANNEL=nixos-18.03
 
 # env are runtime
 ENV \
     TZ=${TZ} \
     LOCALE_TYPE=${LOCALE_TYPE} \
     \
     ## Define type we build, and the instance we'll create
     RHODECODE_TYPE=${RHODECODE_TYPE} \
     RC_TYPE_ID=enterprise-1 \
     \
     ## SETUP ARGS FOR INSTALLATION ##
     ## set version we build on, get from .env or set default ver
     RHODECODE_VERSION=${RHODECODE_VERSION} \
     \
     ## set DB, default sqlite
     RHODECODE_DB=${RHODECODE_DB} \
     \
     ## set app bootstrap required data
     RHODECODE_USER_NAME=${RHODECODE_USER_NAME} \
     RHODECODE_USER_PASS=${RHODECODE_USER_PASS} \
     RHODECODE_USER_EMAIL=${RHODECODE_USER_EMAIL} \
     \
     RC_USER=rhodecode \
     \
     # SVN CONFIG
     MOD_DAV_SVN_CONF_FILE=/etc/rhodecode/conf/svn/mod_dav_svn.conf \
     MOD_DAV_SVN_PORT=8090 \
     \
     # SSHD CONFIG
     SSHD_CONF_FILE=/etc/rhodecode/sshd_config \
     \
     SHARED_CONF_DIR=/etc/rhodecode/conf \
     BUILD_CONF=/etc/rhodecode/conf_build \
     BUILD_BIN_DIR=/usr/local/bin/rhodecode_bin \
     RHODECODE_DATA_DIR=/var/opt/rhodecode_data \
     RHODECODE_REPO_DIR=/var/opt/rhodecode_repo_store \
     RHODECODE_HTTP_PORT=10020 \
     RHODECODE_VCS_PORT=10010 \
     RHODECODE_HOST=0.0.0.0 \
     RHODECODE_VCS_HOST=127.0.0.1
 
 ENV \
     RCCONTROL=/home/$RC_USER/.rccontrol-profile/bin/rccontrol \
     SUPERVISOR_CONF=/home/$RC_USER/.rccontrol/supervisor/supervisord.ini \
     # make application scripts visible
     PATH=$PATH:/home/$RC_USER/.rccontrol-profile/bin
 
 ENV SVN_LOCALE_DEPS apache2 apache2-utils libapache2-mod-svn
 ENV SSH_LOCALE_DEPS openssh-server
 ENV PYTHON_DEPS python2
 ENV EXTRA_DEPS  ""
 
 ENV \
     PATH=$PATH:/nix/var/nix/profiles/per-user/$RC_USER/profile/bin:/home/$RC_USER/rhodecode-enterprise-ee/profile/bin \
     NIX_BLD_USER=nixbld \
     NIX_PATH=/nix/var/nix/profiles/per-user/$RC_USER/channels \
     NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
 
 
 RUN \
 echo "** install base packages **" && \
 set -eux; \
 	\
 	savedAptMark="$(apt-mark showmanual)"; \
 	apt-get update; \
 	DEBIAN_FRONTEND="noninteractive" \
 	apt-get install -y --no-install-recommends \
         tini \
         bash \
         binutils \
         tzdata \
         locales \
         openssl \
         curl \
         sudo \
         gosu \
         bzip2 \
         ca-certificates \
         $PYTHON_DEPS \
         $SSH_LOCALE_DEPS \
         $SVN_LOCALE_DEPS \
         $EXTRA_DEPS \
 	; \
 	rm -rf /var/lib/apt/lists/*;
 
 RUN \
 echo "** Configure the python executable for py2/3 compat **" && \
 IS_PY=$(which python3 || which python2) && \
 if [ -n $IS_PY ] ; then ln -s $IS_PY /usr/bin/python ; fi
 
 RUN \
 echo "** Configure the locales **" && \
     sed -i "s/^# ${LOCALE_TYPE}/${LOCALE_TYPE}/g" /etc/locale.gen && \
     locale-gen
 
 # locale-archive is a fix for old nix glibc2.26 locales available
 ENV \
     LOCALE_ARCHIVE=/var/opt/locale-archive \
     LANG=${LOCALE_TYPE} \
     LANGUAGE=${LOCALE_TYPE} \
     LC_ALL=${LOCALE_TYPE}
 
 # configure the system user
 # explicitly set uid/gid to guarantee that it won't change in the future
 # the values 999:999 are identical to the current user/group id assigned
 RUN \
 echo "** Create system user $RC_USER **" && \
   groupadd --system --gid 999 $RC_USER && \
   useradd --system --gid $RC_USER --uid 999 --shell /bin/bash $RC_USER && \
   usermod -G $RC_USER $RC_USER
 
 RUN \
 echo "** Create nix-build user $NIX_BLD_USER **" && \
     groupadd --system --gid 1099 $NIX_BLD_USER && \
     useradd --system --gid $NIX_BLD_USER --uid 1099 --shell /bin/bash $NIX_BLD_USER && \
     usermod -G $NIX_BLD_USER $NIX_BLD_USER
 
 RUN \
 echo "** disable nix sandboxing **" && \
     mkdir /etc/nix && echo 'sandbox = false' > /etc/nix/nix.conf
 
 
 # set the defult bash shell
 SHELL ["/bin/bash", "-c"]
 
 # Fix and set a timezone
 RUN \
 echo "** configure the timezone **" && \
 rm /etc/localtime && cp /usr/share/zoneinfo/$TZ /etc/localtime && \
 echo $TZ > /etc/timezone
 
 
 RUN \
 echo "** prepare rhodecode store and cache **" && \
   install -d -m 0700 -o $RC_USER -g $RC_USER /nix && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /opt/rhodecode && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /usr/local/bin/rhodecode_bin && \
   install -d -m 0755 -o $RC_USER -g $RC_USER $RHODECODE_REPO_DIR && \
   install -d -m 0755 -o $RC_USER -g $RC_USER $RHODECODE_DATA_DIR && \
   install -d -m 0755 -o $RC_USER -g $RC_USER $BUILD_CONF && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/rhodecode-vcsserver && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/rhodecode-enterprise-ce && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/rhodecode-enterprise-ee && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/ && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/cache && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/bootstrap && \
   install -d -m 0700 -o $RC_USER -g $RC_USER /home/$RC_USER/.ssh && \
   install -d -m 0700 -o $RC_USER -g $RC_USER /home/$RC_USER/.rhoderc
 
 # expose our custom sshd config
 COPY service/sshd/sshd_config $SSHD_CONF_FILE
 
 # Apache SVN setup
 RUN \
     echo "**** Apache config cleanup ****" && \
     rm -f /etc/apache2/conf.d/info.conf \
           /etc/apache2/conf.d/mpm.conf \
           /etc/apache2/conf.d/userdir.conf && \
     rm -f /etc/apache2/sites-enabled/* && \
     rm -f /etc/apache2/sites-available/*
 
 # custom SVN virtualhost
 COPY service/svn/virtualhost.conf /etc/apache2/sites-enabled/
 
 RUN \
 echo "**** Apache config ****" && \
     echo $(strings /usr/lib/apache2/modules/mod_dav_svn.so | grep 'Powered by') > /var/opt/dav.version && \
     mkdir -p /run/apache2 && \
     mkdir -p /var/opt/www && \
     echo "unset HOME" > /etc/apache2/envvars && \
     echo "export APACHE_RUN_USER=${RC_USER}" >> /etc/apache2/envvars && \
     echo "export APACHE_PID_FILE=/var/run/apache2/apache2.pid" >> /etc/apache2/envvars && \
     echo "export APACHE_RUN_DIR=/var/run/apache2" >> /etc/apache2/envvars && \
     echo "export APACHE_LOCK_DIR=/var/lock/apache2" >> /etc/apache2/envvars && \
     echo "export APACHE_RUN_USER=${RC_USER}" >> /etc/apache2/envvars && \
     echo "export APACHE_RUN_GROUP=${RC_USER}" >> /etc/apache2/envvars && \
     sed -i "s/Listen 80/Listen ${MOD_DAV_SVN_PORT}/g" /etc/apache2/ports.conf
 
 
 # Copy artifacts
 
 COPY --chown=$RC_USER:$RC_USER .cache/* /home/$RC_USER/.rccontrol/cache/
 COPY --chown=$RC_USER:$RC_USER config/_shared/rhodecode_enterprise.license /home/$RC_USER/.rccontrol/bootstrap/
 COPY --chown=$RC_USER:$RC_USER service/rhodecode/bootstrap/* /home/$RC_USER/.rccontrol/bootstrap/
 
+RUN \
+echo "**** locale-archive path ****" && \
+    mv -v /home/$RC_USER/.rccontrol/cache/locale-archive /var/opt/locale-archive
+
 COPY --chown=$RC_USER:$RC_USER .source/ /home/$RC_USER/
 RUN \
 echo "** prepare rhodecode dirs **" && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/vcsserver-1 && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/community-1 && \
   install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/enterprise-1 && \
-
-RUN \
-echo "**** locale-archive path ****" && \
-    mv -v /home/$RC_USER/.rccontrol/cache/locale-archive /var/opt/locale-archive
+  echo "Done rhodecode dirs"
 
 # change to non-root user for RUN commands
 USER $RC_USER
 WORKDIR /home/$RC_USER
 
 
 RUN \
 echo "** download and install nix ** from $DEV_NIX_VERSION/install" && \
-    curl -L https://releases.nixos.org/nix/nix-$DEV_NIX_VERSION/install | USER=$RC_USER /bin/bash
+    curl -L https://releases.nixos.org/nix/nix-$DEV_NIX_VERSION/install | USER=$RC_USER /bin/bash && \
+    echo "Done nix install"
 
 RUN \
 echo "** update nix package database and set channel to $DEV_NIX_CHANNEL **" && \
     . /home/rhodecode/.nix-profile/etc/profile.d/nix.sh && \
 	nix-channel --add https://channels.nixos.org/$DEV_NIX_CHANNEL nixpkgs && \
-	nix-channel --update
+	nix-channel --update && \
+    echo "done nix package updates"
 
 
 RUN \
     echo "** save nix config **" && \
     touch /home/$RC_USER/.rhoderc && \
     mkdir -p /home/$RC_USER/.nixpkgs && touch /home/$RC_USER/.nixpkgs/config.nix && \
     printf '{\n  rc = {\n   sources = {\n    rhodecode-vcsserver = "/home/'$RC_USER'/rhodecode-vcsserver";\n    rhodecode-enterprise-ce = "/home/'$RC_USER'/rhodecode-enterprise-ce";\n    rhodecode-enterprise-ee = "/home/'$RC_USER'/rhodecode-enterprise-ee";\n   };\n  };\n}\n' > /home/$RC_USER/.nixpkgs/config.nix
 
 
 RUN \
 echo "** install rhodecode control **" && \
 #  cd /home/$RC_USER/.rccontrol/cache && \
 #  INSTALLER=$(ls -Art /home/$RC_USER/.rccontrol/cache/RhodeCode-installer-* | tail -n 1) && \
 #  chmod +x ${INSTALLER} && \
 #  ${INSTALLER} --accept-license && \
 #  ${RCCONTROL} self-init && \
 #  cp -v /home/$RC_USER/.rccontrol-profile/etc/ca-bundle.crt $BUILD_CONF/ && \
   echo "Done installing rhodecode control"
 
 RUN \
 echo "** install vcsserver ${RHODECODE_VERSION} **" && \
   . /home/rhodecode/.nix-profile/etc/profile.d/nix.sh && \
   nix-build --show-trace --cores 0 --max-jobs 4 --no-build-output --out-link /home/$RC_USER/.rccontrol/vcsserver-1/profile rhodecode-vcsserver/default.nix && \
   nix-shell --command 'echo COMMAND FROM NIX-SHELL TEST' rhodecode-vcsserver/default.nix && \
   VCSSERVER_PATH=/home/$RC_USER/rhodecode-vcsserver && \
   rm -rf $BUILD_BIN_DIR/vcs_bin && \
   cp -rv --preserve=links /home/$RC_USER/.rccontrol/vcsserver-1/profile/bin $BUILD_BIN_DIR/vcs_bin && \
   cp -v ${VCSSERVER_PATH}/configs/production.ini $BUILD_CONF/vcsserver.ini && \
   cp -v ${VCSSERVER_PATH}/configs/gunicorn_config.py $BUILD_CONF/gunicorn_conf_vcs.py && \
   echo "Done installing vcsserver"
 
 RUN \
 echo "** install build Community ${RHODECODE_VERSION} **" && \
   . /home/rhodecode/.nix-profile/etc/profile.d/nix.sh && \
   echo "done"
 
 RUN \
 echo "** install rhodecode ${RHODECODE_VERSION} **" && \
   . /home/rhodecode/.nix-profile/etc/profile.d/nix.sh && \
   nix-build --show-trace --cores 0 --max-jobs 4 --no-build-output --out-link /home/$RC_USER/.rccontrol/enterprise-1/profile rhodecode-enterprise-ee/default.nix && \
   nix-shell --command 'echo ok' rhodecode-enterprise-ee/default.nix && \
   RHODECODE_PATH=/home/$RC_USER/rhodecode-enterprise-ee && \
   rm -rf $BUILD_BIN_DIR/bin && \
   cp -rv --preserve=links /home/$RC_USER/.rccontrol/enterprise-1/profile/bin $BUILD_BIN_DIR/ && \
   cp -v ${RHODECODE_PATH}/configs/production.ini $BUILD_CONF/rhodecode.ini && \
-  cp -v ${RHODECODE_PATH}/configs/gunicorn_config.py $BUILD_CONF/gunicorn_conf.py && \
+  cp -v ${RHODECODE_PATH}/configs/gunicorn_config.py $BUILD_CONF/gunicorn_conf_rc.py && \
   mkdir -p $RHODECODE_DATA_DIR/static && cp -r /home/$RC_USER/.rccontrol/enterprise-1/profile/etc/static/* $RHODECODE_DATA_DIR/static/ && \
   echo "Done installing rhodecode"
 
 
 RUN \
 echo "** configure supervisord **" && \
   #cp -v ${SUPERVISOR_CONF} $BUILD_CONF/ && \
   #sed -i "s/self_managed_supervisor = False/self_managed_supervisor = True/g" /home/$RC_USER/.rccontrol.ini && \
   echo "Done installing supervisord"
 
 USER root
 
 RUN \
 echo "**** cleanup ****" && \
     apt-get remove -y $PYTHON_DEPS && \
     apt-get autoclean -y && \
     rm -f /tmp/* && \
     rm -f /home/$RC_USER/.rccontrol/cache/RhodeCode-installer-* && \
     rm -f /home/$RC_USER/.rccontrol/cache/*.bz2 && \
     rm -rf /var/lib/apt/lists/* && \
     rm -rf /var/cache/apk/* && \
     rm -f ${SUPERVISOR_CONF} && \
     rm -rf /home/$RC_USER/rhodecode-vcsserver && \
     rm -rf /home/$RC_USER/rhodecode-enterprise-ce && \
     rm -rf /home/$RC_USER/rhodecode-enterprise-ee && \
     echo "Done cleanup"
 
 
 # copy entrypoints
 COPY entrypoints.d/entrypoint.sh /opt/entrypoints.d/entrypoint.sh
 
 RUN chmod +x /opt/entrypoints.d/entrypoint.sh
 
 # config volume
 VOLUME /etc/rhodecode/conf
 
 # repo store volume
 VOLUME /var/opt/rhodecode_repo_store
 
 # data volume
 VOLUME /var/opt/rhodecode_data
 
 # logs volume
 VOLUME /var/log/rhodecode
 
 ENTRYPOINT ["/opt/entrypoints.d/entrypoint.sh"]
 
 # compose can override this
-CMD [ "$BUILD_BIN_DIR/bin/gunicorn", "--error-logfile=-", "--paster=/etc/rhodecode/conf_build/run.ini", "--config=/etc/rhodecode/conf_build/gunicorn_conf.py" ]
+CMD [ "$BUILD_BIN_DIR/bin/gunicorn", "--error-logfile=-", "--paster=/etc/rhodecode/conf_build/run.ini", "--config=/etc/rhodecode/conf_build/gunicorn_conf_rc.py" ]