rccontrol: various fixes

- added rccontrol stack-upgrade with 0 downtime replace of services
- fixed entrypoints for svn reboot

docker-compose-apps.source.yaml

@@ -16,8 +16,6 b' services:'
       context: .
       dockerfile: service/rhodecode/rhodecode_source.dockerfile
 
-    image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:-4.28.0}_SOURCE
-
   vcsserver:
     environment:
       HISTFILE: /home/rhodecode/.bash_history_docker
@@ -29,36 +27,26 b' services:'
       context: .
       dockerfile: service/rhodecode/rhodecode_source.dockerfile
 
-    image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:-4.28.0}_SOURCE
-
   celery:
 
     build:
       context: .
       dockerfile: service/rhodecode/rhodecode_source.dockerfile
 
-    image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:-4.28.0}_SOURCE
-
   celery-beat:
 
     build:
       context: .
       dockerfile: service/rhodecode/rhodecode_source.dockerfile
 
-    image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:-4.28.0}_SOURCE
-
   svn:
 
     build:
       context: .
       dockerfile: service/rhodecode/rhodecode_source.dockerfile
 
-    image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:-4.28.0}_SOURCE
-
   sshd:
 
     build:
       context: .
       dockerfile: service/rhodecode/rhodecode_source.dockerfile
-
-    image: rhodecode/rhodecode-${RC_EDITION}:${RC_VERSION:-4.28.0}_SOURCE
  No newline at end of file

docker-compose-apps.yaml

@@ -269,6 +269,7 b' services:'
     restart: always
     env_file:
         - ${RC_ENV_FILE:?must-specify-rc-env-file}
+
     command: ["apachectl", "-D", "FOREGROUND"]
 
     environment:

docker-compose-services.yaml

@@ -212,7 +212,7 b' services:'
       - logvolume:/var/log/rhodecode
 
     profiles:
-      ["postgres"]
+      ["postgres", "database"]
 
     logging:
       *custom-logging

entrypoints.d/entrypoint.sh

@@ -138,15 +138,19 b' if [[ $RC_APP_TYPE = "rhodecode_http" ]]; then'
 fi
 
 
+if [[ $RC_APP_TYPE = "rhodecode_sshd" ]]; then
+  # Fix problem with Missing privilege separation directory error
+  mkdir -p /run/sshd
+fi
+
+if [[ $RC_APP_TYPE = "rhodecode_svn" ]]; then
+  set -- tini -- "$@"
+fi
+
 if [ "$RC_APP_PROC" = 1 ]; then
   # Fix problem with zombie processes when using executables like supervisord/gunicorn
   set -- tini -- "$@"
   set -- gosu $RC_USER "$@"
 fi
 
-if [ "$RC_APP_TYPE" = "rhodecode_sshd" ]; then
-  # Fix problem with Missing privilege separation directory error
-  mkdir -p /run/sshd
-fi
-
 exec "$@"

scripts/rccontrol/rccontrol

@@ -80,6 +80,12 b' rccontrol_usage() {'
     printf "Environment Variables:\n"
 
     # :environment_variable.usage
+    echo "  RC_CLI_VERSION_NAME"
+    printf "    default version to build and install\n"
+    printf "    Default: 4.28.0.REL.2022.12.20.2\n"
+    echo
+
+    # :environment_variable.usage
     echo "  RCC_CONFIG"
     printf "    default config file for rccontrol\n"
     printf "    Default: .rccontrol.ini\n"
@@ -91,12 +97,6 b' rccontrol_usage() {'
     echo
 
     # :environment_variable.usage
-    echo "  RC_CLI_VERSION_NAME"
-    printf "    default version to build and install\n"
-    printf "    Default: 4.27.0\n"
-    echo
-
-    # :environment_variable.usage
     echo "  RC_STACK_ROUTER_EXT"
     printf "\n"
     printf "    Default: .custom/docker-compose-router.override.yaml\n"
@@ -674,8 +674,8 b' rccontrol_stack_status_usage() {'
 
     # :command.usage_flags
     # :flag.usage
-    echo "  --simple, -s"
-    printf "    Display a simple format using a table\n"
+    echo "  --detailed"
+    printf "    Display a detailed format\n"
     echo
 
   fi
@@ -694,8 +694,131 b' rccontrol_stack_upgrade_usage() {'
   fi
 
   printf "Usage:\n"
-  printf "  rccontrol stack-upgrade\n"
-  printf "  rccontrol stack-upgrade --help | -h\n"
+  printf "  rccontrol stack-upgrade COMMAND\n"
+  printf "  rccontrol stack-upgrade [COMMAND] --help | -h\n"
+  echo
+  # :command.usage_commands
+  printf "Commands:\n"
+  echo "  router      upgrade the router stack"
+  echo "  metrics     run the router stack"
+  echo "  services    run the router stack"
+  echo "  rhodecode   run the router stack"
+  echo
+
+  # :command.long_usage
+  if [[ -n $long_usage ]]; then
+    printf "Options:\n"
+
+    # :command.usage_fixed_flags
+    echo "  --help, -h"
+    printf "    Show this help\n"
+    echo
+
+  fi
+}
+
+# :command.usage
+rccontrol_stack_upgrade_router_usage() {
+  if [[ -n $long_usage ]]; then
+    printf "rccontrol stack-upgrade router - upgrade the router stack\n"
+    echo
+
+  else
+    printf "rccontrol stack-upgrade router - upgrade the router stack\n"
+    echo
+
+  fi
+
+  printf "Usage:\n"
+  printf "  rccontrol stack-upgrade router\n"
+  printf "  rccontrol stack-upgrade router --help | -h\n"
+  echo
+
+  # :command.long_usage
+  if [[ -n $long_usage ]]; then
+    printf "Options:\n"
+
+    # :command.usage_fixed_flags
+    echo "  --help, -h"
+    printf "    Show this help\n"
+    echo
+
+  fi
+}
+
+# :command.usage
+rccontrol_stack_upgrade_metrics_usage() {
+  if [[ -n $long_usage ]]; then
+    printf "rccontrol stack-upgrade metrics - run the router stack\n"
+    echo
+
+  else
+    printf "rccontrol stack-upgrade metrics - run the router stack\n"
+    echo
+
+  fi
+
+  printf "Usage:\n"
+  printf "  rccontrol stack-upgrade metrics\n"
+  printf "  rccontrol stack-upgrade metrics --help | -h\n"
+  echo
+
+  # :command.long_usage
+  if [[ -n $long_usage ]]; then
+    printf "Options:\n"
+
+    # :command.usage_fixed_flags
+    echo "  --help, -h"
+    printf "    Show this help\n"
+    echo
+
+  fi
+}
+
+# :command.usage
+rccontrol_stack_upgrade_services_usage() {
+  if [[ -n $long_usage ]]; then
+    printf "rccontrol stack-upgrade services - run the router stack\n"
+    echo
+
+  else
+    printf "rccontrol stack-upgrade services - run the router stack\n"
+    echo
+
+  fi
+
+  printf "Usage:\n"
+  printf "  rccontrol stack-upgrade services\n"
+  printf "  rccontrol stack-upgrade services --help | -h\n"
+  echo
+
+  # :command.long_usage
+  if [[ -n $long_usage ]]; then
+    printf "Options:\n"
+
+    # :command.usage_fixed_flags
+    echo "  --help, -h"
+    printf "    Show this help\n"
+    echo
+
+  fi
+}
+
+# :command.usage
+rccontrol_stack_upgrade_rhodecode_usage() {
+  if [[ -n $long_usage ]]; then
+    printf "rccontrol stack-upgrade rhodecode - run the router stack\n"
+    echo
+
+  else
+    printf "rccontrol stack-upgrade rhodecode - run the router stack\n"
+    echo
+
+  fi
+
+  printf "Usage:\n"
+  printf "  rccontrol stack-upgrade rhodecode [OPTIONS]\n"
+  printf "  rccontrol stack-upgrade rhodecode --help | -h\n"
   echo
 
   # :command.long_usage
@@ -707,6 +830,13 b' rccontrol_stack_upgrade_usage() {'
     printf "    Show this help\n"
     echo
 
+    # :command.usage_flags
+    # :flag.usage
+    echo "  --stop-wait STOP_WAIT"
+    printf "    Number of seconds to wait for stopping old containers\n"
+    printf "    Default: 120\n"
+    echo
+
   fi
 }
 
@@ -1480,24 +1610,61 b' rccontrol_self_update_command() {'
   revision=${args[--revision]}
 
   FETCH_FROM=$SERVER_URL/rhodecode-enterprise-docker/raw/master/scripts/rccontrol/rccontrol
-  TARGET=$PWD/scripts/rccontrol/rccontrol
 
-  version_old=$($TARGET --version)
+  FINAL_TARGET=$PWD/scripts/rccontrol/rccontrol
+  DL_TARGET=$PWD/scripts/rccontrol/rccontrol.tmp
+
+  version_old=$($FINAL_TARGET --version)
+  rc_version_old=$(echo $RC_CLI_VERSION_NAME)
 
   echo "$(green self-update: downloading new rccontrol script from $FETCH_FROM)"
-  curl --fail --header "X-Rc-Auth-Token: $AUTH_TOKEN" -o $TARGET -L $FETCH_FROM
-  chmod +x $TARGET
+  curl --fail --header "X-Rc-Auth-Token: $AUTH_TOKEN" -o $DL_TARGET -L $FETCH_FROM
+  chmod +x $DL_TARGET
+
+  version_dl=$($DL_TARGET --version)
+  rc_version_dl=$(echo $RC_CLI_VERSION_NAME)
 
-  version_dl=$($TARGET --version)
-  echo "self-update: version downloaded: $version_dl"
+  echo "self-update: rccontrol version downloaded: $version_dl"
 
   if [[ $version_old == $version_dl ]]; then
-    echo "$(yellow self-update: no new version available, latest seems to be $version_dl)"
+    echo "$(yellow self-update: downloaded version is the same as current version ($version_dl))"
     exit
+  fi
+
+  .env --file $RUNTIME_ENV
+  .env get RC_VERSION
+
+  ENV_VER=$REPLY
+
+  if [[ $DEBUG ]]; then
+    echo "env version $ENV_VER"
+    echo "Old Version: $rc_version_old"
+    echo "Current RC VERSION: $rc_version_dl"
+  fi
+
+  if [[ $rc_version_old == $rc_version_dl ]]; then
+    echo "$(yellow self-update: latest RC_VERSION=$rc_version_dl seems to be up-to date)"
   else
-    echo "$(green self-update: update to new version: $version_dl)"
+    echo "$(green self-update: setting new RC_VERSION to runtime.env: $rc_version_dl)"
+
+    while true; do
+      read -p "self-update: Would you like to continue with setting RC_VERSION=$rc_version_dl into .runtime.env file? [yn] " yn
+      case $yn in
+        [Yy]*)
+          .env set RC_VERSION=$rc_version_dl
+          echo "self-update: new version set into config"
+          return 2 ;;
+        [Nn]*)
+          exit ;;
+        *) echo "Please answer y or n." ;;
+      esac
+    done
+
   fi
 
+  echo "$(green self-update: update to new version: $version_dl)"
+  rm $FINAL_TARGET && mv -v $DL_TARGET $FINAL_TARGET
+
   get_docker_definitions $revision
 
   exit
@@ -1802,6 +1969,7 b' rccontrol_get_build_source_command() {'
   # src/get_build_source_command.sh
   check_bootstrap
 
+  DEBUG=${args[--debug]}
   AUTH_TOKEN=${args[--auth-token]}
   SERVER_URL=${args[--server-url]}
   revision=${args[--revision]}
@@ -1813,16 +1981,16 b' rccontrol_get_build_source_command() {'
   RHODECODE_EE_HASH=$revision
 
   # download sources
-  echo "** download rhodecode source for build from $SERVER_URL using '$revision' hash**"
+  echo "** download rhodecode source for build from $SERVER_URL using '$revision' hash **"
 
   echo "getting $SERVER_URL/rhodecode-vcsserver/archive/$RHODECODE_VCS_HASH.tgz"
-  curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-vcsserver/archive/$RHODECODE_VCS_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR
+  curl --fail --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-vcsserver/archive/$RHODECODE_VCS_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR
 
   echo "getting $SERVER_URL/rhodecode-enterprise-ce/archive/$RHODECODE_CE_HASH.tgz"
-  curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-ce/archive/$RHODECODE_CE_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR
+  curl --fail --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-ce/archive/$RHODECODE_CE_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR
 
   echo "getting $SERVER_URL/rhodecode-enterprise-ee/archive/$RHODECODE_EE_HASH.tgz"
-  curl --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-ee/archive/$RHODECODE_EE_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR
+  curl --fail --header "X-Rc-Auth-Token: $AUTH_TOKEN" -L $SERVER_URL/rhodecode-enterprise-ee/archive/$RHODECODE_EE_HASH.tgz?with_hash=0 | tar -xz -C $SOURCE_DIR
 
   rm -rf $SOURCE_DIR/rhodecode-vcsserver && \
   mv  $SOURCE_DIR/*rhodecode-vcsserver-plain     $SOURCE_DIR/rhodecode-vcsserver
@@ -1932,7 +2100,7 b' rccontrol_stack_services_command() {'
     RC_STACK_SERVICES_EXT_LCL="-f $RC_STACK_SERVICES_EXT"
   fi
 
-  RC_STACK_PROFILES="--profile postgres --profile redis --profile elasticsearch --profile channelstream"
+  RC_STACK_PROFILES="--profile database --profile redis --profile elasticsearch --profile channelstream"
 
   CMD_SERVICES="\
   RC_ENV_FILE=$ENV_FILE docker compose \
@@ -2016,7 +2184,7 b' rccontrol_stack_status_command() {'
   # src/stack_status_command.sh
   check_bootstrap
   DEBUG=${args[--debug]}
-  SIMPLE=${args[--simple]}
+  DETAILED=${args[--detailed]}
 
   if [[ $DEBUG ]]; then
     echo "---"
@@ -2034,34 +2202,145 b' rccontrol_stack_status_command() {'
     echo "---"
     fi
 
-  if [[ $SIMPLE ]]; then
-    docker ps --filter=name=rc_cluster --format "table {{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}" | (read -r; printf "%s\n" "$REPLY"; sort -k 2 )
-  else
+  if [[ $DETAILED ]]; then
       ps_cmd=$(docker ps --filter=name=rc_cluster --format="{{.ID}}")
 
       for service in $ps_cmd; do
-
         servicename=`docker inspect --format '{{ .Name }}' $service`
         servicename=${servicename:1}
         printf "CONTAINER:\\t$servicename\n"
         printf "LOGS:\\t\\tdocker logs --tail=100 $service\n"
         docker ps --filter=name=$servicename --format="ID:\\t\\t{{.ID}}\nIMAGE:\\t\\t{{.Image}}\nSTATUS:\\t\\t{{.Status}}\nPORTS:\\t\\t{{.Ports}}\n"
         echo ""
-
       done
+  else
+      docker ps --filter=name=rc_cluster --format "table {{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}" | (read -r; printf "%s\n" "$REPLY"; sort -k 2 )
   fi
 
 }
 
 # :command.function
-rccontrol_stack_upgrade_command() {
-  # src/stack_upgrade_command.sh
-  check_bootstrap
+rccontrol_stack_upgrade_router_command() {
+  # src/stack_upgrade_router_command.sh
+  echo "To upgrade this stack run this:"
+  echo "./rccontrol stack router up --force-recreate --build --detach"
+  echo ""
+  echo "To reclaim old image space run: docker image prune -f"
+
+}
+
+# :command.function
+rccontrol_stack_upgrade_metrics_command() {
+  # src/stack_upgrade_metrics_command.sh
+  echo "To upgrade this stack run this:"
+  echo "./rccontrol stack metrics up --force-recreate --build --detach"
+  echo ""
+  echo "To reclaim old image space run: docker image prune -f"
+
+}
 
-  echo "To upgrade particular stack run this:"
-  echo "./rccontrol stack STACK_NAME up --force-recreate --build --detach"
+# :command.function
+rccontrol_stack_upgrade_services_command() {
+  # src/stack_upgrade_services_command.sh
+  echo "To upgrade this stack run this:"
+  echo "./rccontrol stack services up --force-recreate --build --detach"
   echo ""
   echo "To reclaim old image space run: docker image prune -f"
+
+}
+
+# :command.function
+rccontrol_stack_upgrade_rhodecode_command() {
+  # src/stack_upgrade_rhodecode_command.sh
+  check_bootstrap
+
+  stop_wait=${args[--stop-wait]}
+  docker_stop_wait=3600
+
+  # search docker IDs for current service
+  .env --file $RUNTIME_ENV
+
+  .env get RC_VERSION
+  RC_VERSION=$REPLY
+
+  .env get RC_EDITION
+  RC_EDITION=$REPLY
+
+  echo "Pulling new image: rhodecode/rhodecode-ee:${RC_VERSION}"
+  #eval "docker pull rhodecode/rhodecode-ee:${RC_VERSION}"
+
+  stop_containers() {
+    target_container=$1
+    docker_stop_wait=$2
+    stop_wait=$3
+    target_containers_pat="rc_cluster_apps-$target_container-*"
+
+    local docker_ids=$(docker ps --filter name="$target_containers_pat" -q)
+
+    scale=$(echo $docker_ids | wc -w)
+    scale_2=$(($scale*2))
+
+    echo ""
+    echo "Found $scale services for $target_container"
+    echo "docker-ids: $docker_ids"
+    echo ""
+
+    echo "Scaling to $scale_2 containers for $target_container"
+    ./rccontrol stack rhodecode up $target_container --no-recreate --detach --scale $target_container=$scale_2
+
+    echo ""
+    echo "Now running $scale_2 services for upgrade..."
+    echo ""
+    echo "Stopping old $target_container containers..."
+
+    spin='⣾⣽⣻⢿⡿⣟⣯⣷'
+    spin='⠾⠽⠻⠟⠯⠷'
+    charwidth=1
+
+    for container in $docker_ids
+    do
+
+      local i=$(($stop_wait * 10))
+
+      tput civis # cursor invisible
+      while true
+        s=$((i / 10))
+        do
+            i=$(($i-1))
+            if [[ $i -lt 1 ]]; then
+              break
+            fi
+
+            k=$(((k + $charwidth) % ${#spin}))
+            printf "%s stopping container in $s seconds..." "${spin:$k:$charwidth}"
+            printf "\r"
+            sleep .1
+        done
+        tput cnorm
+
+      echo "now running container $container stop with --time $docker_stop_wait"
+      docker stop --time $docker_stop_wait $container
+      echo "Stopped $container"
+    done
+
+  }
+
+  # RHODECODE
+  stop_containers "rhodecode" $docker_stop_wait $stop_wait
+
+  # VCSSERVER
+  stop_containers "vcsserver" $docker_stop_wait $stop_wait
+
+  # SVN
+  stop_containers "svn" $docker_stop_wait $stop_wait
+
+  ./rccontrol stack rhodecode up sshd --force-recreate --build --detach
+  ./rccontrol stack rhodecode up celery --force-recreate --build --detach
+  ./rccontrol stack rhodecode up celery-beat --force-recreate --build --detach
+
+  echo "Upgrade completed"
+  exit
+
 }
 
 # :command.function
@@ -2320,8 +2599,8 b' parse_requirements() {'
 
   # :command.environment_variables_filter
   # :command.environment_variables_default
+  export RC_CLI_VERSION_NAME="${RC_CLI_VERSION_NAME:-4.28.0.REL.2022.12.20.2}"
   export RCC_CONFIG="${RCC_CONFIG:-.rccontrol.ini}"
-  export RC_CLI_VERSION_NAME="${RC_CLI_VERSION_NAME:-4.27.0}"
   export RC_STACK_ROUTER_EXT="${RC_STACK_ROUTER_EXT:-.custom/docker-compose-router.override.yaml}"
   export RC_STACK_METRICS_EXT="${RC_STACK_METRICS_EXT:-.custom/docker-compose-metrics.override.yaml}"
   export RC_STACK_SERVICES_EXT="${RC_STACK_SERVICES_EXT:-.custom/docker-compose-services.override.yaml}"
@@ -3361,10 +3640,10 b' rccontrol_stack_status_parse_requirements() {'
     key="$1"
     case "$key" in
       # :flag.case
-      --simple | -s)
+      --detailed)
 
         # :flag.case_no_arg
-        args[--simple]=1
+        args[--detailed]=1
         shift
         ;;
 
@@ -3405,7 +3684,183 b' rccontrol_stack_upgrade_parse_requirements() {'
   done
 
   # :command.command_filter
-  action="stack-upgrade"
+  action=${1:-}
+
+  case $action in
+    -*) ;;
+
+    router)
+      action="router"
+      shift
+      rccontrol_stack_upgrade_router_parse_requirements "$@"
+      shift $#
+      ;;
+
+    metrics)
+      action="metrics"
+      shift
+      rccontrol_stack_upgrade_metrics_parse_requirements "$@"
+      shift $#
+      ;;
+
+    services)
+      action="services"
+      shift
+      rccontrol_stack_upgrade_services_parse_requirements "$@"
+      shift $#
+      ;;
+
+    rhodecode)
+      action="rhodecode"
+      shift
+      rccontrol_stack_upgrade_rhodecode_parse_requirements "$@"
+      shift $#
+      ;;
+
+    # :command.command_fallback
+    "")
+      rccontrol_stack_upgrade_usage >&2
+      exit 1
+      ;;
+
+    *)
+      printf "invalid command: %s\n" "$action" >&2
+      exit 1
+      ;;
+
+  esac
+
+  # :command.parse_requirements_while
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case "$key" in
+
+      -?*)
+        printf "invalid option: %s\n" "$key" >&2
+        exit 1
+        ;;
+
+      *)
+        # :command.parse_requirements_case
+        # :command.parse_requirements_case_simple
+        printf "invalid argument: %s\n" "$key" >&2
+        exit 1
+
+        ;;
+
+    esac
+  done
+
+}
+
+# :command.parse_requirements
+rccontrol_stack_upgrade_router_parse_requirements() {
+  # :command.fixed_flags_filter
+  while [[ $# -gt 0 ]]; do
+    case "${1:-}" in
+      --help | -h)
+        long_usage=yes
+        rccontrol_stack_upgrade_router_usage
+        exit
+        ;;
+
+      *)
+        break
+        ;;
+
+    esac
+  done
+
+  # :command.command_filter
+  action="stack-upgrade router"
+
+  # :command.parse_requirements_while
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case "$key" in
+
+      -?*)
+        printf "invalid option: %s\n" "$key" >&2
+        exit 1
+        ;;
+
+      *)
+        # :command.parse_requirements_case
+        # :command.parse_requirements_case_simple
+        printf "invalid argument: %s\n" "$key" >&2
+        exit 1
+
+        ;;
+
+    esac
+  done
+
+}
+
+# :command.parse_requirements
+rccontrol_stack_upgrade_metrics_parse_requirements() {
+  # :command.fixed_flags_filter
+  while [[ $# -gt 0 ]]; do
+    case "${1:-}" in
+      --help | -h)
+        long_usage=yes
+        rccontrol_stack_upgrade_metrics_usage
+        exit
+        ;;
+
+      *)
+        break
+        ;;
+
+    esac
+  done
+
+  # :command.command_filter
+  action="stack-upgrade metrics"
+
+  # :command.parse_requirements_while
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case "$key" in
+
+      -?*)
+        printf "invalid option: %s\n" "$key" >&2
+        exit 1
+        ;;
+
+      *)
+        # :command.parse_requirements_case
+        # :command.parse_requirements_case_simple
+        printf "invalid argument: %s\n" "$key" >&2
+        exit 1
+
+        ;;
+
+    esac
+  done
+
+}
+
+# :command.parse_requirements
+rccontrol_stack_upgrade_services_parse_requirements() {
+  # :command.fixed_flags_filter
+  while [[ $# -gt 0 ]]; do
+    case "${1:-}" in
+      --help | -h)
+        long_usage=yes
+        rccontrol_stack_upgrade_services_usage
+        exit
+        ;;
+
+      *)
+        break
+        ;;
+
+    esac
+  done
+
+  # :command.command_filter
+  action="stack-upgrade services"
 
   # :command.parse_requirements_while
   while [[ $# -gt 0 ]]; do
@@ -3431,6 +3886,67 b' rccontrol_stack_upgrade_parse_requirements() {'
 }
 
 # :command.parse_requirements
+rccontrol_stack_upgrade_rhodecode_parse_requirements() {
+  # :command.fixed_flags_filter
+  while [[ $# -gt 0 ]]; do
+    case "${1:-}" in
+      --help | -h)
+        long_usage=yes
+        rccontrol_stack_upgrade_rhodecode_usage
+        exit
+        ;;
+
+      *)
+        break
+        ;;
+
+    esac
+  done
+
+  # :command.command_filter
+  action="stack-upgrade rhodecode"
+
+  # :command.parse_requirements_while
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case "$key" in
+      # :flag.case
+      --stop-wait)
+
+        # :flag.case_arg
+        if [[ -n ${2+x} ]]; then
+
+          args[--stop-wait]="$2"
+          shift
+          shift
+        else
+          printf "%s\n" "--stop-wait requires an argument: --stop-wait STOP_WAIT" >&2
+          exit 1
+        fi
+        ;;
+
+      -?*)
+        printf "invalid option: %s\n" "$key" >&2
+        exit 1
+        ;;
+
+      *)
+        # :command.parse_requirements_case
+        # :command.parse_requirements_case_simple
+        printf "invalid argument: %s\n" "$key" >&2
+        exit 1
+
+        ;;
+
+    esac
+  done
+
+  # :command.default_assignments
+  [[ -n ${args[--stop-wait]:-} ]] || args[--stop-wait]="120"
+
+}
+
+# :command.parse_requirements
 rccontrol_cli_parse_requirements() {
   # :command.fixed_flags_filter
   while [[ $# -gt 0 ]]; do
@@ -3896,13 +4412,13 b' rccontrol__completions_parse_requirements() {'
 
 # :command.initialize
 initialize() {
-  version="4.28.0.rel2022.12.15.3"
+  version="4.28.0.REL.2022.12.20.2"
   long_usage=''
   set -e
 
   # :command.environment_variables_default
+  export RC_CLI_VERSION_NAME="${RC_CLI_VERSION_NAME:-4.28.0.REL.2022.12.20.2}"
   export RCC_CONFIG="${RCC_CONFIG:-.rccontrol.ini}"
-  export RC_CLI_VERSION_NAME="${RC_CLI_VERSION_NAME:-4.27.0}"
   export RC_STACK_ROUTER_EXT="${RC_STACK_ROUTER_EXT:-.custom/docker-compose-router.override.yaml}"
   export RC_STACK_METRICS_EXT="${RC_STACK_METRICS_EXT:-.custom/docker-compose-metrics.override.yaml}"
   export RC_STACK_SERVICES_EXT="${RC_STACK_SERVICES_EXT:-.custom/docker-compose-services.override.yaml}"
@@ -4148,6 +4664,42 b' run() {'
       fi
       ;;
 
+    "stack-upgrade router")
+      if [[ ${args[--help]:-} ]]; then
+        long_usage=yes
+        rccontrol_stack_upgrade_router_usage
+      else
+        rccontrol_stack_upgrade_router_command
+      fi
+      ;;
+
+    "stack-upgrade metrics")
+      if [[ ${args[--help]:-} ]]; then
+        long_usage=yes
+        rccontrol_stack_upgrade_metrics_usage
+      else
+        rccontrol_stack_upgrade_metrics_command
+      fi
+      ;;
+
+    "stack-upgrade services")
+      if [[ ${args[--help]:-} ]]; then
+        long_usage=yes
+        rccontrol_stack_upgrade_services_usage
+      else
+        rccontrol_stack_upgrade_services_command
+      fi
+      ;;
+
+    "stack-upgrade rhodecode")
+      if [[ ${args[--help]:-} ]]; then
+        long_usage=yes
+        rccontrol_stack_upgrade_rhodecode_usage
+      else
+        rccontrol_stack_upgrade_rhodecode_command
+      fi
+      ;;
+
     "cli")
       if [[ ${args[--help]:-} ]]; then
         long_usage=yes