docker: added healthcheck for sshd and added autoheal flags to fix unhealthy containers

docker-compose-apps.yaml

@@ -93,6 +93,7 b' services:'
       *custom-logging
 
     labels:
+      - "autoheal=true"
       - "traefik.enable=true"
       - "traefik.http.routers.rhodecode.entrypoints=http"
       - "traefik.http.routers.rhodecode.rule=Host(`${RC_HOSTNAME:?must-specify-rhodecode-hostname}`)"
@@ -157,6 +158,9 b' services:'
     logging:
       *custom-logging
 
+    labels:
+      - "autoheal=true"
+
   celery:
     networks:
       - rhodecode_network
@@ -298,6 +302,9 b' services:'
     logging:
       *custom-logging
 
+    labels:
+      - "autoheal=true"
+
   sshd:
     networks:
       - rhodecode_network
@@ -316,6 +323,13 b' services:'
       RC_APP_TYPE: rhodecode_sshd
       SSH_BOOTSTRAP: 1
 
+    healthcheck:
+      # we call ssh internally, to find permission denied, if SSH is DOWN it would be connection refused
+      test: [ "CMD-SHELL", "/usr/bin/ssh -p22 -o StrictHostKeyChecking=no -o PasswordAuthentication=No rhodecode@localhost true 2>&1 | grep -c 'Permission denied' > /dev/null" ]
+      timeout: 30s
+      interval: 60s
+      retries: 10
+
 #    ports:
 #      # set from .env file
 #      - "${RC_SSH_PORT:?must-specify-ssh-port}:22"
@@ -330,6 +344,7 b' services:'
       *custom-logging
 
     labels:
+      - "autoheal=true"
       - "traefik.enable=true"
       - "traefik.http.routers.sshd.entrypoints=ssh"
       - "traefik.http.routers.sshd.rule=Host(`*`)"

scripts/rccontrol/rccontrol

@@ -838,7 +838,7 b' rccontrol_stack_upgrade_rhodecode_usage() {'
 
     # :flag.usage
     echo "  --stop-wait STOP_WAIT"
-    printf "    Number of seconds to wait for stopping old containers\n"
+    printf "    Number of seconds to wait for stopping old containers, use low value to\n    quickly restart and recycle containers\n"
     printf "    Default: 120\n"
     echo
 
@@ -1667,7 +1667,7 b' rccontrol_self_update_command() {'
 
   fi
 
-  echo "$(green self-update: update to new version: $version_dl)"
+  echo "$(green self-update: updating rccontrol script to new version: $version_dl)"
   rm $FINAL_TARGET && mv -v $DL_TARGET $FINAL_TARGET
 
   get_docker_definitions $revision
@@ -2273,7 +2273,7 b' rccontrol_stack_upgrade_rhodecode_command() {'
   RC_EDITION=$REPLY
 
   if [[ $restart_unhealthy ]]; then
-    unhealthy=$(docker ps -f health=unhealthy --format "{{.ID}}")
+    unhealthy=$(docker ps --filter health=unhealthy --filter label=autoheal=true --format "{{.ID}}")
 
     if [[ $unhealthy == "" ]]; then
       echo "$(green no unhealthy containers found)"
@@ -2350,12 +2350,12 b' rccontrol_stack_upgrade_rhodecode_command() {'
 
   }
 
-  # RHODECODE
-  stop_containers "rhodecode" $docker_stop_wait $stop_wait
-
   # VCSSERVER
   stop_containers "vcsserver" $docker_stop_wait $stop_wait
 
+  # RHODECODE
+  stop_containers "rhodecode" $docker_stop_wait $stop_wait
+
   # SVN
   stop_containers "svn" $docker_stop_wait $stop_wait
 
@@ -4445,7 +4445,7 b' rccontrol__completions_parse_requirements() {'
 
 # :command.initialize
 initialize() {
-  version="4.28.0.REL.2022.12.21.1"
+  version="4.28.0.REL.2022.12.21.2"
   long_usage=''
   set -e