feat(dev-tools): allow running celery/celeryd for dev in more flexible ways

- mainly control over ce/ee source
- celeryd command allows running in background for tests

scripts/dev-tools/dev-tools

@@ -39,6 +39,7 b' dev_tools_usage() {'
   printf "  %s   run bash console for rhodecode-ce\n" "ce                 "
   printf "  %s   run bash console for svn web\n" "svn                "
   printf "  %s   run bash console for rhodecode-celery\n" "celery             "
+  printf "  %s   run celery instance on top of mounted source code, e.g ./dev-tools celeryd&\n" "celeryd            "
   printf "  %s   run bash console for rhodecode-sshd\n" "sshd               "
   printf "  %s   run bash console for rhodecode-tools-ce\n" "tools              "
   printf "  %s   create dev env required to run vcsserver and rhodecode\n" "dev-env            "
@@ -74,7 +75,7 b' dev_tools_usage() {'
     echo
 
     # :command.footer
-    printf "RhodeCode Inc 2023\n\n"
+    printf "RhodeCode Inc 2024\n\n"
     echo
 
   fi
@@ -221,7 +222,7 b' dev_tools_celery_usage() {'
   fi
 
   printf "%s\n" "Usage:"
-  printf "  dev-tools celery\n"
+  printf "  dev-tools celery [OPTIONS]\n"
   printf "  dev-tools celery --help | -h\n"
   echo
 
@@ -229,6 +230,61 b' dev_tools_celery_usage() {'
   if [[ -n $long_usage ]]; then
     printf "%s\n" "Options:"
 
+    # :command.usage_flags
+    # :flag.usage
+    printf "  %s\n" "--workdir WORKDIR"
+    printf "    workdir to run the pip install -e on top of can be ce or ee only\n"
+    printf "    Default: /home/rhodecode/rhodecode-enterprise-ce\n"
+    echo
+
+    # :flag.usage
+    printf "  %s\n" "--cmd CMD"
+    printf "    cmd script to run, this is on host machine, relative to $WORKSPACE_HOME\n"
+    printf "    Default: .dev/cmd-celery.sh\n"
+    echo
+
+    # :command.usage_fixed_flags
+    printf "  %s\n" "--help, -h"
+    printf "    Show this help\n"
+    echo
+
+  fi
+}
+
+# :command.usage
+dev_tools_celeryd_usage() {
+  if [[ -n $long_usage ]]; then
+    printf "dev-tools celeryd - run celery instance on top of mounted source code, e.g ./dev-tools celeryd&\n"
+    echo
+
+  else
+    printf "dev-tools celeryd - run celery instance on top of mounted source code, e.g ./dev-tools celeryd&\n"
+    echo
+
+  fi
+
+  printf "%s\n" "Usage:"
+  printf "  dev-tools celeryd [OPTIONS]\n"
+  printf "  dev-tools celeryd --help | -h\n"
+  echo
+
+  # :command.long_usage
+  if [[ -n $long_usage ]]; then
+    printf "%s\n" "Options:"
+
+    # :command.usage_flags
+    # :flag.usage
+    printf "  %s\n" "--workdir WORKDIR"
+    printf "    workdir to run the pip install -e on top of can be ce or ee only\n"
+    printf "    Default: /home/rhodecode/rhodecode-enterprise-ce\n"
+    echo
+
+    # :flag.usage
+    printf "  %s\n" "--cmd CMD"
+    printf "    cmd script to run, this is on host machine, relative to $WORKSPACE_HOME\n"
+    printf "    Default: .dev/cmd-celeryd.sh\n"
+    echo
+
     # :command.usage_fixed_flags
     printf "  %s\n" "--help, -h"
     printf "    Show this help\n"
@@ -962,16 +1018,77 b' dev_tools_svn_command() {'
 # :command.function
 dev_tools_celery_command() {
   # src/celery_command.sh
+  code_workdir=${args[--workdir]}
+  cmd_script=${args[--cmd]}
+
+  exec_script=$WORKSPACE_HOME/$cmd_script
   container_name=dev-celery
 
+  echo "Using exec script: $exec_script"
+
+  if [[ ! -f $exec_script ]]; then
+    echo "Generating new $exec_script file..."
+    mkdir -p $(dirname $exec_script)
+
+    cat <<< "
+    # Generated dev-env script for celery
+
+    echo \"Run celery dev using the following command:\"
+    echo \"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\"
+    echo \"celery --no-color --app=rhodecode.lib.celerylib.loader worker --autoscale=4,2 --max-tasks-per-child=100 --task-events --loglevel=DEBUG --ini=.dev/dev.ini\"
+    echo \"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\"
+    make sh
+    " > $exec_script
+    chmod +x $exec_script
+  fi
+
+  docker_exec_script="/var/opt/rhodecode_repo_store/$cmd_script"
+
   docker ps -q --filter "name=$container_name" | grep -q . && echo $container_name found || echo $container_name not found
   docker ps -q --filter "name=$container_name" | grep -q . && docker stop $container_name && docker rm -fv $container_name
 
   eval "
   ./rcstack stack --env-file-path .custom/.dev.env rhodecode -f docker-compose-apps.dev.yaml \
   run -e ENV_FILE= -e RC_ENV_FILE= -e RC_APP_TYPE=rhodecode_sh --rm --use-aliases \
-  --workdir=\"/home/rhodecode/rhodecode-enterprise-ce\" \
-  --name=\"$container_name\" rhodecode bash /home/rhodecode/rhodecode-enterprise-ce/.dev/dev-env-celery.sh"
+  --workdir=\"$code_workdir\" \
+  --name=\"$container_name\" rhodecode bash $docker_exec_script"
+
+}
+
+# :command.function
+dev_tools_celeryd_command() {
+  # src/celeryd_command.sh
+  code_workdir=${args[--workdir]}
+  cmd_script=${args[--cmd]}
+
+  exec_script=$WORKSPACE_HOME/$cmd_script
+  container_name=dev-celeryd
+
+  echo "Using exec script: $exec_script"
+
+  if [[ ! -f $exec_script ]]; then
+    echo "Generating new $exec_script file..."
+    mkdir -p $(dirname $exec_script)
+
+    cat <<< "
+    # Generated dev-env script for celery
+
+    echo \"Run celery dev using the following command:\"
+    make dev-env && celery --no-color --app=rhodecode.lib.celerylib.loader worker --autoscale=4,2 --max-tasks-per-child=100 --task-events --loglevel=DEBUG --ini=.dev/dev.ini
+    " > $exec_script
+    chmod +x $exec_script
+  fi
+
+  docker_exec_script="/var/opt/rhodecode_repo_store/$cmd_script"
+
+  docker ps -q --filter "name=$container_name" | grep -q . && echo $container_name found || echo $container_name not found
+  docker ps -q --filter "name=$container_name" | grep -q . && docker stop $container_name && docker rm -fv $container_name
+
+  eval "
+  ./rcstack stack --env-file-path .custom/.dev.env rhodecode -f docker-compose-apps.dev.yaml \
+  run -e ENV_FILE= -e RC_ENV_FILE= -e RC_APP_TYPE=rhodecode_sh --rm --use-aliases \
+  --workdir=\"$code_workdir\" \
+  --name=\"$container_name\" rhodecode $docker_exec_script"
 
 }
 
@@ -1412,6 +1529,13 b' parse_requirements() {'
       shift $#
       ;;
 
+    celeryd)
+      action="celeryd"
+      shift
+      dev_tools_celeryd_parse_requirements "$@"
+      shift $#
+      ;;
+
     sshd)
       action="sshd"
       shift
@@ -1747,6 +1871,35 b' dev_tools_celery_parse_requirements() {'
   while [[ $# -gt 0 ]]; do
     key="$1"
     case "$key" in
+      # :flag.case
+      --workdir)
+
+        # :flag.case_arg
+        if [[ -n ${2+x} ]]; then
+
+          args['--workdir']="$2"
+          shift
+          shift
+        else
+          printf "%s\n" "--workdir requires an argument: --workdir WORKDIR" >&2
+          exit 1
+        fi
+        ;;
+
+      # :flag.case
+      --cmd)
+
+        # :flag.case_arg
+        if [[ -n ${2+x} ]]; then
+
+          args['--cmd']="$2"
+          shift
+          shift
+        else
+          printf "%s\n" "--cmd requires an argument: --cmd CMD" >&2
+          exit 1
+        fi
+        ;;
 
       -?*)
         printf "invalid option: %s\n" "$key" >&2
@@ -1764,6 +1917,87 b' dev_tools_celery_parse_requirements() {'
     esac
   done
 
+  # :command.default_assignments
+  [[ -n ${args['--workdir']:-} ]] || args['--workdir']="/home/rhodecode/rhodecode-enterprise-ce"
+  [[ -n ${args['--cmd']:-} ]] || args['--cmd']=".dev/cmd-celery.sh"
+
+}
+
+# :command.parse_requirements
+dev_tools_celeryd_parse_requirements() {
+  # :command.fixed_flags_filter
+  while [[ $# -gt 0 ]]; do
+    case "${1:-}" in
+      --help | -h)
+        long_usage=yes
+        dev_tools_celeryd_usage
+        exit
+        ;;
+
+      *)
+        break
+        ;;
+
+    esac
+  done
+
+  # :command.command_filter
+  action="celeryd"
+
+  # :command.parse_requirements_while
+  while [[ $# -gt 0 ]]; do
+    key="$1"
+    case "$key" in
+      # :flag.case
+      --workdir)
+
+        # :flag.case_arg
+        if [[ -n ${2+x} ]]; then
+
+          args['--workdir']="$2"
+          shift
+          shift
+        else
+          printf "%s\n" "--workdir requires an argument: --workdir WORKDIR" >&2
+          exit 1
+        fi
+        ;;
+
+      # :flag.case
+      --cmd)
+
+        # :flag.case_arg
+        if [[ -n ${2+x} ]]; then
+
+          args['--cmd']="$2"
+          shift
+          shift
+        else
+          printf "%s\n" "--cmd requires an argument: --cmd CMD" >&2
+          exit 1
+        fi
+        ;;
+
+      -?*)
+        printf "invalid option: %s\n" "$key" >&2
+        exit 1
+        ;;
+
+      *)
+        # :command.parse_requirements_case
+        # :command.parse_requirements_case_simple
+        printf "invalid argument: %s\n" "$key" >&2
+        exit 1
+
+        ;;
+
+    esac
+  done
+
+  # :command.default_assignments
+  [[ -n ${args['--workdir']:-} ]] || args['--workdir']="/home/rhodecode/rhodecode-enterprise-ce"
+  [[ -n ${args['--cmd']:-} ]] || args['--cmd']=".dev/cmd-celeryd.sh"
+
 }
 
 # :command.parse_requirements
@@ -2631,6 +2865,7 b' run() {'
     "ce") dev_tools_ce_command ;;
     "svn") dev_tools_svn_command ;;
     "celery") dev_tools_celery_command ;;
+    "celeryd") dev_tools_celeryd_command ;;
     "sshd") dev_tools_sshd_command ;;
     "tools") dev_tools_tools_command ;;
     "dev-env") dev_tools_dev_env_command ;;