From ccaf90c4769de33bb51687dba1c0b394ceac057a 2024-02-22 09:32:38 From: RhodeCode Admin Date: 2024-02-22 09:32:38 Subject: [PATCH] feat(dev-tools): allow running celery/celeryd for dev in more flexible ways - mainly control over ce/ee source - celeryd command allows running in background for tests --- diff --git a/scripts/dev-tools/dev-tools b/scripts/dev-tools/dev-tools index 0b182d0..41c794f 100755 --- a/scripts/dev-tools/dev-tools +++ b/scripts/dev-tools/dev-tools @@ -39,6 +39,7 @@ dev_tools_usage() { printf " %s run bash console for rhodecode-ce\n" "ce " printf " %s run bash console for svn web\n" "svn " printf " %s run bash console for rhodecode-celery\n" "celery " + printf " %s run celery instance on top of mounted source code, e.g ./dev-tools celeryd&\n" "celeryd " printf " %s run bash console for rhodecode-sshd\n" "sshd " printf " %s run bash console for rhodecode-tools-ce\n" "tools " printf " %s create dev env required to run vcsserver and rhodecode\n" "dev-env " @@ -74,7 +75,7 @@ dev_tools_usage() { echo # :command.footer - printf "RhodeCode Inc 2023\n\n" + printf "RhodeCode Inc 2024\n\n" echo fi @@ -221,7 +222,7 @@ dev_tools_celery_usage() { fi printf "%s\n" "Usage:" - printf " dev-tools celery\n" + printf " dev-tools celery [OPTIONS]\n" printf " dev-tools celery --help | -h\n" echo @@ -229,6 +230,61 @@ dev_tools_celery_usage() { if [[ -n $long_usage ]]; then printf "%s\n" "Options:" + # :command.usage_flags + # :flag.usage + printf " %s\n" "--workdir WORKDIR" + printf " workdir to run the pip install -e on top of can be ce or ee only\n" + printf " Default: /home/rhodecode/rhodecode-enterprise-ce\n" + echo + + # :flag.usage + printf " %s\n" "--cmd CMD" + printf " cmd script to run, this is on host machine, relative to $WORKSPACE_HOME\n" + printf " Default: .dev/cmd-celery.sh\n" + echo + + # :command.usage_fixed_flags + printf " %s\n" "--help, -h" + printf " Show this help\n" + echo + + fi +} + +# :command.usage +dev_tools_celeryd_usage() { + if [[ -n $long_usage ]]; then + printf "dev-tools celeryd - run celery instance on top of mounted source code, e.g ./dev-tools celeryd&\n" + echo + + else + printf "dev-tools celeryd - run celery instance on top of mounted source code, e.g ./dev-tools celeryd&\n" + echo + + fi + + printf "%s\n" "Usage:" + printf " dev-tools celeryd [OPTIONS]\n" + printf " dev-tools celeryd --help | -h\n" + echo + + # :command.long_usage + if [[ -n $long_usage ]]; then + printf "%s\n" "Options:" + + # :command.usage_flags + # :flag.usage + printf " %s\n" "--workdir WORKDIR" + printf " workdir to run the pip install -e on top of can be ce or ee only\n" + printf " Default: /home/rhodecode/rhodecode-enterprise-ce\n" + echo + + # :flag.usage + printf " %s\n" "--cmd CMD" + printf " cmd script to run, this is on host machine, relative to $WORKSPACE_HOME\n" + printf " Default: .dev/cmd-celeryd.sh\n" + echo + # :command.usage_fixed_flags printf " %s\n" "--help, -h" printf " Show this help\n" @@ -962,16 +1018,77 @@ dev_tools_svn_command() { # :command.function dev_tools_celery_command() { # src/celery_command.sh + code_workdir=${args[--workdir]} + cmd_script=${args[--cmd]} + + exec_script=$WORKSPACE_HOME/$cmd_script container_name=dev-celery + echo "Using exec script: $exec_script" + + if [[ ! -f $exec_script ]]; then + echo "Generating new $exec_script file..." + mkdir -p $(dirname $exec_script) + + cat <<< " + # Generated dev-env script for celery + + echo \"Run celery dev using the following command:\" + echo \"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\" + echo \"celery --no-color --app=rhodecode.lib.celerylib.loader worker --autoscale=4,2 --max-tasks-per-child=100 --task-events --loglevel=DEBUG --ini=.dev/dev.ini\" + echo \"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\" + make sh + " > $exec_script + chmod +x $exec_script + fi + + docker_exec_script="/var/opt/rhodecode_repo_store/$cmd_script" + docker ps -q --filter "name=$container_name" | grep -q . && echo $container_name found || echo $container_name not found docker ps -q --filter "name=$container_name" | grep -q . && docker stop $container_name && docker rm -fv $container_name eval " ./rcstack stack --env-file-path .custom/.dev.env rhodecode -f docker-compose-apps.dev.yaml \ run -e ENV_FILE= -e RC_ENV_FILE= -e RC_APP_TYPE=rhodecode_sh --rm --use-aliases \ - --workdir=\"/home/rhodecode/rhodecode-enterprise-ce\" \ - --name=\"$container_name\" rhodecode bash /home/rhodecode/rhodecode-enterprise-ce/.dev/dev-env-celery.sh" + --workdir=\"$code_workdir\" \ + --name=\"$container_name\" rhodecode bash $docker_exec_script" + +} + +# :command.function +dev_tools_celeryd_command() { + # src/celeryd_command.sh + code_workdir=${args[--workdir]} + cmd_script=${args[--cmd]} + + exec_script=$WORKSPACE_HOME/$cmd_script + container_name=dev-celeryd + + echo "Using exec script: $exec_script" + + if [[ ! -f $exec_script ]]; then + echo "Generating new $exec_script file..." + mkdir -p $(dirname $exec_script) + + cat <<< " + # Generated dev-env script for celery + + echo \"Run celery dev using the following command:\" + make dev-env && celery --no-color --app=rhodecode.lib.celerylib.loader worker --autoscale=4,2 --max-tasks-per-child=100 --task-events --loglevel=DEBUG --ini=.dev/dev.ini + " > $exec_script + chmod +x $exec_script + fi + + docker_exec_script="/var/opt/rhodecode_repo_store/$cmd_script" + + docker ps -q --filter "name=$container_name" | grep -q . && echo $container_name found || echo $container_name not found + docker ps -q --filter "name=$container_name" | grep -q . && docker stop $container_name && docker rm -fv $container_name + + eval " + ./rcstack stack --env-file-path .custom/.dev.env rhodecode -f docker-compose-apps.dev.yaml \ + run -e ENV_FILE= -e RC_ENV_FILE= -e RC_APP_TYPE=rhodecode_sh --rm --use-aliases \ + --workdir=\"$code_workdir\" \ + --name=\"$container_name\" rhodecode $docker_exec_script" } @@ -1412,6 +1529,13 @@ parse_requirements() { shift $# ;; + celeryd) + action="celeryd" + shift + dev_tools_celeryd_parse_requirements "$@" + shift $# + ;; + sshd) action="sshd" shift @@ -1747,6 +1871,35 @@ dev_tools_celery_parse_requirements() { while [[ $# -gt 0 ]]; do key="$1" case "$key" in + # :flag.case + --workdir) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args['--workdir']="$2" + shift + shift + else + printf "%s\n" "--workdir requires an argument: --workdir WORKDIR" >&2 + exit 1 + fi + ;; + + # :flag.case + --cmd) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args['--cmd']="$2" + shift + shift + else + printf "%s\n" "--cmd requires an argument: --cmd CMD" >&2 + exit 1 + fi + ;; -?*) printf "invalid option: %s\n" "$key" >&2 @@ -1764,6 +1917,87 @@ dev_tools_celery_parse_requirements() { esac done + # :command.default_assignments + [[ -n ${args['--workdir']:-} ]] || args['--workdir']="/home/rhodecode/rhodecode-enterprise-ce" + [[ -n ${args['--cmd']:-} ]] || args['--cmd']=".dev/cmd-celery.sh" + +} + +# :command.parse_requirements +dev_tools_celeryd_parse_requirements() { + # :command.fixed_flags_filter + while [[ $# -gt 0 ]]; do + case "${1:-}" in + --help | -h) + long_usage=yes + dev_tools_celeryd_usage + exit + ;; + + *) + break + ;; + + esac + done + + # :command.command_filter + action="celeryd" + + # :command.parse_requirements_while + while [[ $# -gt 0 ]]; do + key="$1" + case "$key" in + # :flag.case + --workdir) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args['--workdir']="$2" + shift + shift + else + printf "%s\n" "--workdir requires an argument: --workdir WORKDIR" >&2 + exit 1 + fi + ;; + + # :flag.case + --cmd) + + # :flag.case_arg + if [[ -n ${2+x} ]]; then + + args['--cmd']="$2" + shift + shift + else + printf "%s\n" "--cmd requires an argument: --cmd CMD" >&2 + exit 1 + fi + ;; + + -?*) + printf "invalid option: %s\n" "$key" >&2 + exit 1 + ;; + + *) + # :command.parse_requirements_case + # :command.parse_requirements_case_simple + printf "invalid argument: %s\n" "$key" >&2 + exit 1 + + ;; + + esac + done + + # :command.default_assignments + [[ -n ${args['--workdir']:-} ]] || args['--workdir']="/home/rhodecode/rhodecode-enterprise-ce" + [[ -n ${args['--cmd']:-} ]] || args['--cmd']=".dev/cmd-celeryd.sh" + } # :command.parse_requirements @@ -2631,6 +2865,7 @@ run() { "ce") dev_tools_ce_command ;; "svn") dev_tools_svn_command ;; "celery") dev_tools_celery_command ;; + "celeryd") dev_tools_celeryd_command ;; "sshd") dev_tools_sshd_command ;; "tools") dev_tools_tools_command ;; "dev-env") dev_tools_dev_env_command ;;