core: various changes

docs/migration-to-docker.md

@@ -0,0 +1,46 b''
+# Quick list of action to migrate old pre 4.28.0 installer based releases to docker stack
+
+### 1. Decide on components use
+
+By default RhodeCode stack uses Redis and Postgres. 
+We recommend using those in Docker as things like
+backup/restore are much easier, and possible to run by our installer
+
+If you plan to use your own redis or DB, uncomment/disable those services in file:
+`docker-compose-services.override.yaml`
+
+Select your DB type by commenting postgres or mysql db in docker-compose-services.override.yaml
+For Sqlite, comment out both DB types
+
+### 2. launch stack based on get started
+    
+Run a quick start guide based on ./rccontrol get-started     
+
+### 3. make a db dump
+If docker-based DB is selected 
+Create a database backup from your db type
+If you plan to use you old running DB, this step can be ommited
+
+### 4. restore db
+If docker-based DB is selected
+Restore the database into docker container running DB
+
+### mv repositories data to .custom/storage
+
+### mv gists data to .custom/storage
+
+### mv artifacts data to .custom/storage
+
+### mv tarball cache data to .custom/storage
+
+### ./rccontrol cli storage
+
+### move repos to /vol/repovolume
+
+### move artifacts data to /vol/datavolume
+
+### move tarballcache data to /vol/datavolume
+
+### cp rhodecode.ini to config/_shared/rhodecode.ini
+
+### cp vcsserver.ini to config/_shared/vcsserver.ini
  No newline at end of file

scripts/dev/bash-ly

@@ -0,0 +1,5 b''
+#!/usr/bin/env bash
+
+cd scripts/rccontrol/
+#alias bashly='docker run --rm -it --user $(id -u):$(id -g) --volume "$PWD:/app" dannyben/bashly:edge'
+docker run --rm -it --user $(id -u):$(id -g) --volume "$PWD:/app" dannyben/bashly:edge generate --watch

config/statsd-exporter/statsd.yaml

@@ -1,31 +1,32 b''
 defaults:
   # metrics expiration
+  # 2 days
   ttl: 48h
 
 mappings:
 
   - match: "*.histogram"
     name: "${1}"
     observer_type: histogram
     histogram_options:
       buckets: [.1, .25, .5, .75, 1, 1.5, 2, 5, 10]
 
   - match: "*.slow_histogram"
     name: "${1}"
     observer_type: histogram
     histogram_options:
       buckets: [.1, .25, .5, 1, 2, 5, 10, 30, 60]
 
   - match: "*.summary"
     name: "${1}"
     observer_type: summary
     summary_options:
         quantiles:
           - quantile: 0.99
             error: 0.001
           - quantile: 0.95
             error: 0.01
           - quantile: 0.9
             error: 0.05
           - quantile: 0.5
             error: 0.005

entrypoints.d/entrypoint.sh

@@ -1,156 +1,162 b''
 #!/usr/bin/env bash
 set -Eeo pipefail
 
 function config_copy() {
   # copy over the configs if they don't exist
+  echo "checking if config files needs bootstrapping"
   for f in /etc/rhodecode/conf_build/*; do
     fname=${f##*/}
     if [ ! -f "/etc/rhodecode/conf/$fname" ]; then
         echo "$fname not exists copying over as default conf..."
         cp -v $f /etc/rhodecode/conf/$fname
     fi
   done
 
 }
 
 function db_upgrade() {
   echo 'ENTRYPOINT: Upgrading database.'
   /usr/local/bin/rhodecode_bin/bin/rc-upgrade-db $MAIN_INI_PATH --force-yes
 }
 
 function db_init() {
 
   gosu $RC_USER \
     /usr/local/bin/rhodecode_bin/bin/rc-setup-app \
     $MAIN_INI_PATH \
     --force-yes \
     --skip-existing-db \
     --user=$RHODECODE_USER_NAME \
     --password=$RHODECODE_USER_PASS \
     --email=$RHODECODE_USER_EMAIL \
     --repos=$RHODECODE_REPO_DIR
 }
 
 function rhodecode_setup() {
   for f in /home/$RC_USER/.rccontrol/bootstrap/*.py; do
     fname=${f##*/}
 
     echo "Running script $fname on $RC_TYPE_ID"
     gosu $RC_USER /usr/local/bin/rhodecode_bin/bin/rc-ishell $MAIN_INI_PATH <<< "%run $f"
 
   done
 }
 
 function generate_ssh_keys() {
   keys_dir=/etc/rhodecode/conf/ssh
 
   if [[ ! -d $keys_dir ]]; then
     echo "Generating $keys_dir/ssh_host_rsa_key ..."
     gosu "$RC_USER" mkdir -p $keys_dir
   fi
 
   # Generate ssh host key for the first time
   if [[ ! -f $keys_dir/ssh_host_rsa_key ]]; then
     echo "Generating $keys_dir/ssh_host_rsa_key ..."
     gosu "$RC_USER" ssh-keygen -f $keys_dir/ssh_host_rsa_key -N '' -t rsa
     gosu "$RC_USER" chmod 0600 $keys_dir/ssh_host_rsa_key
   fi
 
   if [[ ! -f $keys_dir/ssh_host_ecdsa_key ]]; then
     echo "Generating $keys_dir/ssh_host_ecdsa_key ..."
     gosu "$RC_USER" ssh-keygen -f $keys_dir/ssh_host_ecdsa_key -N '' -t ecdsa
     gosu "$RC_USER" chmod 0600 $keys_dir/ssh_host_ecdsa_key
   fi
 
   if [[ ! -f $keys_dir/ssh_host_ed25519_key ]]; then
     echo "Generating $keys_dir/ssh_host_ed25519_key ..."
     gosu "$RC_USER" ssh-keygen -f $keys_dir/ssh_host_ed25519_key -N '' -t ed25519
     gosu "$RC_USER" chmod 0600 $keys_dir/ssh_host_ed25519_key
   fi
 
   if [[ ! -f $keys_dir/authorized_keys ]]; then
     echo "Generating $keys_dir/authorized_keys..."
     gosu "$RC_USER" touch $keys_dir/authorized_keys
   fi
 
   sed -i "s/AllowUsers USER/AllowUsers $RC_USER/" $SSHD_CONF_FILE
 }
 
 
 echo "ENTRYPOINT: Running $RC_APP_TYPE with cmd '$1'"
 
 if [ "$SSH_BOOTSTRAP" = 1 ]; then
   # generate SSH keys
   generate_ssh_keys
 fi
 
 isLikelyWeb=
 case "$1" in
 	supervisord | pserve | gunicorn ) isLikelyWeb=1 ;;
 esac
 
+if [[ $RC_APP_TYPE = "rhodecode_vcsserver" ]]; then
+  # copy over default configuration files
+  config_copy
+fi
+
 if [[ $RC_APP_TYPE = "rhodecode_http" ]]; then
 
+  # copy over default configuration files
+  config_copy
+
   DB_INIT_FILE=/var/opt/rhodecode_data/.dbinit_bootstrapped
   if [ "$FORCE_DB_INIT_FILE" = 1 ]; then
     rm $DB_INIT_FILE
   fi
 
   # Avoid DB_INIT to run multiple times
   if [[ ! -e $DB_INIT_FILE ]]; then
     echo "ENTRYPOINT: Starting $RC_APP_TYPE initial db bootstrap"
 
     db_init
 
     gosu $RC_USER touch "$DB_INIT_FILE"
     echo "ENTRYPOINT: marked as db-bootstrapped at $DB_INIT_FILE"
 
   fi
 
   RC_SETUP_FILE=/var/opt/rhodecode_data/.setup_bootstrapped
   if [ "$FORCE_RC_SETUP_APP" = 1 ]; then
     rm $RC_SETUP_FILE
   fi
 
   # Avoid destroying bootstrapping by simple start/stop
   if [[ ! -e $RC_SETUP_FILE ]]; then
     echo "ENTRYPOINT: Starting $RC_APP_TYPE initial bootstrap"
 
-    # copy over default configuration files
-    config_copy
-
     # setup application with specific options
     if [ "$SETUP_APP" = 1 ]; then
       rhodecode_setup
     fi
 
     gosu $RC_USER touch "$RC_SETUP_FILE"
     echo "ENTRYPOINT: marked as setup-bootstrapped at $RC_SETUP_FILE"
 
   fi
 
   if [ "$DB_UPGRADE" = 1 ]; then
     # run DB migrate
     echo "Found DB_UPGRADE flag, running DB upgrade"
     db_upgrade
   fi
 
 fi
 
 
 if [[ $RC_APP_TYPE = "rhodecode_sshd" ]]; then
   # Fix problem with Missing privilege separation directory error
   mkdir -p /run/sshd
 fi
 
 if [[ $RC_APP_TYPE = "rhodecode_svn" ]]; then
   set -- tini -- "$@"
 fi
 
 if [ "$RC_APP_PROC" = 1 ]; then
   # Fix problem with zombie processes when using executables like supervisord/gunicorn
   set -- tini -- "$@"
   set -- gosu $RC_USER "$@"
 fi
 
 exec "$@"

scripts/dev/dev.sh

@@ -1,47 +1,47 b''
 #!/usr/bin/env bash
 set -euo pipefail
 IFS=$'\n\t'
 
 #export RC_VERSION="4.28.0.REL20221120_SOURCE"
 #docker image tag 94977371485a rhodecode/rhodecode-ee:4.27.0.DEV_BUILD
 
 
 usage="$(basename "$0") ./dev.sh <COMMAND> -- run docker command
 where <COMMAND>:
     v|vcs       -- run bash console for vcsserver
     e|ee        -- run bash console for rhodecode-ee
     c|ce        -- run bash console for rhodecode-ce
     dev-env     -- create dev env required to run vcsserver and rhodecode
     prune       -- clear disk space for docker
     "
 
 case ${1:-} in
 
     v|vcs )
         eval "(docker stop dev-vcsserver || echo skip-stop ) && ./rccontrol stack rhodecode -f docker-compose-apps.dev.yaml run --service-ports --rm --use-aliases --workdir="/home/rhodecode/rhodecode-vcsserver" --name="dev-vcsserver" vcsserver bash"
         exit
         ;;
     e|ee )
         eval "(docker stop dev-enterprise-ee || echo skip-stop ) && ./rccontrol stack rhodecode -f docker-compose-apps.dev.yaml run -e RC_APP_TYPE=rhodecode_bash --publish 10020:10020 --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ee" --name="dev-enterprise-ee" rhodecode bash"
         exit
         ;;
     c|ce )
         eval "(docker stop dev-enterprise-ce || echo skip-stop ) && ./rccontrol stack rhodecode -f docker-compose-apps.dev.yaml run --publish 10022:10020 --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ce" --name="dev-enterprise-ce" rhodecode bash"
         exit
         ;;
     dev-env )
-        eval "./rccontrol stack metrics up --detach statsd-exporter && ./rccontrol stack services up --detach && ./rccontrol stack rhodecode up --detach celery celery-beat && ./rccontrol status -s"
+        eval "./rccontrol stack metrics up --detach statsd-exporter && ./rccontrol stack services up --detach && ./rccontrol stack rhodecode up --detach celery celery-beat && ./rccontrol status"
         exit
         ;;
     prune )
         docker system prune -f
         docker builder prune -a -f
         docker image prune -a -f
         docker system df
         exit
         ;;
     * )
         echo "${usage}"
         exit
         ;;
 esac