docker: added source installation, dev scripts and mysql config

dev.sh

@@ -0,0 +1,63 b''
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+CURRENT_USER_UID=$(id -u)
+CURRENT_USER_GID=$(id -g)
+USER_UID=${USER_UID:-$CURRENT_USER_UID}
+USER_GID=${USER_GID:-$CURRENT_USER_GID}
+CMD="docker-compose -f docker-compose.yml"
+BUILD_CMD="docker-compose -f docker-compose.yaml -f docker-compose.source.yml"
+DEV_CMD="docker-compose -f docker-compose.yaml -f docker-compose.source.yml -f docker-compose.dev.yml"
+
+usage="$(basename "$0") ./dev.sh <COMMAND> -- run docker command
+where <COMMAND>:
+    [v]cs       -- run bash console for vcsserver
+    [r]c        -- run bash console for rhodecode-ee
+    ce          -- run bash console for rhodecode-ce
+    [t]est[s]   -- run bash console for rhodecode-ce tests
+    test-mysql  -- run bash console for rhodecode-ce tests with mysql
+    test-pgsql  -- run bash console for rhodecode-ce tests with postgres
+    dev-build   -- build rhodecode dev app for use without cache use
+    dev-env     -- run the dev stack required for development
+    redis       -- run the redis CLI attached to rc_cluster_redis_1
+    run <cmd>   -- run <cmd>"
+
+case ${1:-} in
+    v|vcs )
+        eval "(docker stop vcsserver-dev || echo skip-stop ) && ${DEV_CMD} run --service-ports --rm --use-aliases --workdir="/home/rhodecode/rhodecode-vcsserver" --name="vcsserver-dev" vcsserver bash"
+        exit
+        ;;
+    r|rc )
+        eval "(docker stop enterprise-ee-dev || echo skip-stop ) && ${DEV_CMD} run --publish 8080:8080 --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ee" --name="enterprise-ee-dev" rhodecode bash"
+        exit
+        ;;
+    ce )
+        eval "(docker stop enterprise-ce-dev || echo skip-stop ) && ${DEV_CMD} run --publish 8081:8080 --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ce" --name="enterprise-ce-dev" rhodecode bash"
+        exit
+        ;;
+    t|test|tests )
+        eval "${DEV_CMD} run --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ce" --name="enterprise-dev-test" rhodecode bash"
+        exit
+        ;;
+    dev-build )
+        eval "${BUILD_CMD} up --detach database && ${BUILD_CMD} build --no-cache --progress=plain rhodecode"
+        exit
+        ;;
+    dev-env )
+        eval "${BUILD_CMD} up --detach database redis channelstream nginx celery beat"
+        exit
+        ;;
+    redis )
+        eval "docker exec --interactive --tty rc_cluster_redis_1 redis-cli"
+        exit
+        ;;
+    run )
+        command=${@:2}
+        eval "${DEV_CMD} run --rm rhodecode ${command}"
+        exit
+        ;;
+    * )
+        echo "${usage}"
+        exit
+        ;;
+esac

docker-compose.source.yml

@@ -0,0 +1,80 b''
+volumes:
+  bashhistory:
+
+
+services:
+
+  rhodecode:
+    environment:
+      HISTFILE: /home/rhodecode/.bash_history_docker
+      DB_UPGRADE: 0  # run the DB upgrade
+
+    volumes:
+      - bashhistory:/home/rhodecode
+
+    build:
+      context: .
+      dockerfile: service/rhodecode/rhodecode_source.dockerfile
+
+    image: rhodecode/app_source:latest
+
+    command: [
+      "/home/rhodecode/rhodecode-enterprise-ee/result/bin/gunicorn",
+      "--name=gunicorn-rhodecode-1",
+      "--error-logfile=-",
+      "--paster=/etc/rhodecode/conf/compose/rhodecode.optimized.ini",
+      "--config=/etc/rhodecode/conf/gunicorn_conf.py"
+    ]
+
+  vcsserver:
+    environment:
+      HISTFILE: /home/rhodecode/.bash_history_docker
+
+    volumes:
+      - bashhistory:/home/rhodecode
+
+    build:
+      context: .
+      dockerfile: service/rhodecode/rhodecode_source.dockerfile
+
+    image: rhodecode/app_source:latest
+
+    command: [
+       "/home/rhodecode/rhodecode-vcsserver/result/bin/gunicorn",
+       "--name=gunicorn-vcsserver-1",
+       "--error-logfile=-",
+       "--paster=/etc/rhodecode/conf/compose/vcsserver.optimized.ini",
+       "--config=/etc/rhodecode/conf/gunicorn_conf.py"
+    ]
+
+  celery:
+
+    build:
+      context: .
+      dockerfile: service/rhodecode/rhodecode_source.dockerfile
+
+    image: rhodecode/app_source:latest
+
+  beat:
+
+    build:
+      context: .
+      dockerfile: service/rhodecode/rhodecode_source.dockerfile
+
+    image: rhodecode/app_source:latest
+
+  svn:
+
+    build:
+      context: .
+      dockerfile: service/rhodecode/rhodecode_source.dockerfile
+
+    image: rhodecode/app_source:latest
+
+  sshd:
+
+    build:
+      context: .
+      dockerfile: service/rhodecode/rhodecode_source.dockerfile
+
+    image: rhodecode/app_source:latest
  No newline at end of file

service/database/mysql_customized.conf

@@ -0,0 +1,3 b''
+character-set-client-handshake = FALSE
+character-set-server = utf8mb4
+collation-server = utf8mb4_unicode_ci

service/database/rhodecode_database_mysql.dockerfile

@@ -0,0 +1,5 b''
+ARG MYSQL_BUILD
+FROM library/mysql:$MYSQL_BUILD
+
+RUN mkdir -p /etc/mysql/conf.d
+COPY service/database/mysql_customized.conf /etc/mysql/conf.d/config-file.cnf

service/rhodecode/rhodecode_source.dockerfile

@@ -0,0 +1,298 b''
+FROM ubuntu:groovy
+MAINTAINER RhodeCode Inc. <[e-mail]>
+
+ARG TZ="UTC"
+ARG LOCALE_TYPE=en_US.UTF-8
+ARG RHODECODE_TYPE=Enterprise
+# source-install
+ARG RHODECODE_VERSION=4.25.0
+
+ARG RHODECODE_DB=sqlite
+ARG RHODECODE_USER_NAME=admin
+ARG RHODECODE_USER_PASS=secret4
+ARG RHODECODE_USER_EMAIL=[e-mail]
+
+# nix ver/channels
+ARG DEV_NIX_VERSION=2.0.4
+ARG DEV_NIX_CHANNEL=nixos-18.03
+
+# env are runtime
+ENV \
+    TZ=${TZ} \
+    LOCALE_TYPE=${LOCALE_TYPE} \
+    \
+    ## Define type we build, and the instance we'll create
+    RHODECODE_TYPE=${RHODECODE_TYPE} \
+    RC_TYPE_ID=enterprise-1 \
+    \
+    ## SETUP ARGS FOR INSTALLATION ##
+    ## set version we build on, get from .env or set default ver
+    RHODECODE_VERSION=${RHODECODE_VERSION} \
+    \
+    ## set DB, default sqlite
+    RHODECODE_DB=${RHODECODE_DB} \
+    \
+    ## set app bootstrap required data
+    RHODECODE_USER_NAME=${RHODECODE_USER_NAME} \
+    RHODECODE_USER_PASS=${RHODECODE_USER_PASS} \
+    RHODECODE_USER_EMAIL=${RHODECODE_USER_EMAIL} \
+    \
+    RC_USER=rhodecode \
+    \
+    # SVN CONFIG
+    MOD_DAV_SVN_CONF_FILE=/etc/rhodecode/conf/svn/mod_dav_svn.conf \
+    MOD_DAV_SVN_PORT=8090 \
+    \
+    # SSHD CONFIG
+    SSHD_CONF_FILE=/etc/rhodecode/sshd_config \
+    \
+    BUILD_CONF=/etc/rhodecode/conf_build \
+    BUILD_BIN_DIR=/var/opt/rhodecode_bin \
+    RHODECODE_DATA_DIR=/var/opt/rhodecode_data \
+    RHODECODE_REPO_DIR=/var/opt/rhodecode_repo_store \
+    RHODECODE_HTTP_PORT=10020 \
+    RHODECODE_VCS_PORT=10010 \
+    RHODECODE_HOST=0.0.0.0 \
+    RHODECODE_VCS_HOST=127.0.0.1
+
+ENV \
+    RCCONTROL=/home/$RC_USER/.rccontrol-profile/bin/rccontrol \
+    SUPERVISOR_CONF=/home/$RC_USER/.rccontrol/supervisor/supervisord.ini \
+    # make application scripts visible
+    PATH=$PATH:/home/$RC_USER/.rccontrol-profile/bin
+
+ENV SVN_LOCALE_DEPS apache2 apache2-utils libapache2-mod-svn
+ENV SSH_LOCALE_DEPS openssh-server
+ENV PYTHON_DEPS python2
+ENV EXTRA_DEPS  vim
+
+ENV \
+    PATH=$PATH:/nix/var/nix/profiles/per-user/$RC_USER/profile/bin:/home/$RC_USER/rhodecode-enterprise-ee/profile/bin \
+    NIX_BLD_USER=nixbld \
+    NIX_PATH=/nix/var/nix/profiles/per-user/$RC_USER/channels
+
+RUN \
+echo "** install base packages **" && \
+set -eux; \
+	\
+	savedAptMark="$(apt-mark showmanual)"; \
+	apt-get update; \
+	DEBIAN_FRONTEND="noninteractive" \
+	apt-get install -y --no-install-recommends \
+        tini \
+        bash \
+        binutils \
+        tzdata \
+        locales \
+        openssl \
+        curl \
+        sudo \
+        gosu \
+        bzip2 \
+        ca-certificates \
+        $PYTHON_DEPS \
+        $SSH_LOCALE_DEPS \
+        $SVN_LOCALE_DEPS \
+        $EXTRA_DEPS \
+	; \
+	rm -rf /var/lib/apt/lists/*;
+
+RUN \
+echo "** Configure the python executable for py2/3 compat **" && \
+ISPY=$(which python3 || which python2) && \
+if [ -n $ISPY ] ; then ln -s $ISPY /usr/bin/python ; fi
+
+RUN \
+echo "** Configure the locales **" && \
+    sed -i "s/^# ${LOCALE_TYPE}/${LOCALE_TYPE}/g" /etc/locale.gen && \
+    locale-gen
+
+# locale-archive is a fix for old nix glibc2.26 locales available
+ENV \
+    LOCALE_ARCHIVE=/var/opt/locale-archive \
+    LANG=${LOCALE_TYPE} \
+    LANGUAGE=${LOCALE_TYPE} \
+    LC_ALL=${LOCALE_TYPE}
+
+# configure the system user
+# explicitly set uid/gid to guarantee that it won't change in the future
+# the values 999:999 are identical to the current user/group id assigned
+RUN \
+echo "** Create system user $RC_USER **" && \
+  groupadd --system --gid 999 $RC_USER && \
+  useradd --system --gid $RC_USER --uid 999 --shell /bin/bash $RC_USER && \
+  usermod -G $RC_USER $RC_USER
+
+RUN \
+echo "** Create nix-build user $NIX_BLD_USER **" && \
+    groupadd --system --gid 1099 $NIX_BLD_USER && \
+    useradd --system --gid $NIX_BLD_USER --uid 1099 --shell /bin/bash $NIX_BLD_USER && \
+    usermod -G $NIX_BLD_USER $NIX_BLD_USER
+
+RUN \
+echo "** disable nix sandboxing **" && \
+    mkdir /etc/nix && echo 'sandbox = false' > /etc/nix/nix.conf
+
+
+# set the defult bash shell
+SHELL ["/bin/bash", "-c"]
+
+# Fix and set a timezone
+RUN \
+echo "** configure the timezone **" && \
+rm /etc/localtime && cp /usr/share/zoneinfo/$TZ /etc/localtime && \
+echo $TZ > /etc/timezone
+
+
+RUN \
+echo "** prepare rhodecode store and cache **" && \
+  install -d -m 0700 -o $RC_USER -g $RC_USER /nix && \
+  install -d -m 0755 -o $RC_USER -g $RC_USER /opt/rhodecode && \
+  install -d -m 0755 -o $RC_USER -g $RC_USER /var/opt/rhodecode_bin && \
+  install -d -m 0755 -o $RC_USER -g $RC_USER $RHODECODE_REPO_DIR && \
+  install -d -m 0755 -o $RC_USER -g $RC_USER $RHODECODE_DATA_DIR && \
+  install -d -m 0755 -o $RC_USER -g $RC_USER $BUILD_CONF && \
+  install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/ && \
+  install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol && \
+  install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/cache && \
+  install -d -m 0755 -o $RC_USER -g $RC_USER /home/$RC_USER/.rccontrol/bootstrap && \
+  install -d -m 0700 -o $RC_USER -g $RC_USER /home/$RC_USER/.ssh
+
+# expose our custom sshd config
+COPY service/sshd/sshd_config $SSHD_CONF_FILE
+
+# Apache SVN setup
+RUN \
+    echo "**** Apache config cleanup ****" && \
+    rm -f /etc/apache2/conf.d/info.conf \
+          /etc/apache2/conf.d/mpm.conf \
+          /etc/apache2/conf.d/userdir.conf && \
+    rm -f /etc/apache2/sites-enabled/* && \
+    rm -f /etc/apache2/sites-available/*
+
+# custom SVN virtualhost
+COPY service/svn/virtualhost.conf /etc/apache2/sites-enabled/
+
+RUN \
+echo "**** Apache config ****" && \
+    echo $(strings /usr/lib/apache2/modules/mod_dav_svn.so | grep 'Powered by') > /var/opt/dav.version && \
+    mkdir -p /run/apache2 && \
+    mkdir -p /var/opt/www && \
+    echo "unset HOME" > /etc/apache2/envvars && \
+    echo "export APACHE_RUN_USER=${RC_USER}" >> /etc/apache2/envvars && \
+    echo "export APACHE_PID_FILE=/var/run/apache2/apache2.pid" >> /etc/apache2/envvars && \
+    echo "export APACHE_RUN_DIR=/var/run/apache2" >> /etc/apache2/envvars && \
+    echo "export APACHE_LOCK_DIR=/var/lock/apache2" >> /etc/apache2/envvars && \
+    echo "export APACHE_RUN_USER=${RC_USER}" >> /etc/apache2/envvars && \
+    echo "export APACHE_RUN_GROUP=${RC_USER}" >> /etc/apache2/envvars && \
+    sed -i "s/Listen 80/Listen ${MOD_DAV_SVN_PORT}/g" /etc/apache2/ports.conf
+
+
+# Copy artifacts
+COPY --chown=$RC_USER:$RC_USER .source/ /home/$RC_USER/
+COPY --chown=$RC_USER:$RC_USER .cache/* /home/$RC_USER/.rccontrol/cache/
+COPY --chown=$RC_USER:$RC_USER config/compose/rhodecode_enterprise.license /home/$RC_USER/.rccontrol/bootstrap/
+COPY --chown=$RC_USER:$RC_USER service/rhodecode/bootstrap/* /home/$RC_USER/.rccontrol/bootstrap/
+
+RUN \
+echo "**** locale-archive path ****" && \
+    mv -v /home/$RC_USER/.rccontrol/cache/locale-archive /var/opt/locale-archive
+
+# change to non-root user for RUN commands
+USER $RC_USER
+WORKDIR /home/$RC_USER
+
+RUN \
+echo "** download and install nix **" && \
+    curl -L https://releases.nixos.org/nix/nix-$DEV_NIX_VERSION/install | USER=$RC_USER /bin/bash
+
+RUN \
+echo "** update nix package database and set channel to $DEV_NIX_CHANNEL **" && \
+    . /home/rhodecode/.nix-profile/etc/profile.d/nix.sh && \
+	nix-channel --add https://nixos.org/channels/$DEV_NIX_CHANNEL nixpkgs && \
+	nix-channel --update
+
+
+RUN \
+    echo "** save nix config **" && \
+    touch /home/$RC_USER/.rhoderc && \
+    mkdir -p /home/$RC_USER/.nixpkgs && touch /home/$RC_USER/.nixpkgs/config.nix && \
+    printf '{\n  rc = {\n   sources = {\n    rhodecode-vcsserver = "/home/'$RC_USER'/rhodecode-vcsserver";\n    rhodecode-enterprise-ce = "/home/'$RC_USER'/rhodecode-enterprise-ce";\n    rhodecode-enterprise-ee = "/home/'$RC_USER'/rhodecode-enterprise-ee";\n   };\n  };\n}\n' > /home/$RC_USER/.nixpkgs/config.nix
+
+
+RUN \
+echo "** install rhodecode control **" && \
+#  cd /home/$RC_USER/.rccontrol/cache && \
+#  INSTALLER=$(ls -Art /home/$RC_USER/.rccontrol/cache/RhodeCode-installer-* | tail -n 1) && \
+#  chmod +x ${INSTALLER} && \
+#  ${INSTALLER} --accept-license && \
+#  ${RCCONTROL} self-init && \
+#  cp -v /home/$RC_USER/.rccontrol-profile/etc/ca-bundle.crt $BUILD_CONF/ && \
+  echo "Done"
+
+RUN \
+echo "** install build vcsserver ${RHODECODE_VERSION} **" && \
+  . /home/rhodecode/.nix-profile/etc/profile.d/nix.sh && \
+  nix-build --show-trace --cores 0 --max-jobs 4 --no-build-output --out-link rhodecode-vcsserver/result rhodecode-vcsserver/default.nix && \
+  nix-shell --command 'echo ok' rhodecode-vcsserver/default.nix && \
+  VCSSERVER_PATH=/home/$RC_USER/rhodecode-vcsserver && \
+  rm -rf $BUILD_BIN_DIR/vcs_bin && ln -s ${VCSSERVER_PATH}/result/bin $BUILD_BIN_DIR/vcs_bin && \
+  cp -v ${VCSSERVER_PATH}/configs/production.ini $BUILD_CONF/vcsserver.ini
+
+RUN \
+echo "** install build Community ${RHODECODE_VERSION} **" && \
+  . /home/rhodecode/.nix-profile/etc/profile.d/nix.sh && \
+  echo "done"
+
+RUN \
+echo "** install build Enterprise ${RHODECODE_VERSION} **" && \
+  . /home/rhodecode/.nix-profile/etc/profile.d/nix.sh && \
+  nix-build --show-trace --cores 0 --max-jobs 4 --no-build-output --out-link rhodecode-enterprise-ee/result rhodecode-enterprise-ee/default.nix && \
+  nix-shell --command 'echo ok' rhodecode-enterprise-ee/default.nix && \
+  RHODECODE_PATH=/home/$RC_USER/rhodecode-enterprise-ee && \
+  rm -rf $BUILD_BIN_DIR/bin && ln -s ${RHODECODE_PATH}/result/bin $BUILD_BIN_DIR/ && \
+  cp -v ${RHODECODE_PATH}/configs/production.ini $BUILD_CONF/rhodecode.ini && \
+  cp -v ${RHODECODE_PATH}/configs/gunicorn_config.py $BUILD_CONF/gunicorn_conf.py && \
+  mkdir -p $RHODECODE_DATA_DIR/static && cp -r ${RHODECODE_PATH}/result/etc/static/* $RHODECODE_DATA_DIR/static/
+
+
+RUN \
+echo "** configure supervisord **" && \
+  #cp -v ${SUPERVISOR_CONF} $BUILD_CONF/ && \
+  #sed -i "s/self_managed_supervisor = False/self_managed_supervisor = True/g" /home/$RC_USER/.rccontrol.ini && \
+  echo "done"
+
+USER root
+
+
+RUN \
+echo "**** cleanup ****" && \
+    apt-get remove -y $PYTHON_DEPS && \
+    apt-get autoclean -y && \
+    rm -f /tmp/* && \
+    rm -f /home/$RC_USER/.rccontrol/cache/RhodeCode-installer-* && \
+    rm -f /home/$RC_USER/.rccontrol/cache/*.bz2 && \
+    rm -rf /var/lib/apt/lists/* \
+    rm -rf /var/cache/apk/* \
+    rm ${SUPERVISOR_CONF}
+
+# copy entrypoints
+COPY entrypoints.d/entrypoint.sh /opt/entrypoints.d/entrypoint.sh
+RUN chmod +x /opt/entrypoints.d/entrypoint.sh
+
+# config volume
+VOLUME /etc/rhodecode/conf
+
+# repo store volume
+VOLUME /var/opt/rhodecode_repo_store
+
+# data volume
+VOLUME /var/opt/rhodecode_data
+
+# logs volume
+VOLUME /var/log/rhodecode
+
+ENTRYPOINT ["/opt/entrypoints.d/entrypoint.sh"]
+
+# compose can override this
+CMD ["supervisord", "--nodaemon", "-c", "/etc/rhodecode/conf/supervisord.ini"]