stack: updated loki/grafana stacks

Added JSON option logs into RhodeCode
Various fixes and optimizations on docker files

config/loki/loki-config.yaml

@@ -0,0 +1,50 b''
+auth_enabled: false
+
+server:
+  http_listen_address: 0.0.0.0
+  grpc_listen_address: 0.0.0.0
+  http_listen_port: 3100
+  grpc_listen_port: 9095
+  log_level: info
+
+common:
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    instance_addr: 127.0.0.1
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  max_streams_per_user: 100000
+
+  ingestion_burst_size_mb: 64
+  ingestion_rate_mb: 512
+
+  per_stream_rate_limit: 512M
+  per_stream_rate_limit_burst: 1024M
+  max_entries_limit_per_query: 1000000
+  max_label_value_length: 20480
+  max_label_name_length: 10240
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+# If you would like to disable reporting, uncomment the following lines:
+#analytics:
+#  reporting_enabled: false
+

config/loki/promtail-config.yaml

@@ -0,0 +1,38 b''
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+# Describes how to save read file offsets to disk
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+    tenant_id: docker
+
+## Definition to what to scrape
+scrape_configs:
+
+- job_name: scraping_nginx
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      __path__: /var/log_volume/nginx/rhodecode*
+      job: nginx
+
+- job_name: scraping_vcsserver
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      __path__: /var/log_volume/apps/rhodecode*
+      job: rhodecode
+
+- job_name: scraping_rhodecode
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      __path__: /var/log_volume/apps/vcsserver*
+      job: vcsserver
  No newline at end of file

config/statsd-exporter/statsd.yaml

@@ -0,0 +1,31 b''
+defaults:
+  # metrics expiration
+  ttl: 48h
+
+mappings:
+
+  - match: "*.histogram"
+    name: "${1}"
+    observer_type: histogram
+    histogram_options:
+      buckets: [.1, .25, .5, .75, 1, 1.5, 2, 5, 10]
+
+  - match: "*.slow_histogram"
+    name: "${1}"
+    observer_type: histogram
+    histogram_options:
+      buckets: [.1, .25, .5, 1, 2, 5, 10, 30, 60]
+
+  - match: "*.summary"
+    name: "${1}"
+    observer_type: summary
+    summary_options:
+        quantiles:
+          - quantile: 0.99
+            error: 0.001
+          - quantile: 0.95
+            error: 0.01
+          - quantile: 0.9
+            error: 0.05
+          - quantile: 0.5
+            error: 0.005

config/prometheus/prometheus.yml

@@ -1,10 +1,17 b''
 global:
-  scrape_interval: 10s
-  evaluation_interval: 10s
+  scrape_interval: 15s
+  evaluation_interval: 15s
 
 scrape_configs:
-  # Scrape statsd exporter
+
+  # Scrape statsd exporter from RhodeCode
   - job_name: "rhodecode_metrics"
     metrics_path: "/metrics"
     static_configs:
       - targets: ["statsd-exporter:9102"]
+
+  # scrape node exporter
+  - job_name: node_metrics
+    metrics_path: "/metrics"
+    static_configs:
+      - targets: ["node-exporter:9100"]

dev.sh

@@ -8,6 +8,8 b' USER_GID=${USER_GID:-$CURRENT_USER_GID}'
 CMD="docker-compose -f docker-compose.yml"
 BUILD_CMD="docker-compose -f docker-compose.yaml -f docker-compose.source.yaml"
 DEV_CMD="docker-compose -f docker-compose.yaml -f docker-compose.source.yaml -f docker-compose.dev.yaml"
+DEV_CMD="docker-compose -f docker-compose.yaml -f docker-compose.source.yaml -f docker-compose.dev.yaml"
+METRICS_CMD="docker-compose -f docker-compose-grafana.yaml"
 
 usage="$(basename "$0") ./dev.sh <COMMAND> -- run docker command
 where <COMMAND>:
@@ -21,19 +23,20 b' where <COMMAND>:'
     dev-env     -- run the dev stack required for development
     redis       -- run the redis CLI attached to rc_cluster_redis_1
     db          -- run the db bash attached to rc_cluster_database_1
+    [m]etrics   -- run the metrics stack
     run <cmd>   -- run <cmd>"
 
 case ${1:-} in
     v|vcs )
-        eval "(docker stop vcsserver-dev || echo skip-stop ) && ${DEV_CMD} run --service-ports --rm --use-aliases --workdir="/home/rhodecode/rhodecode-vcsserver" --name="vcsserver-dev" vcsserver bash"
+        eval "(docker stop dev-vcsserver || echo skip-stop ) && ${DEV_CMD} run --service-ports --rm --use-aliases --workdir="/home/rhodecode/rhodecode-vcsserver" --name="dev-vcsserver" vcsserver bash"
         exit
         ;;
     r|rc )
-        eval "(docker stop enterprise-ee-dev || echo skip-stop ) && ${DEV_CMD} run --publish 8080:8080 --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ee" --name="enterprise-ee-dev" rhodecode bash"
+        eval "(docker stop dev-enterprise-ee || echo skip-stop ) && ${DEV_CMD} run --publish 8080:8080 --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ee" --name="dev-enterprise-ee" rhodecode bash"
         exit
         ;;
     ce )
-        eval "(docker stop enterprise-ce-dev || echo skip-stop ) && ${DEV_CMD} run --publish 8081:8080 --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ce" --name="enterprise-ce-dev" rhodecode bash"
+        eval "(docker stop dev-enterprise-ce || echo skip-stop ) && ${DEV_CMD} run --publish 8081:8080 --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ce" --name="dev-enterprise-ce" rhodecode bash"
         exit
         ;;
     t|test|tests )
@@ -56,6 +59,10 b' case ${1:-} in'
         eval "docker exec --interactive --tty rc_cluster_database_1 /bin/bash -c 'psql -U $$POSTGRES_USER' "
         exit
         ;;
+    m|metrics )
+        eval "${METRICS_CMD} up --detach"
+        exit
+        ;;
     run )
         command=${@:2}
         eval "${DEV_CMD} run --rm rhodecode ${command}"

docker-compose-grafana.yaml

@@ -3,6 +3,12 b" version: '3.9'"
 volumes:
   prometheus_data: {}
   grafana_data: {}
+  logvolume:
+   driver: local
+   driver_opts:
+    type: none
+    o: bind
+    device: $PWD/logs
 
 networks:
   rhodecode_network:
@@ -15,17 +21,37 b' services:'
     image: prom/statsd-exporter:v0.22.8
     restart: unless-stopped
     command: [
-      '--statsd.mapping-config=/statsd/statsd.conf',
+      '--statsd.mapping-config=/statsd/statsd.yaml',
       '--statsd.listen-udp=:9125',
       '--web.listen-address=:9102'
     ]
     networks:
       - rhodecode_network
     ports:
-      - "9125:9125/udp"
-      - "9102:9102"
+      - "127.0.0.1:9125:9125/udp"
+      - "127.0.0.1:9102:9102"
     volumes:
-      - ./config/statsd-exporter/statsd.conf:/statsd/statsd.conf
+      - ./config/statsd-exporter:/statsd
+
+  node-exporter:
+    image: prom/node-exporter:v1.4.0
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.rootfs=/rootfs"
+      - "--path.sysfs=/host/sys"
+      - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
+      - "--collector.systemd"
+      - "--collector.processes"
+    networks:
+      - rhodecode_network
+    pid: host
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:9100:9100"
+    volumes:
+      - "/proc:/host/proc:ro"
+      - "/sys:/host/sys:ro"
+      - "/:/rootfs:ro"
 
   prometheus:
     image: prom/prometheus:v2.39.1
@@ -38,7 +64,7 b' services:'
       - "--storage.tsdb.retention.time=24h"
       - "--web.enable-lifecycle"
     ports:
-      - "9090:9090"
+      - "127.0.0.1:9090:9090"
     networks:
       - rhodecode_network
     volumes:
@@ -46,13 +72,39 b' services:'
       - prometheus_data:/prometheus
 
   grafana:
-    image: grafana/grafana:9.1.8-ubuntu
+    image: grafana/grafana:9.2.1-ubuntu
     restart: unless-stopped
     env_file:
       - ./config/grafana/grafana.env
     ports:
-      - "3000:3000"
+      - "4000:3000"
     volumes:
       - grafana_data:/var/lib/grafana
     networks:
       - rhodecode_network
+
+  loki:
+    image: grafana/loki:2.6.1
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:3100:3100"
+      - "127.0.0.1:9095:9095"
+    command: [
+      "-config.file=/etc/loki/loki-config.yaml"
+    ]
+    networks:
+      - rhodecode_network
+    volumes:
+      - ./config/loki:/etc/loki
+
+  promtail:
+    image: grafana/promtail:latest
+    command: [
+      "-config.file=/etc/promtail/promtail-config.yaml"
+    ]
+    networks:
+      - rhodecode_network
+    volumes:
+      - /var/log:/var/log
+      - logvolume:/var/log_volume
+      - ./config/loki:/etc/promtail

service/database/pg_customized.conf

@@ -430,27 +430,27 b" listen_addresses = '*'"
 
 # - Where to Log -
 
-#log_destination = 'stderr'		# Valid values are combinations of
+log_destination = 'stderr'		# Valid values are combinations of
 					# stderr, csvlog, syslog, and eventlog,
 					# depending on platform.  csvlog
 					# requires logging_collector to be on.
 
 # This is used when logging to stderr:
-#logging_collector = off		# Enable capturing of stderr and csvlog
+logging_collector = on		# Enable capturing of stderr and csvlog
 					# into log files. Required to be on for
 					# csvlogs.
 					# (change requires restart)
 
 # These are only used if logging_collector is on:
-#log_directory = 'log'			# directory where log files are written,
+log_directory = '/var/log/rhodecode/postgres'			# directory where log files are written,
 					# can be absolute or relative to PGDATA
-#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log'	# log file name pattern,
+log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log'	# log file name pattern,
 					# can include strftime() escapes
 #log_file_mode = 0600			# creation mode for log files,
 					# begin with 0 to use octal notation
 #log_rotation_age = 1d			# Automatic rotation of logfiles will
 					# happen after that time.  0 disables.
-#log_rotation_size = 10MB		# Automatic rotation of logfiles will
+log_rotation_size = 512MB		# Automatic rotation of logfiles will
 					# happen after that much log output.
 					# 0 disables.
 #log_truncate_on_rotation = off		# If on, an existing log file with the
@@ -502,7 +502,7 b" listen_addresses = '*'"
 					#   fatal
 					#   panic (effectively off)
 
-#log_min_duration_statement = -1	# -1 is disabled, 0 logs all statements
+log_min_duration_statement = 1000	# -1 is disabled, 0 logs all statements
 					# and their durations, > 0 logs only
 					# statements running at least this number
 					# of milliseconds

service/nginx/http.conf

@@ -35,12 +35,29 b' limit_req_zone $binary_remote_addr zone=http_req_limit:10m rate=1r/s;'
 ## custom log format
 log_format http_log_custom '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time $upstream_response_time $pipe';
 
+log_format json_log_custom escape=json
+  '{'
+    '"source":"nginx",'
+    '"remote_user":"$remote_user",'
+    '"time_local":"$time_local",'
+    '"remote_addr":"$remote_addr",'
+    '"request":"$request",'
+    '"status": "$status",'
+    '"request_method": "$request_method",'
+    '"body_bytes_sent":"$body_bytes_sent",'
+    '"request_time":"$request_time",'
+    '"upstream_response_time":"$upstream_response_time",'
+    '"http_referrer":"$http_referer",'
+    '"http_user_agent":"$http_user_agent"'
+  '}';
+
+
 server {
   listen 80 default;
   # ensure we get the proper Docker DNS resolver for load balancing.
   resolver 127.0.0.11 ipv6=off;
   server_name localhost 127.0.0.1;
-  access_log      /var/log/rhodecode/nginx/rhodecode.http.access.log http_log_custom;
+  access_log      /var/log/rhodecode/nginx/rhodecode.http.access.log json_log_custom;
   error_log       /var/log/rhodecode/nginx/rhodecode.http.error.log;
 
   # uncomment to redirect http traffic to https

service/rhodecode/rhodecode_source.dockerfile

@@ -1,5 +1,5 b''
-FROM ubuntu:latest
-# Using 20.04 LTS Release
+FROM ubuntu:22.04
+# Using 22.04 LTS Release
 MAINTAINER RhodeCode Inc. <[e-mail]>
 
 ARG TZ="UTC"

service/svn/rhodecode_svn.dockerfile

@@ -1,14 +1,16 b''
-FROM debian:buster
+FROM ubuntu:22.04
+# Using 22.04 LTS Release
 MAINTAINER RhodeCode Inc. <[e-mail]>
 
+
 ENV \
     RC_USER=rhodecode \
     MOD_DAV_SVN_CONF_FILE=/etc/rhodecode/conf/svn/mod_dav_svn.conf
 
 RUN \
 echo "** install base svn packages **" && \
-  apk update && \
-  apk add --no-cache \
+  apt-get update && \
+  apt-get install --no-cache \
     tini \
     bash \
     curl \
@@ -29,7 +31,7 b' echo "** Create system user $RC_USER **" && \\'
 
 RUN \
 echo "**** cleanup ****" && \
-    apk del tzdata python2 && \
+    apt-get purge tzdata python2 && \
     rm -f /tmp/* && \
     rm -rf /var/lib/apt/lists/* \
     rm -rf /var/cache/apk/*