diff --git a/config/loki/loki-config.yaml b/config/loki/loki-config.yaml new file mode 100644 index 0000000..657de0d --- /dev/null +++ b/config/loki/loki-config.yaml @@ -0,0 +1,50 @@ +auth_enabled: false + +server: + http_listen_address: 0.0.0.0 + grpc_listen_address: 0.0.0.0 + http_listen_port: 3100 + grpc_listen_port: 9095 + log_level: info + +common: + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h + +limits_config: + max_streams_per_user: 100000 + + ingestion_burst_size_mb: 64 + ingestion_rate_mb: 512 + + per_stream_rate_limit: 512M + per_stream_rate_limit_burst: 1024M + max_entries_limit_per_query: 1000000 + max_label_value_length: 20480 + max_label_name_length: 10240 + +ruler: + alertmanager_url: http://localhost:9093 + +# If you would like to disable reporting, uncomment the following lines: +#analytics: +# reporting_enabled: false + diff --git a/config/loki/promtail-config.yaml b/config/loki/promtail-config.yaml new file mode 100644 index 0000000..6ee78d0 --- /dev/null +++ b/config/loki/promtail-config.yaml @@ -0,0 +1,38 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +# Describes how to save read file offsets to disk +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + tenant_id: docker + +## Definition to what to scrape +scrape_configs: + +- job_name: scraping_nginx + static_configs: + - targets: + - localhost + labels: + __path__: /var/log_volume/nginx/rhodecode* + job: nginx + +- job_name: scraping_vcsserver + static_configs: + - targets: + - localhost + labels: + __path__: /var/log_volume/apps/rhodecode* + job: rhodecode + +- job_name: scraping_rhodecode + static_configs: + - targets: + - localhost + labels: + __path__: /var/log_volume/apps/vcsserver* + job: vcsserver \ No newline at end of file diff --git a/config/prometheus/prometheus.yml b/config/prometheus/prometheus.yml index dc2d832..42e11b4 100644 --- a/config/prometheus/prometheus.yml +++ b/config/prometheus/prometheus.yml @@ -1,10 +1,17 @@ global: - scrape_interval: 10s - evaluation_interval: 10s + scrape_interval: 15s + evaluation_interval: 15s scrape_configs: - # Scrape statsd exporter + + # Scrape statsd exporter from RhodeCode - job_name: "rhodecode_metrics" metrics_path: "/metrics" static_configs: - targets: ["statsd-exporter:9102"] + + # scrape node exporter + - job_name: node_metrics + metrics_path: "/metrics" + static_configs: + - targets: ["node-exporter:9100"] diff --git a/config/statsd-exporter/statsd.yaml b/config/statsd-exporter/statsd.yaml new file mode 100644 index 0000000..3c50650 --- /dev/null +++ b/config/statsd-exporter/statsd.yaml @@ -0,0 +1,31 @@ +defaults: + # metrics expiration + ttl: 48h + +mappings: + + - match: "*.histogram" + name: "${1}" + observer_type: histogram + histogram_options: + buckets: [.1, .25, .5, .75, 1, 1.5, 2, 5, 10] + + - match: "*.slow_histogram" + name: "${1}" + observer_type: histogram + histogram_options: + buckets: [.1, .25, .5, 1, 2, 5, 10, 30, 60] + + - match: "*.summary" + name: "${1}" + observer_type: summary + summary_options: + quantiles: + - quantile: 0.99 + error: 0.001 + - quantile: 0.95 + error: 0.01 + - quantile: 0.9 + error: 0.05 + - quantile: 0.5 + error: 0.005 diff --git a/dev.sh b/dev.sh index 9a42994..e97cc93 100755 --- a/dev.sh +++ b/dev.sh @@ -8,6 +8,8 @@ USER_GID=${USER_GID:-$CURRENT_USER_GID} CMD="docker-compose -f docker-compose.yml" BUILD_CMD="docker-compose -f docker-compose.yaml -f docker-compose.source.yaml" DEV_CMD="docker-compose -f docker-compose.yaml -f docker-compose.source.yaml -f docker-compose.dev.yaml" +DEV_CMD="docker-compose -f docker-compose.yaml -f docker-compose.source.yaml -f docker-compose.dev.yaml" +METRICS_CMD="docker-compose -f docker-compose-grafana.yaml" usage="$(basename "$0") ./dev.sh -- run docker command where : @@ -21,19 +23,20 @@ where : dev-env -- run the dev stack required for development redis -- run the redis CLI attached to rc_cluster_redis_1 db -- run the db bash attached to rc_cluster_database_1 + [m]etrics -- run the metrics stack run -- run " case ${1:-} in v|vcs ) - eval "(docker stop vcsserver-dev || echo skip-stop ) && ${DEV_CMD} run --service-ports --rm --use-aliases --workdir="/home/rhodecode/rhodecode-vcsserver" --name="vcsserver-dev" vcsserver bash" + eval "(docker stop dev-vcsserver || echo skip-stop ) && ${DEV_CMD} run --service-ports --rm --use-aliases --workdir="/home/rhodecode/rhodecode-vcsserver" --name="dev-vcsserver" vcsserver bash" exit ;; r|rc ) - eval "(docker stop enterprise-ee-dev || echo skip-stop ) && ${DEV_CMD} run --publish 8080:8080 --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ee" --name="enterprise-ee-dev" rhodecode bash" + eval "(docker stop dev-enterprise-ee || echo skip-stop ) && ${DEV_CMD} run --publish 8080:8080 --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ee" --name="dev-enterprise-ee" rhodecode bash" exit ;; ce ) - eval "(docker stop enterprise-ce-dev || echo skip-stop ) && ${DEV_CMD} run --publish 8081:8080 --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ce" --name="enterprise-ce-dev" rhodecode bash" + eval "(docker stop dev-enterprise-ce || echo skip-stop ) && ${DEV_CMD} run --publish 8081:8080 --rm --use-aliases --workdir="/home/rhodecode/rhodecode-enterprise-ce" --name="dev-enterprise-ce" rhodecode bash" exit ;; t|test|tests ) @@ -56,6 +59,10 @@ case ${1:-} in eval "docker exec --interactive --tty rc_cluster_database_1 /bin/bash -c 'psql -U $$POSTGRES_USER' " exit ;; + m|metrics ) + eval "${METRICS_CMD} up --detach" + exit + ;; run ) command=${@:2} eval "${DEV_CMD} run --rm rhodecode ${command}" diff --git a/docker-compose-grafana.yaml b/docker-compose-grafana.yaml index 0b03e4e..bbcab7d 100644 --- a/docker-compose-grafana.yaml +++ b/docker-compose-grafana.yaml @@ -3,6 +3,12 @@ version: '3.9' volumes: prometheus_data: {} grafana_data: {} + logvolume: + driver: local + driver_opts: + type: none + o: bind + device: $PWD/logs networks: rhodecode_network: @@ -15,17 +21,37 @@ services: image: prom/statsd-exporter:v0.22.8 restart: unless-stopped command: [ - '--statsd.mapping-config=/statsd/statsd.conf', + '--statsd.mapping-config=/statsd/statsd.yaml', '--statsd.listen-udp=:9125', '--web.listen-address=:9102' ] networks: - rhodecode_network ports: - - "9125:9125/udp" - - "9102:9102" + - "127.0.0.1:9125:9125/udp" + - "127.0.0.1:9102:9102" volumes: - - ./config/statsd-exporter/statsd.conf:/statsd/statsd.conf + - ./config/statsd-exporter:/statsd + + node-exporter: + image: prom/node-exporter:v1.4.0 + command: + - "--path.procfs=/host/proc" + - "--path.rootfs=/rootfs" + - "--path.sysfs=/host/sys" + - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)" + - "--collector.systemd" + - "--collector.processes" + networks: + - rhodecode_network + pid: host + restart: unless-stopped + ports: + - "127.0.0.1:9100:9100" + volumes: + - "/proc:/host/proc:ro" + - "/sys:/host/sys:ro" + - "/:/rootfs:ro" prometheus: image: prom/prometheus:v2.39.1 @@ -38,7 +64,7 @@ services: - "--storage.tsdb.retention.time=24h" - "--web.enable-lifecycle" ports: - - "9090:9090" + - "127.0.0.1:9090:9090" networks: - rhodecode_network volumes: @@ -46,13 +72,39 @@ services: - prometheus_data:/prometheus grafana: - image: grafana/grafana:9.1.8-ubuntu + image: grafana/grafana:9.2.1-ubuntu restart: unless-stopped env_file: - ./config/grafana/grafana.env ports: - - "3000:3000" + - "4000:3000" volumes: - grafana_data:/var/lib/grafana networks: - rhodecode_network + + loki: + image: grafana/loki:2.6.1 + restart: unless-stopped + ports: + - "127.0.0.1:3100:3100" + - "127.0.0.1:9095:9095" + command: [ + "-config.file=/etc/loki/loki-config.yaml" + ] + networks: + - rhodecode_network + volumes: + - ./config/loki:/etc/loki + + promtail: + image: grafana/promtail:latest + command: [ + "-config.file=/etc/promtail/promtail-config.yaml" + ] + networks: + - rhodecode_network + volumes: + - /var/log:/var/log + - logvolume:/var/log_volume + - ./config/loki:/etc/promtail diff --git a/logs/apps/.dirkeep b/logs/apps/.dirkeep new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/logs/apps/.dirkeep diff --git a/service/database/pg_customized.conf b/service/database/pg_customized.conf index 0d68de3..120fc7a 100644 --- a/service/database/pg_customized.conf +++ b/service/database/pg_customized.conf @@ -430,27 +430,27 @@ listen_addresses = '*' # - Where to Log - -#log_destination = 'stderr' # Valid values are combinations of +log_destination = 'stderr' # Valid values are combinations of # stderr, csvlog, syslog, and eventlog, # depending on platform. csvlog # requires logging_collector to be on. # This is used when logging to stderr: -#logging_collector = off # Enable capturing of stderr and csvlog +logging_collector = on # Enable capturing of stderr and csvlog # into log files. Required to be on for # csvlogs. # (change requires restart) # These are only used if logging_collector is on: -#log_directory = 'log' # directory where log files are written, +log_directory = '/var/log/rhodecode/postgres' # directory where log files are written, # can be absolute or relative to PGDATA -#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, +log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, # can include strftime() escapes #log_file_mode = 0600 # creation mode for log files, # begin with 0 to use octal notation #log_rotation_age = 1d # Automatic rotation of logfiles will # happen after that time. 0 disables. -#log_rotation_size = 10MB # Automatic rotation of logfiles will +log_rotation_size = 512MB # Automatic rotation of logfiles will # happen after that much log output. # 0 disables. #log_truncate_on_rotation = off # If on, an existing log file with the @@ -502,7 +502,7 @@ listen_addresses = '*' # fatal # panic (effectively off) -#log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements +log_min_duration_statement = 1000 # -1 is disabled, 0 logs all statements # and their durations, > 0 logs only # statements running at least this number # of milliseconds diff --git a/service/nginx/http.conf b/service/nginx/http.conf index b1eca11..196a73d 100644 --- a/service/nginx/http.conf +++ b/service/nginx/http.conf @@ -35,12 +35,29 @@ limit_req_zone $binary_remote_addr zone=http_req_limit:10m rate=1r/s; ## custom log format log_format http_log_custom '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_time $upstream_response_time $pipe'; +log_format json_log_custom escape=json + '{' + '"source":"nginx",' + '"remote_user":"$remote_user",' + '"time_local":"$time_local",' + '"remote_addr":"$remote_addr",' + '"request":"$request",' + '"status": "$status",' + '"request_method": "$request_method",' + '"body_bytes_sent":"$body_bytes_sent",' + '"request_time":"$request_time",' + '"upstream_response_time":"$upstream_response_time",' + '"http_referrer":"$http_referer",' + '"http_user_agent":"$http_user_agent"' + '}'; + + server { listen 80 default; # ensure we get the proper Docker DNS resolver for load balancing. resolver 127.0.0.11 ipv6=off; server_name localhost 127.0.0.1; - access_log /var/log/rhodecode/nginx/rhodecode.http.access.log http_log_custom; + access_log /var/log/rhodecode/nginx/rhodecode.http.access.log json_log_custom; error_log /var/log/rhodecode/nginx/rhodecode.http.error.log; # uncomment to redirect http traffic to https diff --git a/service/rhodecode/rhodecode_source.dockerfile b/service/rhodecode/rhodecode_source.dockerfile index 5ddd47f..5ef2e4b 100644 --- a/service/rhodecode/rhodecode_source.dockerfile +++ b/service/rhodecode/rhodecode_source.dockerfile @@ -1,5 +1,5 @@ -FROM ubuntu:latest -# Using 20.04 LTS Release +FROM ubuntu:22.04 +# Using 22.04 LTS Release MAINTAINER RhodeCode Inc. ARG TZ="UTC" diff --git a/service/svn/rhodecode_svn.dockerfile b/service/svn/rhodecode_svn.dockerfile index d7300b4..d38b401 100644 --- a/service/svn/rhodecode_svn.dockerfile +++ b/service/svn/rhodecode_svn.dockerfile @@ -1,14 +1,16 @@ -FROM debian:buster +FROM ubuntu:22.04 +# Using 22.04 LTS Release MAINTAINER RhodeCode Inc. + ENV \ RC_USER=rhodecode \ MOD_DAV_SVN_CONF_FILE=/etc/rhodecode/conf/svn/mod_dav_svn.conf RUN \ echo "** install base svn packages **" && \ - apk update && \ - apk add --no-cache \ + apt-get update && \ + apt-get install --no-cache \ tini \ bash \ curl \ @@ -29,7 +31,7 @@ echo "** Create system user $RC_USER **" && \ RUN \ echo "**** cleanup ****" && \ - apk del tzdata python2 && \ + apt-get purge tzdata python2 && \ rm -f /tmp/* && \ rm -rf /var/lib/apt/lists/* \ rm -rf /var/cache/apk/*