release: Release 5.2.0

vcsserver/lib/hash_utils.py

@@ -0,0 +1,53 b''
+# RhodeCode VCSServer provides access to different vcs backends via network.
+# Copyright (C) 2014-2023 RhodeCode GmbH
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+
+import hashlib
+from vcsserver.lib.str_utils import safe_bytes, safe_str
+
+
+def md5(s):
+    return hashlib.md5(s).hexdigest()
+
+
+def md5_safe(s, return_type=''):
+
+    val = md5(safe_bytes(s))
+    if return_type == 'str':
+        val = safe_str(val)
+    return val
+
+
+def sha1(s):
+    return hashlib.sha1(s).hexdigest()
+
+
+def sha1_safe(s, return_type=''):
+    val = sha1(safe_bytes(s))
+    if return_type == 'str':
+        val = safe_str(val)
+    return val
+
+
+def sha256(s):
+    return hashlib.sha256(s).hexdigest()
+
+
+def sha256_safe(s, return_type=''):
+    val = sha256(safe_bytes(s))
+    if return_type == 'str':
+        val = safe_str(val)
+    return val

Makefile

@@ -1,12 +1,49 b''
+.DEFAULT_GOAL := help
+
+# Pretty print values cf. https://misc.flogisoft.com/bash/tip_colors_and_formatting
+RESET := \033[0m # Reset all formatting
+GREEN := \033[0;32m # Resets before setting 16b colour (32 -- green)
+YELLOW := \033[0;33m
+ORANGE := \033[0;38;5;208m # Reset then set 256b colour (208 -- orange)
+PEACH := \033[0;38;5;216m
+
+
+## ---------------------------------------------------------------------------------- ##
+## ------------------------- Help usage builder ------------------------------------- ##
+## ---------------------------------------------------------------------------------- ##
+# use '# >>> Build commands' to create section
+# use '# target: target description' to create help for target
+.PHONY: help
+help:
+	@echo "Usage:"
+	@cat $(MAKEFILE_LIST) | grep -E '^# >>>|^# [A-Za-z0-9_.-]+:' | sed -E 's/^# //' | awk ' \
+		BEGIN { \
+			green="\033[32m"; \
+			yellow="\033[33m"; \
+			reset="\033[0m"; \
+			section=""; \
+		} \
+		/^>>>/ { \
+			section=substr($$0, 5); \
+			printf "\n" green ">>> %s" reset "\n", section; \
+			next; \
+		} \
+		/^([A-Za-z0-9_.-]+):/ { \
+			target=$$1; \
+			gsub(/:$$/, "", target); \
+			description=substr($$0, index($$0, ":") + 2); \
+			if (description == "") { description="-"; } \
+			printf " - " yellow "%-35s" reset " %s\n", target, description; \
+		} \
+	'
+
 # required for pushd to work..
 SHELL = /bin/bash
 
-
-# set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
-OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
+# >>> Tests commands
 
 .PHONY: clean
-## Cleanup compiled and cache py files
+# clean: Cleanup compiled and cache py files
 clean:
 	make test-clean
 	find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
@@ -14,14 +51,14 b' clean:'
 
 
 .PHONY: test
-## run test-clean and tests
+# test: run test-clean and tests
 test:
 	make test-clean
-	make test-only
+	unset RC_SQLALCHEMY_DB1_URL && unset RC_DB_URL && make test-only
 
 
 .PHONY: test-clean
-## run test-clean and tests
+# test-clean: run test-clean and tests
 test-clean:
 	rm -rf coverage.xml htmlcov junit.xml pylint.log result
 	find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
@@ -29,33 +66,19 b' test-clean:'
 
 
 .PHONY: test-only
-## Run tests only without cleanup
+# test-only: Run tests only without cleanup
 test-only:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar \
 	--cov-report=term-missing --cov-report=html \
 	--cov=vcsserver vcsserver
 
-
-.PHONY: ruff-check
-## run a ruff analysis
-ruff-check:
-	ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs  --exclude .dev .
+# >>> Dev commands
 
-.PHONY: pip-packages
-## Show outdated packages
-pip-packages:
-	python ${OUTDATED_PACKAGES}
-
-
-.PHONY: build
-## Build sdist/egg
-build:
-	python -m build
 
 
 .PHONY: dev-sh
-## make dev-sh
+# dev-sh: make dev-sh
 dev-sh:
 	sudo echo "deb [trusted=yes] https://apt.fury.io/rsteube/ /" | sudo tee -a "/etc/apt/sources.list.d/fury.list"
 	sudo apt-get update
@@ -68,14 +91,14 b' dev-sh:'
 
 
 .PHONY: dev-cleanup
-## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
+# dev-cleanup: Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
 dev-cleanup:
 	pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
 	rm -rf /tmp/*
 
 
 .PHONY: dev-env
-## make dev-env based on the requirements files and install develop of packages
+# dev-env: make dev-env based on the requirements files and install develop of packages
 ## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y
 dev-env:
 	sudo -u root chown rhodecode:rhodecode /home/rhodecode/.cache/pip/
@@ -86,7 +109,7 b' dev-env:'
 
 
 .PHONY: sh
-## shortcut for make dev-sh dev-env
+# sh: shortcut for make dev-sh dev-env
 sh:
 	make dev-env
 	make dev-sh
@@ -96,49 +119,12 b' sh:'
 workers?=1
 
 .PHONY: dev-srv
-## run gunicorn web server with reloader, use workers=N to set multiworker mode
+# dev-srv: run gunicorn web server with reloader, use workers=N to set multiworker mode, workers=N allows changes of workers
 dev-srv:
 	gunicorn --paste=.dev/dev.ini --bind=0.0.0.0:10010 --config=.dev/gunicorn_config.py --reload --workers=$(workers)
 
-
-# Default command on calling make
-.DEFAULT_GOAL := show-help
+.PHONY: ruff-check
+# ruff-check: run a ruff analysis
+ruff-check:
+	ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs  --exclude .dev .
 
-.PHONY: show-help
-show-help:
-	@echo "$$(tput bold)Available rules:$$(tput sgr0)"
-	@echo
-	@sed -n -e "/^## / { \
-		h; \
-		s/.*//; \
-		:doc" \
-		-e "H; \
-		n; \
-		s/^## //; \
-		t doc" \
-		-e "s/:.*//; \
-		G; \
-		s/\\n## /---/; \
-		s/\\n/ /g; \
-		p; \
-	}" ${MAKEFILE_LIST} \
-	| LC_ALL='C' sort --ignore-case \
-	| awk -F '---' \
-		-v ncol=$$(tput cols) \
-		-v indent=19 \
-		-v col_on="$$(tput setaf 6)" \
-		-v col_off="$$(tput sgr0)" \
-	'{ \
-		printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
-		n = split($$2, words, " "); \
-		line_length = ncol - indent; \
-		for (i = 1; i <= n; i++) { \
-			line_length -= length(words[i]) + 1; \
-			if (line_length <= 0) { \
-				line_length = ncol - indent - length(words[i]) - 1; \
-				printf "\n%*s ", -indent, " "; \
-			} \
-			printf "%s ", words[i]; \
-		} \
-		printf "\n"; \
-	}'

configs/gunicorn_config.py

@@ -13,6 +13,7 b' import traceback'
 import random
 import socket
 import dataclasses
+import json
 from gunicorn.glogging import Logger
 
 
@@ -37,17 +38,41 b" accesslog = '-'"
 worker_tmp_dir = None
 tmp_upload_dir = None
 
-# use re-use port logic
-#reuse_port = True
+# use re-use port logic to let linux internals load-balance the requests better.
+reuse_port = True
 
 # Custom log format
 #access_log_format = (
 #    '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
 
 # loki format for easier parsing in grafana
-access_log_format = (
+loki_access_log_format = (
     'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
 
+# JSON format
+json_access_log_format = json.dumps({
+    'time': r'%(t)s',
+    'pid': r'%(p)s',
+    'level': 'INFO',
+    'ip': r'%(h)s',
+    'request_time': r'%(L)s',
+    'remote_address': r'%(h)s',
+    'user_name': r'%(u)s',
+    'status': r'%(s)s',
+    'method': r'%(m)s',
+    'url_path': r'%(U)s',
+    'query_string': r'%(q)s',
+    'protocol': r'%(H)s',
+    'response_length': r'%(B)s',
+    'referer': r'%(f)s',
+    'user_agent': r'%(a)s',
+
+})
+
+access_log_format = loki_access_log_format
+if os.environ.get('RC_LOGGING_FORMATTER') == 'json':
+    access_log_format = json_access_log_format
+
 # self adjust workers based on CPU count, to use maximum of CPU and not overquota the resources
 # workers = get_workers()
 

requirements.txt

@@ -11,8 +11,8 b' celery==5.3.6'
     click==8.1.3
   click-repl==0.2.0
     click==8.1.3
-    prompt-toolkit==3.0.38
-      wcwidth==0.2.6
+    prompt_toolkit==3.0.47
+      wcwidth==0.2.13
     six==1.16.0
   kombu==5.3.5
     amqp==5.2.0
@@ -29,22 +29,22 b' dogpile.cache==1.3.3'
     pbr==5.11.1
 dulwich==0.21.6
   urllib3==1.26.14
-fsspec==2024.6.0
-gunicorn==21.2.0
-  packaging==24.0
+fsspec==2024.9.0
+gunicorn==23.0.0
+  packaging==24.1
 hg-evolve==11.1.3
 importlib-metadata==6.0.0
   zipp==3.15.0
 mercurial==6.7.4
 more-itertools==9.1.0
 msgpack==1.0.8
-orjson==3.10.3
+orjson==3.10.7
 psutil==5.9.8
 py==1.11.0
 pygit2==1.13.3
   cffi==1.16.0
     pycparser==2.21
-pygments==2.15.1
+pygments==2.18.0
 pyparsing==3.1.1
 pyramid==2.0.2
   hupper==1.12
@@ -56,11 +56,11 b' pyramid==2.0.2'
   venusian==3.0.0
   webob==1.8.7
   zope.deprecation==5.0.0
-  zope.interface==6.3.0
-redis==5.0.4
+  zope.interface==6.4.post2
+redis==5.1.0
   async-timeout==4.0.3
 repoze.lru==0.7
-s3fs==2024.6.0
+s3fs==2024.9.0
   aiobotocore==2.13.0
     aiohttp==3.9.5
       aiosignal==1.3.1
@@ -87,12 +87,12 b' s3fs==2024.6.0'
     yarl==1.9.4
       idna==3.4
       multidict==6.0.5
-  fsspec==2024.6.0
+  fsspec==2024.9.0
 scandir==1.10.0
 setproctitle==1.3.3
 subvertpy==0.11.0
 waitress==3.0.0
-wcwidth==0.2.6
+wcwidth==0.2.13
 
 
 ## test related requirements

requirements_test.txt

@@ -4,38 +4,38 b' pytest-cov==4.1.0'
   coverage==7.4.3
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
 pytest-env==1.1.3
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
 pytest-profiling==1.7.0
   gprof2dot==2022.7.29
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
   six==1.16.0
 pytest-rerunfailures==13.0
-  packaging==24.0
+  packaging==24.1
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
 pytest-runner==6.0.1
 pytest-sugar==1.0.0
-  packaging==24.0
+  packaging==24.1
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
   termcolor==2.4.0
 pytest-timeout==2.3.1
   pytest==8.1.1
     iniconfig==2.0.0
-    packaging==24.0
+    packaging==24.1
     pluggy==1.4.0
 webtest==3.0.0
   beautifulsoup4==4.12.3

vcsserver/exceptions.py

@@ -53,6 +53,12 b' def ArchiveException(org_exc=None):'
     return _make_exception_wrapper
 
 
+def ClientNotSupportedException(org_exc=None):
+    def _make_exception_wrapper(*args):
+        return _make_exception('client_not_supported', org_exc, *args)
+    return _make_exception_wrapper
+
+
 def LookupException(org_exc=None):
     def _make_exception_wrapper(*args):
         return _make_exception('lookup', org_exc, *args)

vcsserver/git_lfs/app.py

@@ -18,6 +18,7 b''
 import re
 import logging
 
+from gunicorn.http.errors import NoMoreData
 from pyramid.config import Configurator
 from pyramid.response import Response, FileIter
 from pyramid.httpexceptions import (
@@ -166,9 +167,14 b' def lfs_objects_oid_upload(request):'
             # read in chunks as stream comes in from Gunicorn
             # this is a specific Gunicorn support function.
             # might work differently on waitress
-            chunk = body.read(blksize)
+            try:
+                chunk = body.read(blksize)
+            except NoMoreData:
+                chunk = None
+
             if not chunk:
                 break
+
             f.write(chunk)
 
     return {'upload': 'ok'}

vcsserver/hooks.py

@@ -167,6 +167,8 b' def _handle_exception(result):'
 
     if exception_class == 'HTTPLockedRC':
         raise exceptions.RepositoryLockedException()(*result['exception_args'])
+    elif exception_class == 'ClientNotSupportedError':
+        raise exceptions.ClientNotSupportedException()(*result['exception_args'])
     elif exception_class == 'HTTPBranchProtected':
         raise exceptions.RepositoryBranchProtectedException()(*result['exception_args'])
     elif exception_class == 'RepositoryError':

vcsserver/lib/archive_cache/backends/objectstore_cache.py

@@ -58,7 +58,7 b' class S3Shard(BaseShard):'
         # ensure folder in bucket exists
         destination = self.bucket
         if not self.fs.exists(destination):
-            self.fs.mkdir(destination, s3_additional_kwargs={})
+            self.fs.mkdir(destination)
 
         writer = self._get_writer(full_path, mode)
 

vcsserver/lib/rc_cache/backends.py

@@ -37,9 +37,9 b' from dogpile.cache.backends import redis'
 from dogpile.cache.backends.file import FileLock
 from dogpile.cache.util import memoized_property
 
-from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug
-from vcsserver.lib.str_utils import safe_bytes, safe_str
-from vcsserver.lib.type_utils import str2bool
+from ...lib.memory_lru_dict import LRUDict, LRUDictDebug
+from ...lib.str_utils import safe_bytes, safe_str
+from ...lib.type_utils import str2bool
 
 _default_max_size = 1024
 
@@ -166,6 +166,13 b' class FileNamespaceBackend(PickleSeriali'
     def get_store(self):
         return self.filename
 
+    def cleanup_store(self):
+        for ext in ("db", "dat", "pag", "dir"):
+            final_filename = self.filename + os.extsep + ext
+            if os.path.exists(final_filename):
+                os.remove(final_filename)
+                log.warning('Removed dbm file %s', final_filename)
+
 
 class BaseRedisBackend(redis_backend.RedisBackend):
     key_prefix = ''
@@ -257,7 +264,7 b' class RedisMsgPackBackend(MsgPackSeriali'
 
 
 def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False):
-    from vcsserver.lib._vendor import redis_lock
+    from ...lib._vendor import redis_lock
 
     class _RedisLockWrapper:
         """LockWrapper for redis_lock"""

vcsserver/lib/rc_cache/utils.py

@@ -25,9 +25,9 b' import decorator'
 from dogpile.cache import CacheRegion
 
 
-from vcsserver.utils import sha1
-from vcsserver.lib.str_utils import safe_bytes
-from vcsserver.lib.type_utils import str2bool # noqa :required by imports from .utils
+from ...lib.hash_utils import sha1
+from ...lib.str_utils import safe_bytes
+from ...lib.type_utils import str2bool # noqa :required by imports from .utils
 
 from . import region_meta
 

vcsserver/tweens/request_wrapper.py

@@ -64,7 +64,7 b' class RequestWrapperTween:'
 
     def __call__(self, request):
         start = time.time()
-        log.debug('Starting request time measurement')
+        log.debug('Starting request processing')
         response = None
 
         try:
@@ -88,7 +88,7 b' class RequestWrapperTween:'
             total = time.time() - start
 
             log.info(
-                'Req[%4s] IP: %s %s Request to %s time: %.4fs [%s], VCSServer %s',
+                'Finished request processing: reqq[%4s] IP: %s %s Request to %s time: %.4fs [%s], VCSServer %s',
                 count, ip, request.environ.get('REQUEST_METHOD'),
                 _view_path, total, ua, _ver_,
                 extra={"time": total, "ver": _ver_, "code": resp_code,