# HG changeset patch # User Andrii Verbytskyi # Date 2024-10-09 15:07:00 # Node ID 50174967ed203c7e786e30cea0c8f7c2b8541c57 # Parent 096707fc60e87b03175a2542eb4f1731aa662a24 # Parent 708aefaa09a15c0c37ba23742653d1cb9a2b477c release: Release 5.2.0 diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -1,12 +1,49 @@ +.DEFAULT_GOAL := help + +# Pretty print values cf. https://misc.flogisoft.com/bash/tip_colors_and_formatting +RESET := \033[0m # Reset all formatting +GREEN := \033[0;32m # Resets before setting 16b colour (32 -- green) +YELLOW := \033[0;33m +ORANGE := \033[0;38;5;208m # Reset then set 256b colour (208 -- orange) +PEACH := \033[0;38;5;216m + + +## ---------------------------------------------------------------------------------- ## +## ------------------------- Help usage builder ------------------------------------- ## +## ---------------------------------------------------------------------------------- ## +# use '# >>> Build commands' to create section +# use '# target: target description' to create help for target +.PHONY: help +help: + @echo "Usage:" + @cat $(MAKEFILE_LIST) | grep -E '^# >>>|^# [A-Za-z0-9_.-]+:' | sed -E 's/^# //' | awk ' \ + BEGIN { \ + green="\033[32m"; \ + yellow="\033[33m"; \ + reset="\033[0m"; \ + section=""; \ + } \ + /^>>>/ { \ + section=substr($$0, 5); \ + printf "\n" green ">>> %s" reset "\n", section; \ + next; \ + } \ + /^([A-Za-z0-9_.-]+):/ { \ + target=$$1; \ + gsub(/:$$/, "", target); \ + description=substr($$0, index($$0, ":") + 2); \ + if (description == "") { description="-"; } \ + printf " - " yellow "%-35s" reset " %s\n", target, description; \ + } \ + ' + # required for pushd to work.. SHELL = /bin/bash - -# set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py -OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES} +# >>> Tests commands .PHONY: clean -## Cleanup compiled and cache py files +# clean: Cleanup compiled and cache py files clean: make test-clean find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';' @@ -14,14 +51,14 @@ clean: .PHONY: test -## run test-clean and tests +# test: run test-clean and tests test: make test-clean - make test-only + unset RC_SQLALCHEMY_DB1_URL && unset RC_DB_URL && make test-only .PHONY: test-clean -## run test-clean and tests +# test-clean: run test-clean and tests test-clean: rm -rf coverage.xml htmlcov junit.xml pylint.log result find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';' @@ -29,33 +66,19 @@ test-clean: .PHONY: test-only -## Run tests only without cleanup +# test-only: Run tests only without cleanup test-only: PYTHONHASHSEED=random \ py.test -x -vv -r xw -p no:sugar \ --cov-report=term-missing --cov-report=html \ --cov=vcsserver vcsserver - -.PHONY: ruff-check -## run a ruff analysis -ruff-check: - ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs --exclude .dev . +# >>> Dev commands -.PHONY: pip-packages -## Show outdated packages -pip-packages: - python ${OUTDATED_PACKAGES} - - -.PHONY: build -## Build sdist/egg -build: - python -m build .PHONY: dev-sh -## make dev-sh +# dev-sh: make dev-sh dev-sh: sudo echo "deb [trusted=yes] https://apt.fury.io/rsteube/ /" | sudo tee -a "/etc/apt/sources.list.d/fury.list" sudo apt-get update @@ -68,14 +91,14 @@ dev-sh: .PHONY: dev-cleanup -## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y +# dev-cleanup: Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y dev-cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y rm -rf /tmp/* .PHONY: dev-env -## make dev-env based on the requirements files and install develop of packages +# dev-env: make dev-env based on the requirements files and install develop of packages ## Cleanup: pip freeze | grep -v "^-e" | grep -v "@" | xargs pip uninstall -y dev-env: sudo -u root chown rhodecode:rhodecode /home/rhodecode/.cache/pip/ @@ -86,7 +109,7 @@ dev-env: .PHONY: sh -## shortcut for make dev-sh dev-env +# sh: shortcut for make dev-sh dev-env sh: make dev-env make dev-sh @@ -96,49 +119,12 @@ sh: workers?=1 .PHONY: dev-srv -## run gunicorn web server with reloader, use workers=N to set multiworker mode +# dev-srv: run gunicorn web server with reloader, use workers=N to set multiworker mode, workers=N allows changes of workers dev-srv: gunicorn --paste=.dev/dev.ini --bind=0.0.0.0:10010 --config=.dev/gunicorn_config.py --reload --workers=$(workers) - -# Default command on calling make -.DEFAULT_GOAL := show-help +.PHONY: ruff-check +# ruff-check: run a ruff analysis +ruff-check: + ruff check --ignore F401 --ignore I001 --ignore E402 --ignore E501 --ignore F841 --exclude rhodecode/lib/dbmigrate --exclude .eggs --exclude .dev . -.PHONY: show-help -show-help: - @echo "$$(tput bold)Available rules:$$(tput sgr0)" - @echo - @sed -n -e "/^## / { \ - h; \ - s/.*//; \ - :doc" \ - -e "H; \ - n; \ - s/^## //; \ - t doc" \ - -e "s/:.*//; \ - G; \ - s/\\n## /---/; \ - s/\\n/ /g; \ - p; \ - }" ${MAKEFILE_LIST} \ - | LC_ALL='C' sort --ignore-case \ - | awk -F '---' \ - -v ncol=$$(tput cols) \ - -v indent=19 \ - -v col_on="$$(tput setaf 6)" \ - -v col_off="$$(tput sgr0)" \ - '{ \ - printf "%s%*s%s ", col_on, -indent, $$1, col_off; \ - n = split($$2, words, " "); \ - line_length = ncol - indent; \ - for (i = 1; i <= n; i++) { \ - line_length -= length(words[i]) + 1; \ - if (line_length <= 0) { \ - line_length = ncol - indent - length(words[i]) - 1; \ - printf "\n%*s ", -indent, " "; \ - } \ - printf "%s ", words[i]; \ - } \ - printf "\n"; \ - }' diff --git a/configs/gunicorn_config.py b/configs/gunicorn_config.py --- a/configs/gunicorn_config.py +++ b/configs/gunicorn_config.py @@ -13,6 +13,7 @@ import traceback import random import socket import dataclasses +import json from gunicorn.glogging import Logger @@ -37,17 +38,41 @@ accesslog = '-' worker_tmp_dir = None tmp_upload_dir = None -# use re-use port logic -#reuse_port = True +# use re-use port logic to let linux internals load-balance the requests better. +reuse_port = True # Custom log format #access_log_format = ( # '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"') # loki format for easier parsing in grafana -access_log_format = ( +loki_access_log_format = ( 'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"') +# JSON format +json_access_log_format = json.dumps({ + 'time': r'%(t)s', + 'pid': r'%(p)s', + 'level': 'INFO', + 'ip': r'%(h)s', + 'request_time': r'%(L)s', + 'remote_address': r'%(h)s', + 'user_name': r'%(u)s', + 'status': r'%(s)s', + 'method': r'%(m)s', + 'url_path': r'%(U)s', + 'query_string': r'%(q)s', + 'protocol': r'%(H)s', + 'response_length': r'%(B)s', + 'referer': r'%(f)s', + 'user_agent': r'%(a)s', + +}) + +access_log_format = loki_access_log_format +if os.environ.get('RC_LOGGING_FORMATTER') == 'json': + access_log_format = json_access_log_format + # self adjust workers based on CPU count, to use maximum of CPU and not overquota the resources # workers = get_workers() diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -11,8 +11,8 @@ celery==5.3.6 click==8.1.3 click-repl==0.2.0 click==8.1.3 - prompt-toolkit==3.0.38 - wcwidth==0.2.6 + prompt_toolkit==3.0.47 + wcwidth==0.2.13 six==1.16.0 kombu==5.3.5 amqp==5.2.0 @@ -29,22 +29,22 @@ dogpile.cache==1.3.3 pbr==5.11.1 dulwich==0.21.6 urllib3==1.26.14 -fsspec==2024.6.0 -gunicorn==21.2.0 - packaging==24.0 +fsspec==2024.9.0 +gunicorn==23.0.0 + packaging==24.1 hg-evolve==11.1.3 importlib-metadata==6.0.0 zipp==3.15.0 mercurial==6.7.4 more-itertools==9.1.0 msgpack==1.0.8 -orjson==3.10.3 +orjson==3.10.7 psutil==5.9.8 py==1.11.0 pygit2==1.13.3 cffi==1.16.0 pycparser==2.21 -pygments==2.15.1 +pygments==2.18.0 pyparsing==3.1.1 pyramid==2.0.2 hupper==1.12 @@ -56,11 +56,11 @@ pyramid==2.0.2 venusian==3.0.0 webob==1.8.7 zope.deprecation==5.0.0 - zope.interface==6.3.0 -redis==5.0.4 + zope.interface==6.4.post2 +redis==5.1.0 async-timeout==4.0.3 repoze.lru==0.7 -s3fs==2024.6.0 +s3fs==2024.9.0 aiobotocore==2.13.0 aiohttp==3.9.5 aiosignal==1.3.1 @@ -87,12 +87,12 @@ s3fs==2024.6.0 yarl==1.9.4 idna==3.4 multidict==6.0.5 - fsspec==2024.6.0 + fsspec==2024.9.0 scandir==1.10.0 setproctitle==1.3.3 subvertpy==0.11.0 waitress==3.0.0 -wcwidth==0.2.6 +wcwidth==0.2.13 ## test related requirements diff --git a/requirements_test.txt b/requirements_test.txt --- a/requirements_test.txt +++ b/requirements_test.txt @@ -4,38 +4,38 @@ pytest-cov==4.1.0 coverage==7.4.3 pytest==8.1.1 iniconfig==2.0.0 - packaging==24.0 + packaging==24.1 pluggy==1.4.0 pytest-env==1.1.3 pytest==8.1.1 iniconfig==2.0.0 - packaging==24.0 + packaging==24.1 pluggy==1.4.0 pytest-profiling==1.7.0 gprof2dot==2022.7.29 pytest==8.1.1 iniconfig==2.0.0 - packaging==24.0 + packaging==24.1 pluggy==1.4.0 six==1.16.0 pytest-rerunfailures==13.0 - packaging==24.0 + packaging==24.1 pytest==8.1.1 iniconfig==2.0.0 - packaging==24.0 + packaging==24.1 pluggy==1.4.0 pytest-runner==6.0.1 pytest-sugar==1.0.0 - packaging==24.0 + packaging==24.1 pytest==8.1.1 iniconfig==2.0.0 - packaging==24.0 + packaging==24.1 pluggy==1.4.0 termcolor==2.4.0 pytest-timeout==2.3.1 pytest==8.1.1 iniconfig==2.0.0 - packaging==24.0 + packaging==24.1 pluggy==1.4.0 webtest==3.0.0 beautifulsoup4==4.12.3 diff --git a/vcsserver/exceptions.py b/vcsserver/exceptions.py --- a/vcsserver/exceptions.py +++ b/vcsserver/exceptions.py @@ -53,6 +53,12 @@ def ArchiveException(org_exc=None): return _make_exception_wrapper +def ClientNotSupportedException(org_exc=None): + def _make_exception_wrapper(*args): + return _make_exception('client_not_supported', org_exc, *args) + return _make_exception_wrapper + + def LookupException(org_exc=None): def _make_exception_wrapper(*args): return _make_exception('lookup', org_exc, *args) diff --git a/vcsserver/git_lfs/app.py b/vcsserver/git_lfs/app.py --- a/vcsserver/git_lfs/app.py +++ b/vcsserver/git_lfs/app.py @@ -18,6 +18,7 @@ import re import logging +from gunicorn.http.errors import NoMoreData from pyramid.config import Configurator from pyramid.response import Response, FileIter from pyramid.httpexceptions import ( @@ -166,9 +167,14 @@ def lfs_objects_oid_upload(request): # read in chunks as stream comes in from Gunicorn # this is a specific Gunicorn support function. # might work differently on waitress - chunk = body.read(blksize) + try: + chunk = body.read(blksize) + except NoMoreData: + chunk = None + if not chunk: break + f.write(chunk) return {'upload': 'ok'} diff --git a/vcsserver/hooks.py b/vcsserver/hooks.py --- a/vcsserver/hooks.py +++ b/vcsserver/hooks.py @@ -167,6 +167,8 @@ def _handle_exception(result): if exception_class == 'HTTPLockedRC': raise exceptions.RepositoryLockedException()(*result['exception_args']) + elif exception_class == 'ClientNotSupportedError': + raise exceptions.ClientNotSupportedException()(*result['exception_args']) elif exception_class == 'HTTPBranchProtected': raise exceptions.RepositoryBranchProtectedException()(*result['exception_args']) elif exception_class == 'RepositoryError': diff --git a/vcsserver/lib/archive_cache/backends/objectstore_cache.py b/vcsserver/lib/archive_cache/backends/objectstore_cache.py --- a/vcsserver/lib/archive_cache/backends/objectstore_cache.py +++ b/vcsserver/lib/archive_cache/backends/objectstore_cache.py @@ -58,7 +58,7 @@ class S3Shard(BaseShard): # ensure folder in bucket exists destination = self.bucket if not self.fs.exists(destination): - self.fs.mkdir(destination, s3_additional_kwargs={}) + self.fs.mkdir(destination) writer = self._get_writer(full_path, mode) diff --git a/vcsserver/lib/hash_utils.py b/vcsserver/lib/hash_utils.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/hash_utils.py @@ -0,0 +1,53 @@ +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2023 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import hashlib +from vcsserver.lib.str_utils import safe_bytes, safe_str + + +def md5(s): + return hashlib.md5(s).hexdigest() + + +def md5_safe(s, return_type=''): + + val = md5(safe_bytes(s)) + if return_type == 'str': + val = safe_str(val) + return val + + +def sha1(s): + return hashlib.sha1(s).hexdigest() + + +def sha1_safe(s, return_type=''): + val = sha1(safe_bytes(s)) + if return_type == 'str': + val = safe_str(val) + return val + + +def sha256(s): + return hashlib.sha256(s).hexdigest() + + +def sha256_safe(s, return_type=''): + val = sha256(safe_bytes(s)) + if return_type == 'str': + val = safe_str(val) + return val diff --git a/vcsserver/lib/rc_cache/backends.py b/vcsserver/lib/rc_cache/backends.py --- a/vcsserver/lib/rc_cache/backends.py +++ b/vcsserver/lib/rc_cache/backends.py @@ -37,9 +37,9 @@ from dogpile.cache.backends import redis from dogpile.cache.backends.file import FileLock from dogpile.cache.util import memoized_property -from vcsserver.lib.memory_lru_dict import LRUDict, LRUDictDebug -from vcsserver.lib.str_utils import safe_bytes, safe_str -from vcsserver.lib.type_utils import str2bool +from ...lib.memory_lru_dict import LRUDict, LRUDictDebug +from ...lib.str_utils import safe_bytes, safe_str +from ...lib.type_utils import str2bool _default_max_size = 1024 @@ -166,6 +166,13 @@ class FileNamespaceBackend(PickleSeriali def get_store(self): return self.filename + def cleanup_store(self): + for ext in ("db", "dat", "pag", "dir"): + final_filename = self.filename + os.extsep + ext + if os.path.exists(final_filename): + os.remove(final_filename) + log.warning('Removed dbm file %s', final_filename) + class BaseRedisBackend(redis_backend.RedisBackend): key_prefix = '' @@ -257,7 +264,7 @@ class RedisMsgPackBackend(MsgPackSeriali def get_mutex_lock(client, lock_key, lock_timeout, auto_renewal=False): - from vcsserver.lib._vendor import redis_lock + from ...lib._vendor import redis_lock class _RedisLockWrapper: """LockWrapper for redis_lock""" diff --git a/vcsserver/lib/rc_cache/utils.py b/vcsserver/lib/rc_cache/utils.py --- a/vcsserver/lib/rc_cache/utils.py +++ b/vcsserver/lib/rc_cache/utils.py @@ -25,9 +25,9 @@ import decorator from dogpile.cache import CacheRegion -from vcsserver.utils import sha1 -from vcsserver.lib.str_utils import safe_bytes -from vcsserver.lib.type_utils import str2bool # noqa :required by imports from .utils +from ...lib.hash_utils import sha1 +from ...lib.str_utils import safe_bytes +from ...lib.type_utils import str2bool # noqa :required by imports from .utils from . import region_meta diff --git a/vcsserver/tweens/request_wrapper.py b/vcsserver/tweens/request_wrapper.py --- a/vcsserver/tweens/request_wrapper.py +++ b/vcsserver/tweens/request_wrapper.py @@ -64,7 +64,7 @@ class RequestWrapperTween: def __call__(self, request): start = time.time() - log.debug('Starting request time measurement') + log.debug('Starting request processing') response = None try: @@ -88,7 +88,7 @@ class RequestWrapperTween: total = time.time() - start log.info( - 'Req[%4s] IP: %s %s Request to %s time: %.4fs [%s], VCSServer %s', + 'Finished request processing: reqq[%4s] IP: %s %s Request to %s time: %.4fs [%s], VCSServer %s', count, ip, request.environ.get('REQUEST_METHOD'), _view_path, total, ua, _ver_, extra={"time": total, "ver": _ver_, "code": resp_code,