# HG changeset patch # User RhodeCode Admin # Date 2021-10-12 12:47:33 # Node ID 610dc89adb62f6400a94762f33ab54a7eb44eb36 # Parent 29e77558c1fd500d573d0e5e67f4273735ae2f60 # Parent 77c100d1a6d9024286a9d475ff293cdafbc91778 merged with stable diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -78,3 +78,4 @@ 6eaf953da06e468a4c4e5239d3d0e700bda6b163 f8161cbc2d94a935d3c395a0e758d9a094287169 v4.25.0 77fe47b5b39338e71b2c040de2c0359b529b6251 v4.25.1 27475bd8a718b9a00a37a8563c4927120865ad85 v4.25.2 +b4ba10dcb4ab67d02b8c5cff32a3827f6c4fdedb v4.26.0 diff --git a/.release.cfg b/.release.cfg --- a/.release.cfg +++ b/.release.cfg @@ -19,11 +19,12 @@ done = true [task:generate_api_docs] done = true +[task:updated_translation] +done = true + [release] state = prepared -version = 4.25.2 - -[task:updated_translation] +version = 4.26.0 [task:generate_js_routes] diff --git a/docs/admin/repo_admin/repo-admin-tasks.rst b/docs/admin/repo_admin/repo-admin-tasks.rst --- a/docs/admin/repo_admin/repo-admin-tasks.rst +++ b/docs/admin/repo_admin/repo-admin-tasks.rst @@ -101,4 +101,18 @@ 2b) Add user called 'admin' into all rep In [3]: permission_name = 'group.write' In [4]: for repo_group in RepoGroup.get_all(): ...: RepoGroupModel().grant_user_permission(repo_group, user, permission_name) - ...: Session().commit() \ No newline at end of file + ...: Session().commit() + + +Delete a problematic pull request +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + :dedent: 1 + + In [1]: from rhodecode.model.pull_request import PullRequestModel + In [2]: pullrequest_id = 123 + In [3]: pr = PullRequest.get(pullrequest_id) + In [4]: super_admin = User.get_first_super_admin() + In [5]: PullRequestModel().delete(pr, super_admin) + In [6]: Session().commit() diff --git a/rhodecode/apps/ops/__init__.py b/rhodecode/apps/ops/__init__.py --- a/rhodecode/apps/ops/__init__.py +++ b/rhodecode/apps/ops/__init__.py @@ -51,6 +51,14 @@ def admin_routes(config): route_name='ops_redirect_test', request_method='GET', renderer='json_ext') + config.add_route( + name='ops_healthcheck', + pattern='/status') + config.add_view( + OpsView, + attr='ops_healthcheck', + route_name='ops_healthcheck', request_method='GET', + renderer='json_ext') def includeme(config): config.include(admin_routes, route_prefix=ADMIN_PREFIX + '/ops') diff --git a/rhodecode/apps/ops/views.py b/rhodecode/apps/ops/views.py --- a/rhodecode/apps/ops/views.py +++ b/rhodecode/apps/ops/views.py @@ -26,6 +26,8 @@ from pyramid.httpexceptions import HTTPF from rhodecode.apps._base import BaseAppView from rhodecode.lib import helpers as h +from rhodecode.lib.auth import LoginRequired +from rhodecode.model.db import UserApiKeys log = logging.getLogger(__name__) @@ -72,3 +74,24 @@ class OpsView(BaseAppView): """ redirect_to = self.request.GET.get('to') or h.route_path('home') raise HTTPFound(redirect_to) + + @LoginRequired(auth_token_access=[UserApiKeys.ROLE_HTTP]) + def ops_healthcheck(self): + from rhodecode.lib.system_info import load_system_info + + vcsserver_info = load_system_info('vcs_server') + if vcsserver_info: + vcsserver_info = vcsserver_info['human_value'] + + db_info = load_system_info('database_info') + if db_info: + db_info = db_info['human_value'] + + health_spec = { + 'caller_ip': self.request.user.ip_addr, + 'vcsserver': vcsserver_info, + 'db': db_info, + } + + return {'healthcheck': health_spec} + diff --git a/rhodecode/lib/celerylib/__init__.py b/rhodecode/lib/celerylib/__init__.py --- a/rhodecode/lib/celerylib/__init__.py +++ b/rhodecode/lib/celerylib/__init__.py @@ -43,6 +43,9 @@ class ResultWrapper(object): def run_task(task, *args, **kwargs): log.debug('Got task `%s` for execution', task) + if task is None: + raise ValueError('Got non-existing task for execution') + if rhodecode.CELERY_ENABLED: celery_is_up = False try: diff --git a/rhodecode/lib/celerylib/tasks.py b/rhodecode/lib/celerylib/tasks.py --- a/rhodecode/lib/celerylib/tasks.py +++ b/rhodecode/lib/celerylib/tasks.py @@ -386,9 +386,7 @@ def beat_check(*args, **kwargs): return time.time() -@async_task(ignore_result=True) -def sync_last_update(*args, **kwargs): - +def sync_last_update_for_objects(*args, **kwargs): skip_repos = kwargs.get('skip_repos') if not skip_repos: repos = Repository.query() \ @@ -405,3 +403,8 @@ def sync_last_update(*args, **kwargs): for root_gr in repo_groups: for repo_gr in reversed(root_gr.recursive_groups()): repo_gr.update_commit_cache() + + +@async_task(ignore_result=True) +def sync_last_update(*args, **kwargs): + sync_last_update_for_objects(*args, **kwargs) diff --git a/rhodecode/lib/middleware/vcs.py b/rhodecode/lib/middleware/vcs.py --- a/rhodecode/lib/middleware/vcs.py +++ b/rhodecode/lib/middleware/vcs.py @@ -166,8 +166,9 @@ def detect_vcs_request(environ, backends # static files no detection '_static', - # skip ops ping + # skip ops ping, status '_admin/ops/ping', + '_admin/ops/status', # full channelstream connect should be VCS skipped '_admin/channelstream/connect', diff --git a/rhodecode/lib/rc_cache/backends.py b/rhodecode/lib/rc_cache/backends.py --- a/rhodecode/lib/rc_cache/backends.py +++ b/rhodecode/lib/rc_cache/backends.py @@ -300,7 +300,6 @@ class BaseRedisBackend(redis_backend.Red def get_mutex(self, key): if self.distributed_lock: lock_key = redis_backend.u('_lock_{0}').format(key) - log.debug('Trying to acquire Redis lock for key %s', lock_key) return get_mutex_lock(self.client, lock_key, self._lock_timeout, auto_renewal=self._lock_auto_renewal) else: @@ -333,12 +332,22 @@ def get_mutex_lock(client, lock_key, loc strict=True, ) + def __repr__(self): + return "{}:{}".format(self.__class__.__name__, lock_key) + + def __str__(self): + return "{}:{}".format(self.__class__.__name__, lock_key) + def __init__(self): self.lock = self.get_lock() + self.lock_key = lock_key def acquire(self, wait=True): + log.debug('Trying to acquire Redis lock for key %s', self.lock_key) try: - return self.lock.acquire(wait) + acquired = self.lock.acquire(wait) + log.debug('Got lock for key %s, %s', self.lock_key, acquired) + return acquired except redis_lock.AlreadyAcquired: return False except redis_lock.AlreadyStarted: diff --git a/rhodecode/lib/rc_cache/utils.py b/rhodecode/lib/rc_cache/utils.py --- a/rhodecode/lib/rc_cache/utils.py +++ b/rhodecode/lib/rc_cache/utils.py @@ -122,7 +122,11 @@ class RhodeCodeCacheRegion(CacheRegion): if not condition: log.debug('Calling un-cached func:%s', user_func.func_name) - return user_func(*arg, **kw) + start = time.time() + result = user_func(*arg, **kw) + total = time.time() - start + log.debug('un-cached func:%s took %.4fs', user_func.func_name, total) + return result key = key_generator(*arg, **kw) diff --git a/rhodecode/lib/system_info.py b/rhodecode/lib/system_info.py --- a/rhodecode/lib/system_info.py +++ b/rhodecode/lib/system_info.py @@ -24,6 +24,8 @@ import sys import time import platform import collections +from functools import wraps + import pkg_resources import logging import resource @@ -51,6 +53,26 @@ STATE_WARN = 'warning' STATE_OK_DEFAULT = {'message': '', 'type': STATE_OK} +registered_helpers = {} + + +def register_sysinfo(func): + """ + @register_helper + def db_check(): + pass + + db_check == registered_helpers['db_check'] + """ + global registered_helpers + registered_helpers[func.__name__] = func + + @wraps(func) + def _wrapper(*args, **kwargs): + return func(*args, **kwargs) + return _wrapper + + # HELPERS def percentage(part, whole): whole = float(whole) @@ -136,12 +158,14 @@ class SysInfo(object): # SysInfo functions +@register_sysinfo def python_info(): value = dict(version=' '.join(platform._sys_version()), executable=sys.executable) return SysInfoRes(value=value) +@register_sysinfo def py_modules(): mods = dict([(p.project_name, {'version': p.version, 'location': p.location}) for p in pkg_resources.working_set]) @@ -150,6 +174,7 @@ def py_modules(): return SysInfoRes(value=value) +@register_sysinfo def platform_type(): from rhodecode.lib.utils import safe_unicode, generate_platform_uuid @@ -160,6 +185,7 @@ def platform_type(): return SysInfoRes(value=value) +@register_sysinfo def locale_info(): import locale @@ -175,6 +201,7 @@ def locale_info(): return SysInfoRes(value=value, human_value=human_value) +@register_sysinfo def ulimit_info(): data = collections.OrderedDict([ ('cpu time (seconds)', get_resource(resource.RLIMIT_CPU)), @@ -198,6 +225,7 @@ def ulimit_info(): return SysInfoRes(value=value) +@register_sysinfo def uptime(): from rhodecode.lib.helpers import age, time_to_datetime from rhodecode.translation import TranslationString @@ -223,6 +251,7 @@ def uptime(): return SysInfoRes(value=value, human_value=human_value) +@register_sysinfo def memory(): from rhodecode.lib.helpers import format_byte_size_binary value = dict(available=0, used=0, used_real=0, cached=0, percent=0, @@ -262,6 +291,7 @@ def memory(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def machine_load(): value = {'1_min': _NA, '5_min': _NA, '15_min': _NA, 'text': ''} state = STATE_OK_DEFAULT @@ -284,6 +314,7 @@ def machine_load(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def cpu(): value = {'cpu': 0, 'cpu_count': 0, 'cpu_usage': []} state = STATE_OK_DEFAULT @@ -302,6 +333,7 @@ def cpu(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def storage(): from rhodecode.lib.helpers import format_byte_size_binary from rhodecode.model.settings import VcsSettingsModel @@ -337,6 +369,7 @@ def storage(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def storage_inodes(): from rhodecode.model.settings import VcsSettingsModel path = VcsSettingsModel().get_repos_location() @@ -371,6 +404,7 @@ def storage_inodes(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def storage_archives(): import rhodecode from rhodecode.lib.utils import safe_str @@ -414,6 +448,7 @@ def storage_archives(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def storage_gist(): from rhodecode.model.gist import GIST_STORE_LOC from rhodecode.model.settings import VcsSettingsModel @@ -457,6 +492,7 @@ def storage_gist(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def storage_temp(): import tempfile from rhodecode.lib.helpers import format_byte_size_binary @@ -485,6 +521,7 @@ def storage_temp(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def search_info(): import rhodecode from rhodecode.lib.index import searcher_from_config @@ -508,6 +545,7 @@ def search_info(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def git_info(): from rhodecode.lib.vcs.backends import git state = STATE_OK_DEFAULT @@ -521,6 +559,7 @@ def git_info(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def hg_info(): from rhodecode.lib.vcs.backends import hg state = STATE_OK_DEFAULT @@ -533,6 +572,7 @@ def hg_info(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def svn_info(): from rhodecode.lib.vcs.backends import svn state = STATE_OK_DEFAULT @@ -545,6 +585,7 @@ def svn_info(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def vcs_backends(): import rhodecode value = rhodecode.CONFIG.get('vcs.backends') @@ -552,6 +593,7 @@ def vcs_backends(): return SysInfoRes(value=value, human_value=human_value) +@register_sysinfo def vcs_server(): import rhodecode from rhodecode.lib.vcs.backends import get_vcsserver_service_data @@ -595,6 +637,7 @@ def vcs_server(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def vcs_server_config(): from rhodecode.lib.vcs.backends import get_vcsserver_service_data state = STATE_OK_DEFAULT @@ -612,6 +655,7 @@ def vcs_server_config(): return SysInfoRes(value=value, state=state, human_value=human_value) +@register_sysinfo def rhodecode_app_info(): import rhodecode edition = rhodecode.CONFIG.get('rhodecode.edition') @@ -628,6 +672,7 @@ def rhodecode_app_info(): return SysInfoRes(value=value, human_value=human_value) +@register_sysinfo def rhodecode_config(): import rhodecode path = rhodecode.CONFIG.get('__file__') @@ -683,6 +728,7 @@ def rhodecode_config(): 'path': path, 'cert_path': cert_path}) +@register_sysinfo def database_info(): import rhodecode from sqlalchemy.engine import url as engine_url @@ -727,6 +773,7 @@ def database_info(): return SysInfoRes(value=db_info, state=state, human_value=human_value) +@register_sysinfo def server_info(environ): import rhodecode from rhodecode.lib.base import get_server_ip_addr, get_server_port @@ -741,6 +788,7 @@ def server_info(environ): return SysInfoRes(value=value) +@register_sysinfo def usage_info(): from rhodecode.model.db import User, Repository value = { @@ -795,3 +843,11 @@ def get_system_info(environ): 'hg': SysInfo(hg_info)(), 'svn': SysInfo(svn_info)(), } + + +def load_system_info(key): + """ + get_sys_info('vcs_server') + get_sys_info('database') + """ + return SysInfo(registered_helpers[key])()